cluster-tools 0.1__py3.6.egg
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- EGG-INFO/PKG-INFO +37 -0
- EGG-INFO/SOURCES.txt +12 -0
- EGG-INFO/dependency_links.txt +1 -0
- EGG-INFO/requires.txt +2 -0
- EGG-INFO/top_level.txt +1 -0
- EGG-INFO/zip-safe +1 -0
- cluster_tools/__init__.py +367 -0
- cluster_tools/__pycache__/__init__.cpython-36.pyc +0 -0
- cluster_tools/__pycache__/generator-tmp.cpython-36.pyc +0 -0
- cluster_tools/__pycache__/remote.cpython-36.pyc +0 -0
- cluster_tools/__pycache__/slurm.cpython-36.pyc +0 -0
- cluster_tools/__pycache__/util.cpython-36.pyc +0 -0
- cluster_tools/generator-tmp.py +15 -0
- cluster_tools/remote.py +57 -0
- cluster_tools/slurm.py +57 -0
- cluster_tools/util.py +45 -0
EGG-INFO/PKG-INFO
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
Metadata-Version: 1.1
|
|
2
|
+
Name: cluster-tools
|
|
3
|
+
Version: 0.1
|
|
4
|
+
Summary: Utility library for easily distributing code execution on clusters
|
|
5
|
+
Home-page: https://github.com/scalableminds/cluster_tools
|
|
6
|
+
Author: scalableminds
|
|
7
|
+
Author-email: hello@scalableminds.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Description: # Cluster Tools
|
|
10
|
+
|
|
11
|
+
[](https://circleci.com/gh/scalableminds/cluster_tools/tree/master)
|
|
12
|
+
|
|
13
|
+
This package provides python `Executor` classes for distributing tasks on a slurm cluster or via multi processing.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## Example
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
import cluster_tools
|
|
20
|
+
|
|
21
|
+
def square(n):
|
|
22
|
+
return n * n
|
|
23
|
+
|
|
24
|
+
if __name__ == '__main__':
|
|
25
|
+
strategy = "slurm" # other valid values are "multiprocessing" and "sequential"
|
|
26
|
+
with cluster_tools.get_executor(strategy) as executor:
|
|
27
|
+
result = list(executor.map(square, [2, 3, 4]))
|
|
28
|
+
assert result == [4, 9, 16]
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Credits
|
|
32
|
+
|
|
33
|
+
Thanks to [sampsyo/clusterfutures](https://github.com/sampsyo/clusterfutures) for providing the slurm core abstraction and [giovtorres/slurm-docker-cluster](https://github.com/giovtorres/slurm-docker-cluster) for providing the slurm docker environment which we use for CI based testing.
|
|
34
|
+
|
|
35
|
+
Platform: ALL
|
|
36
|
+
Classifier: Intended Audience :: Developers
|
|
37
|
+
Classifier: Intended Audience :: Science/Research
|
EGG-INFO/SOURCES.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
setup.py
|
|
3
|
+
cluster_tools/__init__.py
|
|
4
|
+
cluster_tools/generator-tmp.py
|
|
5
|
+
cluster_tools/remote.py
|
|
6
|
+
cluster_tools/slurm.py
|
|
7
|
+
cluster_tools/util.py
|
|
8
|
+
cluster_tools.egg-info/PKG-INFO
|
|
9
|
+
cluster_tools.egg-info/SOURCES.txt
|
|
10
|
+
cluster_tools.egg-info/dependency_links.txt
|
|
11
|
+
cluster_tools.egg-info/requires.txt
|
|
12
|
+
cluster_tools.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
EGG-INFO/requires.txt
ADDED
EGG-INFO/top_level.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cluster_tools
|
EGG-INFO/zip-safe
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
from concurrent import futures
|
|
2
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from . import slurm
|
|
8
|
+
from .remote import INFILE_FMT, OUTFILE_FMT
|
|
9
|
+
from .util import random_string, local_filename, call
|
|
10
|
+
import cloudpickle
|
|
11
|
+
import logging
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class RemoteException(Exception):
|
|
15
|
+
def __init__(self, error):
|
|
16
|
+
self.error = error
|
|
17
|
+
|
|
18
|
+
def __str__(self):
|
|
19
|
+
return "\n" + self.error.strip()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class FileWaitThread(threading.Thread):
|
|
23
|
+
"""A thread that polls the filesystem waiting for a list of files to
|
|
24
|
+
be created. When a specified file is created, it invokes a callback.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, callback, interval=1):
|
|
28
|
+
"""The callable ``callback`` will be invoked with value
|
|
29
|
+
associated with the filename of each file that is created.
|
|
30
|
+
``interval`` specifies the polling rate.
|
|
31
|
+
"""
|
|
32
|
+
threading.Thread.__init__(self)
|
|
33
|
+
self.callback = callback
|
|
34
|
+
self.interval = interval
|
|
35
|
+
self.waiting = {}
|
|
36
|
+
self.lock = threading.Lock()
|
|
37
|
+
self.shutdown = False
|
|
38
|
+
|
|
39
|
+
def stop(self):
|
|
40
|
+
"""Stop the thread soon."""
|
|
41
|
+
with self.lock:
|
|
42
|
+
self.shutdown = True
|
|
43
|
+
|
|
44
|
+
def wait(self, filename, value):
|
|
45
|
+
"""Adds a new filename (and its associated callback value) to
|
|
46
|
+
the set of files being waited upon.
|
|
47
|
+
"""
|
|
48
|
+
with self.lock:
|
|
49
|
+
self.waiting[filename] = value
|
|
50
|
+
|
|
51
|
+
def run(self):
|
|
52
|
+
while True:
|
|
53
|
+
with self.lock:
|
|
54
|
+
if self.shutdown:
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
# Poll for each file.
|
|
58
|
+
for filename in list(self.waiting):
|
|
59
|
+
job_id = self.waiting[filename]
|
|
60
|
+
|
|
61
|
+
# Let's get the state for the job to check whether it failed.
|
|
62
|
+
# It's important to check the status before checking whether the file exists.
|
|
63
|
+
# Otherwise, we can run into a race condition where the job succeeds
|
|
64
|
+
# after os.path.exists was called and before scontrol is executed.
|
|
65
|
+
stdout, _, exit_code = call("scontrol show job {}".format(job_id))
|
|
66
|
+
|
|
67
|
+
if exit_code != 0:
|
|
68
|
+
logging.error(
|
|
69
|
+
"Couldn't call scontrol to determine job's status. {}. Continuing to poll for output file.".format(
|
|
70
|
+
e
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
|
+
elif "JobState=FAILED" in str(stdout):
|
|
74
|
+
self.callback(job_id, True)
|
|
75
|
+
del self.waiting[filename]
|
|
76
|
+
elif "JobState=COMPLETED" in str(stdout):
|
|
77
|
+
logging.error(
|
|
78
|
+
"Job state is completed, but {} couldn't be found.".format(
|
|
79
|
+
filename
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
self.callback(job_id, True)
|
|
83
|
+
del self.waiting[filename]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
if os.path.exists(filename):
|
|
87
|
+
self.callback(job_id, False)
|
|
88
|
+
del self.waiting[filename]
|
|
89
|
+
|
|
90
|
+
time.sleep(self.interval)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class SlurmExecutor(futures.Executor):
|
|
94
|
+
"""Futures executor for executing jobs on a Slurm cluster."""
|
|
95
|
+
|
|
96
|
+
def __init__(
|
|
97
|
+
self,
|
|
98
|
+
debug=False,
|
|
99
|
+
keep_logs=False,
|
|
100
|
+
job_resources=None,
|
|
101
|
+
job_name=None,
|
|
102
|
+
additional_setup_lines=[],
|
|
103
|
+
**kwargs
|
|
104
|
+
):
|
|
105
|
+
os.makedirs(local_filename(), exist_ok=True)
|
|
106
|
+
self.debug = debug
|
|
107
|
+
self.job_resources = job_resources
|
|
108
|
+
self.additional_setup_lines = additional_setup_lines
|
|
109
|
+
self.job_name = job_name
|
|
110
|
+
self.was_requested_to_shutdown = False
|
|
111
|
+
|
|
112
|
+
# `jobs` maps from job id to future and workerid
|
|
113
|
+
# In case, job arrays are used: job id and workerid are in the format of
|
|
114
|
+
# `job_id-job_index` and `workerid-job_index`.
|
|
115
|
+
self.jobs = {}
|
|
116
|
+
self.job_outfiles = {}
|
|
117
|
+
self.jobs_lock = threading.Lock()
|
|
118
|
+
self.jobs_empty_cond = threading.Condition(self.jobs_lock)
|
|
119
|
+
self.keep_logs = keep_logs
|
|
120
|
+
|
|
121
|
+
self.wait_thread = FileWaitThread(self._completion)
|
|
122
|
+
self.wait_thread.start()
|
|
123
|
+
|
|
124
|
+
self.meta_data = {}
|
|
125
|
+
if "logging_config" in kwargs:
|
|
126
|
+
self.meta_data["logging_config"] = kwargs["logging_config"]
|
|
127
|
+
|
|
128
|
+
def _start(self, workerid, job_count=None):
|
|
129
|
+
"""Start a job with the given worker ID and return an ID
|
|
130
|
+
identifying the new job. The job should run ``python -m
|
|
131
|
+
cfut.remote <workerid>.
|
|
132
|
+
"""
|
|
133
|
+
return slurm.submit(
|
|
134
|
+
"{} -m cluster_tools.remote {}".format(sys.executable, workerid),
|
|
135
|
+
job_resources=self.job_resources,
|
|
136
|
+
job_name=self.job_name,
|
|
137
|
+
additional_setup_lines=self.additional_setup_lines,
|
|
138
|
+
job_count=job_count,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def _cleanup(self, jobid):
|
|
142
|
+
"""Given a job ID as returned by _start, perform any necessary
|
|
143
|
+
cleanup after the job has finished.
|
|
144
|
+
"""
|
|
145
|
+
if self.keep_logs:
|
|
146
|
+
return
|
|
147
|
+
|
|
148
|
+
outf = slurm.OUTFILE_FMT.format(str(jobid))
|
|
149
|
+
try:
|
|
150
|
+
os.unlink(outf)
|
|
151
|
+
except OSError:
|
|
152
|
+
pass
|
|
153
|
+
|
|
154
|
+
def _completion(self, jobid, failed_early):
|
|
155
|
+
"""Called whenever a job finishes."""
|
|
156
|
+
with self.jobs_lock:
|
|
157
|
+
fut, workerid = self.jobs.pop(jobid)
|
|
158
|
+
if not self.jobs:
|
|
159
|
+
self.jobs_empty_cond.notify_all()
|
|
160
|
+
if self.debug:
|
|
161
|
+
print("job completed: {}".format(jobid), file=sys.stderr)
|
|
162
|
+
|
|
163
|
+
if failed_early:
|
|
164
|
+
# If the code which should be executed on a node wasn't even
|
|
165
|
+
# started (e.g., because python isn't installed or the cluster_tools
|
|
166
|
+
# couldn't be found), no output was written to disk. We only noticed
|
|
167
|
+
# this circumstance because the whole slurm job was marked as failed.
|
|
168
|
+
# Therefore, we don't try to deserialize pickle output.
|
|
169
|
+
success = False
|
|
170
|
+
result = "Job submission/execution failed. Please look into the log file at {}".format(
|
|
171
|
+
slurm.OUTFILE_FMT.format(jobid)
|
|
172
|
+
)
|
|
173
|
+
else:
|
|
174
|
+
with open(OUTFILE_FMT % workerid, "rb") as f:
|
|
175
|
+
outdata = f.read()
|
|
176
|
+
success, result = cloudpickle.loads(outdata)
|
|
177
|
+
|
|
178
|
+
if success:
|
|
179
|
+
fut.set_result(result)
|
|
180
|
+
else:
|
|
181
|
+
fut.set_exception(RemoteException(result))
|
|
182
|
+
|
|
183
|
+
# Clean up communication files.
|
|
184
|
+
|
|
185
|
+
if os.path.exists(INFILE_FMT % workerid):
|
|
186
|
+
os.unlink(INFILE_FMT % workerid)
|
|
187
|
+
if os.path.exists(OUTFILE_FMT % workerid):
|
|
188
|
+
os.unlink(OUTFILE_FMT % workerid)
|
|
189
|
+
|
|
190
|
+
self._cleanup(jobid)
|
|
191
|
+
|
|
192
|
+
def ensure_not_shutdown(self):
|
|
193
|
+
if self.was_requested_to_shutdown:
|
|
194
|
+
raise RuntimeError(
|
|
195
|
+
"submit() was invoked on a SlurmExecutor instance even though shutdown() was executed for that instance."
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
def submit(self, fun, *args, **kwargs):
|
|
199
|
+
"""Submit a job to the pool."""
|
|
200
|
+
fut = futures.Future()
|
|
201
|
+
|
|
202
|
+
self.ensure_not_shutdown()
|
|
203
|
+
|
|
204
|
+
# Start the job.
|
|
205
|
+
workerid = random_string()
|
|
206
|
+
|
|
207
|
+
funcser = cloudpickle.dumps((fun, args, kwargs, self.meta_data), True)
|
|
208
|
+
with open(INFILE_FMT % workerid, "wb") as f:
|
|
209
|
+
f.write(funcser)
|
|
210
|
+
|
|
211
|
+
jobid = self._start(workerid)
|
|
212
|
+
|
|
213
|
+
if self.debug:
|
|
214
|
+
print("job submitted: %i" % jobid, file=sys.stderr)
|
|
215
|
+
|
|
216
|
+
# Thread will wait for it to finish.
|
|
217
|
+
self.wait_thread.wait(OUTFILE_FMT % workerid, jobid)
|
|
218
|
+
|
|
219
|
+
with self.jobs_lock:
|
|
220
|
+
self.jobs[jobid] = (fut, workerid)
|
|
221
|
+
|
|
222
|
+
fut.slurm_jobid = jobid
|
|
223
|
+
return fut
|
|
224
|
+
|
|
225
|
+
def map_to_futures(self, fun, allArgs):
|
|
226
|
+
self.ensure_not_shutdown()
|
|
227
|
+
allArgs = list(allArgs)
|
|
228
|
+
|
|
229
|
+
futs = []
|
|
230
|
+
workerid = random_string()
|
|
231
|
+
|
|
232
|
+
get_workerid_with_index = lambda index: workerid + "_" + str(index)
|
|
233
|
+
|
|
234
|
+
# Submit jobs eagerly
|
|
235
|
+
for index, arg in enumerate(allArgs):
|
|
236
|
+
fut = futures.Future()
|
|
237
|
+
|
|
238
|
+
# Start the job.
|
|
239
|
+
funcser = cloudpickle.dumps((fun, [arg], {}, self.meta_data), True)
|
|
240
|
+
infile_name = INFILE_FMT % get_workerid_with_index(index)
|
|
241
|
+
|
|
242
|
+
with open(infile_name, "wb") as f:
|
|
243
|
+
f.write(funcser)
|
|
244
|
+
|
|
245
|
+
futs.append(fut)
|
|
246
|
+
|
|
247
|
+
job_count = len(allArgs)
|
|
248
|
+
jobid = self._start(workerid, job_count)
|
|
249
|
+
get_jobid_with_index = lambda index: str(jobid) + "_" + str(index)
|
|
250
|
+
|
|
251
|
+
if self.debug:
|
|
252
|
+
print(
|
|
253
|
+
"main job submitted: %i. consists of %i subjobs." % (jobid, job_count),
|
|
254
|
+
file=sys.stderr,
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
with self.jobs_lock:
|
|
258
|
+
for index, fut in enumerate(futs):
|
|
259
|
+
jobid_with_index = get_jobid_with_index(index)
|
|
260
|
+
# Thread will wait for it to finish.
|
|
261
|
+
workerid_with_index = get_workerid_with_index(index)
|
|
262
|
+
self.wait_thread.wait(
|
|
263
|
+
OUTFILE_FMT % workerid_with_index, jobid_with_index
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
fut.slurm_jobid = jobid
|
|
267
|
+
fut.slurm_jobindex = index
|
|
268
|
+
|
|
269
|
+
self.jobs[jobid_with_index] = (fut, workerid_with_index)
|
|
270
|
+
|
|
271
|
+
return futs
|
|
272
|
+
|
|
273
|
+
def shutdown(self, wait=True):
|
|
274
|
+
"""Close the pool."""
|
|
275
|
+
self.was_requested_to_shutdown = True
|
|
276
|
+
if wait:
|
|
277
|
+
with self.jobs_lock:
|
|
278
|
+
if self.jobs:
|
|
279
|
+
self.jobs_empty_cond.wait()
|
|
280
|
+
|
|
281
|
+
self.wait_thread.stop()
|
|
282
|
+
self.wait_thread.join()
|
|
283
|
+
|
|
284
|
+
def map(self, func, args, timeout=None, chunksize=None):
|
|
285
|
+
if chunksize is not None:
|
|
286
|
+
logging.warning(
|
|
287
|
+
"The provided chunksize parameter is ignored by SlurmExecutor."
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
start_time = time.time()
|
|
292
|
+
|
|
293
|
+
futs = self.map_to_futures(func, args)
|
|
294
|
+
results = []
|
|
295
|
+
|
|
296
|
+
# Return a separate generator as iterator to avoid that the
|
|
297
|
+
# map() method itself becomes a generator.
|
|
298
|
+
# If map() was a generator, the submit() calls would be invoked
|
|
299
|
+
# lazily which can lead to a shutdown of the executor before
|
|
300
|
+
# the submit calls are performed.
|
|
301
|
+
def result_generator():
|
|
302
|
+
for fut in futs:
|
|
303
|
+
passed_time = time.time() - start_time
|
|
304
|
+
remaining_timeout = (
|
|
305
|
+
None if timeout is None else timeout - passed_time
|
|
306
|
+
)
|
|
307
|
+
yield fut.result(remaining_timeout)
|
|
308
|
+
|
|
309
|
+
return result_generator()
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def map_unordered(self, func, args):
|
|
313
|
+
futs = self.map_to_futures(func, args)
|
|
314
|
+
|
|
315
|
+
# Return a separate generator to avoid that map_unordered
|
|
316
|
+
# is executed lazily.
|
|
317
|
+
def result_generator():
|
|
318
|
+
for fut in futures.as_completed(futs):
|
|
319
|
+
yield fut.result()
|
|
320
|
+
|
|
321
|
+
return result_generator()
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def get_existent_kwargs_subset(white_list, kwargs):
|
|
325
|
+
new_kwargs = {}
|
|
326
|
+
for arg_name in white_list:
|
|
327
|
+
if arg_name in kwargs:
|
|
328
|
+
new_kwargs[arg_name] = kwargs[arg_name]
|
|
329
|
+
|
|
330
|
+
return new_kwargs
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
class WrappedProcessPoolExecutor(ProcessPoolExecutor):
|
|
334
|
+
def __init__(self, **kwargs):
|
|
335
|
+
white_list = ["max_workers", "mp_context", "initializer", "initargs"]
|
|
336
|
+
new_kwargs = get_existent_kwargs_subset(white_list, kwargs)
|
|
337
|
+
|
|
338
|
+
return ProcessPoolExecutor.__init__(self, **new_kwargs)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def map_unordered(self, func, args):
|
|
342
|
+
|
|
343
|
+
futs = [self.submit(func, arg) for arg in args]
|
|
344
|
+
|
|
345
|
+
# Return a separate generator to avoid that map_unordered
|
|
346
|
+
# is executed lazily (otherwise, jobs would be submitted
|
|
347
|
+
# lazily, as well).
|
|
348
|
+
def result_generator():
|
|
349
|
+
for fut in futures.as_completed(futs):
|
|
350
|
+
yield fut.result()
|
|
351
|
+
|
|
352
|
+
return result_generator()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
class SequentialExecutor(WrappedProcessPoolExecutor):
|
|
356
|
+
def __init__(self, **kwargs):
|
|
357
|
+
kwargs["max_workers"] = 1
|
|
358
|
+
return WrappedProcessPoolExecutor.__init__(self, **kwargs)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def get_executor(environment, **kwargs):
|
|
362
|
+
if environment == "slurm":
|
|
363
|
+
return SlurmExecutor(**kwargs)
|
|
364
|
+
elif environment == "multiprocessing":
|
|
365
|
+
return WrappedProcessPoolExecutor(**kwargs)
|
|
366
|
+
elif environment == "sequential":
|
|
367
|
+
return SequentialExecutor(**kwargs)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import concurrent.futures
|
|
3
|
+
from multiprocessing import Pool
|
|
4
|
+
|
|
5
|
+
s = [1, 2, 3]
|
|
6
|
+
|
|
7
|
+
with Pool(2) as p:
|
|
8
|
+
result = p.map(time.sleep, s)
|
|
9
|
+
print(type(result))
|
|
10
|
+
print(result[0])
|
|
11
|
+
|
|
12
|
+
with concurrent.futures.ThreadPoolExecutor(4):
|
|
13
|
+
result = e.map(time.sleep, s)
|
|
14
|
+
print(type(result))
|
|
15
|
+
print(result[0])
|
cluster_tools/remote.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Tools for executing remote commands."""
|
|
2
|
+
import cloudpickle
|
|
3
|
+
import sys
|
|
4
|
+
import os
|
|
5
|
+
import traceback
|
|
6
|
+
from .util import local_filename
|
|
7
|
+
import logging
|
|
8
|
+
|
|
9
|
+
INFILE_FMT = local_filename("cfut.in.%s.pickle")
|
|
10
|
+
OUTFILE_FMT = local_filename("cfut.out.%s.pickle")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def format_remote_exc():
|
|
14
|
+
typ, value, tb = sys.exc_info()
|
|
15
|
+
tb = tb.tb_next # Remove root call to worker().
|
|
16
|
+
return "".join(traceback.format_exception(typ, value, tb))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def worker(workerid):
|
|
20
|
+
"""Called to execute a job on a remote host."""
|
|
21
|
+
try:
|
|
22
|
+
with open(INFILE_FMT % workerid, "rb") as f:
|
|
23
|
+
indata = f.read()
|
|
24
|
+
fun, args, kwargs, meta_data = cloudpickle.loads(indata)
|
|
25
|
+
logging_config = meta_data.get("logging_config", {"level": logging.DEBUG})
|
|
26
|
+
|
|
27
|
+
logging.basicConfig(**logging_config)
|
|
28
|
+
logging.info("Setting up logging.basicConfig with default values (potentially overwriting logging configuration of the main script. Config: {}".format(logging_config))
|
|
29
|
+
logging.info("Starting job computation...")
|
|
30
|
+
result = True, fun(*args, **kwargs)
|
|
31
|
+
logging.info("Job computation completed.")
|
|
32
|
+
out = cloudpickle.dumps(result, True)
|
|
33
|
+
|
|
34
|
+
except Exception as e:
|
|
35
|
+
print(traceback.format_exc())
|
|
36
|
+
|
|
37
|
+
result = False, format_remote_exc()
|
|
38
|
+
logging.info("Job computation failed.")
|
|
39
|
+
out = cloudpickle.dumps(result, False)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
destfile = OUTFILE_FMT % workerid
|
|
43
|
+
tempfile = destfile + ".tmp"
|
|
44
|
+
with open(tempfile, "wb") as f:
|
|
45
|
+
f.write(out)
|
|
46
|
+
logging.info("Pickle file written to {}.".format(tempfile))
|
|
47
|
+
os.rename(tempfile, destfile)
|
|
48
|
+
logging.info("Pickle file renamed to {}.".format(destfile))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
if __name__ == "__main__":
|
|
52
|
+
worker_id = sys.argv[1]
|
|
53
|
+
task_id = os.environ.get("SLURM_ARRAY_TASK_ID", None)
|
|
54
|
+
if task_id is not None:
|
|
55
|
+
worker_id = worker_id + "_" + task_id
|
|
56
|
+
|
|
57
|
+
worker(worker_id)
|
cluster_tools/slurm.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Abstracts access to a Slurm cluster via its command-line tools.
|
|
2
|
+
"""
|
|
3
|
+
import re
|
|
4
|
+
import os
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from .util import chcall, random_string, local_filename
|
|
8
|
+
|
|
9
|
+
LOG_FILE = local_filename("slurmpy.log")
|
|
10
|
+
OUTFILE_FMT = local_filename("slurmpy.stdout.{}.log")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def submit_text(job, job_name):
|
|
14
|
+
"""Submits a Slurm job represented as a job file string. Returns
|
|
15
|
+
the job ID.
|
|
16
|
+
"""
|
|
17
|
+
job_name_arg = ""
|
|
18
|
+
if job_name is not None:
|
|
19
|
+
job_name_arg = '--job-name "{}"'.format(job_name)
|
|
20
|
+
|
|
21
|
+
filename = local_filename("_temp_{}.sh".format(random_string()))
|
|
22
|
+
with open(filename, "w") as f:
|
|
23
|
+
f.write(job)
|
|
24
|
+
jobid, _ = chcall("sbatch {} --parsable {}".format(job_name_arg, filename))
|
|
25
|
+
os.unlink(filename)
|
|
26
|
+
return int(jobid)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def submit(
|
|
30
|
+
cmdline,
|
|
31
|
+
outpath_fmt=OUTFILE_FMT,
|
|
32
|
+
job_resources=None,
|
|
33
|
+
job_name=None,
|
|
34
|
+
additional_setup_lines=[],
|
|
35
|
+
job_count=None,
|
|
36
|
+
):
|
|
37
|
+
"""Starts a Slurm job that runs the specified shell command line.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
outpath = outpath_fmt.format("%j" if job_count is None else "%A.%a")
|
|
41
|
+
|
|
42
|
+
job_resources_lines = []
|
|
43
|
+
if job_resources is not None:
|
|
44
|
+
for resource, value in job_resources.items():
|
|
45
|
+
job_resources_lines += ["#SBATCH --{}={}".format(resource, value)]
|
|
46
|
+
|
|
47
|
+
job_array_line = ""
|
|
48
|
+
if job_count is not None:
|
|
49
|
+
job_array_line = "#SBATCH --array=0-{}".format(job_count - 1)
|
|
50
|
+
|
|
51
|
+
script_lines = (
|
|
52
|
+
["#!/bin/sh", "#SBATCH --output={}".format(outpath), job_array_line]
|
|
53
|
+
+ job_resources_lines
|
|
54
|
+
+ [*additional_setup_lines, "srun {}".format(cmdline)]
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
return submit_text("\n".join(script_lines), job_name)
|
cluster_tools/util.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import random
|
|
3
|
+
import string
|
|
4
|
+
import os
|
|
5
|
+
|
|
6
|
+
def local_filename(filename=""):
|
|
7
|
+
return os.path.join(os.getenv("CFUT_DIR", ".cfut"), filename)
|
|
8
|
+
|
|
9
|
+
def random_string(length=32, chars=(string.ascii_letters + string.digits)):
|
|
10
|
+
return ''.join(random.choice(chars) for i in range(length))
|
|
11
|
+
|
|
12
|
+
def call(command, stdin=None):
|
|
13
|
+
"""Invokes a shell command as a subprocess, optionally with some
|
|
14
|
+
data sent to the standard input. Returns the standard output data,
|
|
15
|
+
the standard error, and the return code.
|
|
16
|
+
"""
|
|
17
|
+
if stdin is not None:
|
|
18
|
+
stdin_flag = subprocess.PIPE
|
|
19
|
+
else:
|
|
20
|
+
stdin_flag = None
|
|
21
|
+
proc = subprocess.Popen(command, shell=True, stdin=stdin_flag,
|
|
22
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
23
|
+
stdout, stderr = proc.communicate(stdin)
|
|
24
|
+
return stdout, stderr, proc.returncode
|
|
25
|
+
|
|
26
|
+
class CommandError(Exception):
|
|
27
|
+
"""Raised when a shell command exits abnormally."""
|
|
28
|
+
def __init__(self, command, code, stderr):
|
|
29
|
+
self.command = command
|
|
30
|
+
self.code = code
|
|
31
|
+
self.stderr = stderr
|
|
32
|
+
|
|
33
|
+
def __str__(self):
|
|
34
|
+
return "%s exited with status %i: %s" % (repr(self.command),
|
|
35
|
+
self.code,
|
|
36
|
+
repr(self.stderr))
|
|
37
|
+
|
|
38
|
+
def chcall(command, stdin=None):
|
|
39
|
+
"""Like ``call`` but raises an exception when the return code is
|
|
40
|
+
nonzero. Only returns the stdout and stderr data.
|
|
41
|
+
"""
|
|
42
|
+
stdout, stderr, code = call(command, stdin)
|
|
43
|
+
if code != 0:
|
|
44
|
+
raise CommandError(command, code, stderr)
|
|
45
|
+
return stdout, stderr
|