cluster-tools 0.1__py3.6.egg

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
EGG-INFO/PKG-INFO ADDED
@@ -0,0 +1,37 @@
1
+ Metadata-Version: 1.1
2
+ Name: cluster-tools
3
+ Version: 0.1
4
+ Summary: Utility library for easily distributing code execution on clusters
5
+ Home-page: https://github.com/scalableminds/cluster_tools
6
+ Author: scalableminds
7
+ Author-email: hello@scalableminds.com
8
+ License: MIT
9
+ Description: # Cluster Tools
10
+
11
+ [![CircleCI](https://circleci.com/gh/scalableminds/cluster_tools/tree/master.svg?style=svg)](https://circleci.com/gh/scalableminds/cluster_tools/tree/master)
12
+
13
+ This package provides python `Executor` classes for distributing tasks on a slurm cluster or via multi processing.
14
+
15
+
16
+ ## Example
17
+
18
+ ```python
19
+ import cluster_tools
20
+
21
+ def square(n):
22
+ return n * n
23
+
24
+ if __name__ == '__main__':
25
+ strategy = "slurm" # other valid values are "multiprocessing" and "sequential"
26
+ with cluster_tools.get_executor(strategy) as executor:
27
+ result = list(executor.map(square, [2, 3, 4]))
28
+ assert result == [4, 9, 16]
29
+ ```
30
+
31
+ ## Credits
32
+
33
+ Thanks to [sampsyo/clusterfutures](https://github.com/sampsyo/clusterfutures) for providing the slurm core abstraction and [giovtorres/slurm-docker-cluster](https://github.com/giovtorres/slurm-docker-cluster) for providing the slurm docker environment which we use for CI based testing.
34
+
35
+ Platform: ALL
36
+ Classifier: Intended Audience :: Developers
37
+ Classifier: Intended Audience :: Science/Research
EGG-INFO/SOURCES.txt ADDED
@@ -0,0 +1,12 @@
1
+ README.md
2
+ setup.py
3
+ cluster_tools/__init__.py
4
+ cluster_tools/generator-tmp.py
5
+ cluster_tools/remote.py
6
+ cluster_tools/slurm.py
7
+ cluster_tools/util.py
8
+ cluster_tools.egg-info/PKG-INFO
9
+ cluster_tools.egg-info/SOURCES.txt
10
+ cluster_tools.egg-info/dependency_links.txt
11
+ cluster_tools.egg-info/requires.txt
12
+ cluster_tools.egg-info/top_level.txt
@@ -0,0 +1 @@
1
+
EGG-INFO/requires.txt ADDED
@@ -0,0 +1,2 @@
1
+ cloudpickle
2
+ futures
EGG-INFO/top_level.txt ADDED
@@ -0,0 +1 @@
1
+ cluster_tools
EGG-INFO/zip-safe ADDED
@@ -0,0 +1 @@
1
+
@@ -0,0 +1,367 @@
1
+ from concurrent import futures
2
+ from concurrent.futures import ProcessPoolExecutor
3
+ import os
4
+ import sys
5
+ import threading
6
+ import time
7
+ from . import slurm
8
+ from .remote import INFILE_FMT, OUTFILE_FMT
9
+ from .util import random_string, local_filename, call
10
+ import cloudpickle
11
+ import logging
12
+
13
+
14
+ class RemoteException(Exception):
15
+ def __init__(self, error):
16
+ self.error = error
17
+
18
+ def __str__(self):
19
+ return "\n" + self.error.strip()
20
+
21
+
22
+ class FileWaitThread(threading.Thread):
23
+ """A thread that polls the filesystem waiting for a list of files to
24
+ be created. When a specified file is created, it invokes a callback.
25
+ """
26
+
27
+ def __init__(self, callback, interval=1):
28
+ """The callable ``callback`` will be invoked with value
29
+ associated with the filename of each file that is created.
30
+ ``interval`` specifies the polling rate.
31
+ """
32
+ threading.Thread.__init__(self)
33
+ self.callback = callback
34
+ self.interval = interval
35
+ self.waiting = {}
36
+ self.lock = threading.Lock()
37
+ self.shutdown = False
38
+
39
+ def stop(self):
40
+ """Stop the thread soon."""
41
+ with self.lock:
42
+ self.shutdown = True
43
+
44
+ def wait(self, filename, value):
45
+ """Adds a new filename (and its associated callback value) to
46
+ the set of files being waited upon.
47
+ """
48
+ with self.lock:
49
+ self.waiting[filename] = value
50
+
51
+ def run(self):
52
+ while True:
53
+ with self.lock:
54
+ if self.shutdown:
55
+ return
56
+
57
+ # Poll for each file.
58
+ for filename in list(self.waiting):
59
+ job_id = self.waiting[filename]
60
+
61
+ # Let's get the state for the job to check whether it failed.
62
+ # It's important to check the status before checking whether the file exists.
63
+ # Otherwise, we can run into a race condition where the job succeeds
64
+ # after os.path.exists was called and before scontrol is executed.
65
+ stdout, _, exit_code = call("scontrol show job {}".format(job_id))
66
+
67
+ if exit_code != 0:
68
+ logging.error(
69
+ "Couldn't call scontrol to determine job's status. {}. Continuing to poll for output file.".format(
70
+ e
71
+ )
72
+ )
73
+ elif "JobState=FAILED" in str(stdout):
74
+ self.callback(job_id, True)
75
+ del self.waiting[filename]
76
+ elif "JobState=COMPLETED" in str(stdout):
77
+ logging.error(
78
+ "Job state is completed, but {} couldn't be found.".format(
79
+ filename
80
+ )
81
+ )
82
+ self.callback(job_id, True)
83
+ del self.waiting[filename]
84
+
85
+
86
+ if os.path.exists(filename):
87
+ self.callback(job_id, False)
88
+ del self.waiting[filename]
89
+
90
+ time.sleep(self.interval)
91
+
92
+
93
+ class SlurmExecutor(futures.Executor):
94
+ """Futures executor for executing jobs on a Slurm cluster."""
95
+
96
+ def __init__(
97
+ self,
98
+ debug=False,
99
+ keep_logs=False,
100
+ job_resources=None,
101
+ job_name=None,
102
+ additional_setup_lines=[],
103
+ **kwargs
104
+ ):
105
+ os.makedirs(local_filename(), exist_ok=True)
106
+ self.debug = debug
107
+ self.job_resources = job_resources
108
+ self.additional_setup_lines = additional_setup_lines
109
+ self.job_name = job_name
110
+ self.was_requested_to_shutdown = False
111
+
112
+ # `jobs` maps from job id to future and workerid
113
+ # In case, job arrays are used: job id and workerid are in the format of
114
+ # `job_id-job_index` and `workerid-job_index`.
115
+ self.jobs = {}
116
+ self.job_outfiles = {}
117
+ self.jobs_lock = threading.Lock()
118
+ self.jobs_empty_cond = threading.Condition(self.jobs_lock)
119
+ self.keep_logs = keep_logs
120
+
121
+ self.wait_thread = FileWaitThread(self._completion)
122
+ self.wait_thread.start()
123
+
124
+ self.meta_data = {}
125
+ if "logging_config" in kwargs:
126
+ self.meta_data["logging_config"] = kwargs["logging_config"]
127
+
128
+ def _start(self, workerid, job_count=None):
129
+ """Start a job with the given worker ID and return an ID
130
+ identifying the new job. The job should run ``python -m
131
+ cfut.remote <workerid>.
132
+ """
133
+ return slurm.submit(
134
+ "{} -m cluster_tools.remote {}".format(sys.executable, workerid),
135
+ job_resources=self.job_resources,
136
+ job_name=self.job_name,
137
+ additional_setup_lines=self.additional_setup_lines,
138
+ job_count=job_count,
139
+ )
140
+
141
+ def _cleanup(self, jobid):
142
+ """Given a job ID as returned by _start, perform any necessary
143
+ cleanup after the job has finished.
144
+ """
145
+ if self.keep_logs:
146
+ return
147
+
148
+ outf = slurm.OUTFILE_FMT.format(str(jobid))
149
+ try:
150
+ os.unlink(outf)
151
+ except OSError:
152
+ pass
153
+
154
+ def _completion(self, jobid, failed_early):
155
+ """Called whenever a job finishes."""
156
+ with self.jobs_lock:
157
+ fut, workerid = self.jobs.pop(jobid)
158
+ if not self.jobs:
159
+ self.jobs_empty_cond.notify_all()
160
+ if self.debug:
161
+ print("job completed: {}".format(jobid), file=sys.stderr)
162
+
163
+ if failed_early:
164
+ # If the code which should be executed on a node wasn't even
165
+ # started (e.g., because python isn't installed or the cluster_tools
166
+ # couldn't be found), no output was written to disk. We only noticed
167
+ # this circumstance because the whole slurm job was marked as failed.
168
+ # Therefore, we don't try to deserialize pickle output.
169
+ success = False
170
+ result = "Job submission/execution failed. Please look into the log file at {}".format(
171
+ slurm.OUTFILE_FMT.format(jobid)
172
+ )
173
+ else:
174
+ with open(OUTFILE_FMT % workerid, "rb") as f:
175
+ outdata = f.read()
176
+ success, result = cloudpickle.loads(outdata)
177
+
178
+ if success:
179
+ fut.set_result(result)
180
+ else:
181
+ fut.set_exception(RemoteException(result))
182
+
183
+ # Clean up communication files.
184
+
185
+ if os.path.exists(INFILE_FMT % workerid):
186
+ os.unlink(INFILE_FMT % workerid)
187
+ if os.path.exists(OUTFILE_FMT % workerid):
188
+ os.unlink(OUTFILE_FMT % workerid)
189
+
190
+ self._cleanup(jobid)
191
+
192
+ def ensure_not_shutdown(self):
193
+ if self.was_requested_to_shutdown:
194
+ raise RuntimeError(
195
+ "submit() was invoked on a SlurmExecutor instance even though shutdown() was executed for that instance."
196
+ )
197
+
198
+ def submit(self, fun, *args, **kwargs):
199
+ """Submit a job to the pool."""
200
+ fut = futures.Future()
201
+
202
+ self.ensure_not_shutdown()
203
+
204
+ # Start the job.
205
+ workerid = random_string()
206
+
207
+ funcser = cloudpickle.dumps((fun, args, kwargs, self.meta_data), True)
208
+ with open(INFILE_FMT % workerid, "wb") as f:
209
+ f.write(funcser)
210
+
211
+ jobid = self._start(workerid)
212
+
213
+ if self.debug:
214
+ print("job submitted: %i" % jobid, file=sys.stderr)
215
+
216
+ # Thread will wait for it to finish.
217
+ self.wait_thread.wait(OUTFILE_FMT % workerid, jobid)
218
+
219
+ with self.jobs_lock:
220
+ self.jobs[jobid] = (fut, workerid)
221
+
222
+ fut.slurm_jobid = jobid
223
+ return fut
224
+
225
+ def map_to_futures(self, fun, allArgs):
226
+ self.ensure_not_shutdown()
227
+ allArgs = list(allArgs)
228
+
229
+ futs = []
230
+ workerid = random_string()
231
+
232
+ get_workerid_with_index = lambda index: workerid + "_" + str(index)
233
+
234
+ # Submit jobs eagerly
235
+ for index, arg in enumerate(allArgs):
236
+ fut = futures.Future()
237
+
238
+ # Start the job.
239
+ funcser = cloudpickle.dumps((fun, [arg], {}, self.meta_data), True)
240
+ infile_name = INFILE_FMT % get_workerid_with_index(index)
241
+
242
+ with open(infile_name, "wb") as f:
243
+ f.write(funcser)
244
+
245
+ futs.append(fut)
246
+
247
+ job_count = len(allArgs)
248
+ jobid = self._start(workerid, job_count)
249
+ get_jobid_with_index = lambda index: str(jobid) + "_" + str(index)
250
+
251
+ if self.debug:
252
+ print(
253
+ "main job submitted: %i. consists of %i subjobs." % (jobid, job_count),
254
+ file=sys.stderr,
255
+ )
256
+
257
+ with self.jobs_lock:
258
+ for index, fut in enumerate(futs):
259
+ jobid_with_index = get_jobid_with_index(index)
260
+ # Thread will wait for it to finish.
261
+ workerid_with_index = get_workerid_with_index(index)
262
+ self.wait_thread.wait(
263
+ OUTFILE_FMT % workerid_with_index, jobid_with_index
264
+ )
265
+
266
+ fut.slurm_jobid = jobid
267
+ fut.slurm_jobindex = index
268
+
269
+ self.jobs[jobid_with_index] = (fut, workerid_with_index)
270
+
271
+ return futs
272
+
273
+ def shutdown(self, wait=True):
274
+ """Close the pool."""
275
+ self.was_requested_to_shutdown = True
276
+ if wait:
277
+ with self.jobs_lock:
278
+ if self.jobs:
279
+ self.jobs_empty_cond.wait()
280
+
281
+ self.wait_thread.stop()
282
+ self.wait_thread.join()
283
+
284
+ def map(self, func, args, timeout=None, chunksize=None):
285
+ if chunksize is not None:
286
+ logging.warning(
287
+ "The provided chunksize parameter is ignored by SlurmExecutor."
288
+ )
289
+
290
+
291
+ start_time = time.time()
292
+
293
+ futs = self.map_to_futures(func, args)
294
+ results = []
295
+
296
+ # Return a separate generator as iterator to avoid that the
297
+ # map() method itself becomes a generator.
298
+ # If map() was a generator, the submit() calls would be invoked
299
+ # lazily which can lead to a shutdown of the executor before
300
+ # the submit calls are performed.
301
+ def result_generator():
302
+ for fut in futs:
303
+ passed_time = time.time() - start_time
304
+ remaining_timeout = (
305
+ None if timeout is None else timeout - passed_time
306
+ )
307
+ yield fut.result(remaining_timeout)
308
+
309
+ return result_generator()
310
+
311
+
312
+ def map_unordered(self, func, args):
313
+ futs = self.map_to_futures(func, args)
314
+
315
+ # Return a separate generator to avoid that map_unordered
316
+ # is executed lazily.
317
+ def result_generator():
318
+ for fut in futures.as_completed(futs):
319
+ yield fut.result()
320
+
321
+ return result_generator()
322
+
323
+
324
+ def get_existent_kwargs_subset(white_list, kwargs):
325
+ new_kwargs = {}
326
+ for arg_name in white_list:
327
+ if arg_name in kwargs:
328
+ new_kwargs[arg_name] = kwargs[arg_name]
329
+
330
+ return new_kwargs
331
+
332
+
333
+ class WrappedProcessPoolExecutor(ProcessPoolExecutor):
334
+ def __init__(self, **kwargs):
335
+ white_list = ["max_workers", "mp_context", "initializer", "initargs"]
336
+ new_kwargs = get_existent_kwargs_subset(white_list, kwargs)
337
+
338
+ return ProcessPoolExecutor.__init__(self, **new_kwargs)
339
+
340
+
341
+ def map_unordered(self, func, args):
342
+
343
+ futs = [self.submit(func, arg) for arg in args]
344
+
345
+ # Return a separate generator to avoid that map_unordered
346
+ # is executed lazily (otherwise, jobs would be submitted
347
+ # lazily, as well).
348
+ def result_generator():
349
+ for fut in futures.as_completed(futs):
350
+ yield fut.result()
351
+
352
+ return result_generator()
353
+
354
+
355
+ class SequentialExecutor(WrappedProcessPoolExecutor):
356
+ def __init__(self, **kwargs):
357
+ kwargs["max_workers"] = 1
358
+ return WrappedProcessPoolExecutor.__init__(self, **kwargs)
359
+
360
+
361
+ def get_executor(environment, **kwargs):
362
+ if environment == "slurm":
363
+ return SlurmExecutor(**kwargs)
364
+ elif environment == "multiprocessing":
365
+ return WrappedProcessPoolExecutor(**kwargs)
366
+ elif environment == "sequential":
367
+ return SequentialExecutor(**kwargs)
@@ -0,0 +1,15 @@
1
+ import time
2
+ import concurrent.futures
3
+ from multiprocessing import Pool
4
+
5
+ s = [1, 2, 3]
6
+
7
+ with Pool(2) as p:
8
+ result = p.map(time.sleep, s)
9
+ print(type(result))
10
+ print(result[0])
11
+
12
+ with concurrent.futures.ThreadPoolExecutor(4):
13
+ result = e.map(time.sleep, s)
14
+ print(type(result))
15
+ print(result[0])
@@ -0,0 +1,57 @@
1
+ """Tools for executing remote commands."""
2
+ import cloudpickle
3
+ import sys
4
+ import os
5
+ import traceback
6
+ from .util import local_filename
7
+ import logging
8
+
9
+ INFILE_FMT = local_filename("cfut.in.%s.pickle")
10
+ OUTFILE_FMT = local_filename("cfut.out.%s.pickle")
11
+
12
+
13
+ def format_remote_exc():
14
+ typ, value, tb = sys.exc_info()
15
+ tb = tb.tb_next # Remove root call to worker().
16
+ return "".join(traceback.format_exception(typ, value, tb))
17
+
18
+
19
+ def worker(workerid):
20
+ """Called to execute a job on a remote host."""
21
+ try:
22
+ with open(INFILE_FMT % workerid, "rb") as f:
23
+ indata = f.read()
24
+ fun, args, kwargs, meta_data = cloudpickle.loads(indata)
25
+ logging_config = meta_data.get("logging_config", {"level": logging.DEBUG})
26
+
27
+ logging.basicConfig(**logging_config)
28
+ logging.info("Setting up logging.basicConfig with default values (potentially overwriting logging configuration of the main script. Config: {}".format(logging_config))
29
+ logging.info("Starting job computation...")
30
+ result = True, fun(*args, **kwargs)
31
+ logging.info("Job computation completed.")
32
+ out = cloudpickle.dumps(result, True)
33
+
34
+ except Exception as e:
35
+ print(traceback.format_exc())
36
+
37
+ result = False, format_remote_exc()
38
+ logging.info("Job computation failed.")
39
+ out = cloudpickle.dumps(result, False)
40
+
41
+
42
+ destfile = OUTFILE_FMT % workerid
43
+ tempfile = destfile + ".tmp"
44
+ with open(tempfile, "wb") as f:
45
+ f.write(out)
46
+ logging.info("Pickle file written to {}.".format(tempfile))
47
+ os.rename(tempfile, destfile)
48
+ logging.info("Pickle file renamed to {}.".format(destfile))
49
+
50
+
51
+ if __name__ == "__main__":
52
+ worker_id = sys.argv[1]
53
+ task_id = os.environ.get("SLURM_ARRAY_TASK_ID", None)
54
+ if task_id is not None:
55
+ worker_id = worker_id + "_" + task_id
56
+
57
+ worker(worker_id)
cluster_tools/slurm.py ADDED
@@ -0,0 +1,57 @@
1
+ """Abstracts access to a Slurm cluster via its command-line tools.
2
+ """
3
+ import re
4
+ import os
5
+ import threading
6
+ import time
7
+ from .util import chcall, random_string, local_filename
8
+
9
+ LOG_FILE = local_filename("slurmpy.log")
10
+ OUTFILE_FMT = local_filename("slurmpy.stdout.{}.log")
11
+
12
+
13
+ def submit_text(job, job_name):
14
+ """Submits a Slurm job represented as a job file string. Returns
15
+ the job ID.
16
+ """
17
+ job_name_arg = ""
18
+ if job_name is not None:
19
+ job_name_arg = '--job-name "{}"'.format(job_name)
20
+
21
+ filename = local_filename("_temp_{}.sh".format(random_string()))
22
+ with open(filename, "w") as f:
23
+ f.write(job)
24
+ jobid, _ = chcall("sbatch {} --parsable {}".format(job_name_arg, filename))
25
+ os.unlink(filename)
26
+ return int(jobid)
27
+
28
+
29
+ def submit(
30
+ cmdline,
31
+ outpath_fmt=OUTFILE_FMT,
32
+ job_resources=None,
33
+ job_name=None,
34
+ additional_setup_lines=[],
35
+ job_count=None,
36
+ ):
37
+ """Starts a Slurm job that runs the specified shell command line.
38
+ """
39
+
40
+ outpath = outpath_fmt.format("%j" if job_count is None else "%A.%a")
41
+
42
+ job_resources_lines = []
43
+ if job_resources is not None:
44
+ for resource, value in job_resources.items():
45
+ job_resources_lines += ["#SBATCH --{}={}".format(resource, value)]
46
+
47
+ job_array_line = ""
48
+ if job_count is not None:
49
+ job_array_line = "#SBATCH --array=0-{}".format(job_count - 1)
50
+
51
+ script_lines = (
52
+ ["#!/bin/sh", "#SBATCH --output={}".format(outpath), job_array_line]
53
+ + job_resources_lines
54
+ + [*additional_setup_lines, "srun {}".format(cmdline)]
55
+ )
56
+
57
+ return submit_text("\n".join(script_lines), job_name)
cluster_tools/util.py ADDED
@@ -0,0 +1,45 @@
1
+ import subprocess
2
+ import random
3
+ import string
4
+ import os
5
+
6
+ def local_filename(filename=""):
7
+ return os.path.join(os.getenv("CFUT_DIR", ".cfut"), filename)
8
+
9
+ def random_string(length=32, chars=(string.ascii_letters + string.digits)):
10
+ return ''.join(random.choice(chars) for i in range(length))
11
+
12
+ def call(command, stdin=None):
13
+ """Invokes a shell command as a subprocess, optionally with some
14
+ data sent to the standard input. Returns the standard output data,
15
+ the standard error, and the return code.
16
+ """
17
+ if stdin is not None:
18
+ stdin_flag = subprocess.PIPE
19
+ else:
20
+ stdin_flag = None
21
+ proc = subprocess.Popen(command, shell=True, stdin=stdin_flag,
22
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
23
+ stdout, stderr = proc.communicate(stdin)
24
+ return stdout, stderr, proc.returncode
25
+
26
+ class CommandError(Exception):
27
+ """Raised when a shell command exits abnormally."""
28
+ def __init__(self, command, code, stderr):
29
+ self.command = command
30
+ self.code = code
31
+ self.stderr = stderr
32
+
33
+ def __str__(self):
34
+ return "%s exited with status %i: %s" % (repr(self.command),
35
+ self.code,
36
+ repr(self.stderr))
37
+
38
+ def chcall(command, stdin=None):
39
+ """Like ``call`` but raises an exception when the return code is
40
+ nonzero. Only returns the stdout and stderr data.
41
+ """
42
+ stdout, stderr, code = call(command, stdin)
43
+ if code != 0:
44
+ raise CommandError(command, code, stderr)
45
+ return stdout, stderr