mqdm 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mqdm-0.0.1/PKG-INFO ADDED
@@ -0,0 +1,79 @@
1
+ Metadata-Version: 2.1
2
+ Name: mqdm
3
+ Version: 0.0.1
4
+ Description-Content-Type: text/markdown
5
+ Requires-Dist: rich
6
+
7
+ # mqdm: progress bars for multiprocessing
8
+ Pretty progress bars with `rich`, in your child processes.
9
+
10
+ ## Install
11
+
12
+ ```bash
13
+ pip install mqdm
14
+ ```
15
+
16
+ ## Worker progress
17
+ ```python
18
+ import mqdm
19
+ import time
20
+
21
+ def my_work(n, sleep, pbar):
22
+ for i in pbar(range(n), description=f'counting to {n}'):
23
+ time.sleep(sleep)
24
+
25
+ # executes my task in a concurrent futures process pool
26
+ mqdm.pool(
27
+ my_work,
28
+ range(1, 10),
29
+ sleep=1,
30
+ n_workers=3,
31
+ )
32
+ ```
33
+
34
+ ![alt text](static/image.png)
35
+
36
+ ## Less high level please
37
+ Basically, the mechanics are this:
38
+ ```python
39
+ # use context manager to start background listener and message queue
40
+ with mqdm.Bars() as pbars:
41
+ # create progress bars and send them to the remote processes
42
+ pool.submit(my_work, 1, pbar=pbars.add())
43
+ pool.submit(my_work, 2, pbar=pbars.add())
44
+ pool.submit(my_work, 3, pbar=pbars.add())
45
+
46
+ # your worker function can look like this
47
+ def my_work(n, sleep, pbar):
48
+ for i in pbar(range(n), description=f'counting to {n}'):
49
+ time.sleep(sleep)
50
+
51
+ # or this
52
+ def my_work(n, pbar, sleep=0.2):
53
+ import time
54
+ with pbar(description=f'counting to {n}', total=n):
55
+ for i in range(n):
56
+ pbar.update(0.5, description=f'Im counting - {n} ')
57
+ time.sleep(sleep/2)
58
+ pbar.update(0.5, description=f'Im counting - {n+0.5}')
59
+ time.sleep(sleep/2)
60
+ ```
61
+
62
+ And you can use it in a pool like this:
63
+
64
+ ```python
65
+ import mqdm
66
+ from concurrent.futures import ProcessPoolExecutor, as_completed
67
+
68
+ items = range(1, 10)
69
+
70
+ with ProcessPoolExecutor(max_workers=n_workers) as pool, mqdm.Bars() as pbars:
71
+ futures = [
72
+ pool.submit(my_work, i, pbar=pbars.add())
73
+ for i in items
74
+ ]
75
+ for f in as_completed(futures):
76
+ print(f.result())
77
+ ```
78
+
79
+ It works by spawning a background thread with a multiprocessing queue. The Bars instance listens for messages from the progress bar proxies in the child processes.
mqdm-0.0.1/README.md ADDED
@@ -0,0 +1,73 @@
1
+ # mqdm: progress bars for multiprocessing
2
+ Pretty progress bars with `rich`, in your child processes.
3
+
4
+ ## Install
5
+
6
+ ```bash
7
+ pip install mqdm
8
+ ```
9
+
10
+ ## Worker progress
11
+ ```python
12
+ import mqdm
13
+ import time
14
+
15
+ def my_work(n, sleep, pbar):
16
+ for i in pbar(range(n), description=f'counting to {n}'):
17
+ time.sleep(sleep)
18
+
19
+ # executes my task in a concurrent futures process pool
20
+ mqdm.pool(
21
+ my_work,
22
+ range(1, 10),
23
+ sleep=1,
24
+ n_workers=3,
25
+ )
26
+ ```
27
+
28
+ ![alt text](static/image.png)
29
+
30
+ ## Less high level please
31
+ Basically, the mechanics are this:
32
+ ```python
33
+ # use context manager to start background listener and message queue
34
+ with mqdm.Bars() as pbars:
35
+ # create progress bars and send them to the remote processes
36
+ pool.submit(my_work, 1, pbar=pbars.add())
37
+ pool.submit(my_work, 2, pbar=pbars.add())
38
+ pool.submit(my_work, 3, pbar=pbars.add())
39
+
40
+ # your worker function can look like this
41
+ def my_work(n, sleep, pbar):
42
+ for i in pbar(range(n), description=f'counting to {n}'):
43
+ time.sleep(sleep)
44
+
45
+ # or this
46
+ def my_work(n, pbar, sleep=0.2):
47
+ import time
48
+ with pbar(description=f'counting to {n}', total=n):
49
+ for i in range(n):
50
+ pbar.update(0.5, description=f'Im counting - {n} ')
51
+ time.sleep(sleep/2)
52
+ pbar.update(0.5, description=f'Im counting - {n+0.5}')
53
+ time.sleep(sleep/2)
54
+ ```
55
+
56
+ And you can use it in a pool like this:
57
+
58
+ ```python
59
+ import mqdm
60
+ from concurrent.futures import ProcessPoolExecutor, as_completed
61
+
62
+ items = range(1, 10)
63
+
64
+ with ProcessPoolExecutor(max_workers=n_workers) as pool, mqdm.Bars() as pbars:
65
+ futures = [
66
+ pool.submit(my_work, i, pbar=pbars.add())
67
+ for i in items
68
+ ]
69
+ for f in as_completed(futures):
70
+ print(f.result())
71
+ ```
72
+
73
+ It works by spawning a background thread with a multiprocessing queue. The Bars instance listens for messages from the progress bar proxies in the child processes.
@@ -0,0 +1,3 @@
1
+ from . import utils
2
+ from .core import Bars, RemoteBar, Bar, args
3
+ pool = Bars.pool
@@ -0,0 +1,4 @@
1
+ from mqdm.core import example_run
2
+
3
+ import fire
4
+ fire.Fire(example_run)
@@ -0,0 +1,119 @@
1
+ '''First draft implementation using manager dict.
2
+
3
+
4
+ '''
5
+
6
+ import random
7
+ from time import sleep
8
+ import threading
9
+ import multiprocessing
10
+ from concurrent.futures import ProcessPoolExecutor
11
+ from rich import progress
12
+
13
+
14
+ class Bars:
15
+ def __init__(self, desc="[green]All jobs progress:") -> None:
16
+ self.desc = desc
17
+ self.pbar = progress.Progress(
18
+ "[progress.description]{task.description}",
19
+ progress.BarColumn(),
20
+ "[progress.percentage]{task.percentage:>3.0f}%",
21
+ progress.TimeRemainingColumn(),
22
+ progress.TimeElapsedColumn(),
23
+ refresh_per_second=8,
24
+ )
25
+ # Create and start a thread for reading from the queue
26
+ reader_thread = threading.Thread(target=queue_reader, args=(queue,))
27
+ reader_thread.start()
28
+ self.manager = multiprocessing.Manager()
29
+
30
+ def __enter__(self):
31
+ self.pbar.__enter__()
32
+ self.manager.__enter__()
33
+ self.tasks = self.manager.dict()
34
+ self.overall_task = self.pbar.add_task(self.desc)
35
+ return self
36
+
37
+ def __exit__(self, c,v,t):
38
+ self.pbar.__exit__(c,v,t)
39
+ self.manager.__exit__(c,v,t)
40
+
41
+ def add(self, title, visible=False, **kw):
42
+ task_id = self.pbar.add_task(title, visible=visible, start=False, **kw)
43
+ return Bar(self.tasks, task_id)
44
+
45
+ def update(self):
46
+ n_finished = 0
47
+ for task_id, data in self.tasks.items():
48
+ data = dict(data)
49
+ complete = data['total'] and data['completed'] >= data['total']
50
+ visible = bool(data['total'] and not complete)
51
+ data.setdefault('visible', visible)
52
+ n_finished += complete
53
+ # update the progress bar for this task:
54
+ self.pbar.update(task_id, **data)
55
+ self.pbar.update(self.overall_task, completed=n_finished, total=len(self.tasks))
56
+
57
+ def as_completed(self, futures, cycle=0.1):
58
+ while futures:
59
+ done = [futures.pop(i) for i, f in enumerate(futures) if f.done()]
60
+ for f in done:
61
+ yield f
62
+ self.update()
63
+ sleep(cycle)
64
+
65
+ @classmethod
66
+ def map(cls, fn, xs, n_workers=8, desc=None, **kw):
67
+ futures = []
68
+ with cls() as pbars, ProcessPoolExecutor(max_workers=n_workers) as executor:
69
+ for i, x in enumerate(xs):
70
+ desc_i = desc(*x, **kw) if callable(desc) else desc or f'task {i}'
71
+ futures.append(executor.submit(fn, *x, pbar=pbars.add(desc_i), **kw))
72
+ for f in pbars.as_completed(futures):
73
+ result = f.result()
74
+
75
+
76
+ class Bar:
77
+ def __init__(self, tasks, task_id):
78
+ self.tasks = tasks
79
+ self.task_id = task_id
80
+ self.current = 0
81
+ self.total = 0
82
+ self.update(0)
83
+
84
+ def __call__(self, iter, total=None):
85
+ try:
86
+ self.total = len(iter)
87
+ except TypeError:
88
+ self.total = total
89
+ def _iter():
90
+
91
+ for x in iter:
92
+ yield x
93
+ self.update()
94
+ return _iter()
95
+
96
+ def set(self, value):
97
+ self.current = value
98
+ self.update(0)
99
+ return self
100
+
101
+ def update(self, n=1):
102
+ self.current += n
103
+ total = self.total if self.current or not self.total else (self.current+1)
104
+ self.tasks[self.task_id] = {"completed": self.current, "total": total}
105
+ return self
106
+
107
+
108
+
109
+ def example_fn(i, pbar):
110
+ import random
111
+ for i in pbar(range(i + 1)):
112
+ sleep(random.random())
113
+
114
+ def pbar_test():
115
+ Bars.map(example_fn, [[i] for i in range(10)], n_workers=5)
116
+ if __name__ == '__main__':
117
+
118
+ import fire
119
+ fire.Fire(pbar_test)
@@ -0,0 +1,347 @@
1
+ ''''''
2
+ import sys
3
+ import queue
4
+ import threading
5
+ import multiprocessing as mp
6
+ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
7
+ from rich import progress, print
8
+ from . import utils
9
+
10
+
11
+
12
+ def get_pbar(pbar=None, bytes=False):
13
+ return pbar or progress.Progress(
14
+ "[progress.description]{task.description}",
15
+ progress.BarColumn(),
16
+ "[progress.percentage]{task.percentage:>3.0f}%",
17
+ *([progress.DownloadColumn()] if bytes else [utils.MofNColumn()]),
18
+ progress.TimeRemainingColumn(),
19
+ progress.TimeElapsedColumn(),
20
+ refresh_per_second=8,
21
+ )
22
+
23
+
24
+ '''
25
+ -- Multi Process:
26
+
27
+ Bars(**kw) -> add_task(overall, **kw)
28
+ Bars.add(**kw) -> add_task(item, **kw)
29
+
30
+ RemoteBar__init__() -> --
31
+ RemoteBar.__call__(**kw) -> RemoteBar.__enter__(**kw)
32
+ RemoteBar.__enter__(**kw) -> start_task(item, **kw)
33
+ iter(RemoteBar(**kw)) -> start_task(item, **kw)
34
+ RemoteBar.update(**kw) -> update(item, **kw)
35
+
36
+ -- Single Process:
37
+
38
+ Bar(**kw) -> add_task(item, **kw)
39
+ Bar.__call__(**kw) -> iter(Bar(**kw))
40
+ Bar.update(**kw) -> update(item, **kw)
41
+
42
+ '''
43
+
44
+
45
+ class Bars:
46
+ def __init__(self, desc="", pbar=None, iter=None, total=None, pool_mode='process', **kw) -> None:
47
+ if isinstance(desc, progress.Progress):
48
+ desc, pbar = None, desc
49
+ self.desc = desc
50
+ self.pbar = get_pbar(pbar)
51
+ self._overall_kw = kw
52
+ self.total = utils.try_len(iter, total)
53
+ self._pq = utils.POOL_QUEUES[pool_mode](self._on_message)
54
+ self._tasks = {}
55
+
56
+ def __enter__(self):
57
+ self.pbar.__enter__()
58
+ self._pq.__enter__()
59
+ self._tasks = {}
60
+ # ---------------------------------------------------------------------------- #
61
+ # add overall task #
62
+ # ---------------------------------------------------------------------------- #
63
+ self.overall_task = self.pbar.add_task(self.desc, total=self.total, **self._overall_kw)
64
+ return self
65
+
66
+ def __exit__(self, c,v,t):
67
+ self._pq.__exit__(c,v,t)
68
+ self.pbar.__exit__(c,v,t)
69
+
70
+ def add(self, title, visible=False, **kw):
71
+ # ---------------------------------------------------------------------------- #
72
+ # add task #
73
+ # ---------------------------------------------------------------------------- #
74
+ task_id = self.pbar.add_task(title, visible=visible, start=False, **kw)
75
+ self._tasks[task_id] = {}
76
+ return RemoteBar(self._pq.queue, task_id)
77
+
78
+ def close(self):
79
+ self.__exit__(None,None,None)
80
+
81
+ def _on_message(self, task_id, method, data):
82
+ if method == 'update':
83
+ self._update(task_id, **data)
84
+ else:
85
+ getattr(self.pbar, method)(**data)
86
+
87
+ def _update(self, task_id, **data):
88
+ # update the task-specific progress bar
89
+ if task_id is not None:
90
+ self._tasks[task_id].update(**data)
91
+ data.pop('complete', None)
92
+ # ---------------------------------------------------------------------------- #
93
+ # update task #
94
+ # ---------------------------------------------------------------------------- #
95
+ self.pbar.update(task_id, **data)
96
+
97
+ # update the overall task progress bar
98
+ n_finished = sum(bool(d and d.get('complete', False)) for d in self._tasks.values())
99
+ # ---------------------------------------------------------------------------- #
100
+ # update overall #
101
+ # ---------------------------------------------------------------------------- #
102
+ self.pbar.update(self.overall_task, completed=n_finished, total=len(self._tasks))
103
+
104
+ @classmethod
105
+ def ipool(cls, fn, xs, *a, n_workers=8, desc=None, mainbar_kw=None, subbar_kw=None, pool_mode='process', **kw):
106
+ """Execute a function in a process pool with a progress bar for each task."""
107
+ # get the arguments for each task
108
+ items = [x if isinstance(x, args) else args(x) for x in xs]
109
+
110
+ # no workers, just run the function
111
+ if n_workers < 2:
112
+ for i, arg in enumerate(items):
113
+ desc_i = arg(desc or f'task {i}')
114
+ yield arg(fn, *a, pbar=Bar, **kw)
115
+ return
116
+
117
+ # run the function in a process pool
118
+ futures = []
119
+ with utils.POOL_EXECUTORS[pool_mode](max_workers=n_workers) as executor, cls(pool_mode=pool_mode, **(mainbar_kw or {})) as pbars:
120
+ for i, arg in enumerate(items):
121
+ desc_i = arg(desc or f'task {i}')
122
+ pbar = pbars.add(desc_i, **(subbar_kw or {}))
123
+ futures.append(executor.submit(fn, *arg.a, *a, pbar=pbar, **dict(kw, **arg.kw)))
124
+ for f in as_completed(futures):
125
+ yield f.result()
126
+
127
+ @classmethod
128
+ def pool(cls, fn, xs, *a, n_workers=8, desc=None, **kw):
129
+ return list(cls.imap(fn, xs, *a, n_workers=n_workers, desc=desc, **kw))
130
+
131
+ # not sure which name is better
132
+ imap = ipool
133
+ map = pool
134
+
135
+
136
+ class args:
137
+ def __init__(self, *a, **kw):
138
+ self.a = a
139
+ self.kw = kw
140
+
141
+ def __call__(self, fn, *a, **kw):
142
+ return fn(*self.a, *a, **dict(self.kw, **kw)) if callable(fn) else fn
143
+
144
+
145
+ class RemoteBar:
146
+ def __init__(self, q, task_id):
147
+ self._queue = q
148
+ self.task_id = task_id
149
+ self.current = 0
150
+ self.total = 0
151
+ self.complete = False
152
+ self._started = False
153
+ self.kw = {}
154
+ self.update(0)
155
+
156
+ def __enter__(self, **kw):
157
+ # ---------------------------------------------------------------------------- #
158
+ # start task #
159
+ # ---------------------------------------------------------------------------- #
160
+ # start the task if it hasn't been started yet
161
+ if not self._started:
162
+ self._call('start_task', task_id=self.task_id, **kw)
163
+ self._started = True
164
+ return self
165
+
166
+ def __exit__(self, exc_type, exc_value, tb):
167
+ self.close()
168
+
169
+ def close(self):
170
+ self.update(0, complete=True)
171
+ # ---------------------------------------------------------------------------- #
172
+ # stop task #
173
+ # ---------------------------------------------------------------------------- #
174
+ if self._started:
175
+ self._call('stop_task', task_id=self.task_id)
176
+ self._started = False
177
+ return
178
+
179
+ def __call__(self, iter=None, total=None, **kw):
180
+ # if the first argument is a string, use it as the description
181
+ if isinstance(iter, str):
182
+ iter, kw['description'] = None, iter
183
+ if iter is None:
184
+ if total is not None:
185
+ self.total = kw['total'] = total
186
+ # if kw or total is not None:
187
+ # self.update(0, **kw)
188
+ # ---------------------------------------------------------------------------- #
189
+ # start task #
190
+ # ---------------------------------------------------------------------------- #
191
+ self.__enter__(**kw)
192
+ return self
193
+
194
+ # --------------------------------- iterable --------------------------------- #
195
+
196
+ self.total = kw['total'] = utils.try_len(iter, total)
197
+ def _iter():
198
+ # ---------------------------------------------------------------------------- #
199
+ # start task #
200
+ # ---------------------------------------------------------------------------- #
201
+ self.__enter__(**kw)
202
+ try:
203
+ for x in iter:
204
+ yield x
205
+ # ---------------------------------------------------------------------------- #
206
+ # update #
207
+ # ---------------------------------------------------------------------------- #
208
+ self.update()
209
+ finally:
210
+ # ---------------------------------------------------------------------------- #
211
+ # stop task #
212
+ # ---------------------------------------------------------------------------- #
213
+ self.__exit__(*sys.exc_info())
214
+ return _iter()
215
+
216
+ def _call(self, method, **kw):
217
+ self._queue.put((self.task_id, method, kw))
218
+
219
+ def set(self, value):
220
+ self.current = value
221
+ self.update(0)
222
+ return self
223
+
224
+ def update(self, n=1, **kw):
225
+ if not self._started:
226
+ self.__enter__(**kw)
227
+
228
+ if 'total' in kw:
229
+ self.total = kw['total']
230
+
231
+ # track all keyword arguments
232
+ self.kw = dict(self.kw, **kw)
233
+ kw = dict(self.kw)
234
+
235
+ # calculate task progress
236
+ self.current += n
237
+ total = self.total if self.current or not self.total else (self.current+1)
238
+ self.complete = total and self.current >= total
239
+ visible = bool(total and not self.complete or not kw.get('transient', True))
240
+ kw.setdefault('visible', visible)
241
+ kw.setdefault('total', total)
242
+ kw.setdefault('complete', self.complete)
243
+
244
+ # ---------------------------------------------------------------------------- #
245
+ # update #
246
+ # ---------------------------------------------------------------------------- #
247
+ self._call('update', completed=self.current, **kw)
248
+ return self
249
+
250
+
251
+
252
+ class Bar:
253
+ def __init__(self, desc=None, bytes=False, pbar=None, total=None, **kw):
254
+ if isinstance(desc, progress.Progress):
255
+ desc, pbar = None, desc
256
+ self.pbar = get_pbar(pbar, bytes=bytes)
257
+ # ---------------------------------------------------------------------------- #
258
+ # add task #
259
+ # ---------------------------------------------------------------------------- #
260
+ self.task_id = self.pbar.add_task(desc, start=total is not None, total=total, **kw)
261
+ self.pbar.__enter__()
262
+
263
+ def __enter__(self):
264
+ return self
265
+
266
+ def __exit__(self, c,v,t):
267
+ self.pbar.__exit__(c,v,t)
268
+
269
+ def __call__(self, iter, total=None, **kw):
270
+ with self.pbar:
271
+ # ---------------------------------------------------------------------------- #
272
+ # update total #
273
+ # ---------------------------------------------------------------------------- #
274
+ # set the initial total
275
+ self.update(0, total=utils.try_len(iter, total), **kw)
276
+ # loop through the elements
277
+ for i, x in enumerate(iter):
278
+ yield x
279
+ # ---------------------------------------------------------------------------- #
280
+ # update #
281
+ # ---------------------------------------------------------------------------- #
282
+ self.update()
283
+
284
+ def update(self, n=1, total=None, **kw):
285
+ # start the task if it hasn't been started yet
286
+ if total is not None:
287
+ # ---------------------------------------------------------------------------- #
288
+ # start task #
289
+ # ---------------------------------------------------------------------------- #
290
+ task = self.pbar._tasks[self.task_id]
291
+ if task.start_time is None:
292
+ self.pbar.start_task(self.task_id)
293
+ print('starting task', total, task, self.task_id)
294
+
295
+ # ---------------------------------------------------------------------------- #
296
+ # update #
297
+ # ---------------------------------------------------------------------------- #
298
+ self.pbar.update(self.task_id, advance=n, total=total, **kw)
299
+ return self
300
+
301
+ def close(self):
302
+ pass
303
+
304
+
305
+
306
+ # ---------------------------------------------------------------------------- #
307
+ # Examples #
308
+ # ---------------------------------------------------------------------------- #
309
+
310
+
311
+ def example_fn(i, pbar):
312
+ import random
313
+ from time import sleep
314
+ for i in pbar(range(i + 1)):
315
+ sleep(random.random()*2)
316
+
317
+ def my_work(n, pbar, sleep=0.2):
318
+ import time
319
+ for i in pbar(range(n), description=f'counting to {n}'):
320
+ time.sleep(sleep)
321
+
322
+
323
+ def my_other_work(n, pbar, sleep=0.2):
324
+ import time
325
+ time.sleep(1)
326
+ with pbar(description=f'counting to {n}', total=n):
327
+ for i in range(n):
328
+ pbar.update(0.5, description=f'Im counting - {n} ')
329
+ time.sleep(sleep/2)
330
+ pbar.update(0.5, description=f'Im counting - {n+0.5}')
331
+ time.sleep(sleep/2)
332
+
333
+
334
+ def example_run():
335
+ Bars.pool(
336
+ example_fn,
337
+ range(10),
338
+ # desc=lambda i: f"wowow {i} :o",
339
+ mainbar_kw={'transient': False},
340
+ subbar_kw={'transient': False},
341
+ n_workers=5)
342
+
343
+
344
+
345
+ if __name__ == '__main__':
346
+ import fire
347
+ fire.Fire(example_run)
@@ -0,0 +1,93 @@
1
+ import queue
2
+ import threading
3
+ import multiprocessing as mp
4
+ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
5
+ from rich import progress
6
+
7
+
8
+ def try_len(it, default):
9
+ try:
10
+ return len(it)
11
+ except TypeError:
12
+ return default
13
+
14
+
15
+ class ThreadQueue:
16
+ '''An event queue to respond to events in a separate thread.'''
17
+ def __init__(self, fn):
18
+ self._fn = fn
19
+ self._closed = False
20
+ self.queue = queue.Queue()
21
+ self._thread = threading.Thread(target=self._monitor, daemon=True)
22
+
23
+ def __enter__(self):
24
+ self._closed = False
25
+ self._thread.start()
26
+ return self
27
+
28
+ def __exit__(self, c,v,t):
29
+ self._closed = True
30
+ self._thread.join()
31
+
32
+ def _monitor(self):
33
+ while not self._closed:
34
+ try:
35
+ xs = self.queue.get(timeout=0.1)
36
+ self._fn(*xs)
37
+ except queue.Empty:
38
+ continue
39
+
40
+ class ProcessQueue:
41
+ '''An event queue to respond to events in a separate process.'''
42
+ def __init__(self, fn):
43
+ self._fn = fn
44
+ self._closed = False
45
+ self._manager = mp.Manager()
46
+ self.queue = self._manager.Queue()
47
+ self._thread = threading.Thread(target=self._monitor, daemon=True)
48
+
49
+ def put(self, *args, **kw):
50
+ self._queue.put(*args, **kw)
51
+
52
+ def get(self, *args, **kw):
53
+ return self._queue.get(*args, **kw)
54
+
55
+ def __enter__(self):
56
+ self._closed = False
57
+ self._manager.__enter__()
58
+ self._thread.start()
59
+ return self
60
+
61
+ def __exit__(self, c,v,t):
62
+ self._closed = True
63
+ self._thread.join()
64
+ self._manager.__exit__(c,v,t)
65
+
66
+ def _monitor(self):
67
+ while not self._closed:
68
+ try:
69
+ xs = self.queue.get(timeout=0.1)
70
+ self._fn(*xs)
71
+ except queue.Empty:
72
+ continue
73
+
74
+ POOL_QUEUES = {
75
+ 'thread': ThreadQueue,
76
+ 'process': ProcessQueue,
77
+ }
78
+
79
+ POOL_EXECUTORS = {
80
+ 'thread': ThreadPoolExecutor,
81
+ 'process': ProcessPoolExecutor,
82
+ }
83
+
84
+
85
+
86
+ class MofNColumn(progress.MofNCompleteColumn):
87
+ '''A progress column that shows the current vs. total count of items.'''
88
+ def render(self, task):
89
+ total = f'{int(task.total):,}' if task.total is not None else "?"
90
+ return progress.Text(
91
+ f"{int(task.completed):,d}{self.separator}{total}",
92
+ style="progress.download",
93
+ )
@@ -0,0 +1,79 @@
1
+ Metadata-Version: 2.1
2
+ Name: mqdm
3
+ Version: 0.0.1
4
+ Description-Content-Type: text/markdown
5
+ Requires-Dist: rich
6
+
7
+ # mqdm: progress bars for multiprocessing
8
+ Pretty progress bars with `rich`, in your child processes.
9
+
10
+ ## Install
11
+
12
+ ```bash
13
+ pip install mqdm
14
+ ```
15
+
16
+ ## Worker progress
17
+ ```python
18
+ import mqdm
19
+ import time
20
+
21
+ def my_work(n, sleep, pbar):
22
+ for i in pbar(range(n), description=f'counting to {n}'):
23
+ time.sleep(sleep)
24
+
25
+ # executes my task in a concurrent futures process pool
26
+ mqdm.pool(
27
+ my_work,
28
+ range(1, 10),
29
+ sleep=1,
30
+ n_workers=3,
31
+ )
32
+ ```
33
+
34
+ ![alt text](static/image.png)
35
+
36
+ ## Less high level please
37
+ Basically, the mechanics are this:
38
+ ```python
39
+ # use context manager to start background listener and message queue
40
+ with mqdm.Bars() as pbars:
41
+ # create progress bars and send them to the remote processes
42
+ pool.submit(my_work, 1, pbar=pbars.add())
43
+ pool.submit(my_work, 2, pbar=pbars.add())
44
+ pool.submit(my_work, 3, pbar=pbars.add())
45
+
46
+ # your worker function can look like this
47
+ def my_work(n, sleep, pbar):
48
+ for i in pbar(range(n), description=f'counting to {n}'):
49
+ time.sleep(sleep)
50
+
51
+ # or this
52
+ def my_work(n, pbar, sleep=0.2):
53
+ import time
54
+ with pbar(description=f'counting to {n}', total=n):
55
+ for i in range(n):
56
+ pbar.update(0.5, description=f'Im counting - {n} ')
57
+ time.sleep(sleep/2)
58
+ pbar.update(0.5, description=f'Im counting - {n+0.5}')
59
+ time.sleep(sleep/2)
60
+ ```
61
+
62
+ And you can use it in a pool like this:
63
+
64
+ ```python
65
+ import mqdm
66
+ from concurrent.futures import ProcessPoolExecutor, as_completed
67
+
68
+ items = range(1, 10)
69
+
70
+ with ProcessPoolExecutor(max_workers=n_workers) as pool, mqdm.Bars() as pbars:
71
+ futures = [
72
+ pool.submit(my_work, i, pbar=pbars.add())
73
+ for i in items
74
+ ]
75
+ for f in as_completed(futures):
76
+ print(f.result())
77
+ ```
78
+
79
+ It works by spawning a background thread with a multiprocessing queue. The Bars instance listens for messages from the progress bar proxies in the child processes.
@@ -0,0 +1,12 @@
1
+ README.md
2
+ setup.py
3
+ mqdm/__init__.py
4
+ mqdm/__main__.py
5
+ mqdm/_alt_dict.py
6
+ mqdm/core.py
7
+ mqdm/utils.py
8
+ mqdm.egg-info/PKG-INFO
9
+ mqdm.egg-info/SOURCES.txt
10
+ mqdm.egg-info/dependency_links.txt
11
+ mqdm.egg-info/requires.txt
12
+ mqdm.egg-info/top_level.txt
@@ -0,0 +1 @@
1
+ rich
@@ -0,0 +1 @@
1
+ mqdm
mqdm-0.0.1/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
mqdm-0.0.1/setup.py ADDED
@@ -0,0 +1,13 @@
1
+ import setuptools
2
+
3
+ setuptools.setup(
4
+ name='mqdm',
5
+ version='0.0.1',
6
+ description='',
7
+ long_description=open('README.md').read().strip(),
8
+ long_description_content_type='text/markdown',
9
+ packages=setuptools.find_packages(),
10
+ install_requires=[
11
+ 'rich',
12
+ ],
13
+ extras_require={})