mqdm 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mqdm-0.0.1/PKG-INFO +79 -0
- mqdm-0.0.1/README.md +73 -0
- mqdm-0.0.1/mqdm/__init__.py +3 -0
- mqdm-0.0.1/mqdm/__main__.py +4 -0
- mqdm-0.0.1/mqdm/_alt_dict.py +119 -0
- mqdm-0.0.1/mqdm/core.py +347 -0
- mqdm-0.0.1/mqdm/utils.py +93 -0
- mqdm-0.0.1/mqdm.egg-info/PKG-INFO +79 -0
- mqdm-0.0.1/mqdm.egg-info/SOURCES.txt +12 -0
- mqdm-0.0.1/mqdm.egg-info/dependency_links.txt +1 -0
- mqdm-0.0.1/mqdm.egg-info/requires.txt +1 -0
- mqdm-0.0.1/mqdm.egg-info/top_level.txt +1 -0
- mqdm-0.0.1/setup.cfg +4 -0
- mqdm-0.0.1/setup.py +13 -0
mqdm-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: mqdm
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Description-Content-Type: text/markdown
|
|
5
|
+
Requires-Dist: rich
|
|
6
|
+
|
|
7
|
+
# mqdm: progress bars for multiprocessing
|
|
8
|
+
Pretty progress bars with `rich`, in your child processes.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install mqdm
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Worker progress
|
|
17
|
+
```python
|
|
18
|
+
import mqdm
|
|
19
|
+
import time
|
|
20
|
+
|
|
21
|
+
def my_work(n, sleep, pbar):
|
|
22
|
+
for i in pbar(range(n), description=f'counting to {n}'):
|
|
23
|
+
time.sleep(sleep)
|
|
24
|
+
|
|
25
|
+
# executes my task in a concurrent futures process pool
|
|
26
|
+
mqdm.pool(
|
|
27
|
+
my_work,
|
|
28
|
+
range(1, 10),
|
|
29
|
+
sleep=1,
|
|
30
|
+
n_workers=3,
|
|
31
|
+
)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+

|
|
35
|
+
|
|
36
|
+
## Less high level please
|
|
37
|
+
Basically, the mechanics are this:
|
|
38
|
+
```python
|
|
39
|
+
# use context manager to start background listener and message queue
|
|
40
|
+
with mqdm.Bars() as pbars:
|
|
41
|
+
# create progress bars and send them to the remote processes
|
|
42
|
+
pool.submit(my_work, 1, pbar=pbars.add())
|
|
43
|
+
pool.submit(my_work, 2, pbar=pbars.add())
|
|
44
|
+
pool.submit(my_work, 3, pbar=pbars.add())
|
|
45
|
+
|
|
46
|
+
# your worker function can look like this
|
|
47
|
+
def my_work(n, sleep, pbar):
|
|
48
|
+
for i in pbar(range(n), description=f'counting to {n}'):
|
|
49
|
+
time.sleep(sleep)
|
|
50
|
+
|
|
51
|
+
# or this
|
|
52
|
+
def my_work(n, pbar, sleep=0.2):
|
|
53
|
+
import time
|
|
54
|
+
with pbar(description=f'counting to {n}', total=n):
|
|
55
|
+
for i in range(n):
|
|
56
|
+
pbar.update(0.5, description=f'Im counting - {n} ')
|
|
57
|
+
time.sleep(sleep/2)
|
|
58
|
+
pbar.update(0.5, description=f'Im counting - {n+0.5}')
|
|
59
|
+
time.sleep(sleep/2)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
And you can use it in a pool like this:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
import mqdm
|
|
66
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
67
|
+
|
|
68
|
+
items = range(1, 10)
|
|
69
|
+
|
|
70
|
+
with ProcessPoolExecutor(max_workers=n_workers) as pool, mqdm.Bars() as pbars:
|
|
71
|
+
futures = [
|
|
72
|
+
pool.submit(my_work, i, pbar=pbars.add())
|
|
73
|
+
for i in items
|
|
74
|
+
]
|
|
75
|
+
for f in as_completed(futures):
|
|
76
|
+
print(f.result())
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
It works by spawning a background thread with a multiprocessing queue. The Bars instance listens for messages from the progress bar proxies in the child processes.
|
mqdm-0.0.1/README.md
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# mqdm: progress bars for multiprocessing
|
|
2
|
+
Pretty progress bars with `rich`, in your child processes.
|
|
3
|
+
|
|
4
|
+
## Install
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
pip install mqdm
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
## Worker progress
|
|
11
|
+
```python
|
|
12
|
+
import mqdm
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
def my_work(n, sleep, pbar):
|
|
16
|
+
for i in pbar(range(n), description=f'counting to {n}'):
|
|
17
|
+
time.sleep(sleep)
|
|
18
|
+
|
|
19
|
+
# executes my task in a concurrent futures process pool
|
|
20
|
+
mqdm.pool(
|
|
21
|
+
my_work,
|
|
22
|
+
range(1, 10),
|
|
23
|
+
sleep=1,
|
|
24
|
+
n_workers=3,
|
|
25
|
+
)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+

|
|
29
|
+
|
|
30
|
+
## Less high level please
|
|
31
|
+
Basically, the mechanics are this:
|
|
32
|
+
```python
|
|
33
|
+
# use context manager to start background listener and message queue
|
|
34
|
+
with mqdm.Bars() as pbars:
|
|
35
|
+
# create progress bars and send them to the remote processes
|
|
36
|
+
pool.submit(my_work, 1, pbar=pbars.add())
|
|
37
|
+
pool.submit(my_work, 2, pbar=pbars.add())
|
|
38
|
+
pool.submit(my_work, 3, pbar=pbars.add())
|
|
39
|
+
|
|
40
|
+
# your worker function can look like this
|
|
41
|
+
def my_work(n, sleep, pbar):
|
|
42
|
+
for i in pbar(range(n), description=f'counting to {n}'):
|
|
43
|
+
time.sleep(sleep)
|
|
44
|
+
|
|
45
|
+
# or this
|
|
46
|
+
def my_work(n, pbar, sleep=0.2):
|
|
47
|
+
import time
|
|
48
|
+
with pbar(description=f'counting to {n}', total=n):
|
|
49
|
+
for i in range(n):
|
|
50
|
+
pbar.update(0.5, description=f'Im counting - {n} ')
|
|
51
|
+
time.sleep(sleep/2)
|
|
52
|
+
pbar.update(0.5, description=f'Im counting - {n+0.5}')
|
|
53
|
+
time.sleep(sleep/2)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
And you can use it in a pool like this:
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
import mqdm
|
|
60
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
61
|
+
|
|
62
|
+
items = range(1, 10)
|
|
63
|
+
|
|
64
|
+
with ProcessPoolExecutor(max_workers=n_workers) as pool, mqdm.Bars() as pbars:
|
|
65
|
+
futures = [
|
|
66
|
+
pool.submit(my_work, i, pbar=pbars.add())
|
|
67
|
+
for i in items
|
|
68
|
+
]
|
|
69
|
+
for f in as_completed(futures):
|
|
70
|
+
print(f.result())
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
It works by spawning a background thread with a multiprocessing queue. The Bars instance listens for messages from the progress bar proxies in the child processes.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
'''First draft implementation using manager dict.
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
'''
|
|
5
|
+
|
|
6
|
+
import random
|
|
7
|
+
from time import sleep
|
|
8
|
+
import threading
|
|
9
|
+
import multiprocessing
|
|
10
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
11
|
+
from rich import progress
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Bars:
|
|
15
|
+
def __init__(self, desc="[green]All jobs progress:") -> None:
|
|
16
|
+
self.desc = desc
|
|
17
|
+
self.pbar = progress.Progress(
|
|
18
|
+
"[progress.description]{task.description}",
|
|
19
|
+
progress.BarColumn(),
|
|
20
|
+
"[progress.percentage]{task.percentage:>3.0f}%",
|
|
21
|
+
progress.TimeRemainingColumn(),
|
|
22
|
+
progress.TimeElapsedColumn(),
|
|
23
|
+
refresh_per_second=8,
|
|
24
|
+
)
|
|
25
|
+
# Create and start a thread for reading from the queue
|
|
26
|
+
reader_thread = threading.Thread(target=queue_reader, args=(queue,))
|
|
27
|
+
reader_thread.start()
|
|
28
|
+
self.manager = multiprocessing.Manager()
|
|
29
|
+
|
|
30
|
+
def __enter__(self):
|
|
31
|
+
self.pbar.__enter__()
|
|
32
|
+
self.manager.__enter__()
|
|
33
|
+
self.tasks = self.manager.dict()
|
|
34
|
+
self.overall_task = self.pbar.add_task(self.desc)
|
|
35
|
+
return self
|
|
36
|
+
|
|
37
|
+
def __exit__(self, c,v,t):
|
|
38
|
+
self.pbar.__exit__(c,v,t)
|
|
39
|
+
self.manager.__exit__(c,v,t)
|
|
40
|
+
|
|
41
|
+
def add(self, title, visible=False, **kw):
|
|
42
|
+
task_id = self.pbar.add_task(title, visible=visible, start=False, **kw)
|
|
43
|
+
return Bar(self.tasks, task_id)
|
|
44
|
+
|
|
45
|
+
def update(self):
|
|
46
|
+
n_finished = 0
|
|
47
|
+
for task_id, data in self.tasks.items():
|
|
48
|
+
data = dict(data)
|
|
49
|
+
complete = data['total'] and data['completed'] >= data['total']
|
|
50
|
+
visible = bool(data['total'] and not complete)
|
|
51
|
+
data.setdefault('visible', visible)
|
|
52
|
+
n_finished += complete
|
|
53
|
+
# update the progress bar for this task:
|
|
54
|
+
self.pbar.update(task_id, **data)
|
|
55
|
+
self.pbar.update(self.overall_task, completed=n_finished, total=len(self.tasks))
|
|
56
|
+
|
|
57
|
+
def as_completed(self, futures, cycle=0.1):
|
|
58
|
+
while futures:
|
|
59
|
+
done = [futures.pop(i) for i, f in enumerate(futures) if f.done()]
|
|
60
|
+
for f in done:
|
|
61
|
+
yield f
|
|
62
|
+
self.update()
|
|
63
|
+
sleep(cycle)
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def map(cls, fn, xs, n_workers=8, desc=None, **kw):
|
|
67
|
+
futures = []
|
|
68
|
+
with cls() as pbars, ProcessPoolExecutor(max_workers=n_workers) as executor:
|
|
69
|
+
for i, x in enumerate(xs):
|
|
70
|
+
desc_i = desc(*x, **kw) if callable(desc) else desc or f'task {i}'
|
|
71
|
+
futures.append(executor.submit(fn, *x, pbar=pbars.add(desc_i), **kw))
|
|
72
|
+
for f in pbars.as_completed(futures):
|
|
73
|
+
result = f.result()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class Bar:
|
|
77
|
+
def __init__(self, tasks, task_id):
|
|
78
|
+
self.tasks = tasks
|
|
79
|
+
self.task_id = task_id
|
|
80
|
+
self.current = 0
|
|
81
|
+
self.total = 0
|
|
82
|
+
self.update(0)
|
|
83
|
+
|
|
84
|
+
def __call__(self, iter, total=None):
|
|
85
|
+
try:
|
|
86
|
+
self.total = len(iter)
|
|
87
|
+
except TypeError:
|
|
88
|
+
self.total = total
|
|
89
|
+
def _iter():
|
|
90
|
+
|
|
91
|
+
for x in iter:
|
|
92
|
+
yield x
|
|
93
|
+
self.update()
|
|
94
|
+
return _iter()
|
|
95
|
+
|
|
96
|
+
def set(self, value):
|
|
97
|
+
self.current = value
|
|
98
|
+
self.update(0)
|
|
99
|
+
return self
|
|
100
|
+
|
|
101
|
+
def update(self, n=1):
|
|
102
|
+
self.current += n
|
|
103
|
+
total = self.total if self.current or not self.total else (self.current+1)
|
|
104
|
+
self.tasks[self.task_id] = {"completed": self.current, "total": total}
|
|
105
|
+
return self
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def example_fn(i, pbar):
|
|
110
|
+
import random
|
|
111
|
+
for i in pbar(range(i + 1)):
|
|
112
|
+
sleep(random.random())
|
|
113
|
+
|
|
114
|
+
def pbar_test():
|
|
115
|
+
Bars.map(example_fn, [[i] for i in range(10)], n_workers=5)
|
|
116
|
+
if __name__ == '__main__':
|
|
117
|
+
|
|
118
|
+
import fire
|
|
119
|
+
fire.Fire(pbar_test)
|
mqdm-0.0.1/mqdm/core.py
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
''''''
|
|
2
|
+
import sys
|
|
3
|
+
import queue
|
|
4
|
+
import threading
|
|
5
|
+
import multiprocessing as mp
|
|
6
|
+
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
|
7
|
+
from rich import progress, print
|
|
8
|
+
from . import utils
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def get_pbar(pbar=None, bytes=False):
|
|
13
|
+
return pbar or progress.Progress(
|
|
14
|
+
"[progress.description]{task.description}",
|
|
15
|
+
progress.BarColumn(),
|
|
16
|
+
"[progress.percentage]{task.percentage:>3.0f}%",
|
|
17
|
+
*([progress.DownloadColumn()] if bytes else [utils.MofNColumn()]),
|
|
18
|
+
progress.TimeRemainingColumn(),
|
|
19
|
+
progress.TimeElapsedColumn(),
|
|
20
|
+
refresh_per_second=8,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
'''
|
|
25
|
+
-- Multi Process:
|
|
26
|
+
|
|
27
|
+
Bars(**kw) -> add_task(overall, **kw)
|
|
28
|
+
Bars.add(**kw) -> add_task(item, **kw)
|
|
29
|
+
|
|
30
|
+
RemoteBar__init__() -> --
|
|
31
|
+
RemoteBar.__call__(**kw) -> RemoteBar.__enter__(**kw)
|
|
32
|
+
RemoteBar.__enter__(**kw) -> start_task(item, **kw)
|
|
33
|
+
iter(RemoteBar(**kw)) -> start_task(item, **kw)
|
|
34
|
+
RemoteBar.update(**kw) -> update(item, **kw)
|
|
35
|
+
|
|
36
|
+
-- Single Process:
|
|
37
|
+
|
|
38
|
+
Bar(**kw) -> add_task(item, **kw)
|
|
39
|
+
Bar.__call__(**kw) -> iter(Bar(**kw))
|
|
40
|
+
Bar.update(**kw) -> update(item, **kw)
|
|
41
|
+
|
|
42
|
+
'''
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Bars:
|
|
46
|
+
def __init__(self, desc="", pbar=None, iter=None, total=None, pool_mode='process', **kw) -> None:
|
|
47
|
+
if isinstance(desc, progress.Progress):
|
|
48
|
+
desc, pbar = None, desc
|
|
49
|
+
self.desc = desc
|
|
50
|
+
self.pbar = get_pbar(pbar)
|
|
51
|
+
self._overall_kw = kw
|
|
52
|
+
self.total = utils.try_len(iter, total)
|
|
53
|
+
self._pq = utils.POOL_QUEUES[pool_mode](self._on_message)
|
|
54
|
+
self._tasks = {}
|
|
55
|
+
|
|
56
|
+
def __enter__(self):
|
|
57
|
+
self.pbar.__enter__()
|
|
58
|
+
self._pq.__enter__()
|
|
59
|
+
self._tasks = {}
|
|
60
|
+
# ---------------------------------------------------------------------------- #
|
|
61
|
+
# add overall task #
|
|
62
|
+
# ---------------------------------------------------------------------------- #
|
|
63
|
+
self.overall_task = self.pbar.add_task(self.desc, total=self.total, **self._overall_kw)
|
|
64
|
+
return self
|
|
65
|
+
|
|
66
|
+
def __exit__(self, c,v,t):
|
|
67
|
+
self._pq.__exit__(c,v,t)
|
|
68
|
+
self.pbar.__exit__(c,v,t)
|
|
69
|
+
|
|
70
|
+
def add(self, title, visible=False, **kw):
|
|
71
|
+
# ---------------------------------------------------------------------------- #
|
|
72
|
+
# add task #
|
|
73
|
+
# ---------------------------------------------------------------------------- #
|
|
74
|
+
task_id = self.pbar.add_task(title, visible=visible, start=False, **kw)
|
|
75
|
+
self._tasks[task_id] = {}
|
|
76
|
+
return RemoteBar(self._pq.queue, task_id)
|
|
77
|
+
|
|
78
|
+
def close(self):
|
|
79
|
+
self.__exit__(None,None,None)
|
|
80
|
+
|
|
81
|
+
def _on_message(self, task_id, method, data):
|
|
82
|
+
if method == 'update':
|
|
83
|
+
self._update(task_id, **data)
|
|
84
|
+
else:
|
|
85
|
+
getattr(self.pbar, method)(**data)
|
|
86
|
+
|
|
87
|
+
def _update(self, task_id, **data):
|
|
88
|
+
# update the task-specific progress bar
|
|
89
|
+
if task_id is not None:
|
|
90
|
+
self._tasks[task_id].update(**data)
|
|
91
|
+
data.pop('complete', None)
|
|
92
|
+
# ---------------------------------------------------------------------------- #
|
|
93
|
+
# update task #
|
|
94
|
+
# ---------------------------------------------------------------------------- #
|
|
95
|
+
self.pbar.update(task_id, **data)
|
|
96
|
+
|
|
97
|
+
# update the overall task progress bar
|
|
98
|
+
n_finished = sum(bool(d and d.get('complete', False)) for d in self._tasks.values())
|
|
99
|
+
# ---------------------------------------------------------------------------- #
|
|
100
|
+
# update overall #
|
|
101
|
+
# ---------------------------------------------------------------------------- #
|
|
102
|
+
self.pbar.update(self.overall_task, completed=n_finished, total=len(self._tasks))
|
|
103
|
+
|
|
104
|
+
@classmethod
|
|
105
|
+
def ipool(cls, fn, xs, *a, n_workers=8, desc=None, mainbar_kw=None, subbar_kw=None, pool_mode='process', **kw):
|
|
106
|
+
"""Execute a function in a process pool with a progress bar for each task."""
|
|
107
|
+
# get the arguments for each task
|
|
108
|
+
items = [x if isinstance(x, args) else args(x) for x in xs]
|
|
109
|
+
|
|
110
|
+
# no workers, just run the function
|
|
111
|
+
if n_workers < 2:
|
|
112
|
+
for i, arg in enumerate(items):
|
|
113
|
+
desc_i = arg(desc or f'task {i}')
|
|
114
|
+
yield arg(fn, *a, pbar=Bar, **kw)
|
|
115
|
+
return
|
|
116
|
+
|
|
117
|
+
# run the function in a process pool
|
|
118
|
+
futures = []
|
|
119
|
+
with utils.POOL_EXECUTORS[pool_mode](max_workers=n_workers) as executor, cls(pool_mode=pool_mode, **(mainbar_kw or {})) as pbars:
|
|
120
|
+
for i, arg in enumerate(items):
|
|
121
|
+
desc_i = arg(desc or f'task {i}')
|
|
122
|
+
pbar = pbars.add(desc_i, **(subbar_kw or {}))
|
|
123
|
+
futures.append(executor.submit(fn, *arg.a, *a, pbar=pbar, **dict(kw, **arg.kw)))
|
|
124
|
+
for f in as_completed(futures):
|
|
125
|
+
yield f.result()
|
|
126
|
+
|
|
127
|
+
@classmethod
|
|
128
|
+
def pool(cls, fn, xs, *a, n_workers=8, desc=None, **kw):
|
|
129
|
+
return list(cls.imap(fn, xs, *a, n_workers=n_workers, desc=desc, **kw))
|
|
130
|
+
|
|
131
|
+
# not sure which name is better
|
|
132
|
+
imap = ipool
|
|
133
|
+
map = pool
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class args:
|
|
137
|
+
def __init__(self, *a, **kw):
|
|
138
|
+
self.a = a
|
|
139
|
+
self.kw = kw
|
|
140
|
+
|
|
141
|
+
def __call__(self, fn, *a, **kw):
|
|
142
|
+
return fn(*self.a, *a, **dict(self.kw, **kw)) if callable(fn) else fn
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class RemoteBar:
|
|
146
|
+
def __init__(self, q, task_id):
|
|
147
|
+
self._queue = q
|
|
148
|
+
self.task_id = task_id
|
|
149
|
+
self.current = 0
|
|
150
|
+
self.total = 0
|
|
151
|
+
self.complete = False
|
|
152
|
+
self._started = False
|
|
153
|
+
self.kw = {}
|
|
154
|
+
self.update(0)
|
|
155
|
+
|
|
156
|
+
def __enter__(self, **kw):
|
|
157
|
+
# ---------------------------------------------------------------------------- #
|
|
158
|
+
# start task #
|
|
159
|
+
# ---------------------------------------------------------------------------- #
|
|
160
|
+
# start the task if it hasn't been started yet
|
|
161
|
+
if not self._started:
|
|
162
|
+
self._call('start_task', task_id=self.task_id, **kw)
|
|
163
|
+
self._started = True
|
|
164
|
+
return self
|
|
165
|
+
|
|
166
|
+
def __exit__(self, exc_type, exc_value, tb):
|
|
167
|
+
self.close()
|
|
168
|
+
|
|
169
|
+
def close(self):
|
|
170
|
+
self.update(0, complete=True)
|
|
171
|
+
# ---------------------------------------------------------------------------- #
|
|
172
|
+
# stop task #
|
|
173
|
+
# ---------------------------------------------------------------------------- #
|
|
174
|
+
if self._started:
|
|
175
|
+
self._call('stop_task', task_id=self.task_id)
|
|
176
|
+
self._started = False
|
|
177
|
+
return
|
|
178
|
+
|
|
179
|
+
def __call__(self, iter=None, total=None, **kw):
|
|
180
|
+
# if the first argument is a string, use it as the description
|
|
181
|
+
if isinstance(iter, str):
|
|
182
|
+
iter, kw['description'] = None, iter
|
|
183
|
+
if iter is None:
|
|
184
|
+
if total is not None:
|
|
185
|
+
self.total = kw['total'] = total
|
|
186
|
+
# if kw or total is not None:
|
|
187
|
+
# self.update(0, **kw)
|
|
188
|
+
# ---------------------------------------------------------------------------- #
|
|
189
|
+
# start task #
|
|
190
|
+
# ---------------------------------------------------------------------------- #
|
|
191
|
+
self.__enter__(**kw)
|
|
192
|
+
return self
|
|
193
|
+
|
|
194
|
+
# --------------------------------- iterable --------------------------------- #
|
|
195
|
+
|
|
196
|
+
self.total = kw['total'] = utils.try_len(iter, total)
|
|
197
|
+
def _iter():
|
|
198
|
+
# ---------------------------------------------------------------------------- #
|
|
199
|
+
# start task #
|
|
200
|
+
# ---------------------------------------------------------------------------- #
|
|
201
|
+
self.__enter__(**kw)
|
|
202
|
+
try:
|
|
203
|
+
for x in iter:
|
|
204
|
+
yield x
|
|
205
|
+
# ---------------------------------------------------------------------------- #
|
|
206
|
+
# update #
|
|
207
|
+
# ---------------------------------------------------------------------------- #
|
|
208
|
+
self.update()
|
|
209
|
+
finally:
|
|
210
|
+
# ---------------------------------------------------------------------------- #
|
|
211
|
+
# stop task #
|
|
212
|
+
# ---------------------------------------------------------------------------- #
|
|
213
|
+
self.__exit__(*sys.exc_info())
|
|
214
|
+
return _iter()
|
|
215
|
+
|
|
216
|
+
def _call(self, method, **kw):
|
|
217
|
+
self._queue.put((self.task_id, method, kw))
|
|
218
|
+
|
|
219
|
+
def set(self, value):
|
|
220
|
+
self.current = value
|
|
221
|
+
self.update(0)
|
|
222
|
+
return self
|
|
223
|
+
|
|
224
|
+
def update(self, n=1, **kw):
|
|
225
|
+
if not self._started:
|
|
226
|
+
self.__enter__(**kw)
|
|
227
|
+
|
|
228
|
+
if 'total' in kw:
|
|
229
|
+
self.total = kw['total']
|
|
230
|
+
|
|
231
|
+
# track all keyword arguments
|
|
232
|
+
self.kw = dict(self.kw, **kw)
|
|
233
|
+
kw = dict(self.kw)
|
|
234
|
+
|
|
235
|
+
# calculate task progress
|
|
236
|
+
self.current += n
|
|
237
|
+
total = self.total if self.current or not self.total else (self.current+1)
|
|
238
|
+
self.complete = total and self.current >= total
|
|
239
|
+
visible = bool(total and not self.complete or not kw.get('transient', True))
|
|
240
|
+
kw.setdefault('visible', visible)
|
|
241
|
+
kw.setdefault('total', total)
|
|
242
|
+
kw.setdefault('complete', self.complete)
|
|
243
|
+
|
|
244
|
+
# ---------------------------------------------------------------------------- #
|
|
245
|
+
# update #
|
|
246
|
+
# ---------------------------------------------------------------------------- #
|
|
247
|
+
self._call('update', completed=self.current, **kw)
|
|
248
|
+
return self
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
class Bar:
|
|
253
|
+
def __init__(self, desc=None, bytes=False, pbar=None, total=None, **kw):
|
|
254
|
+
if isinstance(desc, progress.Progress):
|
|
255
|
+
desc, pbar = None, desc
|
|
256
|
+
self.pbar = get_pbar(pbar, bytes=bytes)
|
|
257
|
+
# ---------------------------------------------------------------------------- #
|
|
258
|
+
# add task #
|
|
259
|
+
# ---------------------------------------------------------------------------- #
|
|
260
|
+
self.task_id = self.pbar.add_task(desc, start=total is not None, total=total, **kw)
|
|
261
|
+
self.pbar.__enter__()
|
|
262
|
+
|
|
263
|
+
def __enter__(self):
|
|
264
|
+
return self
|
|
265
|
+
|
|
266
|
+
def __exit__(self, c,v,t):
|
|
267
|
+
self.pbar.__exit__(c,v,t)
|
|
268
|
+
|
|
269
|
+
def __call__(self, iter, total=None, **kw):
|
|
270
|
+
with self.pbar:
|
|
271
|
+
# ---------------------------------------------------------------------------- #
|
|
272
|
+
# update total #
|
|
273
|
+
# ---------------------------------------------------------------------------- #
|
|
274
|
+
# set the initial total
|
|
275
|
+
self.update(0, total=utils.try_len(iter, total), **kw)
|
|
276
|
+
# loop through the elements
|
|
277
|
+
for i, x in enumerate(iter):
|
|
278
|
+
yield x
|
|
279
|
+
# ---------------------------------------------------------------------------- #
|
|
280
|
+
# update #
|
|
281
|
+
# ---------------------------------------------------------------------------- #
|
|
282
|
+
self.update()
|
|
283
|
+
|
|
284
|
+
def update(self, n=1, total=None, **kw):
|
|
285
|
+
# start the task if it hasn't been started yet
|
|
286
|
+
if total is not None:
|
|
287
|
+
# ---------------------------------------------------------------------------- #
|
|
288
|
+
# start task #
|
|
289
|
+
# ---------------------------------------------------------------------------- #
|
|
290
|
+
task = self.pbar._tasks[self.task_id]
|
|
291
|
+
if task.start_time is None:
|
|
292
|
+
self.pbar.start_task(self.task_id)
|
|
293
|
+
print('starting task', total, task, self.task_id)
|
|
294
|
+
|
|
295
|
+
# ---------------------------------------------------------------------------- #
|
|
296
|
+
# update #
|
|
297
|
+
# ---------------------------------------------------------------------------- #
|
|
298
|
+
self.pbar.update(self.task_id, advance=n, total=total, **kw)
|
|
299
|
+
return self
|
|
300
|
+
|
|
301
|
+
def close(self):
|
|
302
|
+
pass
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
# ---------------------------------------------------------------------------- #
|
|
307
|
+
# Examples #
|
|
308
|
+
# ---------------------------------------------------------------------------- #
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def example_fn(i, pbar):
|
|
312
|
+
import random
|
|
313
|
+
from time import sleep
|
|
314
|
+
for i in pbar(range(i + 1)):
|
|
315
|
+
sleep(random.random()*2)
|
|
316
|
+
|
|
317
|
+
def my_work(n, pbar, sleep=0.2):
|
|
318
|
+
import time
|
|
319
|
+
for i in pbar(range(n), description=f'counting to {n}'):
|
|
320
|
+
time.sleep(sleep)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def my_other_work(n, pbar, sleep=0.2):
|
|
324
|
+
import time
|
|
325
|
+
time.sleep(1)
|
|
326
|
+
with pbar(description=f'counting to {n}', total=n):
|
|
327
|
+
for i in range(n):
|
|
328
|
+
pbar.update(0.5, description=f'Im counting - {n} ')
|
|
329
|
+
time.sleep(sleep/2)
|
|
330
|
+
pbar.update(0.5, description=f'Im counting - {n+0.5}')
|
|
331
|
+
time.sleep(sleep/2)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def example_run():
|
|
335
|
+
Bars.pool(
|
|
336
|
+
example_fn,
|
|
337
|
+
range(10),
|
|
338
|
+
# desc=lambda i: f"wowow {i} :o",
|
|
339
|
+
mainbar_kw={'transient': False},
|
|
340
|
+
subbar_kw={'transient': False},
|
|
341
|
+
n_workers=5)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
if __name__ == '__main__':
|
|
346
|
+
import fire
|
|
347
|
+
fire.Fire(example_run)
|
mqdm-0.0.1/mqdm/utils.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import queue
|
|
2
|
+
import threading
|
|
3
|
+
import multiprocessing as mp
|
|
4
|
+
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
|
5
|
+
from rich import progress
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def try_len(it, default):
|
|
9
|
+
try:
|
|
10
|
+
return len(it)
|
|
11
|
+
except TypeError:
|
|
12
|
+
return default
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ThreadQueue:
|
|
16
|
+
'''An event queue to respond to events in a separate thread.'''
|
|
17
|
+
def __init__(self, fn):
|
|
18
|
+
self._fn = fn
|
|
19
|
+
self._closed = False
|
|
20
|
+
self.queue = queue.Queue()
|
|
21
|
+
self._thread = threading.Thread(target=self._monitor, daemon=True)
|
|
22
|
+
|
|
23
|
+
def __enter__(self):
|
|
24
|
+
self._closed = False
|
|
25
|
+
self._thread.start()
|
|
26
|
+
return self
|
|
27
|
+
|
|
28
|
+
def __exit__(self, c,v,t):
|
|
29
|
+
self._closed = True
|
|
30
|
+
self._thread.join()
|
|
31
|
+
|
|
32
|
+
def _monitor(self):
|
|
33
|
+
while not self._closed:
|
|
34
|
+
try:
|
|
35
|
+
xs = self.queue.get(timeout=0.1)
|
|
36
|
+
self._fn(*xs)
|
|
37
|
+
except queue.Empty:
|
|
38
|
+
continue
|
|
39
|
+
|
|
40
|
+
class ProcessQueue:
|
|
41
|
+
'''An event queue to respond to events in a separate process.'''
|
|
42
|
+
def __init__(self, fn):
|
|
43
|
+
self._fn = fn
|
|
44
|
+
self._closed = False
|
|
45
|
+
self._manager = mp.Manager()
|
|
46
|
+
self.queue = self._manager.Queue()
|
|
47
|
+
self._thread = threading.Thread(target=self._monitor, daemon=True)
|
|
48
|
+
|
|
49
|
+
def put(self, *args, **kw):
|
|
50
|
+
self._queue.put(*args, **kw)
|
|
51
|
+
|
|
52
|
+
def get(self, *args, **kw):
|
|
53
|
+
return self._queue.get(*args, **kw)
|
|
54
|
+
|
|
55
|
+
def __enter__(self):
|
|
56
|
+
self._closed = False
|
|
57
|
+
self._manager.__enter__()
|
|
58
|
+
self._thread.start()
|
|
59
|
+
return self
|
|
60
|
+
|
|
61
|
+
def __exit__(self, c,v,t):
|
|
62
|
+
self._closed = True
|
|
63
|
+
self._thread.join()
|
|
64
|
+
self._manager.__exit__(c,v,t)
|
|
65
|
+
|
|
66
|
+
def _monitor(self):
|
|
67
|
+
while not self._closed:
|
|
68
|
+
try:
|
|
69
|
+
xs = self.queue.get(timeout=0.1)
|
|
70
|
+
self._fn(*xs)
|
|
71
|
+
except queue.Empty:
|
|
72
|
+
continue
|
|
73
|
+
|
|
74
|
+
POOL_QUEUES = {
|
|
75
|
+
'thread': ThreadQueue,
|
|
76
|
+
'process': ProcessQueue,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
POOL_EXECUTORS = {
|
|
80
|
+
'thread': ThreadPoolExecutor,
|
|
81
|
+
'process': ProcessPoolExecutor,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class MofNColumn(progress.MofNCompleteColumn):
|
|
87
|
+
'''A progress column that shows the current vs. total count of items.'''
|
|
88
|
+
def render(self, task):
|
|
89
|
+
total = f'{int(task.total):,}' if task.total is not None else "?"
|
|
90
|
+
return progress.Text(
|
|
91
|
+
f"{int(task.completed):,d}{self.separator}{total}",
|
|
92
|
+
style="progress.download",
|
|
93
|
+
)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: mqdm
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Description-Content-Type: text/markdown
|
|
5
|
+
Requires-Dist: rich
|
|
6
|
+
|
|
7
|
+
# mqdm: progress bars for multiprocessing
|
|
8
|
+
Pretty progress bars with `rich`, in your child processes.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install mqdm
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Worker progress
|
|
17
|
+
```python
|
|
18
|
+
import mqdm
|
|
19
|
+
import time
|
|
20
|
+
|
|
21
|
+
def my_work(n, sleep, pbar):
|
|
22
|
+
for i in pbar(range(n), description=f'counting to {n}'):
|
|
23
|
+
time.sleep(sleep)
|
|
24
|
+
|
|
25
|
+
# executes my task in a concurrent futures process pool
|
|
26
|
+
mqdm.pool(
|
|
27
|
+
my_work,
|
|
28
|
+
range(1, 10),
|
|
29
|
+
sleep=1,
|
|
30
|
+
n_workers=3,
|
|
31
|
+
)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+

|
|
35
|
+
|
|
36
|
+
## Less high level please
|
|
37
|
+
Basically, the mechanics are this:
|
|
38
|
+
```python
|
|
39
|
+
# use context manager to start background listener and message queue
|
|
40
|
+
with mqdm.Bars() as pbars:
|
|
41
|
+
# create progress bars and send them to the remote processes
|
|
42
|
+
pool.submit(my_work, 1, pbar=pbars.add())
|
|
43
|
+
pool.submit(my_work, 2, pbar=pbars.add())
|
|
44
|
+
pool.submit(my_work, 3, pbar=pbars.add())
|
|
45
|
+
|
|
46
|
+
# your worker function can look like this
|
|
47
|
+
def my_work(n, sleep, pbar):
|
|
48
|
+
for i in pbar(range(n), description=f'counting to {n}'):
|
|
49
|
+
time.sleep(sleep)
|
|
50
|
+
|
|
51
|
+
# or this
|
|
52
|
+
def my_work(n, pbar, sleep=0.2):
|
|
53
|
+
import time
|
|
54
|
+
with pbar(description=f'counting to {n}', total=n):
|
|
55
|
+
for i in range(n):
|
|
56
|
+
pbar.update(0.5, description=f'Im counting - {n} ')
|
|
57
|
+
time.sleep(sleep/2)
|
|
58
|
+
pbar.update(0.5, description=f'Im counting - {n+0.5}')
|
|
59
|
+
time.sleep(sleep/2)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
And you can use it in a pool like this:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
import mqdm
|
|
66
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
67
|
+
|
|
68
|
+
items = range(1, 10)
|
|
69
|
+
|
|
70
|
+
with ProcessPoolExecutor(max_workers=n_workers) as pool, mqdm.Bars() as pbars:
|
|
71
|
+
futures = [
|
|
72
|
+
pool.submit(my_work, i, pbar=pbars.add())
|
|
73
|
+
for i in items
|
|
74
|
+
]
|
|
75
|
+
for f in as_completed(futures):
|
|
76
|
+
print(f.result())
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
It works by spawning a background thread with a multiprocessing queue. The Bars instance listens for messages from the progress bar proxies in the child processes.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
setup.py
|
|
3
|
+
mqdm/__init__.py
|
|
4
|
+
mqdm/__main__.py
|
|
5
|
+
mqdm/_alt_dict.py
|
|
6
|
+
mqdm/core.py
|
|
7
|
+
mqdm/utils.py
|
|
8
|
+
mqdm.egg-info/PKG-INFO
|
|
9
|
+
mqdm.egg-info/SOURCES.txt
|
|
10
|
+
mqdm.egg-info/dependency_links.txt
|
|
11
|
+
mqdm.egg-info/requires.txt
|
|
12
|
+
mqdm.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
rich
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mqdm
|
mqdm-0.0.1/setup.cfg
ADDED
mqdm-0.0.1/setup.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import setuptools
|
|
2
|
+
|
|
3
|
+
setuptools.setup(
|
|
4
|
+
name='mqdm',
|
|
5
|
+
version='0.0.1',
|
|
6
|
+
description='',
|
|
7
|
+
long_description=open('README.md').read().strip(),
|
|
8
|
+
long_description_content_type='text/markdown',
|
|
9
|
+
packages=setuptools.find_packages(),
|
|
10
|
+
install_requires=[
|
|
11
|
+
'rich',
|
|
12
|
+
],
|
|
13
|
+
extras_require={})
|