dejaq 0.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dejaq-0.0.6/LICENSE.md +21 -0
- dejaq-0.0.6/PKG-INFO +136 -0
- dejaq-0.0.6/README.md +124 -0
- dejaq-0.0.6/dejaq/__init__.py +2 -0
- dejaq-0.0.6/dejaq/parallel.py +128 -0
- dejaq-0.0.6/dejaq/queues.py +273 -0
- dejaq-0.0.6/dejaq.egg-info/PKG-INFO +136 -0
- dejaq-0.0.6/dejaq.egg-info/SOURCES.txt +10 -0
- dejaq-0.0.6/dejaq.egg-info/dependency_links.txt +1 -0
- dejaq-0.0.6/dejaq.egg-info/top_level.txt +1 -0
- dejaq-0.0.6/pyproject.toml +22 -0
- dejaq-0.0.6/setup.cfg +4 -0
dejaq-0.0.6/LICENSE.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2019-2024 jlab.berlin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
dejaq-0.0.6/PKG-INFO
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: dejaq
|
|
3
|
+
Version: 0.0.6
|
|
4
|
+
Summary: Déjà Queue – A fast multiprocessing queue for Python
|
|
5
|
+
Author: jlab.berlin, Benjamin Judkewitz
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/danionella/dejaq
|
|
8
|
+
Keywords: multiprocessing,queue,shared_memory
|
|
9
|
+
Requires-Python: >=3.8
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE.md
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+

|
|
15
|
+
[](https://pypi.org/project/dejaq/)
|
|
16
|
+
[](https://anaconda.org/danionella/dejaq)
|
|
17
|
+
[](https://opensource.org/licenses/MIT)
|
|
18
|
+

|
|
19
|
+
|
|
20
|
+
# Déjà Queue
|
|
21
|
+
|
|
22
|
+
A drop-in replacement for `multiprocessing.Queue`. Faster, because it takes advantage of a shared memory ring buffer (rather than slow pipes) and [pickle protocol 5 out-of-band data](https://peps.python.org/pep-0574/) to minimize copies. `DejaQueue` supports any type of [picklable](https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled) Python object, including numpy arrays or nested dictionaries with mixed content.
|
|
23
|
+
|
|
24
|
+
The speed of `DejaQueue` enables efficient inter-job communication in data processing pipelines, which can be implemented in a few lines of code with [`dejaq.Parallel`](#dejaqparallel).
|
|
25
|
+
|
|
26
|
+
Auto-generated API documentation: https://danionella.github.io/dejaq
|
|
27
|
+
|
|
28
|
+
## Examples
|
|
29
|
+
### `dejaq.DejaQueue`
|
|
30
|
+
```python
|
|
31
|
+
import numpy as np
|
|
32
|
+
from multiprocessing import Process
|
|
33
|
+
from dejaq import DejaQueue
|
|
34
|
+
|
|
35
|
+
def produce(queue):
|
|
36
|
+
for i in range(20):
|
|
37
|
+
random_shape = np.random.randint(5,10, size=3)
|
|
38
|
+
array = np.random.randn(*random_shape)
|
|
39
|
+
queue.put(array, meta=i)
|
|
40
|
+
print(f'produced {type(array)} {array.shape} {array.dtype}; meta: {i}; hash: {hash(array.tobytes())}\n')
|
|
41
|
+
|
|
42
|
+
def consume(queue, pid):
|
|
43
|
+
while True:
|
|
44
|
+
array, meta = queue.get()
|
|
45
|
+
print(f'consumer {pid} consumed {type(array)} {array.shape} {array.dtype}; meta: {meta}; hash: {hash(array.tobytes())}\n')
|
|
46
|
+
|
|
47
|
+
queue = DejaQueue(bytes=10e6)
|
|
48
|
+
producer = Process(target=produce, args=(queue,))
|
|
49
|
+
consumers = [Process(target=consume, args=(queue, pid)) for pid in range(3)]
|
|
50
|
+
for c in consumers:
|
|
51
|
+
c.start()
|
|
52
|
+
producer.start()
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
### `dejaq.Parallel`
|
|
57
|
+
The following examples show how to use `dejaq.Parallel` to parallelize a function or a class, and how to create job pipelines.
|
|
58
|
+
|
|
59
|
+
Here we execute a function and map iterable inputs across 10 workers. To enable pipelining, the results of each stage are provided as iterable generator. Use the `.compute()` method to get the final result (note that each stage pre-fetches results from `n_workers` calls, so some of the execution already starts before `.compute`)
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from time import sleep
|
|
63
|
+
from dejaq import Parallel
|
|
64
|
+
|
|
65
|
+
def slow_function(arg):
|
|
66
|
+
sleep(1.0)
|
|
67
|
+
return arg + 5
|
|
68
|
+
|
|
69
|
+
input_iterable = range(100)
|
|
70
|
+
slow_function = Parallel(n_workers=10)(slow_function)
|
|
71
|
+
stage = slow_function(input_iterable)
|
|
72
|
+
result = stage.compute() # or list(stage)
|
|
73
|
+
# or shorter:
|
|
74
|
+
result = Parallel(n_workers=10)(slow_function)(input_iterable).compute()
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
You can also use `Parallel` as a function decorator:
|
|
78
|
+
```python
|
|
79
|
+
@Parallel(n_workers=10)
|
|
80
|
+
def slow_function_decorated(arg):
|
|
81
|
+
sleep(1.0)
|
|
82
|
+
return arg + 5
|
|
83
|
+
|
|
84
|
+
result = slow_function_decorated(input_iterable).compute()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Similarly, you can decorate a class. It will be instantiated within a worker. Iterable items will be fed to the `__call__` method. Note how the additional init arguments are provided:
|
|
88
|
+
```python
|
|
89
|
+
@Parallel(n_workers=1)
|
|
90
|
+
class Reader:
|
|
91
|
+
def __init__(self, arg1):
|
|
92
|
+
self.arg1 = arg1
|
|
93
|
+
def __call__(self, item):
|
|
94
|
+
return item + self.arg1
|
|
95
|
+
|
|
96
|
+
result = Reader(arg1=0.5)(input_iterable).compute()
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Finally, you can create pipelines of chained jobs. In this example, we have a single threaded reader and consumer, but a parallel processing stage (an example use case is sequentially reading a file, compressing chunks in parallel and then sequentially writing to an output file):
|
|
100
|
+
```python
|
|
101
|
+
@Parallel(n_workers=1)
|
|
102
|
+
class Producer:
|
|
103
|
+
def __init__(self, arg1):
|
|
104
|
+
self.arg1 = arg1
|
|
105
|
+
def __call__(self, item):
|
|
106
|
+
return item + self.arg1
|
|
107
|
+
|
|
108
|
+
@Parallel(n_workers=10)
|
|
109
|
+
class Processor:
|
|
110
|
+
def __init__(self, arg1):
|
|
111
|
+
self.arg1 = arg1
|
|
112
|
+
def __call__(self, arg):
|
|
113
|
+
sleep(1.0) #simulating a slow function
|
|
114
|
+
return arg * self.arg1
|
|
115
|
+
|
|
116
|
+
@Parallel(n_workers=1)
|
|
117
|
+
class Consumer:
|
|
118
|
+
def __init__(self, arg1):
|
|
119
|
+
self.arg1 = arg1
|
|
120
|
+
def __call__(self, arg):
|
|
121
|
+
return arg - self.arg1
|
|
122
|
+
|
|
123
|
+
input_iterable = range(100)
|
|
124
|
+
stage1 = Producer(0.5)(input_iterable)
|
|
125
|
+
stage2 = Processor(10.0)(stage1)
|
|
126
|
+
stage3 = Consumer(1000)(stage2)
|
|
127
|
+
result = stage3.compute()
|
|
128
|
+
|
|
129
|
+
# or:
|
|
130
|
+
result = Consumer(1000)(Processor(10.0)(Producer(0.5)(input_iterable))).compute()
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
# See also
|
|
134
|
+
- [ArrayQueues](https://github.com/portugueslab/arrayqueues)
|
|
135
|
+
- [joblib.Parallel](https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html)
|
|
136
|
+
- [Déjà Q](https://en.wikipedia.org/wiki/Deja_Q)
|
dejaq-0.0.6/README.md
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
|
|
2
|
+

|
|
3
|
+
[](https://pypi.org/project/dejaq/)
|
|
4
|
+
[](https://anaconda.org/danionella/dejaq)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
# Déjà Queue
|
|
9
|
+
|
|
10
|
+
A drop-in replacement for `multiprocessing.Queue`. Faster, because it takes advantage of a shared memory ring buffer (rather than slow pipes) and [pickle protocol 5 out-of-band data](https://peps.python.org/pep-0574/) to minimize copies. `DejaQueue` supports any type of [picklable](https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled) Python object, including numpy arrays or nested dictionaries with mixed content.
|
|
11
|
+
|
|
12
|
+
The speed of `DejaQueue` enables efficient inter-job communication in data processing pipelines, which can be implemented in a few lines of code with [`dejaq.Parallel`](#dejaqparallel).
|
|
13
|
+
|
|
14
|
+
Auto-generated API documentation: https://danionella.github.io/dejaq
|
|
15
|
+
|
|
16
|
+
## Examples
|
|
17
|
+
### `dejaq.DejaQueue`
|
|
18
|
+
```python
|
|
19
|
+
import numpy as np
|
|
20
|
+
from multiprocessing import Process
|
|
21
|
+
from dejaq import DejaQueue
|
|
22
|
+
|
|
23
|
+
def produce(queue):
|
|
24
|
+
for i in range(20):
|
|
25
|
+
random_shape = np.random.randint(5,10, size=3)
|
|
26
|
+
array = np.random.randn(*random_shape)
|
|
27
|
+
queue.put(array, meta=i)
|
|
28
|
+
print(f'produced {type(array)} {array.shape} {array.dtype}; meta: {i}; hash: {hash(array.tobytes())}\n')
|
|
29
|
+
|
|
30
|
+
def consume(queue, pid):
|
|
31
|
+
while True:
|
|
32
|
+
array, meta = queue.get()
|
|
33
|
+
print(f'consumer {pid} consumed {type(array)} {array.shape} {array.dtype}; meta: {meta}; hash: {hash(array.tobytes())}\n')
|
|
34
|
+
|
|
35
|
+
queue = DejaQueue(bytes=10e6)
|
|
36
|
+
producer = Process(target=produce, args=(queue,))
|
|
37
|
+
consumers = [Process(target=consume, args=(queue, pid)) for pid in range(3)]
|
|
38
|
+
for c in consumers:
|
|
39
|
+
c.start()
|
|
40
|
+
producer.start()
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
### `dejaq.Parallel`
|
|
45
|
+
The following examples show how to use `dejaq.Parallel` to parallelize a function or a class, and how to create job pipelines.
|
|
46
|
+
|
|
47
|
+
Here we execute a function and map iterable inputs across 10 workers. To enable pipelining, the results of each stage are provided as iterable generator. Use the `.compute()` method to get the final result (note that each stage pre-fetches results from `n_workers` calls, so some of the execution already starts before `.compute`)
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from time import sleep
|
|
51
|
+
from dejaq import Parallel
|
|
52
|
+
|
|
53
|
+
def slow_function(arg):
|
|
54
|
+
sleep(1.0)
|
|
55
|
+
return arg + 5
|
|
56
|
+
|
|
57
|
+
input_iterable = range(100)
|
|
58
|
+
slow_function = Parallel(n_workers=10)(slow_function)
|
|
59
|
+
stage = slow_function(input_iterable)
|
|
60
|
+
result = stage.compute() # or list(stage)
|
|
61
|
+
# or shorter:
|
|
62
|
+
result = Parallel(n_workers=10)(slow_function)(input_iterable).compute()
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
You can also use `Parallel` as a function decorator:
|
|
66
|
+
```python
|
|
67
|
+
@Parallel(n_workers=10)
|
|
68
|
+
def slow_function_decorated(arg):
|
|
69
|
+
sleep(1.0)
|
|
70
|
+
return arg + 5
|
|
71
|
+
|
|
72
|
+
result = slow_function_decorated(input_iterable).compute()
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Similarly, you can decorate a class. It will be instantiated within a worker. Iterable items will be fed to the `__call__` method. Note how the additional init arguments are provided:
|
|
76
|
+
```python
|
|
77
|
+
@Parallel(n_workers=1)
|
|
78
|
+
class Reader:
|
|
79
|
+
def __init__(self, arg1):
|
|
80
|
+
self.arg1 = arg1
|
|
81
|
+
def __call__(self, item):
|
|
82
|
+
return item + self.arg1
|
|
83
|
+
|
|
84
|
+
result = Reader(arg1=0.5)(input_iterable).compute()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Finally, you can create pipelines of chained jobs. In this example, we have a single threaded reader and consumer, but a parallel processing stage (an example use case is sequentially reading a file, compressing chunks in parallel and then sequentially writing to an output file):
|
|
88
|
+
```python
|
|
89
|
+
@Parallel(n_workers=1)
|
|
90
|
+
class Producer:
|
|
91
|
+
def __init__(self, arg1):
|
|
92
|
+
self.arg1 = arg1
|
|
93
|
+
def __call__(self, item):
|
|
94
|
+
return item + self.arg1
|
|
95
|
+
|
|
96
|
+
@Parallel(n_workers=10)
|
|
97
|
+
class Processor:
|
|
98
|
+
def __init__(self, arg1):
|
|
99
|
+
self.arg1 = arg1
|
|
100
|
+
def __call__(self, arg):
|
|
101
|
+
sleep(1.0) #simulating a slow function
|
|
102
|
+
return arg * self.arg1
|
|
103
|
+
|
|
104
|
+
@Parallel(n_workers=1)
|
|
105
|
+
class Consumer:
|
|
106
|
+
def __init__(self, arg1):
|
|
107
|
+
self.arg1 = arg1
|
|
108
|
+
def __call__(self, arg):
|
|
109
|
+
return arg - self.arg1
|
|
110
|
+
|
|
111
|
+
input_iterable = range(100)
|
|
112
|
+
stage1 = Producer(0.5)(input_iterable)
|
|
113
|
+
stage2 = Processor(10.0)(stage1)
|
|
114
|
+
stage3 = Consumer(1000)(stage2)
|
|
115
|
+
result = stage3.compute()
|
|
116
|
+
|
|
117
|
+
# or:
|
|
118
|
+
result = Consumer(1000)(Processor(10.0)(Producer(0.5)(input_iterable))).compute()
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
# See also
|
|
122
|
+
- [ArrayQueues](https://github.com/portugueslab/arrayqueues)
|
|
123
|
+
- [joblib.Parallel](https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html)
|
|
124
|
+
- [Déjà Q](https://en.wikipedia.org/wiki/Deja_Q)
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import multiprocessing as mp
|
|
3
|
+
|
|
4
|
+
from . import DejaQueue
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class lazymap:
|
|
8
|
+
''' Returns an Iterable, functionally related to map (except that outputs are calculated by a pool of processes).
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
fcn (callable): function that is being mapped. Signature: fcn(item, **kwargs)
|
|
12
|
+
it (iterable): iterable that maps over the function, providing items as arguments
|
|
13
|
+
n_workers (int): number of workers (default: 1)
|
|
14
|
+
buffer_bytes (int): size of the queue buffer (default: 10e6 bytes)
|
|
15
|
+
**kwargs: optional, being passed to fcn
|
|
16
|
+
|
|
17
|
+
Returns:
|
|
18
|
+
(iterable): an iterable that returns the results of fcn(item) for each item in it
|
|
19
|
+
'''
|
|
20
|
+
|
|
21
|
+
def __init__(self, fcn, it, n_workers=1, buffer_bytes=10e6, **kwargs):
|
|
22
|
+
self.it = it
|
|
23
|
+
self.in_queue = DejaQueue(buffer_bytes)
|
|
24
|
+
self.out_queue = DejaQueue(buffer_bytes)
|
|
25
|
+
self.k = mp.Value("l", 0)
|
|
26
|
+
self.k_changed = mp.Condition()
|
|
27
|
+
self.n_workers = n_workers
|
|
28
|
+
self.workers = [mp.Process(target=self._worker_fcn, args=(pid, fcn), kwargs=kwargs) for pid in range(n_workers)]
|
|
29
|
+
[w.start() for w in self.workers]
|
|
30
|
+
self.generator = self._lazymap_generator(it, n_workers)
|
|
31
|
+
|
|
32
|
+
def __len__(self):
|
|
33
|
+
return len(self.it)
|
|
34
|
+
|
|
35
|
+
def __iter__(self):
|
|
36
|
+
return self.generator
|
|
37
|
+
|
|
38
|
+
def _worker_fcn(self, pid, fcn, **kwargs):
|
|
39
|
+
while not self.in_queue.done:
|
|
40
|
+
item = self.in_queue.get()
|
|
41
|
+
if item is None: break
|
|
42
|
+
res = fcn(item, **kwargs)
|
|
43
|
+
with self.k_changed:
|
|
44
|
+
self.k_changed.wait_for(lambda: self.k.value % self.n_workers == pid)
|
|
45
|
+
self.out_queue.put(res)
|
|
46
|
+
self.k.value += 1
|
|
47
|
+
self.k_changed.notify_all()
|
|
48
|
+
|
|
49
|
+
def _lazymap_generator(self, it, n_workers):
|
|
50
|
+
it = iter(it)
|
|
51
|
+
[self.in_queue.put(next(it)) for i in range(n_workers)]
|
|
52
|
+
k = -1
|
|
53
|
+
for k, item in enumerate(it):
|
|
54
|
+
res = self.out_queue.get()
|
|
55
|
+
self.in_queue.put(item)
|
|
56
|
+
yield res
|
|
57
|
+
for k in range(k + 1, k + 1 + n_workers):
|
|
58
|
+
yield self.out_queue.get()
|
|
59
|
+
self.close()
|
|
60
|
+
|
|
61
|
+
def close(self):
|
|
62
|
+
[self.in_queue.put(None) for _ in self.workers]
|
|
63
|
+
[w.join() for w in self.workers]
|
|
64
|
+
self.out_queue = None
|
|
65
|
+
self.in_queue = None
|
|
66
|
+
|
|
67
|
+
def compute(self, progress=True, ndarray=True, **kwargs):
|
|
68
|
+
''' Computes the results of the lazymap and returns them as a list.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
progress (bool): whether to show a tqdm progress bar (default: True)
|
|
72
|
+
**kwargs: optional, being passed to tqqdm
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
(list): a list of the results of fcn(item) for each item in it
|
|
76
|
+
'''
|
|
77
|
+
iterable = self
|
|
78
|
+
if progress:
|
|
79
|
+
from tqdm.auto import tqdm
|
|
80
|
+
iterable = tqdm(iterable)
|
|
81
|
+
out = list(iterable)
|
|
82
|
+
if ndarray:
|
|
83
|
+
try:
|
|
84
|
+
return np.array(out)
|
|
85
|
+
except:
|
|
86
|
+
pass
|
|
87
|
+
else:
|
|
88
|
+
return out
|
|
89
|
+
|
|
90
|
+
class WorkerWrapper:
|
|
91
|
+
''' A helper class used by the Parallel decorator to wrap a class to make it a parallel worker.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
cls (class): the class to be wrapped
|
|
95
|
+
n_workers (int): number of workers (default: 1)
|
|
96
|
+
buffer_bytes (int): size of the queue buffer (default: 10e6 bytes)
|
|
97
|
+
'''
|
|
98
|
+
def __init__(self, cls, n_workers=1, buffer_bytes=10e6):
|
|
99
|
+
self.instance = None
|
|
100
|
+
self.cls = cls
|
|
101
|
+
self.n_workers = n_workers
|
|
102
|
+
self.buffer_bytes = int(buffer_bytes)
|
|
103
|
+
def __call__(self, *init_args, **init_kwargs):
|
|
104
|
+
def mapper(iterable, **map_kwargs):
|
|
105
|
+
def worker(arg, **kwargs):
|
|
106
|
+
if self.instance is None:
|
|
107
|
+
self.instance = self.cls(*init_args, **init_kwargs)
|
|
108
|
+
return self.instance(arg, **kwargs)
|
|
109
|
+
return lazymap(worker, iterable, self.n_workers, self.buffer_bytes, **map_kwargs)
|
|
110
|
+
return mapper
|
|
111
|
+
|
|
112
|
+
def Parallel(n_workers=1, buffer_bytes=10e6):
|
|
113
|
+
''' A wrapper to make a class or callablle a parallel worker. Can be used as a decorator.
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
n_workers (int): number of workers (default: 1)
|
|
117
|
+
buffer_bytes (int): size of the queue buffer (default: 10e6 bytes)
|
|
118
|
+
'''
|
|
119
|
+
def decorator(cls):
|
|
120
|
+
if isinstance(cls, type):
|
|
121
|
+
return WorkerWrapper(cls, n_workers, buffer_bytes)
|
|
122
|
+
elif callable(cls):
|
|
123
|
+
def wrapped(iterable, **kwargs):
|
|
124
|
+
return lazymap(cls, iterable, n_workers, buffer_bytes, **kwargs)
|
|
125
|
+
return wrapped
|
|
126
|
+
else:
|
|
127
|
+
raise ValueError(f'Invalid type {type(cls)}')
|
|
128
|
+
return decorator
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import multiprocessing as mp
|
|
3
|
+
import pickle
|
|
4
|
+
import dataclasses
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
@dataclasses.dataclass
|
|
8
|
+
class FrameInfo:
|
|
9
|
+
''' A class to store metadata about a data frame in a FIFO queue.'''
|
|
10
|
+
nbytes: int
|
|
11
|
+
head: int
|
|
12
|
+
tail: int
|
|
13
|
+
meta: Any # any picklable object
|
|
14
|
+
|
|
15
|
+
class ByteFIFO:
|
|
16
|
+
""" A FIFO buffer (queue) for bytes. The queue is implemented as a ring buffer in shared memory.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, bytes=10e6):
|
|
20
|
+
"""
|
|
21
|
+
Initializes a ByteFIFO object.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
buffer_bytes (int): The size of the buffer in bytes. Defaults to 10 MiB.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
self.buffer_bytes = int(bytes)
|
|
28
|
+
self.buffer = mp.Array("c", self.buffer_bytes, lock=False)
|
|
29
|
+
self._view = None
|
|
30
|
+
self.queue = mp.Manager().Queue() # manager helps avoid out-of-order problems
|
|
31
|
+
self.get_lock = mp.Lock()
|
|
32
|
+
self.put_lock = mp.Lock()
|
|
33
|
+
self.head_changed = mp.Condition()
|
|
34
|
+
self.head = mp.Value("l", 0)
|
|
35
|
+
self.tail = mp.Value("l", 0)
|
|
36
|
+
self.closed = mp.Value("b", False)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def put(self, array_bytes, meta=None, timeout=None):
|
|
40
|
+
"""
|
|
41
|
+
Puts a byte array into the queue.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
array_bytes (numpy.ndarray or memoryview): The byte array to be put into the queue.
|
|
45
|
+
meta (Any, optional): Additional metadata associated with the byte array.
|
|
46
|
+
|
|
47
|
+
Raises:
|
|
48
|
+
AssertionError: If the size of the byte array exceeds the buffer size.
|
|
49
|
+
"""
|
|
50
|
+
if type(array_bytes) == memoryview:
|
|
51
|
+
array_bytes = np.frombuffer(array_bytes, dtype='byte')
|
|
52
|
+
elif type(array_bytes) == np.ndarray:
|
|
53
|
+
array_bytes = array_bytes.ravel().view('byte')
|
|
54
|
+
nbytes = array_bytes.nbytes
|
|
55
|
+
assert nbytes < self.buffer_bytes, "Array size exceeds buffer size."
|
|
56
|
+
with self.put_lock:
|
|
57
|
+
while self._available_space() < nbytes:
|
|
58
|
+
with self.head_changed:
|
|
59
|
+
if not self.head_changed.wait(timeout=timeout):
|
|
60
|
+
raise TimeoutError("Timeout waiting for available space.")
|
|
61
|
+
head = self.tail.value
|
|
62
|
+
self._write_buffer(array_bytes)
|
|
63
|
+
frame_info = FrameInfo(nbytes=nbytes, head=head, tail=self.tail.value, meta=meta)
|
|
64
|
+
self.queue.put(frame_info)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _write_buffer(self, array_bytes):
|
|
68
|
+
''' Write a byte array into the queue. Warning: this function should be called after acquiring the put_lock.
|
|
69
|
+
'''
|
|
70
|
+
tail = self.tail.value
|
|
71
|
+
nbytes = len(array_bytes)
|
|
72
|
+
if tail + nbytes <= self.buffer_bytes:
|
|
73
|
+
self.view[tail : tail + nbytes] = array_bytes
|
|
74
|
+
self.tail.value = (tail + nbytes) % self.buffer_bytes
|
|
75
|
+
else:
|
|
76
|
+
tail_part_size = self.buffer_bytes - tail
|
|
77
|
+
self.view[tail:] = array_bytes[:tail_part_size]
|
|
78
|
+
self.view[: nbytes - tail_part_size] = array_bytes[tail_part_size:]
|
|
79
|
+
self.tail.value = nbytes - tail_part_size
|
|
80
|
+
return nbytes
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def get(self, callback=None, copy=None, **kwargs):
|
|
84
|
+
""" Gets a byte array from the queue.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
callback (Callable, optional): A callback function to be called with the byte array (pre-copy, potentially unsafe!) and metadata.
|
|
88
|
+
copy (bool, optional): Whether to make a copy of the byte array. Defaults to None: copy if a callback is not provided.
|
|
89
|
+
**kwargs: Additional keyword arguments to be passed to the queue's get method.
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
A tuple containing the byte array and any metadata provided with put OR the return value of the callback function, if provided.
|
|
93
|
+
"""
|
|
94
|
+
with self.get_lock:
|
|
95
|
+
frame_info = self.queue.get(**kwargs)
|
|
96
|
+
head = frame_info.head
|
|
97
|
+
tail = frame_info.tail
|
|
98
|
+
assert head == self.head.value, f"head: {head}, self.head: {self.head.value}"
|
|
99
|
+
if head <= tail:
|
|
100
|
+
array_bytes = self.view[head:tail]
|
|
101
|
+
else:
|
|
102
|
+
array_bytes = np.concatenate((self.view[head:], self.view[:tail]))
|
|
103
|
+
if copy or ((copy is None) and (callback is None)):
|
|
104
|
+
array_bytes = array_bytes.copy()
|
|
105
|
+
if callback is not None:
|
|
106
|
+
return_value = callback(array_bytes, frame_info.meta)
|
|
107
|
+
else:
|
|
108
|
+
return_value = array_bytes, frame_info.meta
|
|
109
|
+
self.head.value = (head + frame_info.nbytes) % self.buffer_bytes
|
|
110
|
+
|
|
111
|
+
with self.head_changed:
|
|
112
|
+
self.head_changed.notify()
|
|
113
|
+
|
|
114
|
+
return return_value
|
|
115
|
+
|
|
116
|
+
def _available_space(self):
|
|
117
|
+
""" Calculates the available space in the buffer.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
int: The available space in bytes.
|
|
121
|
+
"""
|
|
122
|
+
return (self.head.value - self.tail.value - 1) % self.buffer_bytes
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def view(self):
|
|
126
|
+
""" numpy.ndarray: A view of the shared memory array as a numpy array. Lazy initialization to avoid pickling issues.
|
|
127
|
+
"""
|
|
128
|
+
if self._view is None:
|
|
129
|
+
self._view = np.frombuffer(self.buffer, "byte")
|
|
130
|
+
return self._view
|
|
131
|
+
|
|
132
|
+
def __del__(self):
|
|
133
|
+
self._view = None
|
|
134
|
+
|
|
135
|
+
def empty(self):
|
|
136
|
+
""" Checks if the queue is empty.
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
bool: True if the queue is empty, False otherwise.
|
|
140
|
+
"""
|
|
141
|
+
return self.queue.empty()
|
|
142
|
+
|
|
143
|
+
def close(self):
|
|
144
|
+
""" Closes the queue.
|
|
145
|
+
"""
|
|
146
|
+
self.closed.value = True
|
|
147
|
+
|
|
148
|
+
def join(self):
|
|
149
|
+
""" Joins the queue
|
|
150
|
+
"""
|
|
151
|
+
self.queue.join()
|
|
152
|
+
|
|
153
|
+
def task_done(self):
|
|
154
|
+
""" Marks a task as done.
|
|
155
|
+
"""
|
|
156
|
+
self.queue.task_done()
|
|
157
|
+
|
|
158
|
+
@property
|
|
159
|
+
def done(self):
|
|
160
|
+
''' Returns True if the queue is empty and closed.'''
|
|
161
|
+
return self.queue.empty() and self.closed.value
|
|
162
|
+
|
|
163
|
+
def __getstate__(self):
|
|
164
|
+
state = {k:v for k,v in self.__dict__.items() if k != '_view'}
|
|
165
|
+
state['_view'] = None
|
|
166
|
+
return state
|
|
167
|
+
|
|
168
|
+
def __setstate__(self, state):
|
|
169
|
+
self.__dict__ = state
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class ArrayFIFO(ByteFIFO):
|
|
173
|
+
""" A fast queue for numpy arrays. The queue is implemented as a ring buffer in shared memory.
|
|
174
|
+
|
|
175
|
+
Args:
|
|
176
|
+
buffer_bytes (int): The size of the buffer in bytes
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
def put(self, array, meta=None, timeout=None):
|
|
180
|
+
"""
|
|
181
|
+
Puts a numpy array into the queue.
|
|
182
|
+
|
|
183
|
+
Args:
|
|
184
|
+
array (numpy.ndarray): The byte array to be put into the queue.
|
|
185
|
+
meta (Any, optional): Additional custom metadata (will be sent through a regular slow Queue).
|
|
186
|
+
timeout (float, optional): The maximum time to wait for available space in the queue.
|
|
187
|
+
|
|
188
|
+
Raises:
|
|
189
|
+
AssertionError: If the size of the byte array exceeds the buffer size.
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
array_bytes = array.ravel().view('byte')
|
|
193
|
+
meta = dict(dtype=array.dtype.str, shape=array.shape, meta=meta)
|
|
194
|
+
super().put(array_bytes, meta=meta, timeout=timeout)
|
|
195
|
+
|
|
196
|
+
def get(self, callback=None, copy=None, **kwargs):
|
|
197
|
+
"""
|
|
198
|
+
Gets a numpy array from the queue.
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
callback (Callable, optional): A callback function to be called with the byte array (pre-copy, potentially unsafe!) and metadata.
|
|
202
|
+
copy (bool, optional): Whether to make a copy of the byte array. Defaults to None: copy if a callback is not provided.
|
|
203
|
+
**kwargs: Additional keyword arguments to be passed to the queue's get method.
|
|
204
|
+
|
|
205
|
+
Returns:
|
|
206
|
+
tuple: A tuple containing the numpy array and any metadata provided with put.
|
|
207
|
+
"""
|
|
208
|
+
def callback_wrapper(array_bytes, meta):
|
|
209
|
+
array = np.frombuffer(array_bytes, dtype=meta['dtype']).reshape(meta['shape'])
|
|
210
|
+
if copy or ((copy is None) and (callback is None)):
|
|
211
|
+
array = array.copy()
|
|
212
|
+
if callback is not None:
|
|
213
|
+
callback(array, meta)
|
|
214
|
+
return array, meta['meta']
|
|
215
|
+
|
|
216
|
+
array, meta = super().get(copy=False, callback=callback_wrapper, **kwargs)
|
|
217
|
+
return array, meta
|
|
218
|
+
|
|
219
|
+
class DejaQueue(ByteFIFO):
|
|
220
|
+
""" A fast queue for arbitrary (picklable) Python objects. The queue is implemented as a ring buffer in shared memory.
|
|
221
|
+
|
|
222
|
+
Args:
|
|
223
|
+
buffer_bytes (int): The size of the buffer in bytes.
|
|
224
|
+
"""
|
|
225
|
+
|
|
226
|
+
def put(self, obj, timeout=None):
|
|
227
|
+
""" Puts a Python object into the queue.
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
obj (Any): The byte array to be put into the queue.
|
|
231
|
+
timeout (float, optional): The maximum time to wait for available space in the queue.
|
|
232
|
+
"""
|
|
233
|
+
buffers = []
|
|
234
|
+
pkl = pickle.dumps(obj, buffer_callback=buffers.append, protocol=pickle.HIGHEST_PROTOCOL)
|
|
235
|
+
buffer_lengths = [len(pkl),] + [len(it.raw()) for it in buffers]
|
|
236
|
+
nbytes_total = sum(buffer_lengths)
|
|
237
|
+
|
|
238
|
+
assert nbytes_total < self.buffer_bytes, "Array size exceeds buffer size."
|
|
239
|
+
|
|
240
|
+
with self.put_lock:
|
|
241
|
+
while self._available_space() < nbytes_total:
|
|
242
|
+
with self.head_changed:
|
|
243
|
+
if not self.head_changed.wait(timeout=timeout):
|
|
244
|
+
raise TimeoutError("Timeout waiting for available space.")
|
|
245
|
+
|
|
246
|
+
head = self.tail.value
|
|
247
|
+
self._write_buffer(np.frombuffer(pkl, 'byte'))
|
|
248
|
+
for buf in buffers:
|
|
249
|
+
self._write_buffer(buf.raw())
|
|
250
|
+
|
|
251
|
+
frame_info = FrameInfo(nbytes=nbytes_total, head=head, tail=self.tail.value, meta=buffer_lengths)
|
|
252
|
+
self.queue.put(frame_info)
|
|
253
|
+
|
|
254
|
+
def get(self, **kwargs):
|
|
255
|
+
""" Gets an item from the queue.
|
|
256
|
+
|
|
257
|
+
Args:
|
|
258
|
+
**kwargs: Additional keyword arguments to be passed to the underlying queue's get method (e.g. timeout).
|
|
259
|
+
|
|
260
|
+
Returns:
|
|
261
|
+
obj: The object that was put into the queue.
|
|
262
|
+
"""
|
|
263
|
+
def callback(array_bytes, buffer_lengths):
|
|
264
|
+
buffers = []
|
|
265
|
+
offset = 0
|
|
266
|
+
for length in buffer_lengths:
|
|
267
|
+
buffers.append(pickle.PickleBuffer(array_bytes[offset:offset+length]))
|
|
268
|
+
offset += length
|
|
269
|
+
obj = pickle.loads(buffers[0], buffers=buffers[1:])
|
|
270
|
+
return obj
|
|
271
|
+
|
|
272
|
+
obj = super().get(copy=False, callback=callback, **kwargs)
|
|
273
|
+
return obj
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: dejaq
|
|
3
|
+
Version: 0.0.6
|
|
4
|
+
Summary: Déjà Queue – A fast multiprocessing queue for Python
|
|
5
|
+
Author: jlab.berlin, Benjamin Judkewitz
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/danionella/dejaq
|
|
8
|
+
Keywords: multiprocessing,queue,shared_memory
|
|
9
|
+
Requires-Python: >=3.8
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE.md
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+

|
|
15
|
+
[](https://pypi.org/project/dejaq/)
|
|
16
|
+
[](https://anaconda.org/danionella/dejaq)
|
|
17
|
+
[](https://opensource.org/licenses/MIT)
|
|
18
|
+

|
|
19
|
+
|
|
20
|
+
# Déjà Queue
|
|
21
|
+
|
|
22
|
+
A drop-in replacement for `multiprocessing.Queue`. Faster, because it takes advantage of a shared memory ring buffer (rather than slow pipes) and [pickle protocol 5 out-of-band data](https://peps.python.org/pep-0574/) to minimize copies. `DejaQueue` supports any type of [picklable](https://docs.python.org/3/library/pickle.html#what-can-be-pickled-and-unpickled) Python object, including numpy arrays or nested dictionaries with mixed content.
|
|
23
|
+
|
|
24
|
+
The speed of `DejaQueue` enables efficient inter-job communication in data processing pipelines, which can be implemented in a few lines of code with [`dejaq.Parallel`](#dejaqparallel).
|
|
25
|
+
|
|
26
|
+
Auto-generated API documentation: https://danionella.github.io/dejaq
|
|
27
|
+
|
|
28
|
+
## Examples
|
|
29
|
+
### `dejaq.DejaQueue`
|
|
30
|
+
```python
|
|
31
|
+
import numpy as np
|
|
32
|
+
from multiprocessing import Process
|
|
33
|
+
from dejaq import DejaQueue
|
|
34
|
+
|
|
35
|
+
def produce(queue):
|
|
36
|
+
for i in range(20):
|
|
37
|
+
random_shape = np.random.randint(5,10, size=3)
|
|
38
|
+
array = np.random.randn(*random_shape)
|
|
39
|
+
queue.put(array, meta=i)
|
|
40
|
+
print(f'produced {type(array)} {array.shape} {array.dtype}; meta: {i}; hash: {hash(array.tobytes())}\n')
|
|
41
|
+
|
|
42
|
+
def consume(queue, pid):
|
|
43
|
+
while True:
|
|
44
|
+
array, meta = queue.get()
|
|
45
|
+
print(f'consumer {pid} consumed {type(array)} {array.shape} {array.dtype}; meta: {meta}; hash: {hash(array.tobytes())}\n')
|
|
46
|
+
|
|
47
|
+
queue = DejaQueue(bytes=10e6)
|
|
48
|
+
producer = Process(target=produce, args=(queue,))
|
|
49
|
+
consumers = [Process(target=consume, args=(queue, pid)) for pid in range(3)]
|
|
50
|
+
for c in consumers:
|
|
51
|
+
c.start()
|
|
52
|
+
producer.start()
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
### `dejaq.Parallel`
|
|
57
|
+
The following examples show how to use `dejaq.Parallel` to parallelize a function or a class, and how to create job pipelines.
|
|
58
|
+
|
|
59
|
+
Here we execute a function and map iterable inputs across 10 workers. To enable pipelining, the results of each stage are provided as iterable generator. Use the `.compute()` method to get the final result (note that each stage pre-fetches results from `n_workers` calls, so some of the execution already starts before `.compute`)
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from time import sleep
|
|
63
|
+
from dejaq import Parallel
|
|
64
|
+
|
|
65
|
+
def slow_function(arg):
|
|
66
|
+
sleep(1.0)
|
|
67
|
+
return arg + 5
|
|
68
|
+
|
|
69
|
+
input_iterable = range(100)
|
|
70
|
+
slow_function = Parallel(n_workers=10)(slow_function)
|
|
71
|
+
stage = slow_function(input_iterable)
|
|
72
|
+
result = stage.compute() # or list(stage)
|
|
73
|
+
# or shorter:
|
|
74
|
+
result = Parallel(n_workers=10)(slow_function)(input_iterable).compute()
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
You can also use `Parallel` as a function decorator:
|
|
78
|
+
```python
|
|
79
|
+
@Parallel(n_workers=10)
|
|
80
|
+
def slow_function_decorated(arg):
|
|
81
|
+
sleep(1.0)
|
|
82
|
+
return arg + 5
|
|
83
|
+
|
|
84
|
+
result = slow_function_decorated(input_iterable).compute()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Similarly, you can decorate a class. It will be instantiated within a worker. Iterable items will be fed to the `__call__` method. Note how the additional init arguments are provided:
|
|
88
|
+
```python
|
|
89
|
+
@Parallel(n_workers=1)
|
|
90
|
+
class Reader:
|
|
91
|
+
def __init__(self, arg1):
|
|
92
|
+
self.arg1 = arg1
|
|
93
|
+
def __call__(self, item):
|
|
94
|
+
return item + self.arg1
|
|
95
|
+
|
|
96
|
+
result = Reader(arg1=0.5)(input_iterable).compute()
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Finally, you can create pipelines of chained jobs. In this example, we have a single threaded reader and consumer, but a parallel processing stage (an example use case is sequentially reading a file, compressing chunks in parallel and then sequentially writing to an output file):
|
|
100
|
+
```python
|
|
101
|
+
@Parallel(n_workers=1)
|
|
102
|
+
class Producer:
|
|
103
|
+
def __init__(self, arg1):
|
|
104
|
+
self.arg1 = arg1
|
|
105
|
+
def __call__(self, item):
|
|
106
|
+
return item + self.arg1
|
|
107
|
+
|
|
108
|
+
@Parallel(n_workers=10)
|
|
109
|
+
class Processor:
|
|
110
|
+
def __init__(self, arg1):
|
|
111
|
+
self.arg1 = arg1
|
|
112
|
+
def __call__(self, arg):
|
|
113
|
+
sleep(1.0) #simulating a slow function
|
|
114
|
+
return arg * self.arg1
|
|
115
|
+
|
|
116
|
+
@Parallel(n_workers=1)
|
|
117
|
+
class Consumer:
|
|
118
|
+
def __init__(self, arg1):
|
|
119
|
+
self.arg1 = arg1
|
|
120
|
+
def __call__(self, arg):
|
|
121
|
+
return arg - self.arg1
|
|
122
|
+
|
|
123
|
+
input_iterable = range(100)
|
|
124
|
+
stage1 = Producer(0.5)(input_iterable)
|
|
125
|
+
stage2 = Processor(10.0)(stage1)
|
|
126
|
+
stage3 = Consumer(1000)(stage2)
|
|
127
|
+
result = stage3.compute()
|
|
128
|
+
|
|
129
|
+
# or:
|
|
130
|
+
result = Consumer(1000)(Processor(10.0)(Producer(0.5)(input_iterable))).compute()
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
# See also
|
|
134
|
+
- [ArrayQueues](https://github.com/portugueslab/arrayqueues)
|
|
135
|
+
- [joblib.Parallel](https://joblib.readthedocs.io/en/latest/generated/joblib.Parallel.html)
|
|
136
|
+
- [Déjà Q](https://en.wikipedia.org/wiki/Deja_Q)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dejaq
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=42", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "dejaq"
|
|
7
|
+
version = "0.0.6"
|
|
8
|
+
description = "Déjà Queue – A fast multiprocessing queue for Python"
|
|
9
|
+
authors = [
|
|
10
|
+
{ name = "jlab.berlin" },
|
|
11
|
+
{ name = "Benjamin Judkewitz"}
|
|
12
|
+
]
|
|
13
|
+
license = {text = "MIT"}
|
|
14
|
+
requires-python = ">=3.8"
|
|
15
|
+
keywords = ["multiprocessing", "queue", "shared_memory"]
|
|
16
|
+
readme = "README.md"
|
|
17
|
+
|
|
18
|
+
[project.urls]
|
|
19
|
+
Homepage = "https://github.com/danionella/dejaq"
|
|
20
|
+
|
|
21
|
+
[tool.setuptools.packages.find]
|
|
22
|
+
include = ["dejaq", "dejaq.*"]
|
dejaq-0.0.6/setup.cfg
ADDED