fluentmap 0.0.2__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fluentmap-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: fluentmap
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A drop-in Python map replacement featuring parallel and batch processing.
|
|
5
|
+
Author: leavers
|
|
6
|
+
Author-Email: Leavers <leavers930@gmail.com>
|
|
7
|
+
License: MIT
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Software Development
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Topic :: Utilities
|
|
21
|
+
Requires-Python: >=3.8
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# Fluentmap
|
|
25
|
+
|
|
26
|
+
[](https://github.com/leavers/fluentmap/actions/workflows/test-suite.yml)
|
|
27
|
+
[](https://pypi.org/project/fluentmap/)
|
|
28
|
+
[](https://pypi.org/project/fluentmap/)
|
|
29
|
+
|
|
30
|
+
Fluentmap provides a drop-in Python map replacement featuring parallel and batch
|
|
31
|
+
processing.
|
|
32
|
+
|
|
33
|
+
## Features
|
|
34
|
+
|
|
35
|
+
- Use `executor` to run tasks in parallel.
|
|
36
|
+
- Use `batch_size`/`chunk_size` to send parameters in batches/chunks.
|
|
37
|
+
- Use `num_prepare` to prepare data in advance for better performance.
|
|
38
|
+
- Call `on_return` hook to process the return value of each task.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
|
|
42
|
+
Fluentmap is available on [PyPI](https://pypi.org/project/fluentmap/):
|
|
43
|
+
|
|
44
|
+
```shell
|
|
45
|
+
pip install fluentmap
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Usage
|
|
49
|
+
|
|
50
|
+
### Drop-in replacement
|
|
51
|
+
|
|
52
|
+
You can start to use fluentmap just like built-in `map`:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from typing import Any, List
|
|
56
|
+
|
|
57
|
+
from fluentmap import map
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
items: List[str] = [...]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def heavy_task(item: str) -> Any:
|
|
64
|
+
"""Suppose this function represents a computationally expensive task."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def postprocessing(result: Any):
|
|
68
|
+
"""Suppose this function represents a postprocessing task."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
for result in map(heavy_task, items):
|
|
72
|
+
postprocessing(result)
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Parallel processing
|
|
76
|
+
|
|
77
|
+
As `heavy_task` is a computationally expensive task, you can use `executor` to
|
|
78
|
+
run it in parallel.
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
82
|
+
|
|
83
|
+
from fluentmap import map
|
|
84
|
+
|
|
85
|
+
# ......
|
|
86
|
+
|
|
87
|
+
with ProcessPoolExecutor() as executor:
|
|
88
|
+
# each heavy_task invocation runs in a separate process
|
|
89
|
+
for result in map(heavy_task, items, executor=executor):
|
|
90
|
+
postprocessing(result)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### Batch/chunk processing
|
|
94
|
+
|
|
95
|
+
You can use `batch_size`/`chunk_size` to send arguments in batches/chunks.
|
|
96
|
+
|
|
97
|
+
The difference between them is that `batch_size` packs multiple arguments into a batch
|
|
98
|
+
before sending them to the function, therefore the function needs to be modified to
|
|
99
|
+
handle a list of arguments.
|
|
100
|
+
|
|
101
|
+
On the other hand, `chunk_size` packs multiple arguments into a chunks before passing
|
|
102
|
+
them to executor workers, while workers still process each argument sequentially.
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
106
|
+
from typing import Any, List
|
|
107
|
+
|
|
108
|
+
from fluentmap import map
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ......
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def heavy_task_in_batch(item: List[str]) -> Any:
|
|
115
|
+
"""Note that `item` is a list since when `batch_size` is set,
|
|
116
|
+
fluentmap will concatenate multiple items into a batch before sending them to
|
|
117
|
+
the function which is to be invoked.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# An example of using `batch_size`
|
|
122
|
+
with ProcessPoolExecutor() as executor:
|
|
123
|
+
for result in map(
|
|
124
|
+
heavy_task_in_batch,
|
|
125
|
+
items,
|
|
126
|
+
executor=executor,
|
|
127
|
+
batch_size=64,
|
|
128
|
+
):
|
|
129
|
+
postprocessing(result)
|
|
130
|
+
|
|
131
|
+
# An example of using `chunk_size`
|
|
132
|
+
with ProcessPoolExecutor() as executor:
|
|
133
|
+
for result in map(
|
|
134
|
+
heavy_task,
|
|
135
|
+
items,
|
|
136
|
+
executor=executor,
|
|
137
|
+
chunk_size=32,
|
|
138
|
+
):
|
|
139
|
+
postprocessing(result)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# Fluentmap
|
|
2
|
+
|
|
3
|
+
[](https://github.com/leavers/fluentmap/actions/workflows/test-suite.yml)
|
|
4
|
+
[](https://pypi.org/project/fluentmap/)
|
|
5
|
+
[](https://pypi.org/project/fluentmap/)
|
|
6
|
+
|
|
7
|
+
Fluentmap provides a drop-in Python map replacement featuring parallel and batch
|
|
8
|
+
processing.
|
|
9
|
+
|
|
10
|
+
## Features
|
|
11
|
+
|
|
12
|
+
- Use `executor` to run tasks in parallel.
|
|
13
|
+
- Use `batch_size`/`chunk_size` to send parameters in batches/chunks.
|
|
14
|
+
- Use `num_prepare` to prepare data in advance for better performance.
|
|
15
|
+
- Call `on_return` hook to process the return value of each task.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
Fluentmap is available on [PyPI](https://pypi.org/project/fluentmap/):
|
|
20
|
+
|
|
21
|
+
```shell
|
|
22
|
+
pip install fluentmap
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Usage
|
|
26
|
+
|
|
27
|
+
### Drop-in replacement
|
|
28
|
+
|
|
29
|
+
You can start to use fluentmap just like built-in `map`:
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from typing import Any, List
|
|
33
|
+
|
|
34
|
+
from fluentmap import map
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
items: List[str] = [...]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def heavy_task(item: str) -> Any:
|
|
41
|
+
"""Suppose this function represents a computationally expensive task."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def postprocessing(result: Any):
|
|
45
|
+
"""Suppose this function represents a postprocessing task."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
for result in map(heavy_task, items):
|
|
49
|
+
postprocessing(result)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### Parallel processing
|
|
53
|
+
|
|
54
|
+
As `heavy_task` is a computationally expensive task, you can use `executor` to
|
|
55
|
+
run it in parallel.
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
59
|
+
|
|
60
|
+
from fluentmap import map
|
|
61
|
+
|
|
62
|
+
# ......
|
|
63
|
+
|
|
64
|
+
with ProcessPoolExecutor() as executor:
|
|
65
|
+
# each heavy_task invocation runs in a separate process
|
|
66
|
+
for result in map(heavy_task, items, executor=executor):
|
|
67
|
+
postprocessing(result)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Batch/chunk processing
|
|
71
|
+
|
|
72
|
+
You can use `batch_size`/`chunk_size` to send arguments in batches/chunks.
|
|
73
|
+
|
|
74
|
+
The difference between them is that `batch_size` packs multiple arguments into a batch
|
|
75
|
+
before sending them to the function, therefore the function needs to be modified to
|
|
76
|
+
handle a list of arguments.
|
|
77
|
+
|
|
78
|
+
On the other hand, `chunk_size` packs multiple arguments into a chunks before passing
|
|
79
|
+
them to executor workers, while workers still process each argument sequentially.
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
83
|
+
from typing import Any, List
|
|
84
|
+
|
|
85
|
+
from fluentmap import map
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# ......
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def heavy_task_in_batch(item: List[str]) -> Any:
|
|
92
|
+
"""Note that `item` is a list since when `batch_size` is set,
|
|
93
|
+
fluentmap will concatenate multiple items into a batch before sending them to
|
|
94
|
+
the function which is to be invoked.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# An example of using `batch_size`
|
|
99
|
+
with ProcessPoolExecutor() as executor:
|
|
100
|
+
for result in map(
|
|
101
|
+
heavy_task_in_batch,
|
|
102
|
+
items,
|
|
103
|
+
executor=executor,
|
|
104
|
+
batch_size=64,
|
|
105
|
+
):
|
|
106
|
+
postprocessing(result)
|
|
107
|
+
|
|
108
|
+
# An example of using `chunk_size`
|
|
109
|
+
with ProcessPoolExecutor() as executor:
|
|
110
|
+
for result in map(
|
|
111
|
+
heavy_task,
|
|
112
|
+
items,
|
|
113
|
+
executor=executor,
|
|
114
|
+
chunk_size=32,
|
|
115
|
+
):
|
|
116
|
+
postprocessing(result)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""
|
|
2
|
-
Fluentmap is a drop-in replacement for Python map
|
|
2
|
+
Fluentmap is a drop-in replacement for Python map featuring parallel and batch
|
|
3
|
+
processing.
|
|
3
4
|
|
|
4
5
|
Copyright (c) 2020-2024, Leavers.
|
|
5
6
|
License: MIT
|
|
@@ -25,7 +26,7 @@ from typing import (
|
|
|
25
26
|
runtime_checkable,
|
|
26
27
|
)
|
|
27
28
|
|
|
28
|
-
__version__ = "0.0
|
|
29
|
+
__version__ = "0.1.0"
|
|
29
30
|
__all__ = ("Arguments", "map")
|
|
30
31
|
|
|
31
32
|
|
|
@@ -1,14 +1,30 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "fluentmap"
|
|
3
|
-
description = "
|
|
3
|
+
description = "A drop-in Python map replacement featuring parallel and batch processing."
|
|
4
4
|
authors = [
|
|
5
|
+
{ name = "leavers" },
|
|
5
6
|
{ name = "Leavers", email = "leavers930@gmail.com" },
|
|
6
7
|
]
|
|
8
|
+
classifiers = [
|
|
9
|
+
"Intended Audience :: Developers",
|
|
10
|
+
"License :: OSI Approved :: MIT License",
|
|
11
|
+
"Operating System :: OS Independent",
|
|
12
|
+
"Programming Language :: Python",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.8",
|
|
15
|
+
"Programming Language :: Python :: 3.9",
|
|
16
|
+
"Programming Language :: Python :: 3.10",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Topic :: Software Development",
|
|
20
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
21
|
+
"Topic :: Utilities",
|
|
22
|
+
]
|
|
7
23
|
dependencies = []
|
|
8
24
|
requires-python = ">=3.8"
|
|
9
25
|
readme = "README.md"
|
|
10
26
|
dynamic = []
|
|
11
|
-
version = "0.0
|
|
27
|
+
version = "0.1.0"
|
|
12
28
|
|
|
13
29
|
[project.license]
|
|
14
30
|
text = "MIT"
|
|
@@ -34,16 +50,6 @@ excludes = [
|
|
|
34
50
|
source = "file"
|
|
35
51
|
path = "fluentmap.py"
|
|
36
52
|
|
|
37
|
-
[tool.pdm.dev-dependencies]
|
|
38
|
-
dev = [
|
|
39
|
-
"autoflake>=2.3.1",
|
|
40
|
-
"mypy>=1.11.1",
|
|
41
|
-
"nox>=2024.4.15",
|
|
42
|
-
"pytest>=8.3.2",
|
|
43
|
-
"ruff>=0.6.1",
|
|
44
|
-
"rtoml>=0.11.0",
|
|
45
|
-
]
|
|
46
|
-
|
|
47
53
|
[tool.autoflake]
|
|
48
54
|
in-place = true
|
|
49
55
|
recursive = true
|
|
@@ -77,3 +83,18 @@ warn_unused_configs = true
|
|
|
77
83
|
|
|
78
84
|
[tool.pytest.ini_options]
|
|
79
85
|
asyncio_mode = "strict"
|
|
86
|
+
|
|
87
|
+
[dependency-groups]
|
|
88
|
+
dev = [
|
|
89
|
+
"autoflake>=2.3.1",
|
|
90
|
+
"coverage[toml]>=7.6.1",
|
|
91
|
+
"mypy>=1.11.1",
|
|
92
|
+
"nox>=2024.4.15",
|
|
93
|
+
"pytest>=8.3.2",
|
|
94
|
+
"pytest-asyncio>=0.23.8",
|
|
95
|
+
"pytest-cov>=5.0.0",
|
|
96
|
+
"pytest-mock>=3.14.0",
|
|
97
|
+
"pytest-timeout>=2.3.1",
|
|
98
|
+
"ruff>=0.6.1",
|
|
99
|
+
"rtoml>=0.11.0",
|
|
100
|
+
]
|
fluentmap-0.0.2/PKG-INFO
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: fluentmap
|
|
3
|
-
Version: 0.0.2
|
|
4
|
-
Summary: Drop-in replacement for Python map with concurrency support
|
|
5
|
-
Author-Email: Leavers <leavers930@gmail.com>
|
|
6
|
-
License: MIT
|
|
7
|
-
Requires-Python: >=3.8
|
|
8
|
-
Description-Content-Type: text/markdown
|
|
9
|
-
|
|
10
|
-
# Fluentmap
|
|
11
|
-
|
|
12
|
-
Drop-in replacement for Python map with concurrency support.
|
|
13
|
-
|
fluentmap-0.0.2/README.md
DELETED
|
File without changes
|