wool 0.1rc12__tar.gz → 0.1rc14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wool might be problematic. Click here for more details.
- {wool-0.1rc12 → wool-0.1rc14}/PKG-INFO +152 -40
- wool-0.1rc14/README.md +232 -0
- {wool-0.1rc12 → wool-0.1rc14}/pyproject.toml +1 -1
- {wool-0.1rc12 → wool-0.1rc14}/wool/__init__.py +1 -3
- {wool-0.1rc12 → wool-0.1rc14}/wool/_worker.py +1 -1
- {wool-0.1rc12 → wool-0.1rc14}/wool/_worker_discovery.py +6 -23
- {wool-0.1rc12 → wool-0.1rc14}/wool/_worker_pool.py +44 -29
- wool-0.1rc12/README.md +0 -120
- {wool-0.1rc12 → wool-0.1rc14}/.gitignore +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/__init__.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/exception.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/task.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/task_pb2.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/task_pb2.pyi +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/task_pb2_grpc.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/worker.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/worker_pb2.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/worker_pb2.pyi +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_protobuf/worker_pb2_grpc.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_resource_pool.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_typing.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_work.py +0 -0
- {wool-0.1rc12 → wool-0.1rc14}/wool/_worker_proxy.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wool
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.1rc14
|
|
4
4
|
Summary: A Python framework for distributed multiprocessing.
|
|
5
5
|
Author-email: Conrad Bzura <conrad@wool.io>
|
|
6
6
|
Maintainer-email: maintainers@wool.io
|
|
@@ -227,9 +227,9 @@ Requires-Dist: pytest-mock; extra == 'dev'
|
|
|
227
227
|
Requires-Dist: ruff; extra == 'dev'
|
|
228
228
|
Description-Content-Type: text/markdown
|
|
229
229
|
|
|
230
|
-
|
|
230
|
+

|
|
231
231
|
|
|
232
|
-
Wool is a native Python package for transparently executing tasks in a horizontally scalable, distributed network of agnostic worker processes. Any picklable async function or method can be converted into a task with a simple decorator and a client connection.
|
|
232
|
+
**Wool** is a native Python package for transparently executing tasks in a horizontally scalable, distributed network of agnostic worker processes. Any picklable async function or method can be converted into a task with a simple decorator and a client connection.
|
|
233
233
|
|
|
234
234
|
## Installation
|
|
235
235
|
|
|
@@ -253,19 +253,73 @@ cd wool
|
|
|
253
253
|
|
|
254
254
|
## Usage
|
|
255
255
|
|
|
256
|
-
###
|
|
256
|
+
### Declaring tasks
|
|
257
257
|
|
|
258
|
-
Wool
|
|
258
|
+
Wool tasks are coroutine functions that are executed in a remote `asyncio` event loop within a worker process. To declare a task, use the `@wool.task` decorator:
|
|
259
259
|
|
|
260
|
-
|
|
260
|
+
```python
|
|
261
|
+
import wool
|
|
261
262
|
|
|
262
|
-
|
|
263
|
-
|
|
263
|
+
@wool.task
|
|
264
|
+
async def sample_task(x, y):
|
|
265
|
+
return x + y
|
|
264
266
|
```
|
|
265
267
|
|
|
266
|
-
|
|
268
|
+
Tasks must be picklable, stateless, and idempotent. Avoid passing unpicklable objects as arguments or return values.
|
|
269
|
+
|
|
270
|
+
### Worker pools
|
|
271
|
+
|
|
272
|
+
Worker pools are responsible for executing tasks. Wool provides two types of pools:
|
|
273
|
+
|
|
274
|
+
#### Ephemeral pools
|
|
275
|
+
|
|
276
|
+
Ephemeral pools are created and destroyed within the scope of a context manager. Use `wool.pool` to declare an ephemeral pool:
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
import asyncio, wool
|
|
280
|
+
|
|
281
|
+
@wool.task
|
|
282
|
+
async def sample_task(x, y):
|
|
283
|
+
return x + y
|
|
284
|
+
|
|
285
|
+
async def main():
|
|
286
|
+
with wool.pool():
|
|
287
|
+
result = await sample_task(1, 2)
|
|
288
|
+
print(f"Result: {result}")
|
|
289
|
+
|
|
290
|
+
asyncio.run(main())
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
#### Durable pools
|
|
294
|
+
|
|
295
|
+
Durable pools are started independently and persist beyond the scope of a single application. Use the `wool` CLI to manage durable pools:
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
wool pool up --port 5050 --authkey deadbeef --module tasks
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Connect to a durable pool using `wool.session`:
|
|
302
|
+
|
|
303
|
+
```python
|
|
304
|
+
import asyncio, wool
|
|
305
|
+
|
|
306
|
+
@wool.task
|
|
307
|
+
async def sample_task(x, y):
|
|
308
|
+
return x + y
|
|
309
|
+
|
|
310
|
+
async def main():
|
|
311
|
+
with wool.session(port=5050, authkey=b"deadbeef"):
|
|
312
|
+
result = await sample_task(1, 2)
|
|
313
|
+
print(f"Result: {result}")
|
|
314
|
+
|
|
315
|
+
asyncio.run(main())
|
|
316
|
+
```
|
|
267
317
|
|
|
268
|
-
|
|
318
|
+
### CLI commands
|
|
319
|
+
|
|
320
|
+
Wool provides a command-line interface (CLI) for managing worker pools.
|
|
321
|
+
|
|
322
|
+
#### Start the worker pool
|
|
269
323
|
|
|
270
324
|
```sh
|
|
271
325
|
wool pool up --host <host> --port <port> --authkey <authkey> --breadth <breadth> --module <module>
|
|
@@ -275,11 +329,9 @@ wool pool up --host <host> --port <port> --authkey <authkey> --breadth <breadth>
|
|
|
275
329
|
- `--port`: The port number (default: `0`).
|
|
276
330
|
- `--authkey`: The authentication key (default: `b""`).
|
|
277
331
|
- `--breadth`: The number of worker processes (default: number of CPU cores).
|
|
278
|
-
- `--module`: Python module containing Wool task definitions
|
|
279
|
-
|
|
280
|
-
#### Stop the Worker Pool
|
|
332
|
+
- `--module`: Python module containing Wool task definitions (optional, can be specified multiple times).
|
|
281
333
|
|
|
282
|
-
|
|
334
|
+
#### Stop the worker pool
|
|
283
335
|
|
|
284
336
|
```sh
|
|
285
337
|
wool pool down --host <host> --port <port> --authkey <authkey> --wait
|
|
@@ -290,9 +342,7 @@ wool pool down --host <host> --port <port> --authkey <authkey> --wait
|
|
|
290
342
|
- `--authkey`: The authentication key (default: `b""`).
|
|
291
343
|
- `--wait`: Wait for in-flight tasks to complete before shutting down.
|
|
292
344
|
|
|
293
|
-
#### Ping the
|
|
294
|
-
|
|
295
|
-
To ping the worker pool, use the `ping` command:
|
|
345
|
+
#### Ping the worker pool
|
|
296
346
|
|
|
297
347
|
```sh
|
|
298
348
|
wool ping --host <host> --port <port> --authkey <authkey>
|
|
@@ -302,46 +352,108 @@ wool ping --host <host> --port <port> --authkey <authkey>
|
|
|
302
352
|
- `--port`: The port number (required).
|
|
303
353
|
- `--authkey`: The authentication key (default: `b""`).
|
|
304
354
|
|
|
305
|
-
###
|
|
355
|
+
### Advanced usage
|
|
356
|
+
|
|
357
|
+
#### Nested pools and sessions
|
|
306
358
|
|
|
307
|
-
|
|
359
|
+
Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts:
|
|
308
360
|
|
|
309
|
-
Module defining remote tasks:
|
|
310
|
-
`tasks.py`
|
|
311
361
|
```python
|
|
312
362
|
import asyncio, wool
|
|
313
363
|
|
|
314
|
-
# Decorate an async function using the `task` decorator
|
|
315
364
|
@wool.task
|
|
316
|
-
async def
|
|
365
|
+
async def task_a():
|
|
317
366
|
await asyncio.sleep(1)
|
|
318
|
-
|
|
367
|
+
|
|
368
|
+
@wool.task
|
|
369
|
+
async def task_b():
|
|
370
|
+
with wool.pool(port=5051):
|
|
371
|
+
await task_a()
|
|
372
|
+
|
|
373
|
+
async def main():
|
|
374
|
+
with wool.pool(port=5050):
|
|
375
|
+
await task_a()
|
|
376
|
+
await task_b()
|
|
377
|
+
|
|
378
|
+
asyncio.run(main())
|
|
319
379
|
```
|
|
320
380
|
|
|
321
|
-
|
|
322
|
-
|
|
381
|
+
In this example, `task_a` is executed by two different pools, while `task_b` is executed by the pool on port 5050.
|
|
382
|
+
|
|
383
|
+
### Best practices
|
|
384
|
+
|
|
385
|
+
#### Sizing worker pools
|
|
386
|
+
|
|
387
|
+
When configuring worker pools, it is important to balance the number of processes with the available system resources:
|
|
388
|
+
|
|
389
|
+
- **CPU-bound tasks**: Size the worker pool to match the number of CPU cores. This is the default behavior when spawning a pool.
|
|
390
|
+
- **I/O-bound tasks**: For workloads involving significant I/O, consider oversizing the pool slightly to maximize the system's I/O capacity utilization.
|
|
391
|
+
- **Mixed workloads**: Monitor memory usage and system load to avoid oversubscription, especially for memory-intensive tasks. Use profiling tools to determine the optimal pool size.
|
|
392
|
+
|
|
393
|
+
#### Defining tasks
|
|
394
|
+
|
|
395
|
+
Wool tasks are coroutine functions that execute asynchronously in a remote `asyncio` event loop. To ensure smooth execution and scalability, prioritize:
|
|
396
|
+
|
|
397
|
+
- **Picklability**: Ensure all task arguments and return values are picklable. Avoid passing unpicklable objects such as open file handles, database connections, or lambda functions.
|
|
398
|
+
- **Statelessness and idempotency**: Design tasks to be stateless and idempotent. Avoid relying on global variables or shared mutable state. This ensures predictable behavior and safe retries.
|
|
399
|
+
- **Non-blocking operations**: To achieve higher concurrency, avoid blocking calls within tasks. Use `asyncio`-compatible libraries for I/O operations.
|
|
400
|
+
- **Inter-process synchronization**: Use Wool's synchronization primitives (e.g., `wool.locking`) for inter-worker and inter-pool coordination. Standard `asyncio` primitives will not behave as expected in a multi-process environment.
|
|
401
|
+
|
|
402
|
+
#### Debugging and logging
|
|
403
|
+
|
|
404
|
+
- Enable detailed logging during development to trace task execution and worker pool behavior:
|
|
405
|
+
```python
|
|
406
|
+
import logging
|
|
407
|
+
logging.basicConfig(level=logging.DEBUG)
|
|
408
|
+
```
|
|
409
|
+
- Use Wool's built-in logging configuration to capture worker-specific logs.
|
|
410
|
+
|
|
411
|
+
#### Nested pools and sessions
|
|
412
|
+
|
|
413
|
+
Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts. This is useful for workflows requiring task segregation or resource isolation.
|
|
414
|
+
|
|
415
|
+
Example:
|
|
323
416
|
```python
|
|
324
417
|
import asyncio, wool
|
|
325
|
-
from tasks import sample_task
|
|
326
418
|
|
|
327
|
-
|
|
419
|
+
@wool.task
|
|
420
|
+
async def task_a():
|
|
421
|
+
await asyncio.sleep(1)
|
|
422
|
+
|
|
423
|
+
@wool.task
|
|
424
|
+
async def task_b():
|
|
425
|
+
with wool.pool(port=5051):
|
|
426
|
+
await task_a()
|
|
427
|
+
|
|
328
428
|
async def main():
|
|
329
|
-
with wool.
|
|
330
|
-
|
|
331
|
-
|
|
429
|
+
with wool.pool(port=5050):
|
|
430
|
+
await task_a()
|
|
431
|
+
await task_b()
|
|
332
432
|
|
|
333
|
-
asyncio.
|
|
433
|
+
asyncio.run(main())
|
|
334
434
|
```
|
|
335
435
|
|
|
336
|
-
|
|
337
|
-
```bash
|
|
338
|
-
wool pool up --port 5050 --authkey deadbeef --breadth 1 --module tasks
|
|
339
|
-
```
|
|
436
|
+
#### Performance optimization
|
|
340
437
|
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
438
|
+
- Minimize the size of arguments and return values to reduce serialization overhead.
|
|
439
|
+
- For large datasets, consider using shared memory or passing references (e.g., file paths) instead of transferring the entire data.
|
|
440
|
+
- Profile tasks to identify and optimize performance bottlenecks.
|
|
441
|
+
|
|
442
|
+
#### Task cancellation
|
|
443
|
+
|
|
444
|
+
- Handle task cancellations gracefully by cleaning up resources and rolling back partial changes.
|
|
445
|
+
- Use `asyncio.CancelledError` to detect and respond to cancellations.
|
|
446
|
+
|
|
447
|
+
#### Error propagation
|
|
448
|
+
|
|
449
|
+
- Wool propagates exceptions raised within tasks to the caller. Use this feature to handle errors centrally in your application.
|
|
450
|
+
|
|
451
|
+
Example:
|
|
452
|
+
```python
|
|
453
|
+
try:
|
|
454
|
+
result = await some_task()
|
|
455
|
+
except Exception as e:
|
|
456
|
+
print(f"Task failed with error: {e}")
|
|
345
457
|
```
|
|
346
458
|
|
|
347
459
|
## License
|
wool-0.1rc14/README.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
**Wool** is a native Python package for transparently executing tasks in a horizontally scalable, distributed network of agnostic worker processes. Any picklable async function or method can be converted into a task with a simple decorator and a client connection.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
### Using pip
|
|
8
|
+
|
|
9
|
+
To install the package using pip, run the following command:
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
[uv] pip install --pre wool
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
### Cloning from GitHub
|
|
16
|
+
|
|
17
|
+
To install the package by cloning from GitHub, run the following commands:
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
git clone https://github.com/wool-labs/wool.git
|
|
21
|
+
cd wool
|
|
22
|
+
[uv] pip install ./wool
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Usage
|
|
26
|
+
|
|
27
|
+
### Declaring tasks
|
|
28
|
+
|
|
29
|
+
Wool tasks are coroutine functions that are executed in a remote `asyncio` event loop within a worker process. To declare a task, use the `@wool.task` decorator:
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import wool
|
|
33
|
+
|
|
34
|
+
@wool.task
|
|
35
|
+
async def sample_task(x, y):
|
|
36
|
+
return x + y
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Tasks must be picklable, stateless, and idempotent. Avoid passing unpicklable objects as arguments or return values.
|
|
40
|
+
|
|
41
|
+
### Worker pools
|
|
42
|
+
|
|
43
|
+
Worker pools are responsible for executing tasks. Wool provides two types of pools:
|
|
44
|
+
|
|
45
|
+
#### Ephemeral pools
|
|
46
|
+
|
|
47
|
+
Ephemeral pools are created and destroyed within the scope of a context manager. Use `wool.pool` to declare an ephemeral pool:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
import asyncio, wool
|
|
51
|
+
|
|
52
|
+
@wool.task
|
|
53
|
+
async def sample_task(x, y):
|
|
54
|
+
return x + y
|
|
55
|
+
|
|
56
|
+
async def main():
|
|
57
|
+
with wool.pool():
|
|
58
|
+
result = await sample_task(1, 2)
|
|
59
|
+
print(f"Result: {result}")
|
|
60
|
+
|
|
61
|
+
asyncio.run(main())
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
#### Durable pools
|
|
65
|
+
|
|
66
|
+
Durable pools are started independently and persist beyond the scope of a single application. Use the `wool` CLI to manage durable pools:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
wool pool up --port 5050 --authkey deadbeef --module tasks
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Connect to a durable pool using `wool.session`:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
import asyncio, wool
|
|
76
|
+
|
|
77
|
+
@wool.task
|
|
78
|
+
async def sample_task(x, y):
|
|
79
|
+
return x + y
|
|
80
|
+
|
|
81
|
+
async def main():
|
|
82
|
+
with wool.session(port=5050, authkey=b"deadbeef"):
|
|
83
|
+
result = await sample_task(1, 2)
|
|
84
|
+
print(f"Result: {result}")
|
|
85
|
+
|
|
86
|
+
asyncio.run(main())
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### CLI commands
|
|
90
|
+
|
|
91
|
+
Wool provides a command-line interface (CLI) for managing worker pools.
|
|
92
|
+
|
|
93
|
+
#### Start the worker pool
|
|
94
|
+
|
|
95
|
+
```sh
|
|
96
|
+
wool pool up --host <host> --port <port> --authkey <authkey> --breadth <breadth> --module <module>
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
- `--host`: The host address (default: `localhost`).
|
|
100
|
+
- `--port`: The port number (default: `0`).
|
|
101
|
+
- `--authkey`: The authentication key (default: `b""`).
|
|
102
|
+
- `--breadth`: The number of worker processes (default: number of CPU cores).
|
|
103
|
+
- `--module`: Python module containing Wool task definitions (optional, can be specified multiple times).
|
|
104
|
+
|
|
105
|
+
#### Stop the worker pool
|
|
106
|
+
|
|
107
|
+
```sh
|
|
108
|
+
wool pool down --host <host> --port <port> --authkey <authkey> --wait
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
- `--host`: The host address (default: `localhost`).
|
|
112
|
+
- `--port`: The port number (required).
|
|
113
|
+
- `--authkey`: The authentication key (default: `b""`).
|
|
114
|
+
- `--wait`: Wait for in-flight tasks to complete before shutting down.
|
|
115
|
+
|
|
116
|
+
#### Ping the worker pool
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
wool ping --host <host> --port <port> --authkey <authkey>
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
- `--host`: The host address (default: `localhost`).
|
|
123
|
+
- `--port`: The port number (required).
|
|
124
|
+
- `--authkey`: The authentication key (default: `b""`).
|
|
125
|
+
|
|
126
|
+
### Advanced usage
|
|
127
|
+
|
|
128
|
+
#### Nested pools and sessions
|
|
129
|
+
|
|
130
|
+
Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
import asyncio, wool
|
|
134
|
+
|
|
135
|
+
@wool.task
|
|
136
|
+
async def task_a():
|
|
137
|
+
await asyncio.sleep(1)
|
|
138
|
+
|
|
139
|
+
@wool.task
|
|
140
|
+
async def task_b():
|
|
141
|
+
with wool.pool(port=5051):
|
|
142
|
+
await task_a()
|
|
143
|
+
|
|
144
|
+
async def main():
|
|
145
|
+
with wool.pool(port=5050):
|
|
146
|
+
await task_a()
|
|
147
|
+
await task_b()
|
|
148
|
+
|
|
149
|
+
asyncio.run(main())
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
In this example, `task_a` is executed by two different pools, while `task_b` is executed by the pool on port 5050.
|
|
153
|
+
|
|
154
|
+
### Best practices
|
|
155
|
+
|
|
156
|
+
#### Sizing worker pools
|
|
157
|
+
|
|
158
|
+
When configuring worker pools, it is important to balance the number of processes with the available system resources:
|
|
159
|
+
|
|
160
|
+
- **CPU-bound tasks**: Size the worker pool to match the number of CPU cores. This is the default behavior when spawning a pool.
|
|
161
|
+
- **I/O-bound tasks**: For workloads involving significant I/O, consider oversizing the pool slightly to maximize the system's I/O capacity utilization.
|
|
162
|
+
- **Mixed workloads**: Monitor memory usage and system load to avoid oversubscription, especially for memory-intensive tasks. Use profiling tools to determine the optimal pool size.
|
|
163
|
+
|
|
164
|
+
#### Defining tasks
|
|
165
|
+
|
|
166
|
+
Wool tasks are coroutine functions that execute asynchronously in a remote `asyncio` event loop. To ensure smooth execution and scalability, prioritize:
|
|
167
|
+
|
|
168
|
+
- **Picklability**: Ensure all task arguments and return values are picklable. Avoid passing unpicklable objects such as open file handles, database connections, or lambda functions.
|
|
169
|
+
- **Statelessness and idempotency**: Design tasks to be stateless and idempotent. Avoid relying on global variables or shared mutable state. This ensures predictable behavior and safe retries.
|
|
170
|
+
- **Non-blocking operations**: To achieve higher concurrency, avoid blocking calls within tasks. Use `asyncio`-compatible libraries for I/O operations.
|
|
171
|
+
- **Inter-process synchronization**: Use Wool's synchronization primitives (e.g., `wool.locking`) for inter-worker and inter-pool coordination. Standard `asyncio` primitives will not behave as expected in a multi-process environment.
|
|
172
|
+
|
|
173
|
+
#### Debugging and logging
|
|
174
|
+
|
|
175
|
+
- Enable detailed logging during development to trace task execution and worker pool behavior:
|
|
176
|
+
```python
|
|
177
|
+
import logging
|
|
178
|
+
logging.basicConfig(level=logging.DEBUG)
|
|
179
|
+
```
|
|
180
|
+
- Use Wool's built-in logging configuration to capture worker-specific logs.
|
|
181
|
+
|
|
182
|
+
#### Nested pools and sessions
|
|
183
|
+
|
|
184
|
+
Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts. This is useful for workflows requiring task segregation or resource isolation.
|
|
185
|
+
|
|
186
|
+
Example:
|
|
187
|
+
```python
|
|
188
|
+
import asyncio, wool
|
|
189
|
+
|
|
190
|
+
@wool.task
|
|
191
|
+
async def task_a():
|
|
192
|
+
await asyncio.sleep(1)
|
|
193
|
+
|
|
194
|
+
@wool.task
|
|
195
|
+
async def task_b():
|
|
196
|
+
with wool.pool(port=5051):
|
|
197
|
+
await task_a()
|
|
198
|
+
|
|
199
|
+
async def main():
|
|
200
|
+
with wool.pool(port=5050):
|
|
201
|
+
await task_a()
|
|
202
|
+
await task_b()
|
|
203
|
+
|
|
204
|
+
asyncio.run(main())
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
#### Performance optimization
|
|
208
|
+
|
|
209
|
+
- Minimize the size of arguments and return values to reduce serialization overhead.
|
|
210
|
+
- For large datasets, consider using shared memory or passing references (e.g., file paths) instead of transferring the entire data.
|
|
211
|
+
- Profile tasks to identify and optimize performance bottlenecks.
|
|
212
|
+
|
|
213
|
+
#### Task cancellation
|
|
214
|
+
|
|
215
|
+
- Handle task cancellations gracefully by cleaning up resources and rolling back partial changes.
|
|
216
|
+
- Use `asyncio.CancelledError` to detect and respond to cancellations.
|
|
217
|
+
|
|
218
|
+
#### Error propagation
|
|
219
|
+
|
|
220
|
+
- Wool propagates exceptions raised within tasks to the caller. Use this feature to handle errors centrally in your application.
|
|
221
|
+
|
|
222
|
+
Example:
|
|
223
|
+
```python
|
|
224
|
+
try:
|
|
225
|
+
result = await some_task()
|
|
226
|
+
except Exception as e:
|
|
227
|
+
print(f"Task failed with error: {e}")
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
## License
|
|
231
|
+
|
|
232
|
+
This project is licensed under the Apache License Version 2.0.
|
|
@@ -54,9 +54,7 @@ try:
|
|
|
54
54
|
except PackageNotFoundError:
|
|
55
55
|
__version__ = "unknown"
|
|
56
56
|
|
|
57
|
-
__proxy__: Final[ContextVar[WorkerProxy | None]] = ContextVar(
|
|
58
|
-
"__proxy__", default=None
|
|
59
|
-
)
|
|
57
|
+
__proxy__: Final[ContextVar[WorkerProxy | None]] = ContextVar("__proxy__", default=None)
|
|
60
58
|
|
|
61
59
|
__proxy_pool__: Final[ContextVar[ResourcePool[WorkerProxy] | None]] = ContextVar(
|
|
62
60
|
"__proxy_pool__", default=None
|
|
@@ -43,7 +43,7 @@ if TYPE_CHECKING:
|
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
@contextmanager
|
|
46
|
-
def _signal_handlers(service:
|
|
46
|
+
def _signal_handlers(service: WorkerService):
|
|
47
47
|
"""Context manager for setting up signal handlers for graceful shutdown.
|
|
48
48
|
|
|
49
49
|
Installs SIGTERM and SIGINT handlers that gracefully shut down the worker
|
|
@@ -991,46 +991,28 @@ class LocalRegistrar(Registrar):
|
|
|
991
991
|
|
|
992
992
|
_shared_memory: multiprocessing.shared_memory.SharedMemory | None = None
|
|
993
993
|
_uri: str
|
|
994
|
-
_created_shared_memory: bool = False
|
|
995
994
|
|
|
996
995
|
def __init__(self, uri: str):
|
|
997
996
|
super().__init__()
|
|
998
997
|
self._uri = uri
|
|
999
|
-
self._created_shared_memory = False
|
|
1000
998
|
|
|
1001
999
|
async def _start(self) -> None:
|
|
1002
1000
|
"""Initialize shared memory for worker registration."""
|
|
1003
1001
|
if self._shared_memory is None:
|
|
1004
1002
|
# Try to connect to existing shared memory first, create if it doesn't exist
|
|
1005
1003
|
shared_memory_name = hashlib.sha256(self._uri.encode()).hexdigest()[:12]
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
)
|
|
1010
|
-
except FileNotFoundError:
|
|
1011
|
-
# Create new shared memory if it doesn't exist
|
|
1012
|
-
self._shared_memory = multiprocessing.shared_memory.SharedMemory(
|
|
1013
|
-
name=shared_memory_name,
|
|
1014
|
-
create=True,
|
|
1015
|
-
size=1024, # 1024 bytes = 256 worker slots (4 bytes per port)
|
|
1016
|
-
)
|
|
1017
|
-
self._created_shared_memory = True
|
|
1018
|
-
# Initialize all slots to 0 (empty)
|
|
1019
|
-
for i in range(len(self._shared_memory.buf)):
|
|
1020
|
-
self._shared_memory.buf[i] = 0
|
|
1004
|
+
self._shared_memory = multiprocessing.shared_memory.SharedMemory(
|
|
1005
|
+
name=shared_memory_name
|
|
1006
|
+
)
|
|
1021
1007
|
|
|
1022
1008
|
async def _stop(self) -> None:
|
|
1023
1009
|
"""Clean up shared memory resources."""
|
|
1024
1010
|
if self._shared_memory:
|
|
1025
1011
|
try:
|
|
1026
1012
|
self._shared_memory.close()
|
|
1027
|
-
# Unlink the shared memory if this registrar created it
|
|
1028
|
-
if self._created_shared_memory:
|
|
1029
|
-
self._shared_memory.unlink()
|
|
1030
1013
|
except Exception:
|
|
1031
1014
|
pass
|
|
1032
1015
|
self._shared_memory = None
|
|
1033
|
-
self._created_shared_memory = False
|
|
1034
1016
|
|
|
1035
1017
|
async def _register(self, worker_info: WorkerInfo) -> None:
|
|
1036
1018
|
"""Register a worker by writing its port to shared memory.
|
|
@@ -1138,7 +1120,6 @@ class LocalDiscovery(Discovery):
|
|
|
1138
1120
|
async def _start(self) -> None:
|
|
1139
1121
|
"""Starts monitoring shared memory for worker registrations."""
|
|
1140
1122
|
if self._shared_memory is None:
|
|
1141
|
-
# Try to connect to existing shared memory first
|
|
1142
1123
|
self._shared_memory = multiprocessing.shared_memory.SharedMemory(
|
|
1143
1124
|
name=hashlib.sha256(self._uri.encode()).hexdigest()[:12]
|
|
1144
1125
|
)
|
|
@@ -1172,7 +1153,9 @@ class LocalDiscovery(Discovery):
|
|
|
1172
1153
|
# Read current state from shared memory
|
|
1173
1154
|
if self._shared_memory:
|
|
1174
1155
|
for i in range(0, len(self._shared_memory.buf), 4):
|
|
1175
|
-
port = struct.unpack(
|
|
1156
|
+
port = struct.unpack(
|
|
1157
|
+
"I", bytes(self._shared_memory.buf[i : i + 4])
|
|
1158
|
+
)[0]
|
|
1176
1159
|
if port > 0: # Active worker
|
|
1177
1160
|
worker_info = WorkerInfo(
|
|
1178
1161
|
uid=f"worker-{port}",
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import atexit
|
|
4
5
|
import hashlib
|
|
5
6
|
import os
|
|
6
7
|
import uuid
|
|
8
|
+
from contextlib import asynccontextmanager
|
|
7
9
|
from multiprocessing.shared_memory import SharedMemory
|
|
8
10
|
from typing import Final
|
|
9
11
|
from typing import overload
|
|
@@ -172,7 +174,6 @@ class WorkerPool:
|
|
|
172
174
|
"""
|
|
173
175
|
|
|
174
176
|
_workers: Final[list[Worker]]
|
|
175
|
-
_shared_memory = None
|
|
176
177
|
|
|
177
178
|
@overload
|
|
178
179
|
def __init__(
|
|
@@ -226,19 +227,27 @@ class WorkerPool:
|
|
|
226
227
|
|
|
227
228
|
uri = f"pool-{uuid.uuid4().hex}"
|
|
228
229
|
|
|
230
|
+
@asynccontextmanager
|
|
229
231
|
async def create_proxy():
|
|
230
|
-
|
|
232
|
+
shared_memory_size = (size + 1) * 4
|
|
233
|
+
shared_memory = SharedMemory(
|
|
231
234
|
name=hashlib.sha256(uri.encode()).hexdigest()[:12],
|
|
232
235
|
create=True,
|
|
233
|
-
size=
|
|
234
|
-
)
|
|
235
|
-
for i in range(1024):
|
|
236
|
-
self._shared_memory.buf[i] = 0
|
|
237
|
-
await self._spawn_workers(uri, *tags, size=size, factory=worker)
|
|
238
|
-
return WorkerProxy(
|
|
239
|
-
discovery=LocalDiscovery(uri),
|
|
240
|
-
loadbalancer=loadbalancer,
|
|
236
|
+
size=shared_memory_size,
|
|
241
237
|
)
|
|
238
|
+
cleanup = atexit.register(lambda: shared_memory.unlink())
|
|
239
|
+
try:
|
|
240
|
+
for i in range(shared_memory_size):
|
|
241
|
+
shared_memory.buf[i] = 0
|
|
242
|
+
await self._spawn_workers(uri, *tags, size=size, factory=worker)
|
|
243
|
+
async with WorkerProxy(
|
|
244
|
+
discovery=LocalDiscovery(uri),
|
|
245
|
+
loadbalancer=loadbalancer,
|
|
246
|
+
):
|
|
247
|
+
yield
|
|
248
|
+
finally:
|
|
249
|
+
shared_memory.unlink()
|
|
250
|
+
atexit.unregister(cleanup)
|
|
242
251
|
|
|
243
252
|
case (size, None) if size is not None:
|
|
244
253
|
if size == 0:
|
|
@@ -251,27 +260,37 @@ class WorkerPool:
|
|
|
251
260
|
|
|
252
261
|
uri = f"pool-{uuid.uuid4().hex}"
|
|
253
262
|
|
|
263
|
+
@asynccontextmanager
|
|
254
264
|
async def create_proxy():
|
|
255
|
-
|
|
265
|
+
shared_memory_size = (size + 1) * 4
|
|
266
|
+
shared_memory = SharedMemory(
|
|
256
267
|
name=hashlib.sha256(uri.encode()).hexdigest()[:12],
|
|
257
268
|
create=True,
|
|
258
|
-
size=
|
|
259
|
-
)
|
|
260
|
-
for i in range(1024):
|
|
261
|
-
self._shared_memory.buf[i] = 0
|
|
262
|
-
await self._spawn_workers(uri, *tags, size=size, factory=worker)
|
|
263
|
-
return WorkerProxy(
|
|
264
|
-
discovery=LocalDiscovery(uri),
|
|
265
|
-
loadbalancer=loadbalancer,
|
|
269
|
+
size=shared_memory_size,
|
|
266
270
|
)
|
|
271
|
+
cleanup = atexit.register(lambda: shared_memory.unlink())
|
|
272
|
+
try:
|
|
273
|
+
for i in range(shared_memory_size):
|
|
274
|
+
shared_memory.buf[i] = 0
|
|
275
|
+
await self._spawn_workers(uri, *tags, size=size, factory=worker)
|
|
276
|
+
async with WorkerProxy(
|
|
277
|
+
discovery=LocalDiscovery(uri),
|
|
278
|
+
loadbalancer=loadbalancer,
|
|
279
|
+
):
|
|
280
|
+
yield
|
|
281
|
+
finally:
|
|
282
|
+
shared_memory.unlink()
|
|
283
|
+
atexit.unregister(cleanup)
|
|
267
284
|
|
|
268
285
|
case (None, discovery) if discovery is not None:
|
|
269
286
|
|
|
287
|
+
@asynccontextmanager
|
|
270
288
|
async def create_proxy():
|
|
271
|
-
|
|
289
|
+
async with WorkerProxy(
|
|
272
290
|
discovery=discovery,
|
|
273
291
|
loadbalancer=loadbalancer,
|
|
274
|
-
)
|
|
292
|
+
):
|
|
293
|
+
yield
|
|
275
294
|
|
|
276
295
|
case _:
|
|
277
296
|
raise RuntimeError
|
|
@@ -287,18 +306,14 @@ class WorkerPool:
|
|
|
287
306
|
:returns:
|
|
288
307
|
The :py:class:`WorkerPool` instance itself for method chaining.
|
|
289
308
|
"""
|
|
290
|
-
self.
|
|
291
|
-
await self.
|
|
309
|
+
self._proxy_context = self._proxy_factory()
|
|
310
|
+
await self._proxy_context.__aenter__()
|
|
292
311
|
return self
|
|
293
312
|
|
|
294
313
|
async def __aexit__(self, *args):
|
|
295
314
|
"""Stops all workers and tears down the pool and its services."""
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
await self._proxy.__aexit__(*args)
|
|
299
|
-
finally:
|
|
300
|
-
if self._shared_memory is not None:
|
|
301
|
-
self._shared_memory.unlink()
|
|
315
|
+
await self._stop_workers()
|
|
316
|
+
await self._proxy_context.__aexit__(*args)
|
|
302
317
|
|
|
303
318
|
async def _spawn_workers(
|
|
304
319
|
self, uri, *tags: str, size: int, factory: WorkerFactory | None
|
wool-0.1rc12/README.md
DELETED
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
# Wool
|
|
2
|
-
|
|
3
|
-
Wool is a native Python package for transparently executing tasks in a horizontally scalable, distributed network of agnostic worker processes. Any picklable async function or method can be converted into a task with a simple decorator and a client connection.
|
|
4
|
-
|
|
5
|
-
## Installation
|
|
6
|
-
|
|
7
|
-
### Using pip
|
|
8
|
-
|
|
9
|
-
To install the package using pip, run the following command:
|
|
10
|
-
|
|
11
|
-
```sh
|
|
12
|
-
[uv] pip install --pre wool
|
|
13
|
-
```
|
|
14
|
-
|
|
15
|
-
### Cloning from GitHub
|
|
16
|
-
|
|
17
|
-
To install the package by cloning from GitHub, run the following commands:
|
|
18
|
-
|
|
19
|
-
```sh
|
|
20
|
-
git clone https://github.com/wool-labs/wool.git
|
|
21
|
-
cd wool
|
|
22
|
-
[uv] pip install ./wool
|
|
23
|
-
```
|
|
24
|
-
|
|
25
|
-
## Usage
|
|
26
|
-
|
|
27
|
-
### CLI Commands
|
|
28
|
-
|
|
29
|
-
Wool provides a command-line interface (CLI) for managing the worker pool.
|
|
30
|
-
|
|
31
|
-
To list the available commands:
|
|
32
|
-
|
|
33
|
-
```sh
|
|
34
|
-
wool --help
|
|
35
|
-
```
|
|
36
|
-
|
|
37
|
-
#### Start the Worker Pool
|
|
38
|
-
|
|
39
|
-
To start the worker pool, use the `up` command:
|
|
40
|
-
|
|
41
|
-
```sh
|
|
42
|
-
wool pool up --host <host> --port <port> --authkey <authkey> --breadth <breadth> --module <module>
|
|
43
|
-
```
|
|
44
|
-
|
|
45
|
-
- `--host`: The host address (default: `localhost`).
|
|
46
|
-
- `--port`: The port number (default: `0`).
|
|
47
|
-
- `--authkey`: The authentication key (default: `b""`).
|
|
48
|
-
- `--breadth`: The number of worker processes (default: number of CPU cores).
|
|
49
|
-
- `--module`: Python module containing Wool task definitions to be executed by this pool (optional, can be specified multiple times).
|
|
50
|
-
|
|
51
|
-
#### Stop the Worker Pool
|
|
52
|
-
|
|
53
|
-
To stop the worker pool, use the `down` command:
|
|
54
|
-
|
|
55
|
-
```sh
|
|
56
|
-
wool pool down --host <host> --port <port> --authkey <authkey> --wait
|
|
57
|
-
```
|
|
58
|
-
|
|
59
|
-
- `--host`: The host address (default: `localhost`).
|
|
60
|
-
- `--port`: The port number (required).
|
|
61
|
-
- `--authkey`: The authentication key (default: `b""`).
|
|
62
|
-
- `--wait`: Wait for in-flight tasks to complete before shutting down.
|
|
63
|
-
|
|
64
|
-
#### Ping the Worker Pool
|
|
65
|
-
|
|
66
|
-
To ping the worker pool, use the `ping` command:
|
|
67
|
-
|
|
68
|
-
```sh
|
|
69
|
-
wool ping --host <host> --port <port> --authkey <authkey>
|
|
70
|
-
```
|
|
71
|
-
|
|
72
|
-
- `--host`: The host address (default: `localhost`).
|
|
73
|
-
- `--port`: The port number (required).
|
|
74
|
-
- `--authkey`: The authentication key (default: `b""`).
|
|
75
|
-
|
|
76
|
-
### Sample Python Application
|
|
77
|
-
|
|
78
|
-
Below is an example of how to create a Wool client connection, decorate an async function using the `task` decorator, and execute the function remotely:
|
|
79
|
-
|
|
80
|
-
Module defining remote tasks:
|
|
81
|
-
`tasks.py`
|
|
82
|
-
```python
|
|
83
|
-
import asyncio, wool
|
|
84
|
-
|
|
85
|
-
# Decorate an async function using the `task` decorator
|
|
86
|
-
@wool.task
|
|
87
|
-
async def sample_task(x, y):
|
|
88
|
-
await asyncio.sleep(1)
|
|
89
|
-
return x + y
|
|
90
|
-
```
|
|
91
|
-
|
|
92
|
-
Module executing remote workflow:
|
|
93
|
-
`main.py`
|
|
94
|
-
```python
|
|
95
|
-
import asyncio, wool
|
|
96
|
-
from tasks import sample_task
|
|
97
|
-
|
|
98
|
-
# Execute the decorated function in an external worker pool
|
|
99
|
-
async def main():
|
|
100
|
-
with wool.PoolSession(port=5050, authkey=b"deadbeef"):
|
|
101
|
-
result = await sample_task(1, 2)
|
|
102
|
-
print(f"Result: {result}")
|
|
103
|
-
|
|
104
|
-
asyncio.new_event_loop().run_until_complete(main())
|
|
105
|
-
```
|
|
106
|
-
|
|
107
|
-
To run the demo, first start a worker pool specifying the module defining the tasks to be executed:
|
|
108
|
-
```bash
|
|
109
|
-
wool pool up --port 5050 --authkey deadbeef --breadth 1 --module tasks
|
|
110
|
-
```
|
|
111
|
-
|
|
112
|
-
Next, in a separate terminal, execute the application defined in `main.py` and, finally, stop the worker pool:
|
|
113
|
-
```bash
|
|
114
|
-
python main.py
|
|
115
|
-
wool pool down --port 5050 --authkey deadbeef
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
## License
|
|
119
|
-
|
|
120
|
-
This project is licensed under the Apache License Version 2.0.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|