wool 0.1rc20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,231 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ from contextlib import contextmanager
5
+ from typing import AsyncIterator
6
+
7
+ import cloudpickle
8
+ from grpc import StatusCode
9
+ from grpc.aio import ServicerContext
10
+
11
+ import wool
12
+ from wool._work import WoolTask
13
+ from wool._work import WoolTaskEvent
14
+ from wool.core import protobuf as pb
15
+
16
+
17
+ class ReadOnlyEvent:
18
+ """A read-only wrapper around :class:`asyncio.Event`.
19
+
20
+ Provides access to check if an event is set and wait for it to be
21
+ set, but prevents external code from setting or clearing the event.
22
+
23
+ :param event:
24
+ The underlying :class:`asyncio.Event` to wrap.
25
+ """
26
+
27
+ def __init__(self, event: asyncio.Event):
28
+ self._event = event
29
+
30
+ def is_set(self) -> bool:
31
+ """Check if the event is set.
32
+
33
+ :returns:
34
+ ``True`` if the event is set, ``False`` otherwise.
35
+ """
36
+ return self._event.is_set()
37
+
38
+ async def wait(self) -> None:
39
+ """Wait until the underlying event is set."""
40
+ await self._event.wait()
41
+
42
+
43
+ class WorkerService(pb.worker.WorkerServicer):
44
+ """gRPC service for task execution.
45
+
46
+ Implements the worker gRPC interface for receiving and executing
47
+ tasks. Runs tasks in the current asyncio event loop and streams
48
+ results back to the client.
49
+
50
+ Handles graceful shutdown by rejecting new tasks while allowing
51
+ in-flight tasks to complete. Exposes :attr:`stopping` and
52
+ :attr:`stopped` events for lifecycle monitoring.
53
+ """
54
+
55
+ _tasks: set[asyncio.Task]
56
+ _stopped: asyncio.Event
57
+ _stopping: asyncio.Event
58
+ _task_completed: asyncio.Event
59
+
60
+ def __init__(self):
61
+ self._stopped = asyncio.Event()
62
+ self._stopping = asyncio.Event()
63
+ self._task_completed = asyncio.Event()
64
+ self._tasks = set()
65
+
66
+ @property
67
+ def stopping(self) -> ReadOnlyEvent:
68
+ """Read-only event signaling that the service is stopping.
69
+
70
+ :returns:
71
+ A :class:`ReadOnlyEvent`.
72
+ """
73
+ return ReadOnlyEvent(self._stopping)
74
+
75
+ @property
76
+ def stopped(self) -> ReadOnlyEvent:
77
+ """Read-only event signaling that the service has stopped.
78
+
79
+ :returns:
80
+ A :class:`ReadOnlyEvent`.
81
+ """
82
+ return ReadOnlyEvent(self._stopped)
83
+
84
+ async def dispatch(
85
+ self, request: pb.task.Task, context: ServicerContext
86
+ ) -> AsyncIterator[pb.worker.Response]:
87
+ """Execute a task in the current event loop.
88
+
89
+ Deserializes the incoming task into a :class:`WoolTask`
90
+ instance, schedules it for execution in the current asyncio
91
+ event loop, and yields responses for acknowledgment and result.
92
+
93
+ :param request:
94
+ The protobuf task message containing the serialized task
95
+ data.
96
+ :param context:
97
+ The :class:`grpc.aio.ServicerContext` for this request.
98
+ :yields:
99
+ First yields an Ack Response when task processing begins,
100
+ then yields a Response containing the task result.
101
+
102
+ .. note::
103
+ Emits a :class:`WoolTaskEvent` when the task is
104
+ scheduled for execution.
105
+ """
106
+ if self._stopping.is_set():
107
+ await context.abort(
108
+ StatusCode.UNAVAILABLE, "Worker service is shutting down"
109
+ )
110
+
111
+ with self._tracker(WoolTask.from_protobuf(request)) as task:
112
+ try:
113
+ yield pb.worker.Response(ack=pb.worker.Ack())
114
+ try:
115
+ result = pb.task.Result(dump=cloudpickle.dumps(await task))
116
+ yield pb.worker.Response(result=result)
117
+ except Exception as e:
118
+ exception = pb.task.Exception(dump=cloudpickle.dumps(e))
119
+ yield pb.worker.Response(exception=exception)
120
+ except asyncio.CancelledError as e:
121
+ exception = pb.task.Exception(dump=cloudpickle.dumps(e))
122
+ yield pb.worker.Response(exception=exception)
123
+
124
+ async def stop(
125
+ self, request: pb.worker.StopRequest, context: ServicerContext | None
126
+ ) -> pb.worker.Void:
127
+ """Stop the worker service and its thread.
128
+
129
+ Gracefully shuts down the worker thread and signals the server
130
+ to stop accepting new requests. This method is idempotent and
131
+ can be called multiple times safely.
132
+
133
+ :param request:
134
+ The protobuf stop request containing the wait timeout.
135
+ :param context:
136
+ The :class:`grpc.aio.ServicerContext` for this request.
137
+ :returns:
138
+ An empty protobuf response indicating completion.
139
+ """
140
+ if self._stopping.is_set():
141
+ return pb.worker.Void()
142
+ await self._stop(timeout=request.timeout)
143
+ return pb.worker.Void()
144
+
145
+ @contextmanager
146
+ def _tracker(self, wool_task: WoolTask):
147
+ """Context manager for tracking running tasks.
148
+
149
+ Manages the lifecycle of a task execution, adding it to the
150
+ active tasks set and emitting appropriate events. Ensures
151
+ proper cleanup when the task completes or fails.
152
+
153
+ :param wool_task:
154
+ The :class:`WoolTask` instance to execute and track.
155
+ :yields:
156
+ The :class:`asyncio.Task` created for the wool task.
157
+
158
+ .. note::
159
+ Emits a :class:`WoolTaskEvent` with type "task-scheduled"
160
+ when the task begins execution.
161
+ """
162
+ WoolTaskEvent("task-scheduled", task=wool_task).emit()
163
+ task = asyncio.create_task(wool_task.run())
164
+ self._tasks.add(task)
165
+ try:
166
+ yield task
167
+ finally:
168
+ self._tasks.remove(task)
169
+
170
+ async def _stop(self, *, timeout: float | None = 0) -> None:
171
+ self._stopping.set()
172
+ await self._await_or_cancel_tasks(timeout=timeout)
173
+ try:
174
+ if proxy_pool := wool.__proxy_pool__.get():
175
+ await proxy_pool.clear()
176
+ finally:
177
+ self._stopped.set()
178
+
179
+ async def _await_or_cancel_tasks(self, *, timeout: float | None = 0) -> None:
180
+ """Stop the worker service gracefully.
181
+
182
+ Gracefully shuts down the worker service by canceling or waiting
183
+ for running tasks. This method is idempotent and can be called
184
+ multiple times safely.
185
+
186
+ :param timeout:
187
+ Maximum time to wait for tasks to complete. If 0 (default),
188
+ tasks are canceled immediately. If None, waits indefinitely.
189
+ If a positive number, waits for that many seconds before
190
+ canceling tasks.
191
+
192
+ .. note::
193
+ If a timeout occurs while waiting for tasks to complete,
194
+ the method recursively calls itself with a timeout of 0
195
+ to cancel all remaining tasks immediately.
196
+ """
197
+ if self._tasks and timeout == 0:
198
+ await self._cancel(*self._tasks)
199
+ elif self._tasks:
200
+ try:
201
+ await asyncio.wait_for(
202
+ asyncio.gather(*self._tasks, return_exceptions=True),
203
+ timeout=timeout,
204
+ )
205
+ except asyncio.TimeoutError:
206
+ return await self._await_or_cancel_tasks(timeout=0)
207
+
208
+ async def _cancel(self, *tasks: asyncio.Task):
209
+ """Cancel multiple tasks safely.
210
+
211
+ Cancels the provided tasks while performing safety checks to
212
+ avoid canceling the current task or already completed tasks.
213
+ Waits for all cancelled tasks to complete in parallel and handles
214
+ cancellation exceptions.
215
+
216
+ :param tasks:
217
+ The :class:`asyncio.Task` instances to cancel.
218
+
219
+ .. note::
220
+ This method performs the following safety checks:
221
+ - Avoids canceling the current task (would cause deadlock)
222
+ - Only cancels tasks that are not already done
223
+ - Properly handles :exc:`asyncio.CancelledError`
224
+ exceptions.
225
+ """
226
+ current = asyncio.current_task()
227
+ to_cancel = [task for task in tasks if not task.done() and task != current]
228
+ for task in to_cancel:
229
+ task.cancel()
230
+ if to_cancel:
231
+ await asyncio.gather(*to_cancel, return_exceptions=True)
@@ -0,0 +1,463 @@
1
+ Metadata-Version: 2.4
2
+ Name: wool
3
+ Version: 0.1rc20
4
+ Summary: A Python framework for distributed multiprocessing.
5
+ Author-email: Conrad Bzura <conrad@wool.io>
6
+ Maintainer-email: maintainers@wool.io
7
+ License: Apache License
8
+ Version 2.0, January 2004
9
+ http://www.apache.org/licenses/
10
+
11
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
12
+
13
+ 1. Definitions.
14
+
15
+ "License" shall mean the terms and conditions for use, reproduction,
16
+ and distribution as defined by Sections 1 through 9 of this document.
17
+
18
+ "Licensor" shall mean the copyright owner or entity authorized by
19
+ the copyright owner that is granting the License.
20
+
21
+ "Legal Entity" shall mean the union of the acting entity and all
22
+ other entities that control, are controlled by, or are under common
23
+ control with that entity. For the purposes of this definition,
24
+ "control" means (i) the power, direct or indirect, to cause the
25
+ direction or management of such entity, whether by contract or
26
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
27
+ outstanding shares, or (iii) beneficial ownership of such entity.
28
+
29
+ "You" (or "Your") shall mean an individual or Legal Entity
30
+ exercising permissions granted by this License.
31
+
32
+ "Source" form shall mean the preferred form for making modifications,
33
+ including but not limited to software source code, documentation
34
+ source, and configuration files.
35
+
36
+ "Object" form shall mean any form resulting from mechanical
37
+ transformation or translation of a Source form, including but
38
+ not limited to compiled object code, generated documentation,
39
+ and conversions to other media types.
40
+
41
+ "Work" shall mean the work of authorship, whether in Source or
42
+ Object form, made available under the License, as indicated by a
43
+ copyright notice that is included in or attached to the work
44
+ (an example is provided in the Appendix below).
45
+
46
+ "Derivative Works" shall mean any work, whether in Source or Object
47
+ form, that is based on (or derived from) the Work and for which the
48
+ editorial revisions, annotations, elaborations, or other modifications
49
+ represent, as a whole, an original work of authorship. For the purposes
50
+ of this License, Derivative Works shall not include works that remain
51
+ separable from, or merely link (or bind by name) to the interfaces of,
52
+ the Work and Derivative Works thereof.
53
+
54
+ "Contribution" shall mean any work of authorship, including
55
+ the original version of the Work and any modifications or additions
56
+ to that Work or Derivative Works thereof, that is intentionally
57
+ submitted to Licensor for inclusion in the Work by the copyright owner
58
+ or by an individual or Legal Entity authorized to submit on behalf of
59
+ the copyright owner. For the purposes of this definition, "submitted"
60
+ means any form of electronic, verbal, or written communication sent
61
+ to the Licensor or its representatives, including but not limited to
62
+ communication on electronic mailing lists, source code control systems,
63
+ and issue tracking systems that are managed by, or on behalf of, the
64
+ Licensor for the purpose of discussing and improving the Work, but
65
+ excluding communication that is conspicuously marked or otherwise
66
+ designated in writing by the copyright owner as "Not a Contribution."
67
+
68
+ "Contributor" shall mean Licensor and any individual or Legal Entity
69
+ on behalf of whom a Contribution has been received by Licensor and
70
+ subsequently incorporated within the Work.
71
+
72
+ 2. Grant of Copyright License. Subject to the terms and conditions of
73
+ this License, each Contributor hereby grants to You a perpetual,
74
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
75
+ copyright license to reproduce, prepare Derivative Works of,
76
+ publicly display, publicly perform, sublicense, and distribute the
77
+ Work and such Derivative Works in Source or Object form.
78
+
79
+ 3. Grant of Patent License. Subject to the terms and conditions of
80
+ this License, each Contributor hereby grants to You a perpetual,
81
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
82
+ (except as stated in this section) patent license to make, have made,
83
+ use, offer to sell, sell, import, and otherwise transfer the Work,
84
+ where such license applies only to those patent claims licensable
85
+ by such Contributor that are necessarily infringed by their
86
+ Contribution(s) alone or by combination of their Contribution(s)
87
+ with the Work to which such Contribution(s) was submitted. If You
88
+ institute patent litigation against any entity (including a
89
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
90
+ or a Contribution incorporated within the Work constitutes direct
91
+ or contributory patent infringement, then any patent licenses
92
+ granted to You under this License for that Work shall terminate
93
+ as of the date such litigation is filed.
94
+
95
+ 4. Redistribution. You may reproduce and distribute copies of the
96
+ Work or Derivative Works thereof in any medium, with or without
97
+ modifications, and in Source or Object form, provided that You
98
+ meet the following conditions:
99
+
100
+ (a) You must give any other recipients of the Work or
101
+ Derivative Works a copy of this License; and
102
+
103
+ (b) You must cause any modified files to carry prominent notices
104
+ stating that You changed the files; and
105
+
106
+ (c) You must retain, in the Source form of any Derivative Works
107
+ that You distribute, all copyright, patent, trademark, and
108
+ attribution notices from the Source form of the Work,
109
+ excluding those notices that do not pertain to any part of
110
+ the Derivative Works; and
111
+
112
+ (d) If the Work includes a "NOTICE" text file as part of its
113
+ distribution, then any Derivative Works that You distribute must
114
+ include a readable copy of the attribution notices contained
115
+ within such NOTICE file, excluding those notices that do not
116
+ pertain to any part of the Derivative Works, in at least one
117
+ of the following places: within a NOTICE text file distributed
118
+ as part of the Derivative Works; within the Source form or
119
+ documentation, if provided along with the Derivative Works; or,
120
+ within a display generated by the Derivative Works, if and
121
+ wherever such third-party notices normally appear. The contents
122
+ of the NOTICE file are for informational purposes only and
123
+ do not modify the License. You may add Your own attribution
124
+ notices within Derivative Works that You distribute, alongside
125
+ or as an addendum to the NOTICE text from the Work, provided
126
+ that such additional attribution notices cannot be construed
127
+ as modifying the License.
128
+
129
+ You may add Your own copyright statement to Your modifications and
130
+ may provide additional or different license terms and conditions
131
+ for use, reproduction, or distribution of Your modifications, or
132
+ for any such Derivative Works as a whole, provided Your use,
133
+ reproduction, and distribution of the Work otherwise complies with
134
+ the conditions stated in this License.
135
+
136
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
137
+ any Contribution intentionally submitted for inclusion in the Work
138
+ by You to the Licensor shall be under the terms and conditions of
139
+ this License, without any additional terms or conditions.
140
+ Notwithstanding the above, nothing herein shall supersede or modify
141
+ the terms of any separate license agreement you may have executed
142
+ with Licensor regarding such Contributions.
143
+
144
+ 6. Trademarks. This License does not grant permission to use the trade
145
+ names, trademarks, service marks, or product names of the Licensor,
146
+ except as required for reasonable and customary use in describing the
147
+ origin of the Work and reproducing the content of the NOTICE file.
148
+
149
+ 7. Disclaimer of Warranty. Unless required by applicable law or
150
+ agreed to in writing, Licensor provides the Work (and each
151
+ Contributor provides its Contributions) on an "AS IS" BASIS,
152
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
153
+ implied, including, without limitation, any warranties or conditions
154
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
155
+ PARTICULAR PURPOSE. You are solely responsible for determining the
156
+ appropriateness of using or redistributing the Work and assume any
157
+ risks associated with Your exercise of permissions under this License.
158
+
159
+ 8. Limitation of Liability. In no event and under no legal theory,
160
+ whether in tort (including negligence), contract, or otherwise,
161
+ unless required by applicable law (such as deliberate and grossly
162
+ negligent acts) or agreed to in writing, shall any Contributor be
163
+ liable to You for damages, including any direct, indirect, special,
164
+ incidental, or consequential damages of any character arising as a
165
+ result of this License or out of the use or inability to use the
166
+ Work (including but not limited to damages for loss of goodwill,
167
+ work stoppage, computer failure or malfunction, or any and all
168
+ other commercial damages or losses), even if such Contributor
169
+ has been advised of the possibility of such damages.
170
+
171
+ 9. Accepting Warranty or Additional Liability. While redistributing
172
+ the Work or Derivative Works thereof, You may choose to offer,
173
+ and charge a fee for, acceptance of support, warranty, indemnity,
174
+ or other liability obligations and/or rights consistent with this
175
+ License. However, in accepting such obligations, You may act only
176
+ on Your own behalf and on Your sole responsibility, not on behalf
177
+ of any other Contributor, and only if You agree to indemnify,
178
+ defend, and hold each Contributor harmless for any liability
179
+ incurred by, or claims asserted against, such Contributor by reason
180
+ of your accepting any such warranty or additional liability.
181
+
182
+ END OF TERMS AND CONDITIONS
183
+
184
+ APPENDIX: How to apply the Apache License to your work.
185
+
186
+ To apply the Apache License to your work, attach the following
187
+ boilerplate notice, with the fields enclosed by brackets "[]"
188
+ replaced with your own identifying information. (Don't include
189
+ the brackets!) The text should be enclosed in the appropriate
190
+ comment syntax for the file format. We also recommend that a
191
+ file or class name and description of purpose be included on the
192
+ same "printed page" as the copyright notice for easier
193
+ identification within third-party archives.
194
+
195
+ Copyright 2025 Wool Labs LLC
196
+
197
+ Licensed under the Apache License, Version 2.0 (the "License");
198
+ you may not use this file except in compliance with the License.
199
+ You may obtain a copy of the License at
200
+
201
+ http://www.apache.org/licenses/LICENSE-2.0
202
+
203
+ Unless required by applicable law or agreed to in writing, software
204
+ distributed under the License is distributed on an "AS IS" BASIS,
205
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
206
+ See the License for the specific language governing permissions and
207
+ limitations under the License.
208
+ Classifier: Intended Audience :: Developers
209
+ Classifier: Operating System :: MacOS :: MacOS X
210
+ Classifier: Operating System :: POSIX :: Linux
211
+ Requires-Python: >=3.11
212
+ Requires-Dist: cloudpickle
213
+ Requires-Dist: grpcio>=1.76.0
214
+ Requires-Dist: portalocker
215
+ Requires-Dist: protobuf
216
+ Requires-Dist: shortuuid
217
+ Requires-Dist: tblib
218
+ Requires-Dist: typing-extensions
219
+ Requires-Dist: watchdog
220
+ Requires-Dist: zeroconf
221
+ Provides-Extra: dev
222
+ Requires-Dist: debugpy; extra == 'dev'
223
+ Requires-Dist: hypothesis; extra == 'dev'
224
+ Requires-Dist: pytest; extra == 'dev'
225
+ Requires-Dist: pytest-asyncio; extra == 'dev'
226
+ Requires-Dist: pytest-cov; extra == 'dev'
227
+ Requires-Dist: pytest-grpc-aio~=0.3.0; extra == 'dev'
228
+ Requires-Dist: pytest-mock; extra == 'dev'
229
+ Requires-Dist: ruff; extra == 'dev'
230
+ Description-Content-Type: text/markdown
231
+
232
+ ![](https://raw.githubusercontent.com/wool-labs/wool/refs/heads/main/assets/woolly-transparent-bg-2048.png)
233
+
234
+ **Wool** is a native Python package for transparently executing tasks in a horizontally scalable, distributed network of agnostic worker processes. Any picklable async function or method can be converted into a task with a simple decorator and a client connection.
235
+
236
+ ## Installation
237
+
238
+ ### Using pip
239
+
240
+ To install the package using pip, run the following command:
241
+
242
+ ```sh
243
+ pip install --pre wool
244
+ ```
245
+
246
+ ### Cloning from GitHub
247
+
248
+ To install the package by cloning from GitHub, run the following commands:
249
+
250
+ ```sh
251
+ git clone https://github.com/wool-labs/wool.git
252
+ cd wool
253
+ pip install ./wool
254
+ ```
255
+
256
+ ## Features
257
+
258
+ ### Declaring tasks
259
+
260
+ Wool tasks are coroutine functions that are executed in a remote `asyncio` event loop within a worker process. To declare a task, use the `@wool.task` decorator:
261
+
262
+ ```python
263
+ import wool
264
+
265
+ @wool.task
266
+ async def sample_task(x, y):
267
+ return x + y
268
+ ```
269
+
270
+ Tasks must be picklable, stateless, and idempotent. Avoid passing unpicklable objects as arguments or return values.
271
+
272
+ ### Worker pools
273
+
274
+ Worker pools are responsible for executing tasks. Wool provides two types of pools:
275
+
276
+ #### Ephemeral pools
277
+
278
+ Ephemeral pools are created and destroyed within the scope of a context manager. Use `wool.pool` to declare an ephemeral pool:
279
+
280
+ ```python
281
+ import asyncio, wool
282
+
283
+ @wool.task
284
+ async def sample_task(x, y):
285
+ return x + y
286
+
287
+ async def main():
288
+ with wool.pool():
289
+ result = await sample_task(1, 2)
290
+ print(f"Result: {result}")
291
+
292
+ asyncio.run(main())
293
+ ```
294
+
295
+ #### Durable pools
296
+
297
+ Durable pools are started independently and persist beyond the scope of a single application. Use the `wool` CLI to manage durable pools:
298
+
299
+ ```bash
300
+ wool pool up --port 5050 --authkey deadbeef --module tasks
301
+ ```
302
+
303
+ Connect to a durable pool using `wool.session`:
304
+
305
+ ```python
306
+ import asyncio, wool
307
+
308
+ @wool.task
309
+ async def sample_task(x, y):
310
+ return x + y
311
+
312
+ async def main():
313
+ with wool.session(port=5050, authkey=b"deadbeef"):
314
+ result = await sample_task(1, 2)
315
+ print(f"Result: {result}")
316
+
317
+ asyncio.run(main())
318
+ ```
319
+
320
+ ### CLI commands
321
+
322
+ Wool provides a command-line interface (CLI) for managing worker pools.
323
+
324
+ #### Start the worker pool
325
+
326
+ ```sh
327
+ wool pool up --host <host> --port <port> --authkey <authkey> --breadth <breadth> --module <module>
328
+ ```
329
+
330
+ - `--host`: The host address (default: `localhost`).
331
+ - `--port`: The port number (default: `0`).
332
+ - `--authkey`: The authentication key (default: `b""`).
333
+ - `--breadth`: The number of worker processes (default: number of CPU cores).
334
+ - `--module`: Python module containing Wool task definitions (optional, can be specified multiple times).
335
+
336
+ #### Stop the worker pool
337
+
338
+ ```sh
339
+ wool pool down --host <host> --port <port> --authkey <authkey> --wait
340
+ ```
341
+
342
+ - `--host`: The host address (default: `localhost`).
343
+ - `--port`: The port number (required).
344
+ - `--authkey`: The authentication key (default: `b""`).
345
+ - `--wait`: Wait for in-flight tasks to complete before shutting down.
346
+
347
+ #### Ping the worker pool
348
+
349
+ ```sh
350
+ wool ping --host <host> --port <port> --authkey <authkey>
351
+ ```
352
+
353
+ - `--host`: The host address (default: `localhost`).
354
+ - `--port`: The port number (required).
355
+ - `--authkey`: The authentication key (default: `b""`).
356
+
357
+ ### Advanced usage
358
+
359
+ #### Nested pools and sessions
360
+
361
+ Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts:
362
+
363
+ ```python
364
+ import wool
365
+
366
+ @wool.task
367
+ async def task_a():
368
+ await asyncio.sleep(1)
369
+
370
+ @wool.task
371
+ async def task_b():
372
+ with wool.pool(port=5051):
373
+ await task_a()
374
+
375
+ async def main():
376
+ with wool.pool(port=5050):
377
+ await task_a()
378
+ await task_b()
379
+
380
+ asyncio.run(main())
381
+ ```
382
+
383
+ In this example, `task_a` is executed by two different pools, while `task_b` is executed by the pool on port 5050.
384
+
385
+ ### Best practices
386
+
387
+ #### Sizing worker pools
388
+
389
+ When configuring worker pools, it is important to balance the number of processes with the available system resources:
390
+
391
+ - **CPU-bound tasks**: Size the worker pool to match the number of CPU cores. This is the default behavior when spawning a pool.
392
+ - **I/O-bound tasks**: For workloads involving significant I/O, consider oversizing the pool slightly to maximize the system's I/O capacity utilization.
393
+ - **Mixed workloads**: Monitor memory usage and system load to avoid oversubscription, especially for memory-intensive tasks. Use profiling tools to determine the optimal pool size.
394
+
395
+ #### Defining tasks
396
+
397
+ Wool tasks are coroutine functions that execute asynchronously in a remote `asyncio` event loop. To ensure smooth execution and scalability, prioritize:
398
+
399
+ - **Picklability**: Ensure all task arguments and return values are picklable. Avoid passing unpicklable objects such as open file handles, database connections, or lambda functions.
400
+ - **Statelessness and idempotency**: Design tasks to be stateless and idempotent. Avoid relying on global variables or shared mutable state. This ensures predictable behavior and safe retries.
401
+ - **Non-blocking operations**: To achieve higher concurrency, avoid blocking calls within tasks. Use `asyncio`-compatible libraries for I/O operations.
402
+ - **Inter-process synchronization**: Use Wool's synchronization primitives (e.g., `wool.locking`) for inter-worker and inter-pool coordination. Standard `asyncio` primitives will not behave as expected in a multi-process environment.
403
+
404
+ #### Debugging and logging
405
+
406
+ - Enable detailed logging during development to trace task execution and worker pool behavior:
407
+ ```python
408
+ import logging
409
+ logging.basicConfig(level=logging.DEBUG)
410
+ ```
411
+ - Use Wool's built-in logging configuration to capture worker-specific logs.
412
+
413
+ #### Nested pools and sessions
414
+
415
+ Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts. This is useful for workflows requiring task segregation or resource isolation.
416
+
417
+ Example:
418
+ ```python
419
+ import asyncio, wool
420
+
421
+ @wool.task
422
+ async def task_a():
423
+ await asyncio.sleep(1)
424
+
425
+ @wool.task
426
+ async def task_b():
427
+ with wool.pool(port=5051):
428
+ await task_a()
429
+
430
+ async def main():
431
+ with wool.pool(port=5050):
432
+ await task_a()
433
+ await task_b()
434
+
435
+ asyncio.run(main())
436
+ ```
437
+
438
+ #### Performance optimization
439
+
440
+ - Minimize the size of arguments and return values to reduce serialization overhead.
441
+ - For large datasets, consider using shared memory or passing references (e.g., file paths) instead of transferring the entire data.
442
+ - Profile tasks to identify and optimize performance bottlenecks.
443
+
444
+ #### Task cancellation
445
+
446
+ - Handle task cancellations gracefully by cleaning up resources and rolling back partial changes.
447
+ - Use `asyncio.CancelledError` to detect and respond to cancellations.
448
+
449
+ #### Error propagation
450
+
451
+ - Wool propagates exceptions raised within tasks to the caller. Use this feature to handle errors centrally in your application.
452
+
453
+ Example:
454
+ ```python
455
+ try:
456
+ result = await some_task()
457
+ except Exception as e:
458
+ print(f"Task failed with error: {e}")
459
+ ```
460
+
461
+ ## License
462
+
463
+ This project is licensed under the Apache License Version 2.0.