wool 0.1rc20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wool/__init__.py +122 -0
- wool/_context.py +29 -0
- wool/_protobuf/worker.py +26 -0
- wool/_resource_pool.py +376 -0
- wool/_typing.py +7 -0
- wool/_undefined.py +11 -0
- wool/_work.py +554 -0
- wool/core/__init__.py +0 -0
- wool/core/discovery/__init__.py +0 -0
- wool/core/discovery/base.py +249 -0
- wool/core/discovery/lan.py +534 -0
- wool/core/discovery/local.py +822 -0
- wool/core/loadbalancer/__init__.py +0 -0
- wool/core/loadbalancer/base.py +125 -0
- wool/core/loadbalancer/roundrobin.py +101 -0
- wool/core/protobuf/__init__.py +18 -0
- wool/core/protobuf/exception.py +3 -0
- wool/core/protobuf/task.py +11 -0
- wool/core/protobuf/task_pb2.py +42 -0
- wool/core/protobuf/task_pb2.pyi +43 -0
- wool/core/protobuf/task_pb2_grpc.py +24 -0
- wool/core/protobuf/worker.py +26 -0
- wool/core/protobuf/worker_pb2.py +53 -0
- wool/core/protobuf/worker_pb2.pyi +65 -0
- wool/core/protobuf/worker_pb2_grpc.py +141 -0
- wool/core/typing.py +22 -0
- wool/core/worker/__init__.py +0 -0
- wool/core/worker/base.py +300 -0
- wool/core/worker/connection.py +250 -0
- wool/core/worker/local.py +148 -0
- wool/core/worker/pool.py +386 -0
- wool/core/worker/process.py +249 -0
- wool/core/worker/proxy.py +427 -0
- wool/core/worker/service.py +231 -0
- wool-0.1rc20.dist-info/METADATA +463 -0
- wool-0.1rc20.dist-info/RECORD +38 -0
- wool-0.1rc20.dist-info/WHEEL +4 -0
- wool-0.1rc20.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from contextlib import contextmanager
|
|
5
|
+
from typing import AsyncIterator
|
|
6
|
+
|
|
7
|
+
import cloudpickle
|
|
8
|
+
from grpc import StatusCode
|
|
9
|
+
from grpc.aio import ServicerContext
|
|
10
|
+
|
|
11
|
+
import wool
|
|
12
|
+
from wool._work import WoolTask
|
|
13
|
+
from wool._work import WoolTaskEvent
|
|
14
|
+
from wool.core import protobuf as pb
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ReadOnlyEvent:
|
|
18
|
+
"""A read-only wrapper around :class:`asyncio.Event`.
|
|
19
|
+
|
|
20
|
+
Provides access to check if an event is set and wait for it to be
|
|
21
|
+
set, but prevents external code from setting or clearing the event.
|
|
22
|
+
|
|
23
|
+
:param event:
|
|
24
|
+
The underlying :class:`asyncio.Event` to wrap.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, event: asyncio.Event):
|
|
28
|
+
self._event = event
|
|
29
|
+
|
|
30
|
+
def is_set(self) -> bool:
|
|
31
|
+
"""Check if the event is set.
|
|
32
|
+
|
|
33
|
+
:returns:
|
|
34
|
+
``True`` if the event is set, ``False`` otherwise.
|
|
35
|
+
"""
|
|
36
|
+
return self._event.is_set()
|
|
37
|
+
|
|
38
|
+
async def wait(self) -> None:
|
|
39
|
+
"""Wait until the underlying event is set."""
|
|
40
|
+
await self._event.wait()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class WorkerService(pb.worker.WorkerServicer):
|
|
44
|
+
"""gRPC service for task execution.
|
|
45
|
+
|
|
46
|
+
Implements the worker gRPC interface for receiving and executing
|
|
47
|
+
tasks. Runs tasks in the current asyncio event loop and streams
|
|
48
|
+
results back to the client.
|
|
49
|
+
|
|
50
|
+
Handles graceful shutdown by rejecting new tasks while allowing
|
|
51
|
+
in-flight tasks to complete. Exposes :attr:`stopping` and
|
|
52
|
+
:attr:`stopped` events for lifecycle monitoring.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
_tasks: set[asyncio.Task]
|
|
56
|
+
_stopped: asyncio.Event
|
|
57
|
+
_stopping: asyncio.Event
|
|
58
|
+
_task_completed: asyncio.Event
|
|
59
|
+
|
|
60
|
+
def __init__(self):
|
|
61
|
+
self._stopped = asyncio.Event()
|
|
62
|
+
self._stopping = asyncio.Event()
|
|
63
|
+
self._task_completed = asyncio.Event()
|
|
64
|
+
self._tasks = set()
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def stopping(self) -> ReadOnlyEvent:
|
|
68
|
+
"""Read-only event signaling that the service is stopping.
|
|
69
|
+
|
|
70
|
+
:returns:
|
|
71
|
+
A :class:`ReadOnlyEvent`.
|
|
72
|
+
"""
|
|
73
|
+
return ReadOnlyEvent(self._stopping)
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def stopped(self) -> ReadOnlyEvent:
|
|
77
|
+
"""Read-only event signaling that the service has stopped.
|
|
78
|
+
|
|
79
|
+
:returns:
|
|
80
|
+
A :class:`ReadOnlyEvent`.
|
|
81
|
+
"""
|
|
82
|
+
return ReadOnlyEvent(self._stopped)
|
|
83
|
+
|
|
84
|
+
async def dispatch(
|
|
85
|
+
self, request: pb.task.Task, context: ServicerContext
|
|
86
|
+
) -> AsyncIterator[pb.worker.Response]:
|
|
87
|
+
"""Execute a task in the current event loop.
|
|
88
|
+
|
|
89
|
+
Deserializes the incoming task into a :class:`WoolTask`
|
|
90
|
+
instance, schedules it for execution in the current asyncio
|
|
91
|
+
event loop, and yields responses for acknowledgment and result.
|
|
92
|
+
|
|
93
|
+
:param request:
|
|
94
|
+
The protobuf task message containing the serialized task
|
|
95
|
+
data.
|
|
96
|
+
:param context:
|
|
97
|
+
The :class:`grpc.aio.ServicerContext` for this request.
|
|
98
|
+
:yields:
|
|
99
|
+
First yields an Ack Response when task processing begins,
|
|
100
|
+
then yields a Response containing the task result.
|
|
101
|
+
|
|
102
|
+
.. note::
|
|
103
|
+
Emits a :class:`WoolTaskEvent` when the task is
|
|
104
|
+
scheduled for execution.
|
|
105
|
+
"""
|
|
106
|
+
if self._stopping.is_set():
|
|
107
|
+
await context.abort(
|
|
108
|
+
StatusCode.UNAVAILABLE, "Worker service is shutting down"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
with self._tracker(WoolTask.from_protobuf(request)) as task:
|
|
112
|
+
try:
|
|
113
|
+
yield pb.worker.Response(ack=pb.worker.Ack())
|
|
114
|
+
try:
|
|
115
|
+
result = pb.task.Result(dump=cloudpickle.dumps(await task))
|
|
116
|
+
yield pb.worker.Response(result=result)
|
|
117
|
+
except Exception as e:
|
|
118
|
+
exception = pb.task.Exception(dump=cloudpickle.dumps(e))
|
|
119
|
+
yield pb.worker.Response(exception=exception)
|
|
120
|
+
except asyncio.CancelledError as e:
|
|
121
|
+
exception = pb.task.Exception(dump=cloudpickle.dumps(e))
|
|
122
|
+
yield pb.worker.Response(exception=exception)
|
|
123
|
+
|
|
124
|
+
async def stop(
|
|
125
|
+
self, request: pb.worker.StopRequest, context: ServicerContext | None
|
|
126
|
+
) -> pb.worker.Void:
|
|
127
|
+
"""Stop the worker service and its thread.
|
|
128
|
+
|
|
129
|
+
Gracefully shuts down the worker thread and signals the server
|
|
130
|
+
to stop accepting new requests. This method is idempotent and
|
|
131
|
+
can be called multiple times safely.
|
|
132
|
+
|
|
133
|
+
:param request:
|
|
134
|
+
The protobuf stop request containing the wait timeout.
|
|
135
|
+
:param context:
|
|
136
|
+
The :class:`grpc.aio.ServicerContext` for this request.
|
|
137
|
+
:returns:
|
|
138
|
+
An empty protobuf response indicating completion.
|
|
139
|
+
"""
|
|
140
|
+
if self._stopping.is_set():
|
|
141
|
+
return pb.worker.Void()
|
|
142
|
+
await self._stop(timeout=request.timeout)
|
|
143
|
+
return pb.worker.Void()
|
|
144
|
+
|
|
145
|
+
@contextmanager
|
|
146
|
+
def _tracker(self, wool_task: WoolTask):
|
|
147
|
+
"""Context manager for tracking running tasks.
|
|
148
|
+
|
|
149
|
+
Manages the lifecycle of a task execution, adding it to the
|
|
150
|
+
active tasks set and emitting appropriate events. Ensures
|
|
151
|
+
proper cleanup when the task completes or fails.
|
|
152
|
+
|
|
153
|
+
:param wool_task:
|
|
154
|
+
The :class:`WoolTask` instance to execute and track.
|
|
155
|
+
:yields:
|
|
156
|
+
The :class:`asyncio.Task` created for the wool task.
|
|
157
|
+
|
|
158
|
+
.. note::
|
|
159
|
+
Emits a :class:`WoolTaskEvent` with type "task-scheduled"
|
|
160
|
+
when the task begins execution.
|
|
161
|
+
"""
|
|
162
|
+
WoolTaskEvent("task-scheduled", task=wool_task).emit()
|
|
163
|
+
task = asyncio.create_task(wool_task.run())
|
|
164
|
+
self._tasks.add(task)
|
|
165
|
+
try:
|
|
166
|
+
yield task
|
|
167
|
+
finally:
|
|
168
|
+
self._tasks.remove(task)
|
|
169
|
+
|
|
170
|
+
async def _stop(self, *, timeout: float | None = 0) -> None:
|
|
171
|
+
self._stopping.set()
|
|
172
|
+
await self._await_or_cancel_tasks(timeout=timeout)
|
|
173
|
+
try:
|
|
174
|
+
if proxy_pool := wool.__proxy_pool__.get():
|
|
175
|
+
await proxy_pool.clear()
|
|
176
|
+
finally:
|
|
177
|
+
self._stopped.set()
|
|
178
|
+
|
|
179
|
+
async def _await_or_cancel_tasks(self, *, timeout: float | None = 0) -> None:
|
|
180
|
+
"""Stop the worker service gracefully.
|
|
181
|
+
|
|
182
|
+
Gracefully shuts down the worker service by canceling or waiting
|
|
183
|
+
for running tasks. This method is idempotent and can be called
|
|
184
|
+
multiple times safely.
|
|
185
|
+
|
|
186
|
+
:param timeout:
|
|
187
|
+
Maximum time to wait for tasks to complete. If 0 (default),
|
|
188
|
+
tasks are canceled immediately. If None, waits indefinitely.
|
|
189
|
+
If a positive number, waits for that many seconds before
|
|
190
|
+
canceling tasks.
|
|
191
|
+
|
|
192
|
+
.. note::
|
|
193
|
+
If a timeout occurs while waiting for tasks to complete,
|
|
194
|
+
the method recursively calls itself with a timeout of 0
|
|
195
|
+
to cancel all remaining tasks immediately.
|
|
196
|
+
"""
|
|
197
|
+
if self._tasks and timeout == 0:
|
|
198
|
+
await self._cancel(*self._tasks)
|
|
199
|
+
elif self._tasks:
|
|
200
|
+
try:
|
|
201
|
+
await asyncio.wait_for(
|
|
202
|
+
asyncio.gather(*self._tasks, return_exceptions=True),
|
|
203
|
+
timeout=timeout,
|
|
204
|
+
)
|
|
205
|
+
except asyncio.TimeoutError:
|
|
206
|
+
return await self._await_or_cancel_tasks(timeout=0)
|
|
207
|
+
|
|
208
|
+
async def _cancel(self, *tasks: asyncio.Task):
|
|
209
|
+
"""Cancel multiple tasks safely.
|
|
210
|
+
|
|
211
|
+
Cancels the provided tasks while performing safety checks to
|
|
212
|
+
avoid canceling the current task or already completed tasks.
|
|
213
|
+
Waits for all cancelled tasks to complete in parallel and handles
|
|
214
|
+
cancellation exceptions.
|
|
215
|
+
|
|
216
|
+
:param tasks:
|
|
217
|
+
The :class:`asyncio.Task` instances to cancel.
|
|
218
|
+
|
|
219
|
+
.. note::
|
|
220
|
+
This method performs the following safety checks:
|
|
221
|
+
- Avoids canceling the current task (would cause deadlock)
|
|
222
|
+
- Only cancels tasks that are not already done
|
|
223
|
+
- Properly handles :exc:`asyncio.CancelledError`
|
|
224
|
+
exceptions.
|
|
225
|
+
"""
|
|
226
|
+
current = asyncio.current_task()
|
|
227
|
+
to_cancel = [task for task in tasks if not task.done() and task != current]
|
|
228
|
+
for task in to_cancel:
|
|
229
|
+
task.cancel()
|
|
230
|
+
if to_cancel:
|
|
231
|
+
await asyncio.gather(*to_cancel, return_exceptions=True)
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wool
|
|
3
|
+
Version: 0.1rc20
|
|
4
|
+
Summary: A Python framework for distributed multiprocessing.
|
|
5
|
+
Author-email: Conrad Bzura <conrad@wool.io>
|
|
6
|
+
Maintainer-email: maintainers@wool.io
|
|
7
|
+
License: Apache License
|
|
8
|
+
Version 2.0, January 2004
|
|
9
|
+
http://www.apache.org/licenses/
|
|
10
|
+
|
|
11
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
12
|
+
|
|
13
|
+
1. Definitions.
|
|
14
|
+
|
|
15
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
16
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
17
|
+
|
|
18
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
19
|
+
the copyright owner that is granting the License.
|
|
20
|
+
|
|
21
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
22
|
+
other entities that control, are controlled by, or are under common
|
|
23
|
+
control with that entity. For the purposes of this definition,
|
|
24
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
25
|
+
direction or management of such entity, whether by contract or
|
|
26
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
27
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
28
|
+
|
|
29
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
30
|
+
exercising permissions granted by this License.
|
|
31
|
+
|
|
32
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
33
|
+
including but not limited to software source code, documentation
|
|
34
|
+
source, and configuration files.
|
|
35
|
+
|
|
36
|
+
"Object" form shall mean any form resulting from mechanical
|
|
37
|
+
transformation or translation of a Source form, including but
|
|
38
|
+
not limited to compiled object code, generated documentation,
|
|
39
|
+
and conversions to other media types.
|
|
40
|
+
|
|
41
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
42
|
+
Object form, made available under the License, as indicated by a
|
|
43
|
+
copyright notice that is included in or attached to the work
|
|
44
|
+
(an example is provided in the Appendix below).
|
|
45
|
+
|
|
46
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
47
|
+
form, that is based on (or derived from) the Work and for which the
|
|
48
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
49
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
50
|
+
of this License, Derivative Works shall not include works that remain
|
|
51
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
52
|
+
the Work and Derivative Works thereof.
|
|
53
|
+
|
|
54
|
+
"Contribution" shall mean any work of authorship, including
|
|
55
|
+
the original version of the Work and any modifications or additions
|
|
56
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
57
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
58
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
59
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
60
|
+
means any form of electronic, verbal, or written communication sent
|
|
61
|
+
to the Licensor or its representatives, including but not limited to
|
|
62
|
+
communication on electronic mailing lists, source code control systems,
|
|
63
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
64
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
65
|
+
excluding communication that is conspicuously marked or otherwise
|
|
66
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
67
|
+
|
|
68
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
69
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
70
|
+
subsequently incorporated within the Work.
|
|
71
|
+
|
|
72
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
73
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
74
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
75
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
76
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
77
|
+
Work and such Derivative Works in Source or Object form.
|
|
78
|
+
|
|
79
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
80
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
81
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
82
|
+
(except as stated in this section) patent license to make, have made,
|
|
83
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
84
|
+
where such license applies only to those patent claims licensable
|
|
85
|
+
by such Contributor that are necessarily infringed by their
|
|
86
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
87
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
88
|
+
institute patent litigation against any entity (including a
|
|
89
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
90
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
91
|
+
or contributory patent infringement, then any patent licenses
|
|
92
|
+
granted to You under this License for that Work shall terminate
|
|
93
|
+
as of the date such litigation is filed.
|
|
94
|
+
|
|
95
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
96
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
97
|
+
modifications, and in Source or Object form, provided that You
|
|
98
|
+
meet the following conditions:
|
|
99
|
+
|
|
100
|
+
(a) You must give any other recipients of the Work or
|
|
101
|
+
Derivative Works a copy of this License; and
|
|
102
|
+
|
|
103
|
+
(b) You must cause any modified files to carry prominent notices
|
|
104
|
+
stating that You changed the files; and
|
|
105
|
+
|
|
106
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
107
|
+
that You distribute, all copyright, patent, trademark, and
|
|
108
|
+
attribution notices from the Source form of the Work,
|
|
109
|
+
excluding those notices that do not pertain to any part of
|
|
110
|
+
the Derivative Works; and
|
|
111
|
+
|
|
112
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
113
|
+
distribution, then any Derivative Works that You distribute must
|
|
114
|
+
include a readable copy of the attribution notices contained
|
|
115
|
+
within such NOTICE file, excluding those notices that do not
|
|
116
|
+
pertain to any part of the Derivative Works, in at least one
|
|
117
|
+
of the following places: within a NOTICE text file distributed
|
|
118
|
+
as part of the Derivative Works; within the Source form or
|
|
119
|
+
documentation, if provided along with the Derivative Works; or,
|
|
120
|
+
within a display generated by the Derivative Works, if and
|
|
121
|
+
wherever such third-party notices normally appear. The contents
|
|
122
|
+
of the NOTICE file are for informational purposes only and
|
|
123
|
+
do not modify the License. You may add Your own attribution
|
|
124
|
+
notices within Derivative Works that You distribute, alongside
|
|
125
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
126
|
+
that such additional attribution notices cannot be construed
|
|
127
|
+
as modifying the License.
|
|
128
|
+
|
|
129
|
+
You may add Your own copyright statement to Your modifications and
|
|
130
|
+
may provide additional or different license terms and conditions
|
|
131
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
132
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
133
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
134
|
+
the conditions stated in this License.
|
|
135
|
+
|
|
136
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
137
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
138
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
139
|
+
this License, without any additional terms or conditions.
|
|
140
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
141
|
+
the terms of any separate license agreement you may have executed
|
|
142
|
+
with Licensor regarding such Contributions.
|
|
143
|
+
|
|
144
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
145
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
146
|
+
except as required for reasonable and customary use in describing the
|
|
147
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
148
|
+
|
|
149
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
150
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
151
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
152
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
153
|
+
implied, including, without limitation, any warranties or conditions
|
|
154
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
155
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
156
|
+
appropriateness of using or redistributing the Work and assume any
|
|
157
|
+
risks associated with Your exercise of permissions under this License.
|
|
158
|
+
|
|
159
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
160
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
161
|
+
unless required by applicable law (such as deliberate and grossly
|
|
162
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
163
|
+
liable to You for damages, including any direct, indirect, special,
|
|
164
|
+
incidental, or consequential damages of any character arising as a
|
|
165
|
+
result of this License or out of the use or inability to use the
|
|
166
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
167
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
168
|
+
other commercial damages or losses), even if such Contributor
|
|
169
|
+
has been advised of the possibility of such damages.
|
|
170
|
+
|
|
171
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
172
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
173
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
174
|
+
or other liability obligations and/or rights consistent with this
|
|
175
|
+
License. However, in accepting such obligations, You may act only
|
|
176
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
177
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
178
|
+
defend, and hold each Contributor harmless for any liability
|
|
179
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
180
|
+
of your accepting any such warranty or additional liability.
|
|
181
|
+
|
|
182
|
+
END OF TERMS AND CONDITIONS
|
|
183
|
+
|
|
184
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
185
|
+
|
|
186
|
+
To apply the Apache License to your work, attach the following
|
|
187
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
188
|
+
replaced with your own identifying information. (Don't include
|
|
189
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
190
|
+
comment syntax for the file format. We also recommend that a
|
|
191
|
+
file or class name and description of purpose be included on the
|
|
192
|
+
same "printed page" as the copyright notice for easier
|
|
193
|
+
identification within third-party archives.
|
|
194
|
+
|
|
195
|
+
Copyright 2025 Wool Labs LLC
|
|
196
|
+
|
|
197
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
198
|
+
you may not use this file except in compliance with the License.
|
|
199
|
+
You may obtain a copy of the License at
|
|
200
|
+
|
|
201
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
202
|
+
|
|
203
|
+
Unless required by applicable law or agreed to in writing, software
|
|
204
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
205
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
206
|
+
See the License for the specific language governing permissions and
|
|
207
|
+
limitations under the License.
|
|
208
|
+
Classifier: Intended Audience :: Developers
|
|
209
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
210
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
211
|
+
Requires-Python: >=3.11
|
|
212
|
+
Requires-Dist: cloudpickle
|
|
213
|
+
Requires-Dist: grpcio>=1.76.0
|
|
214
|
+
Requires-Dist: portalocker
|
|
215
|
+
Requires-Dist: protobuf
|
|
216
|
+
Requires-Dist: shortuuid
|
|
217
|
+
Requires-Dist: tblib
|
|
218
|
+
Requires-Dist: typing-extensions
|
|
219
|
+
Requires-Dist: watchdog
|
|
220
|
+
Requires-Dist: zeroconf
|
|
221
|
+
Provides-Extra: dev
|
|
222
|
+
Requires-Dist: debugpy; extra == 'dev'
|
|
223
|
+
Requires-Dist: hypothesis; extra == 'dev'
|
|
224
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
225
|
+
Requires-Dist: pytest-asyncio; extra == 'dev'
|
|
226
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
227
|
+
Requires-Dist: pytest-grpc-aio~=0.3.0; extra == 'dev'
|
|
228
|
+
Requires-Dist: pytest-mock; extra == 'dev'
|
|
229
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
230
|
+
Description-Content-Type: text/markdown
|
|
231
|
+
|
|
232
|
+

|
|
233
|
+
|
|
234
|
+
**Wool** is a native Python package for transparently executing tasks in a horizontally scalable, distributed network of agnostic worker processes. Any picklable async function or method can be converted into a task with a simple decorator and a client connection.
|
|
235
|
+
|
|
236
|
+
## Installation
|
|
237
|
+
|
|
238
|
+
### Using pip
|
|
239
|
+
|
|
240
|
+
To install the package using pip, run the following command:
|
|
241
|
+
|
|
242
|
+
```sh
|
|
243
|
+
pip install --pre wool
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
### Cloning from GitHub
|
|
247
|
+
|
|
248
|
+
To install the package by cloning from GitHub, run the following commands:
|
|
249
|
+
|
|
250
|
+
```sh
|
|
251
|
+
git clone https://github.com/wool-labs/wool.git
|
|
252
|
+
cd wool
|
|
253
|
+
pip install ./wool
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
## Features
|
|
257
|
+
|
|
258
|
+
### Declaring tasks
|
|
259
|
+
|
|
260
|
+
Wool tasks are coroutine functions that are executed in a remote `asyncio` event loop within a worker process. To declare a task, use the `@wool.task` decorator:
|
|
261
|
+
|
|
262
|
+
```python
|
|
263
|
+
import wool
|
|
264
|
+
|
|
265
|
+
@wool.task
|
|
266
|
+
async def sample_task(x, y):
|
|
267
|
+
return x + y
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
Tasks must be picklable, stateless, and idempotent. Avoid passing unpicklable objects as arguments or return values.
|
|
271
|
+
|
|
272
|
+
### Worker pools
|
|
273
|
+
|
|
274
|
+
Worker pools are responsible for executing tasks. Wool provides two types of pools:
|
|
275
|
+
|
|
276
|
+
#### Ephemeral pools
|
|
277
|
+
|
|
278
|
+
Ephemeral pools are created and destroyed within the scope of a context manager. Use `wool.pool` to declare an ephemeral pool:
|
|
279
|
+
|
|
280
|
+
```python
|
|
281
|
+
import asyncio, wool
|
|
282
|
+
|
|
283
|
+
@wool.task
|
|
284
|
+
async def sample_task(x, y):
|
|
285
|
+
return x + y
|
|
286
|
+
|
|
287
|
+
async def main():
|
|
288
|
+
with wool.pool():
|
|
289
|
+
result = await sample_task(1, 2)
|
|
290
|
+
print(f"Result: {result}")
|
|
291
|
+
|
|
292
|
+
asyncio.run(main())
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
#### Durable pools
|
|
296
|
+
|
|
297
|
+
Durable pools are started independently and persist beyond the scope of a single application. Use the `wool` CLI to manage durable pools:
|
|
298
|
+
|
|
299
|
+
```bash
|
|
300
|
+
wool pool up --port 5050 --authkey deadbeef --module tasks
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
Connect to a durable pool using `wool.session`:
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
import asyncio, wool
|
|
307
|
+
|
|
308
|
+
@wool.task
|
|
309
|
+
async def sample_task(x, y):
|
|
310
|
+
return x + y
|
|
311
|
+
|
|
312
|
+
async def main():
|
|
313
|
+
with wool.session(port=5050, authkey=b"deadbeef"):
|
|
314
|
+
result = await sample_task(1, 2)
|
|
315
|
+
print(f"Result: {result}")
|
|
316
|
+
|
|
317
|
+
asyncio.run(main())
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
### CLI commands
|
|
321
|
+
|
|
322
|
+
Wool provides a command-line interface (CLI) for managing worker pools.
|
|
323
|
+
|
|
324
|
+
#### Start the worker pool
|
|
325
|
+
|
|
326
|
+
```sh
|
|
327
|
+
wool pool up --host <host> --port <port> --authkey <authkey> --breadth <breadth> --module <module>
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
- `--host`: The host address (default: `localhost`).
|
|
331
|
+
- `--port`: The port number (default: `0`).
|
|
332
|
+
- `--authkey`: The authentication key (default: `b""`).
|
|
333
|
+
- `--breadth`: The number of worker processes (default: number of CPU cores).
|
|
334
|
+
- `--module`: Python module containing Wool task definitions (optional, can be specified multiple times).
|
|
335
|
+
|
|
336
|
+
#### Stop the worker pool
|
|
337
|
+
|
|
338
|
+
```sh
|
|
339
|
+
wool pool down --host <host> --port <port> --authkey <authkey> --wait
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
- `--host`: The host address (default: `localhost`).
|
|
343
|
+
- `--port`: The port number (required).
|
|
344
|
+
- `--authkey`: The authentication key (default: `b""`).
|
|
345
|
+
- `--wait`: Wait for in-flight tasks to complete before shutting down.
|
|
346
|
+
|
|
347
|
+
#### Ping the worker pool
|
|
348
|
+
|
|
349
|
+
```sh
|
|
350
|
+
wool ping --host <host> --port <port> --authkey <authkey>
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
- `--host`: The host address (default: `localhost`).
|
|
354
|
+
- `--port`: The port number (required).
|
|
355
|
+
- `--authkey`: The authentication key (default: `b""`).
|
|
356
|
+
|
|
357
|
+
### Advanced usage
|
|
358
|
+
|
|
359
|
+
#### Nested pools and sessions
|
|
360
|
+
|
|
361
|
+
Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts:
|
|
362
|
+
|
|
363
|
+
```python
|
|
364
|
+
import wool
|
|
365
|
+
|
|
366
|
+
@wool.task
|
|
367
|
+
async def task_a():
|
|
368
|
+
await asyncio.sleep(1)
|
|
369
|
+
|
|
370
|
+
@wool.task
|
|
371
|
+
async def task_b():
|
|
372
|
+
with wool.pool(port=5051):
|
|
373
|
+
await task_a()
|
|
374
|
+
|
|
375
|
+
async def main():
|
|
376
|
+
with wool.pool(port=5050):
|
|
377
|
+
await task_a()
|
|
378
|
+
await task_b()
|
|
379
|
+
|
|
380
|
+
asyncio.run(main())
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
In this example, `task_a` is executed by two different pools, while `task_b` is executed by the pool on port 5050.
|
|
384
|
+
|
|
385
|
+
### Best practices
|
|
386
|
+
|
|
387
|
+
#### Sizing worker pools
|
|
388
|
+
|
|
389
|
+
When configuring worker pools, it is important to balance the number of processes with the available system resources:
|
|
390
|
+
|
|
391
|
+
- **CPU-bound tasks**: Size the worker pool to match the number of CPU cores. This is the default behavior when spawning a pool.
|
|
392
|
+
- **I/O-bound tasks**: For workloads involving significant I/O, consider oversizing the pool slightly to maximize the system's I/O capacity utilization.
|
|
393
|
+
- **Mixed workloads**: Monitor memory usage and system load to avoid oversubscription, especially for memory-intensive tasks. Use profiling tools to determine the optimal pool size.
|
|
394
|
+
|
|
395
|
+
#### Defining tasks
|
|
396
|
+
|
|
397
|
+
Wool tasks are coroutine functions that execute asynchronously in a remote `asyncio` event loop. To ensure smooth execution and scalability, prioritize:
|
|
398
|
+
|
|
399
|
+
- **Picklability**: Ensure all task arguments and return values are picklable. Avoid passing unpicklable objects such as open file handles, database connections, or lambda functions.
|
|
400
|
+
- **Statelessness and idempotency**: Design tasks to be stateless and idempotent. Avoid relying on global variables or shared mutable state. This ensures predictable behavior and safe retries.
|
|
401
|
+
- **Non-blocking operations**: To achieve higher concurrency, avoid blocking calls within tasks. Use `asyncio`-compatible libraries for I/O operations.
|
|
402
|
+
- **Inter-process synchronization**: Use Wool's synchronization primitives (e.g., `wool.locking`) for inter-worker and inter-pool coordination. Standard `asyncio` primitives will not behave as expected in a multi-process environment.
|
|
403
|
+
|
|
404
|
+
#### Debugging and logging
|
|
405
|
+
|
|
406
|
+
- Enable detailed logging during development to trace task execution and worker pool behavior:
|
|
407
|
+
```python
|
|
408
|
+
import logging
|
|
409
|
+
logging.basicConfig(level=logging.DEBUG)
|
|
410
|
+
```
|
|
411
|
+
- Use Wool's built-in logging configuration to capture worker-specific logs.
|
|
412
|
+
|
|
413
|
+
#### Nested pools and sessions
|
|
414
|
+
|
|
415
|
+
Wool supports nesting pools and sessions to achieve complex workflows. Tasks can be dispatched to specific pools by nesting contexts. This is useful for workflows requiring task segregation or resource isolation.
|
|
416
|
+
|
|
417
|
+
Example:
|
|
418
|
+
```python
|
|
419
|
+
import asyncio, wool
|
|
420
|
+
|
|
421
|
+
@wool.task
|
|
422
|
+
async def task_a():
|
|
423
|
+
await asyncio.sleep(1)
|
|
424
|
+
|
|
425
|
+
@wool.task
|
|
426
|
+
async def task_b():
|
|
427
|
+
with wool.pool(port=5051):
|
|
428
|
+
await task_a()
|
|
429
|
+
|
|
430
|
+
async def main():
|
|
431
|
+
with wool.pool(port=5050):
|
|
432
|
+
await task_a()
|
|
433
|
+
await task_b()
|
|
434
|
+
|
|
435
|
+
asyncio.run(main())
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
#### Performance optimization
|
|
439
|
+
|
|
440
|
+
- Minimize the size of arguments and return values to reduce serialization overhead.
|
|
441
|
+
- For large datasets, consider using shared memory or passing references (e.g., file paths) instead of transferring the entire data.
|
|
442
|
+
- Profile tasks to identify and optimize performance bottlenecks.
|
|
443
|
+
|
|
444
|
+
#### Task cancellation
|
|
445
|
+
|
|
446
|
+
- Handle task cancellations gracefully by cleaning up resources and rolling back partial changes.
|
|
447
|
+
- Use `asyncio.CancelledError` to detect and respond to cancellations.
|
|
448
|
+
|
|
449
|
+
#### Error propagation
|
|
450
|
+
|
|
451
|
+
- Wool propagates exceptions raised within tasks to the caller. Use this feature to handle errors centrally in your application.
|
|
452
|
+
|
|
453
|
+
Example:
|
|
454
|
+
```python
|
|
455
|
+
try:
|
|
456
|
+
result = await some_task()
|
|
457
|
+
except Exception as e:
|
|
458
|
+
print(f"Task failed with error: {e}")
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
## License
|
|
462
|
+
|
|
463
|
+
This project is licensed under the Apache License Version 2.0.
|