wool 0.1rc8__py3-none-any.whl → 0.1rc10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wool might be problematic. Click here for more details.
- wool/__init__.py +71 -50
- wool/_protobuf/__init__.py +14 -0
- wool/_protobuf/exception.py +3 -0
- wool/_protobuf/task.py +11 -0
- wool/_protobuf/task_pb2.py +42 -0
- wool/_protobuf/task_pb2.pyi +43 -0
- wool/_protobuf/{mempool/metadata/metadata_pb2_grpc.py → task_pb2_grpc.py} +2 -2
- wool/_protobuf/worker.py +24 -0
- wool/_protobuf/worker_pb2.py +47 -0
- wool/_protobuf/worker_pb2.pyi +39 -0
- wool/_protobuf/worker_pb2_grpc.py +141 -0
- wool/_resource_pool.py +376 -0
- wool/_typing.py +0 -10
- wool/_work.py +553 -0
- wool/_worker.py +843 -169
- wool/_worker_discovery.py +1223 -0
- wool/_worker_pool.py +331 -0
- wool/_worker_proxy.py +515 -0
- {wool-0.1rc8.dist-info → wool-0.1rc10.dist-info}/METADATA +8 -7
- wool-0.1rc10.dist-info/RECORD +22 -0
- wool-0.1rc10.dist-info/entry_points.txt +2 -0
- wool/_cli.py +0 -262
- wool/_event.py +0 -109
- wool/_future.py +0 -171
- wool/_logging.py +0 -44
- wool/_manager.py +0 -181
- wool/_mempool/__init__.py +0 -4
- wool/_mempool/_mempool.py +0 -311
- wool/_mempool/_metadata.py +0 -39
- wool/_mempool/_service.py +0 -225
- wool/_pool.py +0 -524
- wool/_protobuf/mempool/mempool_pb2.py +0 -66
- wool/_protobuf/mempool/mempool_pb2.pyi +0 -108
- wool/_protobuf/mempool/mempool_pb2_grpc.py +0 -312
- wool/_protobuf/mempool/metadata/metadata_pb2.py +0 -36
- wool/_protobuf/mempool/metadata/metadata_pb2.pyi +0 -17
- wool/_queue.py +0 -32
- wool/_session.py +0 -429
- wool/_task.py +0 -366
- wool/_utils.py +0 -63
- wool-0.1rc8.dist-info/RECORD +0 -28
- wool-0.1rc8.dist-info/entry_points.txt +0 -2
- {wool-0.1rc8.dist-info → wool-0.1rc10.dist-info}/WHEEL +0 -0
wool/_mempool/_service.py
DELETED
|
@@ -1,225 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import asyncio
|
|
4
|
-
from typing import AsyncGenerator
|
|
5
|
-
from typing import Final
|
|
6
|
-
from weakref import WeakSet
|
|
7
|
-
from weakref import WeakValueDictionary
|
|
8
|
-
|
|
9
|
-
import shortuuid
|
|
10
|
-
from grpc.aio import ServicerContext
|
|
11
|
-
|
|
12
|
-
from wool._mempool import MemoryPool
|
|
13
|
-
from wool._protobuf.mempool import mempool_pb2 as proto
|
|
14
|
-
from wool._protobuf.mempool import mempool_pb2_grpc as rpc
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class Session:
|
|
18
|
-
"""
|
|
19
|
-
A session represents a client connection to the memory pool service and
|
|
20
|
-
serves as the scope for any shared references acquired over its duration.
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
id: Final[str]
|
|
24
|
-
queue: Final[asyncio.Queue[proto.SessionResponse]]
|
|
25
|
-
references: Final[set[Reference]]
|
|
26
|
-
sessions: Final[WeakValueDictionary[str, Session]] = WeakValueDictionary()
|
|
27
|
-
|
|
28
|
-
@classmethod
|
|
29
|
-
def get(cls, id: str) -> Session | None:
|
|
30
|
-
return cls.sessions.get(id)
|
|
31
|
-
|
|
32
|
-
def __init__(self):
|
|
33
|
-
self.id = shortuuid.uuid()
|
|
34
|
-
self.queue = asyncio.Queue()
|
|
35
|
-
self.references = set()
|
|
36
|
-
self.sessions[self.id] = self
|
|
37
|
-
|
|
38
|
-
def __eq__(self, other) -> bool:
|
|
39
|
-
if isinstance(other, Session):
|
|
40
|
-
return self.id == other.id
|
|
41
|
-
return False
|
|
42
|
-
|
|
43
|
-
def __hash__(self) -> int:
|
|
44
|
-
return hash(self.id)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class Reference:
|
|
48
|
-
id: Final[str]
|
|
49
|
-
mempool: Final[MemoryPool]
|
|
50
|
-
sessions: Final[WeakSet[Session]]
|
|
51
|
-
|
|
52
|
-
_references: Final[WeakValueDictionary[str, Reference]] = (
|
|
53
|
-
WeakValueDictionary()
|
|
54
|
-
)
|
|
55
|
-
_to_delete: Final[set[str]] = set()
|
|
56
|
-
_initialized: bool = False
|
|
57
|
-
|
|
58
|
-
@classmethod
|
|
59
|
-
def get(cls, id: str) -> Reference | None:
|
|
60
|
-
return cls._references.get(id)
|
|
61
|
-
|
|
62
|
-
@classmethod
|
|
63
|
-
def new(cls, id: str, *, mempool: MemoryPool) -> Reference:
|
|
64
|
-
if id in cls._references:
|
|
65
|
-
raise ValueError(f"Reference {id} already exists")
|
|
66
|
-
return cls(id, mempool=mempool)
|
|
67
|
-
|
|
68
|
-
def __new__(cls, id: str, *, mempool: MemoryPool):
|
|
69
|
-
if id in cls._references:
|
|
70
|
-
if id in cls._to_delete:
|
|
71
|
-
cls._to_delete.remove(id)
|
|
72
|
-
return cls._references[id]
|
|
73
|
-
return super().__new__(cls)
|
|
74
|
-
|
|
75
|
-
def __init__(self, id: str, *, mempool: MemoryPool):
|
|
76
|
-
if not self._initialized:
|
|
77
|
-
self.id = id
|
|
78
|
-
self.mempool = mempool
|
|
79
|
-
self.sessions = WeakSet()
|
|
80
|
-
self._references[id] = self
|
|
81
|
-
self._initialized = True
|
|
82
|
-
|
|
83
|
-
def __eq__(self, other) -> bool:
|
|
84
|
-
if isinstance(other, Reference):
|
|
85
|
-
return self.id == other.id
|
|
86
|
-
return False
|
|
87
|
-
|
|
88
|
-
def __hash__(self) -> int:
|
|
89
|
-
return hash(self.id)
|
|
90
|
-
|
|
91
|
-
def __del__(self):
|
|
92
|
-
self._to_delete.add(self.id)
|
|
93
|
-
|
|
94
|
-
id = self.id
|
|
95
|
-
to_delete = self._to_delete
|
|
96
|
-
mempool = self.mempool
|
|
97
|
-
|
|
98
|
-
async def _delete():
|
|
99
|
-
if id in to_delete:
|
|
100
|
-
try:
|
|
101
|
-
to_delete.remove(id)
|
|
102
|
-
await mempool.delete(id)
|
|
103
|
-
except FileNotFoundError:
|
|
104
|
-
pass
|
|
105
|
-
|
|
106
|
-
try:
|
|
107
|
-
asyncio.get_running_loop().create_task(_delete())
|
|
108
|
-
except RuntimeError:
|
|
109
|
-
asyncio.new_event_loop().run_until_complete(_delete())
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
class MemoryPoolService(rpc.MemoryPoolServicer):
|
|
113
|
-
def __init__(self, mempool: MemoryPool | None = None):
|
|
114
|
-
self._mempool = mempool or MemoryPool()
|
|
115
|
-
self._shutdown = asyncio.Event()
|
|
116
|
-
|
|
117
|
-
async def session(
|
|
118
|
-
self, request: proto.SessionRequest, context: ServicerContext
|
|
119
|
-
) -> AsyncGenerator[proto.SessionResponse]:
|
|
120
|
-
session = Session()
|
|
121
|
-
yield proto.SessionResponse(session=proto.Session(id=session.id))
|
|
122
|
-
while True:
|
|
123
|
-
yield await session.queue.get()
|
|
124
|
-
|
|
125
|
-
async def acquire(
|
|
126
|
-
self, request: proto.AcquireRequest, context: ServicerContext
|
|
127
|
-
) -> proto.AcquireResponse:
|
|
128
|
-
if not (session := Session.get(request.session.id)):
|
|
129
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
130
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
131
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
132
|
-
session.references.add(reference)
|
|
133
|
-
reference.sessions.add(session)
|
|
134
|
-
return proto.AcquireResponse()
|
|
135
|
-
|
|
136
|
-
async def map(
|
|
137
|
-
self, request: proto.AcquireRequest, context: ServicerContext
|
|
138
|
-
) -> proto.AcquireResponse:
|
|
139
|
-
if not (session := Session.get(request.session.id)):
|
|
140
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
141
|
-
await self._mempool.map(request.reference.id)
|
|
142
|
-
reference = Reference(request.reference.id, mempool=self._mempool)
|
|
143
|
-
await self.acquire(
|
|
144
|
-
proto.AcquireRequest(
|
|
145
|
-
session=proto.Session(id=session.id),
|
|
146
|
-
reference=proto.Reference(id=reference.id),
|
|
147
|
-
),
|
|
148
|
-
context,
|
|
149
|
-
)
|
|
150
|
-
return proto.AcquireResponse()
|
|
151
|
-
|
|
152
|
-
async def put(
|
|
153
|
-
self, request: proto.PutRequest, context: ServicerContext
|
|
154
|
-
) -> proto.PutResponse:
|
|
155
|
-
if not (session := Session.get(request.session.id)):
|
|
156
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
157
|
-
reference = Reference(
|
|
158
|
-
id=await self._mempool.put(request.dump, mutable=request.mutable),
|
|
159
|
-
mempool=self._mempool,
|
|
160
|
-
)
|
|
161
|
-
await self.acquire(
|
|
162
|
-
proto.AcquireRequest(
|
|
163
|
-
session=proto.Session(id=session.id),
|
|
164
|
-
reference=proto.Reference(id=reference.id),
|
|
165
|
-
),
|
|
166
|
-
context,
|
|
167
|
-
)
|
|
168
|
-
return proto.PutResponse(reference=proto.Reference(id=reference.id))
|
|
169
|
-
|
|
170
|
-
async def get(
|
|
171
|
-
self, request: proto.GetRequest, context: ServicerContext
|
|
172
|
-
) -> proto.GetResponse:
|
|
173
|
-
if not (session := Session.get(request.session.id)):
|
|
174
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
175
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
176
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
177
|
-
if reference not in session.references:
|
|
178
|
-
await self.acquire(
|
|
179
|
-
proto.AcquireRequest(
|
|
180
|
-
session=proto.Session(id=session.id),
|
|
181
|
-
reference=proto.Reference(id=reference.id),
|
|
182
|
-
),
|
|
183
|
-
context,
|
|
184
|
-
)
|
|
185
|
-
dump = await self._mempool.get(reference.id)
|
|
186
|
-
return proto.GetResponse(dump=dump)
|
|
187
|
-
|
|
188
|
-
async def post(
|
|
189
|
-
self, request: proto.PostRequest, context: ServicerContext
|
|
190
|
-
) -> proto.PostResponse:
|
|
191
|
-
if not (session := Session.get(request.session.id)):
|
|
192
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
193
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
194
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
195
|
-
if reference not in session.references:
|
|
196
|
-
await self.acquire(
|
|
197
|
-
proto.AcquireRequest(
|
|
198
|
-
session=proto.Session(id=session.id),
|
|
199
|
-
reference=proto.Reference(id=reference.id),
|
|
200
|
-
),
|
|
201
|
-
context,
|
|
202
|
-
)
|
|
203
|
-
updated = await self._mempool.post(request.reference.id, request.dump)
|
|
204
|
-
if updated:
|
|
205
|
-
for session in Reference(
|
|
206
|
-
id=request.reference.id, mempool=self._mempool
|
|
207
|
-
).sessions:
|
|
208
|
-
if session.id is not request.session.id:
|
|
209
|
-
event = proto.Event(
|
|
210
|
-
reference=request.reference,
|
|
211
|
-
event_type="post",
|
|
212
|
-
)
|
|
213
|
-
await session.queue.put(proto.SessionResponse(event=event))
|
|
214
|
-
return proto.PostResponse(updated=updated)
|
|
215
|
-
|
|
216
|
-
async def release(
|
|
217
|
-
self, request: proto.ReleaseRequest, context: ServicerContext
|
|
218
|
-
) -> proto.ReleaseResponse:
|
|
219
|
-
if not (session := Session.get(request.session.id)):
|
|
220
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
221
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
222
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
223
|
-
session.references.remove(reference)
|
|
224
|
-
reference.sessions.remove(session)
|
|
225
|
-
return proto.ReleaseResponse()
|