wool 0.1rc8__tar.gz → 0.1rc9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wool might be problematic. Click here for more details.
- {wool-0.1rc8 → wool-0.1rc9}/PKG-INFO +3 -2
- {wool-0.1rc8 → wool-0.1rc9}/pyproject.toml +7 -1
- wool-0.1rc9/wool/_mempool/_client.py +167 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_mempool/_mempool.py +8 -8
- {wool-0.1rc8 → wool-0.1rc9}/wool/_mempool/_metadata.py +8 -12
- wool-0.1rc9/wool/_mempool/_service.py +227 -0
- wool-0.1rc9/wool/_protobuf/__init__.py +11 -0
- {wool-0.1rc8/wool/_protobuf/mempool/metadata → wool-0.1rc9/wool/_protobuf/mempool}/metadata_pb2.py +6 -6
- {wool-0.1rc8/wool/_protobuf/mempool/metadata → wool-0.1rc9/wool/_protobuf/mempool}/metadata_pb2.pyi +1 -1
- {wool-0.1rc8/wool/_protobuf/mempool/metadata → wool-0.1rc9/wool/_protobuf/mempool}/metadata_pb2_grpc.py +1 -1
- wool-0.1rc8/wool/_protobuf/mempool/mempool_pb2.py → wool-0.1rc9/wool/_protobuf/mempool/service_pb2.py +5 -5
- wool-0.1rc8/wool/_protobuf/mempool/mempool_pb2_grpc.py → wool-0.1rc9/wool/_protobuf/mempool/service_pb2_grpc.py +81 -38
- wool-0.1rc8/wool/_mempool/_service.py +0 -225
- wool-0.1rc8/wool/_protobuf/__init__.py +0 -4
- {wool-0.1rc8 → wool-0.1rc9}/.gitignore +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/README.md +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/__init__.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_cli.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_event.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_future.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_logging.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_manager.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_mempool/__init__.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_pool.py +0 -0
- /wool-0.1rc8/wool/_protobuf/mempool/mempool_pb2.pyi → /wool-0.1rc9/wool/_protobuf/mempool/service_pb2.pyi +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_queue.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_session.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_task.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_typing.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_utils.py +0 -0
- {wool-0.1rc8 → wool-0.1rc9}/wool/_worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wool
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.1rc9
|
|
4
4
|
Summary: A Python framework for distributed multiprocessing.
|
|
5
5
|
Author-email: Conrad Bzura <conrad@wool.io>
|
|
6
6
|
Maintainer-email: maintainers@wool.io
|
|
@@ -221,9 +221,10 @@ Provides-Extra: dev
|
|
|
221
221
|
Requires-Dist: pytest; extra == 'dev'
|
|
222
222
|
Requires-Dist: pytest-asyncio; extra == 'dev'
|
|
223
223
|
Requires-Dist: pytest-grpc-aio~=0.2.0; extra == 'dev'
|
|
224
|
+
Requires-Dist: pytest-mock; extra == 'dev'
|
|
224
225
|
Requires-Dist: ruff; extra == 'dev'
|
|
225
226
|
Provides-Extra: locking
|
|
226
|
-
Requires-Dist: wool-locking==0.
|
|
227
|
+
Requires-Dist: wool-locking==0.1rc9; extra == 'locking'
|
|
227
228
|
Description-Content-Type: text/markdown
|
|
228
229
|
|
|
229
230
|
# Wool
|
|
@@ -29,7 +29,13 @@ readme = "README.md"
|
|
|
29
29
|
requires-python = ">=3.10"
|
|
30
30
|
|
|
31
31
|
[project.optional-dependencies]
|
|
32
|
-
dev = [
|
|
32
|
+
dev = [
|
|
33
|
+
"pytest",
|
|
34
|
+
"pytest-asyncio",
|
|
35
|
+
"pytest-grpc-aio~=0.2.0",
|
|
36
|
+
"pytest-mock",
|
|
37
|
+
"ruff",
|
|
38
|
+
]
|
|
33
39
|
locking = ["wool-locking"]
|
|
34
40
|
|
|
35
41
|
[project.scripts]
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from typing import Final
|
|
5
|
+
from typing import Protocol
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from typing import Self
|
|
9
|
+
except ImportError:
|
|
10
|
+
from typing_extensions import Self
|
|
11
|
+
|
|
12
|
+
import grpc
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from wool._protobuf.mempool import service_pb2 as pb
|
|
16
|
+
from wool._protobuf.mempool import service_pb2_grpc as rpc
|
|
17
|
+
except ImportError as e:
|
|
18
|
+
from wool._protobuf import ProtobufImportError
|
|
19
|
+
|
|
20
|
+
raise ProtobufImportError(e) from e
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class EventHandler(Protocol):
|
|
24
|
+
@staticmethod
|
|
25
|
+
def __call__(client: MemoryPoolClient, response: pb.Event) -> None: ...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class MemoryPoolClient:
|
|
29
|
+
_event_stream: Final[grpc.aio.UnaryStreamCall]
|
|
30
|
+
_listener: asyncio.Task
|
|
31
|
+
_session: pb.Session | None
|
|
32
|
+
|
|
33
|
+
def __init__(self, channel: grpc.aio.Channel, event_handler: EventHandler):
|
|
34
|
+
self._stub = rpc.MemoryPoolStub(channel)
|
|
35
|
+
self._event_stream = self._stub.session(pb.SessionRequest())
|
|
36
|
+
self._event_handler = event_handler
|
|
37
|
+
self._session = None
|
|
38
|
+
|
|
39
|
+
async def __aenter__(self) -> Self:
|
|
40
|
+
if self._session:
|
|
41
|
+
raise RuntimeError(
|
|
42
|
+
"Client session may not be re-entered. "
|
|
43
|
+
"Use 'async with MemoryPoolClient(...)' only once."
|
|
44
|
+
)
|
|
45
|
+
response = await self._event_stream.read()
|
|
46
|
+
assert isinstance(response, pb.SessionResponse)
|
|
47
|
+
self._session = response.session
|
|
48
|
+
self._listener = asyncio.create_task(self._listen())
|
|
49
|
+
return self
|
|
50
|
+
|
|
51
|
+
async def __aexit__(self, *_):
|
|
52
|
+
if not self._session:
|
|
53
|
+
raise RuntimeError(
|
|
54
|
+
"Client session has not been entered. "
|
|
55
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
56
|
+
)
|
|
57
|
+
assert self._listener
|
|
58
|
+
self._listener.cancel()
|
|
59
|
+
try:
|
|
60
|
+
await self._listener
|
|
61
|
+
except asyncio.CancelledError:
|
|
62
|
+
pass
|
|
63
|
+
|
|
64
|
+
def __del__(self):
|
|
65
|
+
try:
|
|
66
|
+
self._event_stream.cancel()
|
|
67
|
+
except Exception:
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def id(self) -> str:
|
|
72
|
+
if not self._session:
|
|
73
|
+
raise RuntimeError(
|
|
74
|
+
"Client session has not been entered. "
|
|
75
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
76
|
+
)
|
|
77
|
+
return self._session.id
|
|
78
|
+
|
|
79
|
+
async def map(self, ref: str):
|
|
80
|
+
if not self._session:
|
|
81
|
+
raise RuntimeError(
|
|
82
|
+
"Client session has not been entered. "
|
|
83
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
84
|
+
)
|
|
85
|
+
request = pb.AcquireRequest(
|
|
86
|
+
session=self._session,
|
|
87
|
+
reference=pb.Reference(id=ref),
|
|
88
|
+
)
|
|
89
|
+
await self._stub.map(request)
|
|
90
|
+
|
|
91
|
+
async def get(self, ref: str) -> bytes:
|
|
92
|
+
if not self._session:
|
|
93
|
+
raise RuntimeError(
|
|
94
|
+
"Client session has not been entered. "
|
|
95
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
96
|
+
)
|
|
97
|
+
request = pb.GetRequest(
|
|
98
|
+
session=self._session,
|
|
99
|
+
reference=pb.Reference(id=ref),
|
|
100
|
+
)
|
|
101
|
+
response: pb.GetResponse = await self._stub.get(request)
|
|
102
|
+
return response.dump
|
|
103
|
+
|
|
104
|
+
async def put(self, dump: bytes, *, mutable: bool = False) -> str:
|
|
105
|
+
if not self._session:
|
|
106
|
+
raise RuntimeError(
|
|
107
|
+
"Client session has not been entered. "
|
|
108
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
109
|
+
)
|
|
110
|
+
request = pb.PutRequest(
|
|
111
|
+
session=self._session,
|
|
112
|
+
dump=dump,
|
|
113
|
+
mutable=mutable,
|
|
114
|
+
)
|
|
115
|
+
response: pb.PutResponse = await self._stub.put(request)
|
|
116
|
+
return response.reference.id
|
|
117
|
+
|
|
118
|
+
async def post(self, ref: str, dump: bytes) -> bool:
|
|
119
|
+
if not self._session:
|
|
120
|
+
raise RuntimeError(
|
|
121
|
+
"Client session has not been entered. "
|
|
122
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
123
|
+
)
|
|
124
|
+
request = pb.PostRequest(
|
|
125
|
+
session=self._session,
|
|
126
|
+
reference=pb.Reference(id=ref),
|
|
127
|
+
dump=dump,
|
|
128
|
+
)
|
|
129
|
+
response: pb.PostResponse = await self._stub.post(request)
|
|
130
|
+
return response.updated
|
|
131
|
+
|
|
132
|
+
async def acquire(self, ref: str):
|
|
133
|
+
if not self._session:
|
|
134
|
+
raise RuntimeError(
|
|
135
|
+
"Client session has not been entered. "
|
|
136
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
137
|
+
)
|
|
138
|
+
request = pb.AcquireRequest(
|
|
139
|
+
session=self._session,
|
|
140
|
+
reference=pb.Reference(id=ref),
|
|
141
|
+
)
|
|
142
|
+
await self._stub.acquire(request)
|
|
143
|
+
|
|
144
|
+
async def release(self, ref: str):
|
|
145
|
+
if not self._session:
|
|
146
|
+
raise RuntimeError(
|
|
147
|
+
"Client session has not been entered. "
|
|
148
|
+
"Use 'async with MemoryPoolClient(...)'."
|
|
149
|
+
)
|
|
150
|
+
request = pb.ReleaseRequest(
|
|
151
|
+
session=self._session,
|
|
152
|
+
reference=pb.Reference(id=ref),
|
|
153
|
+
)
|
|
154
|
+
await self._stub.release(request)
|
|
155
|
+
|
|
156
|
+
async def _listen(self):
|
|
157
|
+
assert self._event_stream
|
|
158
|
+
try:
|
|
159
|
+
while True:
|
|
160
|
+
if (response := await self._event_stream.read()) is None:
|
|
161
|
+
break
|
|
162
|
+
assert isinstance(response, pb.SessionResponse), (
|
|
163
|
+
f"Unexpected event type: {type(response)}"
|
|
164
|
+
)
|
|
165
|
+
self._event_handler(self, response.event)
|
|
166
|
+
except asyncio.CancelledError:
|
|
167
|
+
self._event_stream.cancel()
|
|
@@ -2,11 +2,11 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import hashlib
|
|
5
|
-
import mmap
|
|
6
5
|
import os
|
|
7
6
|
import pathlib
|
|
8
7
|
import shutil
|
|
9
8
|
from contextlib import asynccontextmanager
|
|
9
|
+
from mmap import mmap
|
|
10
10
|
from typing import BinaryIO
|
|
11
11
|
|
|
12
12
|
try:
|
|
@@ -23,7 +23,7 @@ class SharedObject:
|
|
|
23
23
|
_id: str
|
|
24
24
|
_mempool: MemoryPool
|
|
25
25
|
_file: BinaryIO
|
|
26
|
-
_mmap: mmap
|
|
26
|
+
_mmap: mmap
|
|
27
27
|
_size: int
|
|
28
28
|
_md5: bytes
|
|
29
29
|
|
|
@@ -31,7 +31,7 @@ class SharedObject:
|
|
|
31
31
|
self._id = id
|
|
32
32
|
self._mempool = mempool
|
|
33
33
|
self._file = open(self._path / "dump", "r+b")
|
|
34
|
-
self._mmap = mmap
|
|
34
|
+
self._mmap = mmap(self._file.fileno(), 0)
|
|
35
35
|
self._size = self.metadata.size
|
|
36
36
|
self._md5 = self.metadata.md5
|
|
37
37
|
|
|
@@ -50,7 +50,7 @@ class SharedObject:
|
|
|
50
50
|
return SharedObjectMetadata(self.id, mempool=self._mempool)
|
|
51
51
|
|
|
52
52
|
@property
|
|
53
|
-
def mmap(self) -> mmap
|
|
53
|
+
def mmap(self) -> mmap:
|
|
54
54
|
return self._mmap
|
|
55
55
|
|
|
56
56
|
@property
|
|
@@ -67,7 +67,7 @@ class SharedObject:
|
|
|
67
67
|
self._mmap.close()
|
|
68
68
|
self._file.close()
|
|
69
69
|
self._file = open(self._path / "dump", "r+b")
|
|
70
|
-
self._mmap = mmap
|
|
70
|
+
self._mmap = mmap(self._file.fileno(), 0)
|
|
71
71
|
self._size = self.metadata.size
|
|
72
72
|
self._md5 = self.metadata.md5
|
|
73
73
|
return self
|
|
@@ -77,7 +77,7 @@ class SharedObjectMetadata:
|
|
|
77
77
|
_id: str
|
|
78
78
|
_mempool: MemoryPool
|
|
79
79
|
_file: BinaryIO
|
|
80
|
-
_mmap: mmap
|
|
80
|
+
_mmap: mmap
|
|
81
81
|
_instances: dict[str, SharedObjectMetadata] = {}
|
|
82
82
|
|
|
83
83
|
def __new__(cls, id: str, *, mempool: MemoryPool):
|
|
@@ -89,7 +89,7 @@ class SharedObjectMetadata:
|
|
|
89
89
|
self._id = id
|
|
90
90
|
self._mempool = mempool
|
|
91
91
|
self._file = open(self._path / "meta", "r+b")
|
|
92
|
-
self._mmap = mmap
|
|
92
|
+
self._mmap = mmap(self._file.fileno(), 0)
|
|
93
93
|
self._instances[id] = self
|
|
94
94
|
|
|
95
95
|
def __del__(self):
|
|
@@ -115,7 +115,7 @@ class SharedObjectMetadata:
|
|
|
115
115
|
return self._metadata.md5
|
|
116
116
|
|
|
117
117
|
@property
|
|
118
|
-
def mmap(self) -> mmap
|
|
118
|
+
def mmap(self) -> mmap:
|
|
119
119
|
return self._mmap
|
|
120
120
|
|
|
121
121
|
@property
|
|
@@ -1,16 +1,14 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
from dataclasses import asdict
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
|
|
6
6
|
try:
|
|
7
|
-
from wool._protobuf.mempool
|
|
8
|
-
except ImportError:
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
)
|
|
13
|
-
raise
|
|
7
|
+
from wool._protobuf.mempool import metadata_pb2 as pb
|
|
8
|
+
except ImportError as e:
|
|
9
|
+
from wool._protobuf import ProtobufImportError
|
|
10
|
+
|
|
11
|
+
raise ProtobufImportError(e) from e
|
|
14
12
|
|
|
15
13
|
|
|
16
14
|
@dataclass
|
|
@@ -22,7 +20,7 @@ class MetadataMessage:
|
|
|
22
20
|
|
|
23
21
|
@classmethod
|
|
24
22
|
def loads(cls, data: bytes) -> MetadataMessage:
|
|
25
|
-
(metadata :=
|
|
23
|
+
(metadata := pb.MetadataMessage()).ParseFromString(data)
|
|
26
24
|
return cls(
|
|
27
25
|
ref=metadata.ref,
|
|
28
26
|
mutable=metadata.mutable,
|
|
@@ -31,9 +29,7 @@ class MetadataMessage:
|
|
|
31
29
|
)
|
|
32
30
|
|
|
33
31
|
def dumps(self) -> bytes:
|
|
34
|
-
return
|
|
35
|
-
ref=self.ref, mutable=self.mutable, size=self.size, md5=self.md5
|
|
36
|
-
).SerializeToString()
|
|
32
|
+
return pb.MetadataMessage(**asdict(self)).SerializeToString()
|
|
37
33
|
|
|
38
34
|
|
|
39
35
|
__all__ = ["MetadataMessage"]
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from typing import AsyncGenerator
|
|
5
|
+
from typing import Final
|
|
6
|
+
from weakref import WeakSet
|
|
7
|
+
from weakref import WeakValueDictionary
|
|
8
|
+
|
|
9
|
+
import shortuuid
|
|
10
|
+
from grpc.aio import ServicerContext
|
|
11
|
+
|
|
12
|
+
from wool._mempool import MemoryPool
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from wool._protobuf.mempool import service_pb2 as pb
|
|
16
|
+
from wool._protobuf.mempool import service_pb2_grpc as rpc
|
|
17
|
+
except ImportError as e:
|
|
18
|
+
from wool._protobuf import ProtobufImportError
|
|
19
|
+
|
|
20
|
+
raise ProtobufImportError(e) from e
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class _Session:
|
|
24
|
+
id: Final[str]
|
|
25
|
+
queue: Final[asyncio.Queue[pb.SessionResponse]]
|
|
26
|
+
references: Final[set[_Reference]]
|
|
27
|
+
|
|
28
|
+
_sessions: Final[WeakValueDictionary[str, _Session]] = (
|
|
29
|
+
WeakValueDictionary()
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def get(cls, id: str) -> _Session | None:
|
|
34
|
+
return cls._sessions.get(id)
|
|
35
|
+
|
|
36
|
+
def __init__(self):
|
|
37
|
+
self.id = shortuuid.uuid()
|
|
38
|
+
self.queue = asyncio.Queue()
|
|
39
|
+
self.references = set()
|
|
40
|
+
self._sessions[self.id] = self
|
|
41
|
+
|
|
42
|
+
def __eq__(self, other) -> bool:
|
|
43
|
+
if isinstance(other, _Session):
|
|
44
|
+
return self.id == other.id
|
|
45
|
+
return False
|
|
46
|
+
|
|
47
|
+
def __hash__(self) -> int:
|
|
48
|
+
return hash(self.id)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _Reference:
|
|
52
|
+
id: Final[str]
|
|
53
|
+
sessions: Final[WeakSet[_Session]]
|
|
54
|
+
|
|
55
|
+
_mempool: Final[MemoryPool]
|
|
56
|
+
_references: Final[WeakValueDictionary[str, _Reference]] = (
|
|
57
|
+
WeakValueDictionary()
|
|
58
|
+
)
|
|
59
|
+
_to_delete: Final[set[str]] = set()
|
|
60
|
+
_initialized: bool = False
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def get(cls, id: str) -> _Reference | None:
|
|
64
|
+
return cls._references.get(id)
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def new(cls, id: str, *, mempool: MemoryPool) -> _Reference:
|
|
68
|
+
if id in cls._references:
|
|
69
|
+
raise ValueError(f"Reference {id} already exists")
|
|
70
|
+
return cls(id, mempool=mempool)
|
|
71
|
+
|
|
72
|
+
def __new__(cls, id: str, *, mempool: MemoryPool):
|
|
73
|
+
if id in cls._references:
|
|
74
|
+
if id in cls._to_delete:
|
|
75
|
+
cls._to_delete.remove(id)
|
|
76
|
+
return cls._references[id]
|
|
77
|
+
return super().__new__(cls)
|
|
78
|
+
|
|
79
|
+
def __init__(self, id: str, *, mempool: MemoryPool):
|
|
80
|
+
if not self._initialized:
|
|
81
|
+
self.id = id
|
|
82
|
+
self.sessions = WeakSet()
|
|
83
|
+
self._mempool = mempool
|
|
84
|
+
self._references[id] = self
|
|
85
|
+
self._initialized = True
|
|
86
|
+
|
|
87
|
+
def __eq__(self, other) -> bool:
|
|
88
|
+
if isinstance(other, _Reference):
|
|
89
|
+
return self.id == other.id
|
|
90
|
+
return False
|
|
91
|
+
|
|
92
|
+
def __hash__(self) -> int:
|
|
93
|
+
return hash(self.id)
|
|
94
|
+
|
|
95
|
+
def __del__(self):
|
|
96
|
+
self._to_delete.add(self.id)
|
|
97
|
+
|
|
98
|
+
async def _delete(id, to_delete, mempool):
|
|
99
|
+
if id in to_delete:
|
|
100
|
+
try:
|
|
101
|
+
to_delete.remove(id)
|
|
102
|
+
await mempool.delete(id)
|
|
103
|
+
except FileNotFoundError:
|
|
104
|
+
pass
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
asyncio.get_running_loop().create_task(
|
|
108
|
+
_delete(self.id, self._to_delete, self._mempool)
|
|
109
|
+
)
|
|
110
|
+
except RuntimeError:
|
|
111
|
+
asyncio.new_event_loop().run_until_complete(
|
|
112
|
+
_delete(self.id, self._to_delete, self._mempool)
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class MemoryPoolService(rpc.MemoryPoolServicer):
|
|
117
|
+
def __init__(self, mempool: MemoryPool | None = None):
|
|
118
|
+
self._mempool = mempool or MemoryPool()
|
|
119
|
+
self._shutdown = asyncio.Event()
|
|
120
|
+
|
|
121
|
+
async def session(
|
|
122
|
+
self, request: pb.SessionRequest, context: ServicerContext
|
|
123
|
+
) -> AsyncGenerator[pb.SessionResponse]:
|
|
124
|
+
session = _Session()
|
|
125
|
+
yield pb.SessionResponse(session=pb.Session(id=session.id))
|
|
126
|
+
while True:
|
|
127
|
+
yield await session.queue.get()
|
|
128
|
+
|
|
129
|
+
async def map(
|
|
130
|
+
self, request: pb.AcquireRequest, context: ServicerContext
|
|
131
|
+
) -> pb.AcquireResponse:
|
|
132
|
+
if not (session := _Session.get(request.session.id)):
|
|
133
|
+
raise ValueError(f"Session {request.session.id} not found")
|
|
134
|
+
await self._mempool.map(request.reference.id or None)
|
|
135
|
+
reference = _Reference(request.reference.id, mempool=self._mempool)
|
|
136
|
+
await self.acquire(
|
|
137
|
+
pb.AcquireRequest(
|
|
138
|
+
session=pb.Session(id=session.id),
|
|
139
|
+
reference=pb.Reference(id=reference.id),
|
|
140
|
+
),
|
|
141
|
+
context,
|
|
142
|
+
)
|
|
143
|
+
return pb.AcquireResponse()
|
|
144
|
+
|
|
145
|
+
async def put(
|
|
146
|
+
self, request: pb.PutRequest, context: ServicerContext
|
|
147
|
+
) -> pb.PutResponse:
|
|
148
|
+
if not (session := _Session.get(request.session.id)):
|
|
149
|
+
raise ValueError(f"Session {request.session.id} not found")
|
|
150
|
+
reference = _Reference(
|
|
151
|
+
id=await self._mempool.put(request.dump, mutable=request.mutable),
|
|
152
|
+
mempool=self._mempool,
|
|
153
|
+
)
|
|
154
|
+
await self.acquire(
|
|
155
|
+
pb.AcquireRequest(
|
|
156
|
+
session=pb.Session(id=session.id),
|
|
157
|
+
reference=pb.Reference(id=reference.id),
|
|
158
|
+
),
|
|
159
|
+
context,
|
|
160
|
+
)
|
|
161
|
+
return pb.PutResponse(reference=pb.Reference(id=reference.id))
|
|
162
|
+
|
|
163
|
+
async def get(
|
|
164
|
+
self, request: pb.GetRequest, context: ServicerContext
|
|
165
|
+
) -> pb.GetResponse:
|
|
166
|
+
if not (session := _Session.get(request.session.id)):
|
|
167
|
+
raise ValueError(f"Session {request.session.id} not found")
|
|
168
|
+
if _Reference.get(request.reference.id) is None:
|
|
169
|
+
await self.acquire(
|
|
170
|
+
pb.AcquireRequest(
|
|
171
|
+
session=pb.Session(id=session.id),
|
|
172
|
+
reference=pb.Reference(id=request.reference.id),
|
|
173
|
+
),
|
|
174
|
+
context,
|
|
175
|
+
)
|
|
176
|
+
dump = await self._mempool.get(request.reference.id)
|
|
177
|
+
return pb.GetResponse(dump=dump)
|
|
178
|
+
|
|
179
|
+
async def post(
|
|
180
|
+
self, request: pb.PostRequest, context: ServicerContext
|
|
181
|
+
) -> pb.PostResponse:
|
|
182
|
+
if not (session := _Session.get(request.session.id)):
|
|
183
|
+
raise ValueError(f"Session {request.session.id} not found")
|
|
184
|
+
if not (reference := _Reference.get(request.reference.id)):
|
|
185
|
+
raise ValueError(f"Reference {request.reference.id} not found")
|
|
186
|
+
if reference not in session.references:
|
|
187
|
+
await self.acquire(
|
|
188
|
+
pb.AcquireRequest(
|
|
189
|
+
session=pb.Session(id=session.id),
|
|
190
|
+
reference=pb.Reference(id=reference.id),
|
|
191
|
+
),
|
|
192
|
+
context,
|
|
193
|
+
)
|
|
194
|
+
updated = await self._mempool.post(request.reference.id, request.dump)
|
|
195
|
+
if updated:
|
|
196
|
+
for session in _Reference(
|
|
197
|
+
id=request.reference.id, mempool=self._mempool
|
|
198
|
+
).sessions:
|
|
199
|
+
if session.id is not request.session.id:
|
|
200
|
+
event = pb.Event(
|
|
201
|
+
reference=request.reference,
|
|
202
|
+
event_type="post",
|
|
203
|
+
)
|
|
204
|
+
await session.queue.put(pb.SessionResponse(event=event))
|
|
205
|
+
return pb.PostResponse(updated=updated)
|
|
206
|
+
|
|
207
|
+
async def acquire(
|
|
208
|
+
self, request: pb.AcquireRequest, context: ServicerContext
|
|
209
|
+
) -> pb.AcquireResponse:
|
|
210
|
+
if not (session := _Session.get(request.session.id)):
|
|
211
|
+
raise ValueError(f"Session {request.session.id} not found")
|
|
212
|
+
if not (reference := _Reference.get(request.reference.id)):
|
|
213
|
+
raise ValueError(f"Reference {request.reference.id} not found")
|
|
214
|
+
session.references.add(reference)
|
|
215
|
+
reference.sessions.add(session)
|
|
216
|
+
return pb.AcquireResponse()
|
|
217
|
+
|
|
218
|
+
async def release(
|
|
219
|
+
self, request: pb.ReleaseRequest, context: ServicerContext
|
|
220
|
+
) -> pb.ReleaseResponse:
|
|
221
|
+
if not (session := _Session.get(request.session.id)):
|
|
222
|
+
raise ValueError(f"Session {request.session.id} not found")
|
|
223
|
+
if not (reference := _Reference.get(request.reference.id)):
|
|
224
|
+
raise ValueError(f"Reference {request.reference.id} not found")
|
|
225
|
+
session.references.remove(reference)
|
|
226
|
+
reference.sessions.remove(session)
|
|
227
|
+
return pb.ReleaseResponse()
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ProtobufImportError(ImportError):
|
|
8
|
+
def __init__(self, exception: ImportError):
|
|
9
|
+
super().__init__(
|
|
10
|
+
f"{str(exception)} - ensure protocol buffers are compiled."
|
|
11
|
+
)
|
{wool-0.1rc8/wool/_protobuf/mempool/metadata → wool-0.1rc9/wool/_protobuf/mempool}/metadata_pb2.py
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
3
|
# NO CHECKED-IN PROTOBUF GENCODE
|
|
4
|
-
# source: mempool/metadata
|
|
4
|
+
# source: mempool/metadata.proto
|
|
5
5
|
# Protobuf Python Version: 6.31.0
|
|
6
6
|
"""Generated protocol buffer code."""
|
|
7
7
|
from google.protobuf import descriptor as _descriptor
|
|
@@ -15,7 +15,7 @@ _runtime_version.ValidateProtobufRuntimeVersion(
|
|
|
15
15
|
31,
|
|
16
16
|
0,
|
|
17
17
|
'',
|
|
18
|
-
'mempool/metadata
|
|
18
|
+
'mempool/metadata.proto'
|
|
19
19
|
)
|
|
20
20
|
# @@protoc_insertion_point(imports)
|
|
21
21
|
|
|
@@ -24,13 +24,13 @@ _sym_db = _symbol_database.Default()
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\
|
|
27
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16mempool/metadata.proto\x12\x16wool._protobuf.mempool\"J\n\x0fMetadataMessage\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12\x0f\n\x07mutable\x18\x02 \x01(\x08\x12\x0c\n\x04size\x18\x03 \x01(\x03\x12\x0b\n\x03md5\x18\x04 \x01(\x0c\x62\x06proto3')
|
|
28
28
|
|
|
29
29
|
_globals = globals()
|
|
30
30
|
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
31
|
-
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'mempool.
|
|
31
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'mempool.metadata_pb2', _globals)
|
|
32
32
|
if not _descriptor._USE_C_DESCRIPTORS:
|
|
33
33
|
DESCRIPTOR._loaded_options = None
|
|
34
|
-
_globals['
|
|
35
|
-
_globals['
|
|
34
|
+
_globals['_METADATAMESSAGE']._serialized_start=50
|
|
35
|
+
_globals['_METADATAMESSAGE']._serialized_end=124
|
|
36
36
|
# @@protoc_insertion_point(module_scope)
|
{wool-0.1rc8/wool/_protobuf/mempool/metadata → wool-0.1rc9/wool/_protobuf/mempool}/metadata_pb2.pyi
RENAMED
|
@@ -4,7 +4,7 @@ from typing import ClassVar as _ClassVar, Optional as _Optional
|
|
|
4
4
|
|
|
5
5
|
DESCRIPTOR: _descriptor.FileDescriptor
|
|
6
6
|
|
|
7
|
-
class
|
|
7
|
+
class MetadataMessage(_message.Message):
|
|
8
8
|
__slots__ = ("ref", "mutable", "size", "md5")
|
|
9
9
|
REF_FIELD_NUMBER: _ClassVar[int]
|
|
10
10
|
MUTABLE_FIELD_NUMBER: _ClassVar[int]
|
|
@@ -17,7 +17,7 @@ except ImportError:
|
|
|
17
17
|
if _version_not_supported:
|
|
18
18
|
raise RuntimeError(
|
|
19
19
|
f'The grpc package installed is at version {GRPC_VERSION},'
|
|
20
|
-
+ f' but the generated code in mempool/
|
|
20
|
+
+ f' but the generated code in mempool/metadata_pb2_grpc.py depends on'
|
|
21
21
|
+ f' grpcio>={GRPC_GENERATED_VERSION}.'
|
|
22
22
|
+ f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
|
|
23
23
|
+ f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
3
|
# NO CHECKED-IN PROTOBUF GENCODE
|
|
4
|
-
# source: mempool/
|
|
4
|
+
# source: mempool/service.proto
|
|
5
5
|
# Protobuf Python Version: 6.31.0
|
|
6
6
|
"""Generated protocol buffer code."""
|
|
7
7
|
from google.protobuf import descriptor as _descriptor
|
|
@@ -15,7 +15,7 @@ _runtime_version.ValidateProtobufRuntimeVersion(
|
|
|
15
15
|
31,
|
|
16
16
|
0,
|
|
17
17
|
'',
|
|
18
|
-
'mempool/
|
|
18
|
+
'mempool/service.proto'
|
|
19
19
|
)
|
|
20
20
|
# @@protoc_insertion_point(imports)
|
|
21
21
|
|
|
@@ -24,11 +24,11 @@ _sym_db = _symbol_database.Default()
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15mempool/
|
|
27
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15mempool/service.proto\x12\x16wool._protobuf.mempool\"\x10\n\x0eSessionRequest\"\x81\x01\n\x0fSessionResponse\x12\x32\n\x07session\x18\x01 \x01(\x0b\x32\x1f.wool._protobuf.mempool.SessionH\x00\x12.\n\x05\x65vent\x18\x02 \x01(\x0b\x32\x1d.wool._protobuf.mempool.EventH\x00\x42\n\n\x08response\"x\n\x0e\x41\x63quireRequest\x12\x34\n\treference\x18\x01 \x01(\x0b\x32!.wool._protobuf.mempool.Reference\x12\x30\n\x07session\x18\x02 \x01(\x0b\x32\x1f.wool._protobuf.mempool.Session\"\x11\n\x0f\x41\x63quireResponse\"]\n\nPutRequest\x12\x30\n\x07session\x18\x01 \x01(\x0b\x32\x1f.wool._protobuf.mempool.Session\x12\x0f\n\x07mutable\x18\x02 \x01(\x08\x12\x0c\n\x04\x64ump\x18\x03 \x01(\x0c\"C\n\x0bPutResponse\x12\x34\n\treference\x18\x01 \x01(\x0b\x32!.wool._protobuf.mempool.Reference\"\x83\x01\n\x0bPostRequest\x12\x30\n\x07session\x18\x01 \x01(\x0b\x32\x1f.wool._protobuf.mempool.Session\x12\x34\n\treference\x18\x02 \x01(\x0b\x32!.wool._protobuf.mempool.Reference\x12\x0c\n\x04\x64ump\x18\x03 \x01(\x0c\"\x1f\n\x0cPostResponse\x12\x0f\n\x07updated\x18\x01 \x01(\x08\"t\n\nGetRequest\x12\x34\n\treference\x18\x01 \x01(\x0b\x32!.wool._protobuf.mempool.Reference\x12\x30\n\x07session\x18\x02 \x01(\x0b\x32\x1f.wool._protobuf.mempool.Session\"\x1b\n\x0bGetResponse\x12\x0c\n\x04\x64ump\x18\x01 \x01(\x0c\"x\n\x0eReleaseRequest\x12\x34\n\treference\x18\x01 \x01(\x0b\x32!.wool._protobuf.mempool.Reference\x12\x30\n\x07session\x18\x02 \x01(\x0b\x32\x1f.wool._protobuf.mempool.Session\"\x11\n\x0fReleaseResponse\"\x17\n\tReference\x12\n\n\x02id\x18\x01 \x01(\t\"\x15\n\x07Session\x12\n\n\x02id\x18\x01 \x01(\t\"Q\n\x05\x45vent\x12\x34\n\treference\x18\x01 \x01(\x0b\x32!.wool._protobuf.mempool.Reference\x12\x12\n\nevent_type\x18\x02 \x01(\t2\xed\x04\n\nMemoryPool\x12\\\n\x07session\x12&.wool._protobuf.mempool.SessionRequest\x1a\'.wool._protobuf.mempool.SessionResponse0\x01\x12Z\n\x07\x61\x63quire\x12&.wool._protobuf.mempool.AcquireRequest\x1a\'.wool._protobuf.mempool.AcquireResponse\x12V\n\x03map\x12&.wool._protobuf.mempool.AcquireRequest\x1a\'.wool._protobuf.mempool.AcquireResponse\x12N\n\x03put\x12\".wool._protobuf.mempool.PutRequest\x1a#.wool._protobuf.mempool.PutResponse\x12Q\n\x04post\x12#.wool._protobuf.mempool.PostRequest\x1a$.wool._protobuf.mempool.PostResponse\x12N\n\x03get\x12\".wool._protobuf.mempool.GetRequest\x1a#.wool._protobuf.mempool.GetResponse\x12Z\n\x07release\x12&.wool._protobuf.mempool.ReleaseRequest\x1a\'.wool._protobuf.mempool.ReleaseResponseb\x06proto3')
|
|
28
28
|
|
|
29
29
|
_globals = globals()
|
|
30
30
|
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
31
|
-
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'mempool.
|
|
31
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'mempool.service_pb2', _globals)
|
|
32
32
|
if not _descriptor._USE_C_DESCRIPTORS:
|
|
33
33
|
DESCRIPTOR._loaded_options = None
|
|
34
34
|
_globals['_SESSIONREQUEST']._serialized_start=49
|
|
@@ -62,5 +62,5 @@ if not _descriptor._USE_C_DESCRIPTORS:
|
|
|
62
62
|
_globals['_EVENT']._serialized_start=1007
|
|
63
63
|
_globals['_EVENT']._serialized_end=1088
|
|
64
64
|
_globals['_MEMORYPOOL']._serialized_start=1091
|
|
65
|
-
_globals['_MEMORYPOOL']._serialized_end=
|
|
65
|
+
_globals['_MEMORYPOOL']._serialized_end=1712
|
|
66
66
|
# @@protoc_insertion_point(module_scope)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import grpc
|
|
4
4
|
import warnings
|
|
5
5
|
|
|
6
|
-
from mempool import
|
|
6
|
+
from mempool import service_pb2 as mempool_dot_service__pb2
|
|
7
7
|
|
|
8
8
|
GRPC_GENERATED_VERSION = '1.73.1'
|
|
9
9
|
GRPC_VERSION = grpc.__version__
|
|
@@ -18,7 +18,7 @@ except ImportError:
|
|
|
18
18
|
if _version_not_supported:
|
|
19
19
|
raise RuntimeError(
|
|
20
20
|
f'The grpc package installed is at version {GRPC_VERSION},'
|
|
21
|
-
+ f' but the generated code in mempool/
|
|
21
|
+
+ f' but the generated code in mempool/service_pb2_grpc.py depends on'
|
|
22
22
|
+ f' grpcio>={GRPC_GENERATED_VERSION}.'
|
|
23
23
|
+ f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
|
|
24
24
|
+ f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
|
|
@@ -36,33 +36,38 @@ class MemoryPoolStub(object):
|
|
|
36
36
|
"""
|
|
37
37
|
self.session = channel.unary_stream(
|
|
38
38
|
'/wool._protobuf.mempool.MemoryPool/session',
|
|
39
|
-
request_serializer=
|
|
40
|
-
response_deserializer=
|
|
39
|
+
request_serializer=mempool_dot_service__pb2.SessionRequest.SerializeToString,
|
|
40
|
+
response_deserializer=mempool_dot_service__pb2.SessionResponse.FromString,
|
|
41
41
|
_registered_method=True)
|
|
42
42
|
self.acquire = channel.unary_unary(
|
|
43
43
|
'/wool._protobuf.mempool.MemoryPool/acquire',
|
|
44
|
-
request_serializer=
|
|
45
|
-
response_deserializer=
|
|
44
|
+
request_serializer=mempool_dot_service__pb2.AcquireRequest.SerializeToString,
|
|
45
|
+
response_deserializer=mempool_dot_service__pb2.AcquireResponse.FromString,
|
|
46
|
+
_registered_method=True)
|
|
47
|
+
self.map = channel.unary_unary(
|
|
48
|
+
'/wool._protobuf.mempool.MemoryPool/map',
|
|
49
|
+
request_serializer=mempool_dot_service__pb2.AcquireRequest.SerializeToString,
|
|
50
|
+
response_deserializer=mempool_dot_service__pb2.AcquireResponse.FromString,
|
|
46
51
|
_registered_method=True)
|
|
47
52
|
self.put = channel.unary_unary(
|
|
48
53
|
'/wool._protobuf.mempool.MemoryPool/put',
|
|
49
|
-
request_serializer=
|
|
50
|
-
response_deserializer=
|
|
54
|
+
request_serializer=mempool_dot_service__pb2.PutRequest.SerializeToString,
|
|
55
|
+
response_deserializer=mempool_dot_service__pb2.PutResponse.FromString,
|
|
51
56
|
_registered_method=True)
|
|
52
57
|
self.post = channel.unary_unary(
|
|
53
58
|
'/wool._protobuf.mempool.MemoryPool/post',
|
|
54
|
-
request_serializer=
|
|
55
|
-
response_deserializer=
|
|
59
|
+
request_serializer=mempool_dot_service__pb2.PostRequest.SerializeToString,
|
|
60
|
+
response_deserializer=mempool_dot_service__pb2.PostResponse.FromString,
|
|
56
61
|
_registered_method=True)
|
|
57
62
|
self.get = channel.unary_unary(
|
|
58
63
|
'/wool._protobuf.mempool.MemoryPool/get',
|
|
59
|
-
request_serializer=
|
|
60
|
-
response_deserializer=
|
|
64
|
+
request_serializer=mempool_dot_service__pb2.GetRequest.SerializeToString,
|
|
65
|
+
response_deserializer=mempool_dot_service__pb2.GetResponse.FromString,
|
|
61
66
|
_registered_method=True)
|
|
62
67
|
self.release = channel.unary_unary(
|
|
63
68
|
'/wool._protobuf.mempool.MemoryPool/release',
|
|
64
|
-
request_serializer=
|
|
65
|
-
response_deserializer=
|
|
69
|
+
request_serializer=mempool_dot_service__pb2.ReleaseRequest.SerializeToString,
|
|
70
|
+
response_deserializer=mempool_dot_service__pb2.ReleaseResponse.FromString,
|
|
66
71
|
_registered_method=True)
|
|
67
72
|
|
|
68
73
|
|
|
@@ -81,6 +86,12 @@ class MemoryPoolServicer(object):
|
|
|
81
86
|
context.set_details('Method not implemented!')
|
|
82
87
|
raise NotImplementedError('Method not implemented!')
|
|
83
88
|
|
|
89
|
+
def map(self, request, context):
|
|
90
|
+
"""Missing associated documentation comment in .proto file."""
|
|
91
|
+
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
|
92
|
+
context.set_details('Method not implemented!')
|
|
93
|
+
raise NotImplementedError('Method not implemented!')
|
|
94
|
+
|
|
84
95
|
def put(self, request, context):
|
|
85
96
|
"""Missing associated documentation comment in .proto file."""
|
|
86
97
|
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
|
@@ -110,33 +121,38 @@ def add_MemoryPoolServicer_to_server(servicer, server):
|
|
|
110
121
|
rpc_method_handlers = {
|
|
111
122
|
'session': grpc.unary_stream_rpc_method_handler(
|
|
112
123
|
servicer.session,
|
|
113
|
-
request_deserializer=
|
|
114
|
-
response_serializer=
|
|
124
|
+
request_deserializer=mempool_dot_service__pb2.SessionRequest.FromString,
|
|
125
|
+
response_serializer=mempool_dot_service__pb2.SessionResponse.SerializeToString,
|
|
115
126
|
),
|
|
116
127
|
'acquire': grpc.unary_unary_rpc_method_handler(
|
|
117
128
|
servicer.acquire,
|
|
118
|
-
request_deserializer=
|
|
119
|
-
response_serializer=
|
|
129
|
+
request_deserializer=mempool_dot_service__pb2.AcquireRequest.FromString,
|
|
130
|
+
response_serializer=mempool_dot_service__pb2.AcquireResponse.SerializeToString,
|
|
131
|
+
),
|
|
132
|
+
'map': grpc.unary_unary_rpc_method_handler(
|
|
133
|
+
servicer.map,
|
|
134
|
+
request_deserializer=mempool_dot_service__pb2.AcquireRequest.FromString,
|
|
135
|
+
response_serializer=mempool_dot_service__pb2.AcquireResponse.SerializeToString,
|
|
120
136
|
),
|
|
121
137
|
'put': grpc.unary_unary_rpc_method_handler(
|
|
122
138
|
servicer.put,
|
|
123
|
-
request_deserializer=
|
|
124
|
-
response_serializer=
|
|
139
|
+
request_deserializer=mempool_dot_service__pb2.PutRequest.FromString,
|
|
140
|
+
response_serializer=mempool_dot_service__pb2.PutResponse.SerializeToString,
|
|
125
141
|
),
|
|
126
142
|
'post': grpc.unary_unary_rpc_method_handler(
|
|
127
143
|
servicer.post,
|
|
128
|
-
request_deserializer=
|
|
129
|
-
response_serializer=
|
|
144
|
+
request_deserializer=mempool_dot_service__pb2.PostRequest.FromString,
|
|
145
|
+
response_serializer=mempool_dot_service__pb2.PostResponse.SerializeToString,
|
|
130
146
|
),
|
|
131
147
|
'get': grpc.unary_unary_rpc_method_handler(
|
|
132
148
|
servicer.get,
|
|
133
|
-
request_deserializer=
|
|
134
|
-
response_serializer=
|
|
149
|
+
request_deserializer=mempool_dot_service__pb2.GetRequest.FromString,
|
|
150
|
+
response_serializer=mempool_dot_service__pb2.GetResponse.SerializeToString,
|
|
135
151
|
),
|
|
136
152
|
'release': grpc.unary_unary_rpc_method_handler(
|
|
137
153
|
servicer.release,
|
|
138
|
-
request_deserializer=
|
|
139
|
-
response_serializer=
|
|
154
|
+
request_deserializer=mempool_dot_service__pb2.ReleaseRequest.FromString,
|
|
155
|
+
response_serializer=mempool_dot_service__pb2.ReleaseResponse.SerializeToString,
|
|
140
156
|
),
|
|
141
157
|
}
|
|
142
158
|
generic_handler = grpc.method_handlers_generic_handler(
|
|
@@ -164,8 +180,8 @@ class MemoryPool(object):
|
|
|
164
180
|
request,
|
|
165
181
|
target,
|
|
166
182
|
'/wool._protobuf.mempool.MemoryPool/session',
|
|
167
|
-
|
|
168
|
-
|
|
183
|
+
mempool_dot_service__pb2.SessionRequest.SerializeToString,
|
|
184
|
+
mempool_dot_service__pb2.SessionResponse.FromString,
|
|
169
185
|
options,
|
|
170
186
|
channel_credentials,
|
|
171
187
|
insecure,
|
|
@@ -191,8 +207,35 @@ class MemoryPool(object):
|
|
|
191
207
|
request,
|
|
192
208
|
target,
|
|
193
209
|
'/wool._protobuf.mempool.MemoryPool/acquire',
|
|
194
|
-
|
|
195
|
-
|
|
210
|
+
mempool_dot_service__pb2.AcquireRequest.SerializeToString,
|
|
211
|
+
mempool_dot_service__pb2.AcquireResponse.FromString,
|
|
212
|
+
options,
|
|
213
|
+
channel_credentials,
|
|
214
|
+
insecure,
|
|
215
|
+
call_credentials,
|
|
216
|
+
compression,
|
|
217
|
+
wait_for_ready,
|
|
218
|
+
timeout,
|
|
219
|
+
metadata,
|
|
220
|
+
_registered_method=True)
|
|
221
|
+
|
|
222
|
+
@staticmethod
|
|
223
|
+
def map(request,
|
|
224
|
+
target,
|
|
225
|
+
options=(),
|
|
226
|
+
channel_credentials=None,
|
|
227
|
+
call_credentials=None,
|
|
228
|
+
insecure=False,
|
|
229
|
+
compression=None,
|
|
230
|
+
wait_for_ready=None,
|
|
231
|
+
timeout=None,
|
|
232
|
+
metadata=None):
|
|
233
|
+
return grpc.experimental.unary_unary(
|
|
234
|
+
request,
|
|
235
|
+
target,
|
|
236
|
+
'/wool._protobuf.mempool.MemoryPool/map',
|
|
237
|
+
mempool_dot_service__pb2.AcquireRequest.SerializeToString,
|
|
238
|
+
mempool_dot_service__pb2.AcquireResponse.FromString,
|
|
196
239
|
options,
|
|
197
240
|
channel_credentials,
|
|
198
241
|
insecure,
|
|
@@ -218,8 +261,8 @@ class MemoryPool(object):
|
|
|
218
261
|
request,
|
|
219
262
|
target,
|
|
220
263
|
'/wool._protobuf.mempool.MemoryPool/put',
|
|
221
|
-
|
|
222
|
-
|
|
264
|
+
mempool_dot_service__pb2.PutRequest.SerializeToString,
|
|
265
|
+
mempool_dot_service__pb2.PutResponse.FromString,
|
|
223
266
|
options,
|
|
224
267
|
channel_credentials,
|
|
225
268
|
insecure,
|
|
@@ -245,8 +288,8 @@ class MemoryPool(object):
|
|
|
245
288
|
request,
|
|
246
289
|
target,
|
|
247
290
|
'/wool._protobuf.mempool.MemoryPool/post',
|
|
248
|
-
|
|
249
|
-
|
|
291
|
+
mempool_dot_service__pb2.PostRequest.SerializeToString,
|
|
292
|
+
mempool_dot_service__pb2.PostResponse.FromString,
|
|
250
293
|
options,
|
|
251
294
|
channel_credentials,
|
|
252
295
|
insecure,
|
|
@@ -272,8 +315,8 @@ class MemoryPool(object):
|
|
|
272
315
|
request,
|
|
273
316
|
target,
|
|
274
317
|
'/wool._protobuf.mempool.MemoryPool/get',
|
|
275
|
-
|
|
276
|
-
|
|
318
|
+
mempool_dot_service__pb2.GetRequest.SerializeToString,
|
|
319
|
+
mempool_dot_service__pb2.GetResponse.FromString,
|
|
277
320
|
options,
|
|
278
321
|
channel_credentials,
|
|
279
322
|
insecure,
|
|
@@ -299,8 +342,8 @@ class MemoryPool(object):
|
|
|
299
342
|
request,
|
|
300
343
|
target,
|
|
301
344
|
'/wool._protobuf.mempool.MemoryPool/release',
|
|
302
|
-
|
|
303
|
-
|
|
345
|
+
mempool_dot_service__pb2.ReleaseRequest.SerializeToString,
|
|
346
|
+
mempool_dot_service__pb2.ReleaseResponse.FromString,
|
|
304
347
|
options,
|
|
305
348
|
channel_credentials,
|
|
306
349
|
insecure,
|
|
@@ -1,225 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import asyncio
|
|
4
|
-
from typing import AsyncGenerator
|
|
5
|
-
from typing import Final
|
|
6
|
-
from weakref import WeakSet
|
|
7
|
-
from weakref import WeakValueDictionary
|
|
8
|
-
|
|
9
|
-
import shortuuid
|
|
10
|
-
from grpc.aio import ServicerContext
|
|
11
|
-
|
|
12
|
-
from wool._mempool import MemoryPool
|
|
13
|
-
from wool._protobuf.mempool import mempool_pb2 as proto
|
|
14
|
-
from wool._protobuf.mempool import mempool_pb2_grpc as rpc
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class Session:
|
|
18
|
-
"""
|
|
19
|
-
A session represents a client connection to the memory pool service and
|
|
20
|
-
serves as the scope for any shared references acquired over its duration.
|
|
21
|
-
"""
|
|
22
|
-
|
|
23
|
-
id: Final[str]
|
|
24
|
-
queue: Final[asyncio.Queue[proto.SessionResponse]]
|
|
25
|
-
references: Final[set[Reference]]
|
|
26
|
-
sessions: Final[WeakValueDictionary[str, Session]] = WeakValueDictionary()
|
|
27
|
-
|
|
28
|
-
@classmethod
|
|
29
|
-
def get(cls, id: str) -> Session | None:
|
|
30
|
-
return cls.sessions.get(id)
|
|
31
|
-
|
|
32
|
-
def __init__(self):
|
|
33
|
-
self.id = shortuuid.uuid()
|
|
34
|
-
self.queue = asyncio.Queue()
|
|
35
|
-
self.references = set()
|
|
36
|
-
self.sessions[self.id] = self
|
|
37
|
-
|
|
38
|
-
def __eq__(self, other) -> bool:
|
|
39
|
-
if isinstance(other, Session):
|
|
40
|
-
return self.id == other.id
|
|
41
|
-
return False
|
|
42
|
-
|
|
43
|
-
def __hash__(self) -> int:
|
|
44
|
-
return hash(self.id)
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
class Reference:
|
|
48
|
-
id: Final[str]
|
|
49
|
-
mempool: Final[MemoryPool]
|
|
50
|
-
sessions: Final[WeakSet[Session]]
|
|
51
|
-
|
|
52
|
-
_references: Final[WeakValueDictionary[str, Reference]] = (
|
|
53
|
-
WeakValueDictionary()
|
|
54
|
-
)
|
|
55
|
-
_to_delete: Final[set[str]] = set()
|
|
56
|
-
_initialized: bool = False
|
|
57
|
-
|
|
58
|
-
@classmethod
|
|
59
|
-
def get(cls, id: str) -> Reference | None:
|
|
60
|
-
return cls._references.get(id)
|
|
61
|
-
|
|
62
|
-
@classmethod
|
|
63
|
-
def new(cls, id: str, *, mempool: MemoryPool) -> Reference:
|
|
64
|
-
if id in cls._references:
|
|
65
|
-
raise ValueError(f"Reference {id} already exists")
|
|
66
|
-
return cls(id, mempool=mempool)
|
|
67
|
-
|
|
68
|
-
def __new__(cls, id: str, *, mempool: MemoryPool):
|
|
69
|
-
if id in cls._references:
|
|
70
|
-
if id in cls._to_delete:
|
|
71
|
-
cls._to_delete.remove(id)
|
|
72
|
-
return cls._references[id]
|
|
73
|
-
return super().__new__(cls)
|
|
74
|
-
|
|
75
|
-
def __init__(self, id: str, *, mempool: MemoryPool):
|
|
76
|
-
if not self._initialized:
|
|
77
|
-
self.id = id
|
|
78
|
-
self.mempool = mempool
|
|
79
|
-
self.sessions = WeakSet()
|
|
80
|
-
self._references[id] = self
|
|
81
|
-
self._initialized = True
|
|
82
|
-
|
|
83
|
-
def __eq__(self, other) -> bool:
|
|
84
|
-
if isinstance(other, Reference):
|
|
85
|
-
return self.id == other.id
|
|
86
|
-
return False
|
|
87
|
-
|
|
88
|
-
def __hash__(self) -> int:
|
|
89
|
-
return hash(self.id)
|
|
90
|
-
|
|
91
|
-
def __del__(self):
|
|
92
|
-
self._to_delete.add(self.id)
|
|
93
|
-
|
|
94
|
-
id = self.id
|
|
95
|
-
to_delete = self._to_delete
|
|
96
|
-
mempool = self.mempool
|
|
97
|
-
|
|
98
|
-
async def _delete():
|
|
99
|
-
if id in to_delete:
|
|
100
|
-
try:
|
|
101
|
-
to_delete.remove(id)
|
|
102
|
-
await mempool.delete(id)
|
|
103
|
-
except FileNotFoundError:
|
|
104
|
-
pass
|
|
105
|
-
|
|
106
|
-
try:
|
|
107
|
-
asyncio.get_running_loop().create_task(_delete())
|
|
108
|
-
except RuntimeError:
|
|
109
|
-
asyncio.new_event_loop().run_until_complete(_delete())
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
class MemoryPoolService(rpc.MemoryPoolServicer):
|
|
113
|
-
def __init__(self, mempool: MemoryPool | None = None):
|
|
114
|
-
self._mempool = mempool or MemoryPool()
|
|
115
|
-
self._shutdown = asyncio.Event()
|
|
116
|
-
|
|
117
|
-
async def session(
|
|
118
|
-
self, request: proto.SessionRequest, context: ServicerContext
|
|
119
|
-
) -> AsyncGenerator[proto.SessionResponse]:
|
|
120
|
-
session = Session()
|
|
121
|
-
yield proto.SessionResponse(session=proto.Session(id=session.id))
|
|
122
|
-
while True:
|
|
123
|
-
yield await session.queue.get()
|
|
124
|
-
|
|
125
|
-
async def acquire(
|
|
126
|
-
self, request: proto.AcquireRequest, context: ServicerContext
|
|
127
|
-
) -> proto.AcquireResponse:
|
|
128
|
-
if not (session := Session.get(request.session.id)):
|
|
129
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
130
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
131
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
132
|
-
session.references.add(reference)
|
|
133
|
-
reference.sessions.add(session)
|
|
134
|
-
return proto.AcquireResponse()
|
|
135
|
-
|
|
136
|
-
async def map(
|
|
137
|
-
self, request: proto.AcquireRequest, context: ServicerContext
|
|
138
|
-
) -> proto.AcquireResponse:
|
|
139
|
-
if not (session := Session.get(request.session.id)):
|
|
140
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
141
|
-
await self._mempool.map(request.reference.id)
|
|
142
|
-
reference = Reference(request.reference.id, mempool=self._mempool)
|
|
143
|
-
await self.acquire(
|
|
144
|
-
proto.AcquireRequest(
|
|
145
|
-
session=proto.Session(id=session.id),
|
|
146
|
-
reference=proto.Reference(id=reference.id),
|
|
147
|
-
),
|
|
148
|
-
context,
|
|
149
|
-
)
|
|
150
|
-
return proto.AcquireResponse()
|
|
151
|
-
|
|
152
|
-
async def put(
|
|
153
|
-
self, request: proto.PutRequest, context: ServicerContext
|
|
154
|
-
) -> proto.PutResponse:
|
|
155
|
-
if not (session := Session.get(request.session.id)):
|
|
156
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
157
|
-
reference = Reference(
|
|
158
|
-
id=await self._mempool.put(request.dump, mutable=request.mutable),
|
|
159
|
-
mempool=self._mempool,
|
|
160
|
-
)
|
|
161
|
-
await self.acquire(
|
|
162
|
-
proto.AcquireRequest(
|
|
163
|
-
session=proto.Session(id=session.id),
|
|
164
|
-
reference=proto.Reference(id=reference.id),
|
|
165
|
-
),
|
|
166
|
-
context,
|
|
167
|
-
)
|
|
168
|
-
return proto.PutResponse(reference=proto.Reference(id=reference.id))
|
|
169
|
-
|
|
170
|
-
async def get(
|
|
171
|
-
self, request: proto.GetRequest, context: ServicerContext
|
|
172
|
-
) -> proto.GetResponse:
|
|
173
|
-
if not (session := Session.get(request.session.id)):
|
|
174
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
175
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
176
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
177
|
-
if reference not in session.references:
|
|
178
|
-
await self.acquire(
|
|
179
|
-
proto.AcquireRequest(
|
|
180
|
-
session=proto.Session(id=session.id),
|
|
181
|
-
reference=proto.Reference(id=reference.id),
|
|
182
|
-
),
|
|
183
|
-
context,
|
|
184
|
-
)
|
|
185
|
-
dump = await self._mempool.get(reference.id)
|
|
186
|
-
return proto.GetResponse(dump=dump)
|
|
187
|
-
|
|
188
|
-
async def post(
|
|
189
|
-
self, request: proto.PostRequest, context: ServicerContext
|
|
190
|
-
) -> proto.PostResponse:
|
|
191
|
-
if not (session := Session.get(request.session.id)):
|
|
192
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
193
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
194
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
195
|
-
if reference not in session.references:
|
|
196
|
-
await self.acquire(
|
|
197
|
-
proto.AcquireRequest(
|
|
198
|
-
session=proto.Session(id=session.id),
|
|
199
|
-
reference=proto.Reference(id=reference.id),
|
|
200
|
-
),
|
|
201
|
-
context,
|
|
202
|
-
)
|
|
203
|
-
updated = await self._mempool.post(request.reference.id, request.dump)
|
|
204
|
-
if updated:
|
|
205
|
-
for session in Reference(
|
|
206
|
-
id=request.reference.id, mempool=self._mempool
|
|
207
|
-
).sessions:
|
|
208
|
-
if session.id is not request.session.id:
|
|
209
|
-
event = proto.Event(
|
|
210
|
-
reference=request.reference,
|
|
211
|
-
event_type="post",
|
|
212
|
-
)
|
|
213
|
-
await session.queue.put(proto.SessionResponse(event=event))
|
|
214
|
-
return proto.PostResponse(updated=updated)
|
|
215
|
-
|
|
216
|
-
async def release(
|
|
217
|
-
self, request: proto.ReleaseRequest, context: ServicerContext
|
|
218
|
-
) -> proto.ReleaseResponse:
|
|
219
|
-
if not (session := Session.get(request.session.id)):
|
|
220
|
-
raise ValueError(f"Session {request.session.id} not found")
|
|
221
|
-
if not (reference := Reference.get(request.reference.id)):
|
|
222
|
-
raise ValueError(f"Reference {request.reference.id} not found")
|
|
223
|
-
session.references.remove(reference)
|
|
224
|
-
reference.sessions.remove(session)
|
|
225
|
-
return proto.ReleaseResponse()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|