volucer-worker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- volucer_worker/__init__.py +112 -0
- volucer_worker/grpc.py +102 -0
- volucer_worker/message.py +172 -0
- volucer_worker/protocol.py +279 -0
- volucer_worker/py.typed +0 -0
- volucer_worker/rpc.py +782 -0
- volucer_worker/supervisor.py +341 -0
- volucer_worker/worker.py +1001 -0
- volucer_worker/wsgi.py +229 -0
- volucer_worker-0.1.0.dist-info/METADATA +448 -0
- volucer_worker-0.1.0.dist-info/RECORD +14 -0
- volucer_worker-0.1.0.dist-info/WHEEL +5 -0
- volucer_worker-0.1.0.dist-info/licenses/LICENSE +21 -0
- volucer_worker-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""A Python worker SDK for volucer's worker protocol.
|
|
2
|
+
|
|
3
|
+
The specification is ``docs/worker-protocol.md`` in `sailantis/volucer`; this package is
|
|
4
|
+
the other end of it, beside the PHP one in ``sailantis/volucer-php-sdk``.
|
|
5
|
+
|
|
6
|
+
The ordinary use is three lines::
|
|
7
|
+
|
|
8
|
+
from volucer_worker import Worker, wsgi
|
|
9
|
+
from myapp import application
|
|
10
|
+
|
|
11
|
+
Worker.connect().run(wsgi.call(application))
|
|
12
|
+
|
|
13
|
+
For an `attach` backend — where volucer dials a relay instead of spawning a process — the
|
|
14
|
+
supervisor half is here too::
|
|
15
|
+
|
|
16
|
+
from volucer_worker import supervisor, wsgi
|
|
17
|
+
|
|
18
|
+
supervisor.serve("unix:/run/app/workers.sock", wsgi.call(application), children=16)
|
|
19
|
+
|
|
20
|
+
Importing this package **rebinds ``sys.stdout``**, and that is deliberate rather than
|
|
21
|
+
surprising once the reason is known: the worker protocol uses the child process's stdout
|
|
22
|
+
as its channel, so anything an application prints would land inside a frame and
|
|
23
|
+
desynchronise the stream permanently. The PHP SDK solves the same problem with an output
|
|
24
|
+
buffer; Python's answer is to move stdout somewhere harmless and keep the real stream
|
|
25
|
+
private to the worker.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import sys
|
|
31
|
+
|
|
32
|
+
from .message import Job, Request, Response
|
|
33
|
+
from .protocol import ProtocolError
|
|
34
|
+
from .worker import Worker, real_stdout, runtime, socket_address, unix_path
|
|
35
|
+
from .rpc import Rpc
|
|
36
|
+
from . import grpc, supervisor, wsgi
|
|
37
|
+
|
|
38
|
+
__version__ = "0.1.0"
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"Job",
|
|
42
|
+
"ProtocolError",
|
|
43
|
+
"Request",
|
|
44
|
+
"Response",
|
|
45
|
+
"Rpc",
|
|
46
|
+
"Worker",
|
|
47
|
+
"grpc",
|
|
48
|
+
"runtime",
|
|
49
|
+
"real_stdout",
|
|
50
|
+
"socket_address",
|
|
51
|
+
"unix_path",
|
|
52
|
+
"supervisor",
|
|
53
|
+
"wsgi",
|
|
54
|
+
"__version__",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _protect_stdout() -> None:
|
|
59
|
+
"""Rebind ``sys.stdout`` away from the protocol channel.
|
|
60
|
+
|
|
61
|
+
Called at import, before a :class:`Worker` is constructed, because the constructor reads
|
|
62
|
+
``sys.stdout.buffer`` as the channel: rebinding first and constructing second is the
|
|
63
|
+
order that makes the two impossible to confuse.
|
|
64
|
+
|
|
65
|
+
The replacement writes to ``sys.stderr``, which is not part of the protocol and is
|
|
66
|
+
drained by the supervisor. An application's ``print`` therefore still reaches a human,
|
|
67
|
+
just not the wire.
|
|
68
|
+
"""
|
|
69
|
+
if getattr(sys.stdout, "_volucer_guard", False):
|
|
70
|
+
return
|
|
71
|
+
|
|
72
|
+
class _GuardedStdout:
|
|
73
|
+
"""A ``sys.stdout`` that cannot corrupt the worker channel."""
|
|
74
|
+
|
|
75
|
+
_volucer_guard = True
|
|
76
|
+
|
|
77
|
+
def write(self, text: str) -> int:
|
|
78
|
+
sys.stderr.write(text)
|
|
79
|
+
return len(text)
|
|
80
|
+
|
|
81
|
+
def writelines(self, lines) -> None:
|
|
82
|
+
for line in lines:
|
|
83
|
+
self.write(line)
|
|
84
|
+
|
|
85
|
+
def flush(self) -> None:
|
|
86
|
+
sys.stderr.flush()
|
|
87
|
+
|
|
88
|
+
def isatty(self) -> bool:
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
def fileno(self) -> int:
|
|
92
|
+
# Deliberately the *real* stderr descriptor rather than stdout's, so a library
|
|
93
|
+
# that does fd-level writes (a logging handler wrapping the stream, say) lands
|
|
94
|
+
# in the same place as everything else.
|
|
95
|
+
return sys.stderr.fileno()
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def encoding(self) -> str:
|
|
99
|
+
return getattr(sys.stderr, "encoding", "utf-8") or "utf-8"
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def errors(self) -> str:
|
|
103
|
+
return getattr(sys.stderr, "errors", "strict") or "strict"
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def buffer(self): # type: ignore[no-untyped-def]
|
|
107
|
+
return sys.stderr.buffer
|
|
108
|
+
|
|
109
|
+
sys.stdout = _GuardedStdout() # type: ignore[assignment]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
_protect_stdout()
|
volucer_worker/grpc.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""The outcome of one gRPC call.
|
|
2
|
+
|
|
3
|
+
A gRPC reply is not an HTTP response, and modelling it as one is the mistake this class
|
|
4
|
+
exists to prevent. Its **outcome travels in the trailer section** — after the body, not in
|
|
5
|
+
the status line — because a streaming reply has to be able to send headers before its
|
|
6
|
+
outcome exists. So a handler says what happened by naming a status here, and the server
|
|
7
|
+
moves it to where a client reads it.
|
|
8
|
+
|
|
9
|
+
``payload`` is the **body**: the raw, length-delimited framed bytes the client receives.
|
|
10
|
+
The server never parses or re-frames it, so a handler owns the framing entirely — write the
|
|
11
|
+
compression flag and the big-endian length yourself, or let whatever protobuf library you
|
|
12
|
+
already use write it. This SDK does not ship an encoder, because doing so would put a
|
|
13
|
+
schema in the transport.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
OK = 0
|
|
19
|
+
CANCELLED = 1
|
|
20
|
+
UNKNOWN = 2
|
|
21
|
+
INVALID_ARGUMENT = 3
|
|
22
|
+
DEADLINE_EXCEEDED = 4
|
|
23
|
+
NOT_FOUND = 5
|
|
24
|
+
ALREADY_EXISTS = 6
|
|
25
|
+
PERMISSION_DENIED = 7
|
|
26
|
+
RESOURCE_EXHAUSTED = 8
|
|
27
|
+
FAILED_PRECONDITION = 9
|
|
28
|
+
ABORTED = 10
|
|
29
|
+
OUT_OF_RANGE = 11
|
|
30
|
+
UNIMPLEMENTED = 12
|
|
31
|
+
INTERNAL = 13
|
|
32
|
+
UNAVAILABLE = 14
|
|
33
|
+
DATA_LOSS = 15
|
|
34
|
+
UNAUTHENTICATED = 16
|
|
35
|
+
|
|
36
|
+
_NAMES = {
|
|
37
|
+
OK: "OK",
|
|
38
|
+
CANCELLED: "CANCELLED",
|
|
39
|
+
UNKNOWN: "UNKNOWN",
|
|
40
|
+
INVALID_ARGUMENT: "INVALID_ARGUMENT",
|
|
41
|
+
DEADLINE_EXCEEDED: "DEADLINE_EXCEEDED",
|
|
42
|
+
NOT_FOUND: "NOT_FOUND",
|
|
43
|
+
ALREADY_EXISTS: "ALREADY_EXISTS",
|
|
44
|
+
PERMISSION_DENIED: "PERMISSION_DENIED",
|
|
45
|
+
RESOURCE_EXHAUSTED: "RESOURCE_EXHAUSTED",
|
|
46
|
+
FAILED_PRECONDITION: "FAILED_PRECONDITION",
|
|
47
|
+
ABORTED: "ABORTED",
|
|
48
|
+
OUT_OF_RANGE: "OUT_OF_RANGE",
|
|
49
|
+
UNIMPLEMENTED: "UNIMPLEMENTED",
|
|
50
|
+
INTERNAL: "INTERNAL",
|
|
51
|
+
UNAVAILABLE: "UNAVAILABLE",
|
|
52
|
+
DATA_LOSS: "DATA_LOSS",
|
|
53
|
+
UNAUTHENTICATED: "UNAUTHENTICATED",
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class GrpcResponse:
|
|
58
|
+
"""A gRPC outcome: a status, the framed body, and a reason for a developer."""
|
|
59
|
+
|
|
60
|
+
__slots__ = ("status", "payload", "reason", "content_type")
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
status: int = OK,
|
|
65
|
+
payload: bytes = b"",
|
|
66
|
+
reason: str = "",
|
|
67
|
+
content_type: str = "application/grpc+proto",
|
|
68
|
+
) -> None:
|
|
69
|
+
self.status = status
|
|
70
|
+
self.payload = payload
|
|
71
|
+
self.reason = reason
|
|
72
|
+
self.content_type = content_type
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def message(
|
|
76
|
+
cls, payload: bytes, content_type: str = "application/grpc+proto"
|
|
77
|
+
) -> "GrpcResponse":
|
|
78
|
+
"""A successful call carrying `payload` — the framed bytes a client will decode."""
|
|
79
|
+
return cls(OK, payload, "", content_type)
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def ok(cls, content_type: str = "application/grpc+proto") -> "GrpcResponse":
|
|
83
|
+
"""A successful call with no payload."""
|
|
84
|
+
return cls(OK, b"", "", content_type)
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def fail(cls, status: int, reason: str = "") -> "GrpcResponse":
|
|
88
|
+
"""A failed call, with `reason` as the sentence a developer will read.
|
|
89
|
+
|
|
90
|
+
`reason` is **not** payload bytes, and there are none: a failed call carries no
|
|
91
|
+
message. The server percent-encodes the sentence into the ``grpc-message`` trailer,
|
|
92
|
+
so it may contain spaces, quotes and newlines.
|
|
93
|
+
"""
|
|
94
|
+
return cls(status, b"", reason)
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def is_ok(self) -> bool:
|
|
98
|
+
return self.status == OK
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def status_name(self) -> str:
|
|
102
|
+
return _NAMES.get(self.status, "")
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""The request and response objects a Python worker's handler sees.
|
|
2
|
+
|
|
3
|
+
These are the SDK's own types rather than a framework's, because the SDK must work with
|
|
4
|
+
every Python web stack and none of them owns the wire. :func:`volucer_worker.wsgi.environ`
|
|
5
|
+
turns a :class:`Request` into a WSGI environment; a framework adapter is that environment,
|
|
6
|
+
not these objects.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Iterable
|
|
13
|
+
from urllib.parse import parse_qsl
|
|
14
|
+
|
|
15
|
+
#: Header names that may legitimately appear more than once on a response.
|
|
16
|
+
#:
|
|
17
|
+
#: ``Set-Cookie`` is the famous one, and it only matters in the response direction: a
|
|
18
|
+
#: request's repeated headers mean "joined with a comma", which is what HTTP says a
|
|
19
|
+
#: repeated field means. Collapsing two cookies into one would silently break a session.
|
|
20
|
+
_REPEATABLE = frozenset({"set-cookie"})
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(slots=True)
|
|
24
|
+
class Request:
|
|
25
|
+
"""One request, as it arrived on the channel.
|
|
26
|
+
|
|
27
|
+
`headers` is a list of pairs rather than a dict, because the same name may arrive
|
|
28
|
+
twice and the order is the client's. :meth:`header` is the ordered lookup, and
|
|
29
|
+
:attr:`header_map` is the collapsed form a WSGI environment needs.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
method: str = "GET"
|
|
33
|
+
uri: str = "/"
|
|
34
|
+
protocol: str = "HTTP/1.1"
|
|
35
|
+
headers: list[tuple[str, str]] = field(default_factory=list)
|
|
36
|
+
remote_addr: str = ""
|
|
37
|
+
script_filename: str = ""
|
|
38
|
+
https: bool = False
|
|
39
|
+
body: bytes = b""
|
|
40
|
+
|
|
41
|
+
def header(self, name: str) -> str | None:
|
|
42
|
+
"""The first value of a header, case-insensitively, or None."""
|
|
43
|
+
lowered = name.lower()
|
|
44
|
+
for key, value in self.headers:
|
|
45
|
+
if key.lower() == lowered:
|
|
46
|
+
return value
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def header_map(self) -> dict[str, str]:
|
|
51
|
+
"""Every header as a name → value map, repeated names joined with a comma.
|
|
52
|
+
|
|
53
|
+
Repeated names are joined rather than overwritten, because that is what a repeated
|
|
54
|
+
HTTP field means. `Set-Cookie` is the exception and it is a *response* header, so
|
|
55
|
+
the response side keeps the pair list instead.
|
|
56
|
+
"""
|
|
57
|
+
collapsed: dict[str, str] = {}
|
|
58
|
+
for key, value in self.headers:
|
|
59
|
+
if key in collapsed:
|
|
60
|
+
collapsed[key] = f"{collapsed[key]}, {value}"
|
|
61
|
+
else:
|
|
62
|
+
collapsed[key] = value
|
|
63
|
+
return collapsed
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def path(self) -> str:
|
|
67
|
+
"""The path, without the query string."""
|
|
68
|
+
return self.uri.split("?", 1)[0]
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def query_string(self) -> str:
|
|
72
|
+
"""Everything after the first `?`, or an empty string."""
|
|
73
|
+
_, _, query = self.uri.partition("?")
|
|
74
|
+
return query
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def query(self) -> dict[str, str]:
|
|
78
|
+
"""The query string as a dict, later values winning."""
|
|
79
|
+
return dict(parse_qsl(self.query_string, keep_blank_values=True))
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def content_type(self) -> str | None:
|
|
83
|
+
return self.header("content-type")
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def is_grpc(self) -> bool:
|
|
87
|
+
"""Whether this request is a gRPC call.
|
|
88
|
+
|
|
89
|
+
The media type decides, on both sides of the channel — see
|
|
90
|
+
`docs/worker-protocol.md`. A worker that serves both HTTP routes and gRPC services
|
|
91
|
+
is the ordinary shape, so this is the question an adapter asks rather than
|
|
92
|
+
something the loop decides for it.
|
|
93
|
+
"""
|
|
94
|
+
media_type = (self.content_type or "").strip().lower()
|
|
95
|
+
if not media_type.startswith("application/grpc"):
|
|
96
|
+
return False
|
|
97
|
+
rest = media_type.removeprefix("application/grpc")
|
|
98
|
+
return rest == "" or rest[0] in ";+-"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass(slots=True)
|
|
102
|
+
class Response:
|
|
103
|
+
"""A response the SDK will send.
|
|
104
|
+
|
|
105
|
+
Header order is preserved as given, because the server relays it verbatim and two
|
|
106
|
+
`Set-Cookie` headers are two cookies.
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
status: int = 200
|
|
110
|
+
headers: list[tuple[str, str]] = field(default_factory=list)
|
|
111
|
+
body: bytes = b""
|
|
112
|
+
|
|
113
|
+
@classmethod
|
|
114
|
+
def text(cls, body: str, status: int = 200) -> "Response":
|
|
115
|
+
return cls(status, [("Content-Type", "text/plain; charset=utf-8")], body.encode())
|
|
116
|
+
|
|
117
|
+
@classmethod
|
|
118
|
+
def html(cls, body: str, status: int = 200) -> "Response":
|
|
119
|
+
return cls(status, [("Content-Type", "text/html; charset=utf-8")], body.encode())
|
|
120
|
+
|
|
121
|
+
@classmethod
|
|
122
|
+
def json(cls, value: object, status: int = 200) -> "Response":
|
|
123
|
+
import json
|
|
124
|
+
|
|
125
|
+
return cls(
|
|
126
|
+
status,
|
|
127
|
+
[("Content-Type", "application/json")],
|
|
128
|
+
json.dumps(value, separators=(",", ":")).encode(),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
def with_header(self, name: str, value: str) -> "Response":
|
|
132
|
+
self.headers.append((name, value))
|
|
133
|
+
return self
|
|
134
|
+
|
|
135
|
+
def with_status(self, status: int) -> "Response":
|
|
136
|
+
self.status = status
|
|
137
|
+
return self
|
|
138
|
+
|
|
139
|
+
def with_body(self, body: bytes) -> "Response":
|
|
140
|
+
self.body = body
|
|
141
|
+
return self
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(slots=True)
|
|
145
|
+
class Job:
|
|
146
|
+
"""One unit of queued work, as a consumer sees it.
|
|
147
|
+
|
|
148
|
+
A job is a separate exchange from a request rather than a flag on one, because the two
|
|
149
|
+
differ in what an application may assume: a request handler is expected to reset what
|
|
150
|
+
the last one left behind, and a consumer is expected to keep its connection pool and
|
|
151
|
+
warm caches across jobs.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
pipeline: str = ""
|
|
155
|
+
id: str = ""
|
|
156
|
+
task: str = ""
|
|
157
|
+
headers: list[tuple[str, str]] = field(default_factory=list)
|
|
158
|
+
attempts: int = 0
|
|
159
|
+
body: bytes = b""
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def iter_headers(headers: Iterable[tuple[str, str]]) -> Iterable[tuple[str, str]]:
|
|
163
|
+
"""Validate a header list from application code, dropping malformed pairs.
|
|
164
|
+
|
|
165
|
+
A non-string name or value is *cast* rather than dropped, because a producer that wrote
|
|
166
|
+
`3` as a header name meant `"3"`, and failing a whole response over it would be a worse
|
|
167
|
+
answer than sending it.
|
|
168
|
+
"""
|
|
169
|
+
for pair in headers:
|
|
170
|
+
if len(pair) != 2:
|
|
171
|
+
continue
|
|
172
|
+
yield str(pair[0]), str(pair[1])
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
"""The volucer worker protocol, as a Python client implements it.
|
|
2
|
+
|
|
3
|
+
The specification is `docs/worker-protocol.md` in `sailantis/volucer`; this module is
|
|
4
|
+
the other end of it. The counterpart in PHP is `sailantis/volucer-php-sdk`, and the two
|
|
5
|
+
implement the same document rather than each other.
|
|
6
|
+
|
|
7
|
+
Everything here is deliberately dependency-free. A worker SDK is loaded before an
|
|
8
|
+
application's own dependencies are known to work, and a protocol codec that needs a
|
|
9
|
+
package is a codec that can fail for a reason unrelated to the protocol.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import struct
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
#: The only protocol version that exists so far.
|
|
19
|
+
PROTOCOL_VERSION = 1
|
|
20
|
+
|
|
21
|
+
#: The size of a frame header on the wire.
|
|
22
|
+
HEADER_LEN = 16
|
|
23
|
+
|
|
24
|
+
#: Refuse absurd payloads before allocating, matching `MAX_PAYLOAD_LEN` in Rust.
|
|
25
|
+
MAX_PAYLOAD_LEN = 64 * 1024 * 1024
|
|
26
|
+
|
|
27
|
+
#: The largest body slice sent in one frame, matching `MAX_BODY_CHUNK` in Rust.
|
|
28
|
+
MAX_BODY_CHUNK = 64 * 1024
|
|
29
|
+
|
|
30
|
+
#: The request id reserved for unsolicited frames.
|
|
31
|
+
UNSOLICITED = 0
|
|
32
|
+
|
|
33
|
+
#: The message type table. The numbering *is* the protocol: never reorder, never reuse.
|
|
34
|
+
HELLO = 1
|
|
35
|
+
HELLO_ACK = 2
|
|
36
|
+
REQUEST = 3
|
|
37
|
+
RESPONSE = 4
|
|
38
|
+
BODY = 5
|
|
39
|
+
EVENT = 6
|
|
40
|
+
LOG = 7
|
|
41
|
+
METRIC = 8
|
|
42
|
+
CONTROL = 9
|
|
43
|
+
CONTROL_ACK = 10
|
|
44
|
+
END = 11
|
|
45
|
+
PING = 12
|
|
46
|
+
PONG = 13
|
|
47
|
+
ERROR = 14
|
|
48
|
+
CALL = 15
|
|
49
|
+
RETURN = 16
|
|
50
|
+
JOB = 17
|
|
51
|
+
ACK = 18
|
|
52
|
+
|
|
53
|
+
#: Bits in the `flags` field.
|
|
54
|
+
FLAG_COMPRESSED = 0x0001
|
|
55
|
+
FLAG_CHUNKED = 0x0002
|
|
56
|
+
FLAG_LAST = 0x0004
|
|
57
|
+
FLAG_ERROR = 0x0008
|
|
58
|
+
FLAG_NO_CHECKSUM = 0x0010
|
|
59
|
+
|
|
60
|
+
_NAMES = {
|
|
61
|
+
HELLO: "HELLO",
|
|
62
|
+
HELLO_ACK: "HELLO_ACK",
|
|
63
|
+
REQUEST: "REQUEST",
|
|
64
|
+
RESPONSE: "RESPONSE",
|
|
65
|
+
BODY: "BODY",
|
|
66
|
+
EVENT: "EVENT",
|
|
67
|
+
LOG: "LOG",
|
|
68
|
+
METRIC: "METRIC",
|
|
69
|
+
CONTROL: "CONTROL",
|
|
70
|
+
CONTROL_ACK: "CONTROL_ACK",
|
|
71
|
+
END: "END",
|
|
72
|
+
PING: "PING",
|
|
73
|
+
PONG: "PONG",
|
|
74
|
+
ERROR: "ERROR",
|
|
75
|
+
CALL: "CALL",
|
|
76
|
+
RETURN: "RETURN",
|
|
77
|
+
JOB: "JOB",
|
|
78
|
+
ACK: "ACK",
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def type_name(message_type: int) -> str:
|
|
83
|
+
"""The wire name of a type, for diagnostics.
|
|
84
|
+
|
|
85
|
+
Mirrors `MessageType::name()` in Rust so both ends describe a frame the same way.
|
|
86
|
+
"""
|
|
87
|
+
return _NAMES.get(message_type, f"unknown ({message_type})")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class ProtocolError(Exception):
|
|
91
|
+
"""A framing or message failure.
|
|
92
|
+
|
|
93
|
+
The names are the protocol's own: a *framing* failure means the reader can no longer
|
|
94
|
+
trust its position in the stream and the worker must be discarded, while a *message*
|
|
95
|
+
failure (a wrong request id, a remote error) leaves the stream aligned.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(self, message: str, *, fatal: bool = True) -> None:
|
|
99
|
+
super().__init__(message)
|
|
100
|
+
self.fatal = fatal
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# --------------------------------------------------------------------- CRC-32C
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _build_crc32c_table() -> tuple[int, ...]:
|
|
107
|
+
"""The Castagnoli table, computed once at import.
|
|
108
|
+
|
|
109
|
+
A pure-Python table rather than :func:`zlib.crc32`, which is CRC-32B: a different
|
|
110
|
+
polynomial, so using it would make every frame fail its checksum against a correct
|
|
111
|
+
peer. The check value is the standard one —
|
|
112
|
+
``crc32c(b"123456789") == 0xE3069283`` — and the test suite pins it.
|
|
113
|
+
"""
|
|
114
|
+
polynomial = 0x82F63B78
|
|
115
|
+
table = []
|
|
116
|
+
for index in range(256):
|
|
117
|
+
value = index
|
|
118
|
+
for _ in range(8):
|
|
119
|
+
value = (value >> 1) ^ polynomial if value & 1 else value >> 1
|
|
120
|
+
table.append(value)
|
|
121
|
+
return tuple(table)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
_CRC32C_TABLE = _build_crc32c_table()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def crc32c(data: bytes) -> int:
|
|
128
|
+
"""CRC-32C (Castagnoli) of `data`, as the 32-bit value the header carries."""
|
|
129
|
+
crc = 0xFFFFFFFF
|
|
130
|
+
for byte in data:
|
|
131
|
+
crc = _CRC32C_TABLE[(crc ^ byte) & 0xFF] ^ (crc >> 8)
|
|
132
|
+
return crc ^ 0xFFFFFFFF
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# ---------------------------------------------------------------------- frames
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class Header:
|
|
139
|
+
"""A decoded frame header: the 16 bytes that precede a payload.
|
|
140
|
+
|
|
141
|
+
A direct transliteration of ``Header::decode`` in
|
|
142
|
+
``crates/volucer-worker/src/header.rs``. Layout, little-endian::
|
|
143
|
+
|
|
144
|
+
offset size field
|
|
145
|
+
0 4 payload_len
|
|
146
|
+
4 1 protocol_version
|
|
147
|
+
5 1 message_type
|
|
148
|
+
6 2 flags
|
|
149
|
+
8 4 request_id
|
|
150
|
+
12 4 crc32c(payload)
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
__slots__ = ("payload_len", "message_type", "flags", "request_id", "checksum")
|
|
154
|
+
|
|
155
|
+
def __init__(
|
|
156
|
+
self,
|
|
157
|
+
payload_len: int,
|
|
158
|
+
message_type: int,
|
|
159
|
+
flags: int,
|
|
160
|
+
request_id: int,
|
|
161
|
+
checksum: int,
|
|
162
|
+
) -> None:
|
|
163
|
+
self.payload_len = payload_len
|
|
164
|
+
self.message_type = message_type
|
|
165
|
+
self.flags = flags
|
|
166
|
+
self.request_id = request_id
|
|
167
|
+
self.checksum = checksum
|
|
168
|
+
|
|
169
|
+
@classmethod
|
|
170
|
+
def decode(cls, raw: bytes) -> "Header":
|
|
171
|
+
"""Parse a header, which must be exactly 16 bytes.
|
|
172
|
+
|
|
173
|
+
Exactly, not "at least". A stream that ended mid-header is an error rather than a
|
|
174
|
+
clean close: treating it as one would hide a truncated frame.
|
|
175
|
+
"""
|
|
176
|
+
if len(raw) != HEADER_LEN:
|
|
177
|
+
raise ProtocolError(
|
|
178
|
+
f"a frame header is exactly {HEADER_LEN} bytes, got {len(raw)}"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
payload_len, version, message_type, flags, request_id, checksum = struct.unpack(
|
|
182
|
+
"<IBBHII", raw
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
if version != PROTOCOL_VERSION:
|
|
186
|
+
raise ProtocolError(
|
|
187
|
+
f"unsupported protocol version {version} (this SDK speaks {PROTOCOL_VERSION})"
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
if not HELLO <= message_type <= ACK:
|
|
191
|
+
raise ProtocolError(
|
|
192
|
+
f"unknown message type {message_type} (this SDK defines 1..18)"
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
# Refused rather than misread: with no codec negotiated, a "compressed" payload
|
|
196
|
+
# would be handed to json as if it were plain text.
|
|
197
|
+
if flags & FLAG_COMPRESSED:
|
|
198
|
+
raise ProtocolError("refusing a compressed frame: no codec is negotiated")
|
|
199
|
+
|
|
200
|
+
return cls(payload_len, message_type, flags, request_id, checksum)
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def is_last(self) -> bool:
|
|
204
|
+
return bool(self.flags & FLAG_LAST)
|
|
205
|
+
|
|
206
|
+
@property
|
|
207
|
+
def declares_no_checksum(self) -> bool:
|
|
208
|
+
"""The writer's *claim*, not a permission.
|
|
209
|
+
|
|
210
|
+
Whether it may be believed is the reader's decision, and
|
|
211
|
+
:meth:`Worker.read_payload` refuses one the handshake never agreed to.
|
|
212
|
+
"""
|
|
213
|
+
return bool(self.flags & FLAG_NO_CHECKSUM)
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def type_name(self) -> str:
|
|
217
|
+
return type_name(self.message_type)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class Frame:
|
|
221
|
+
"""A frame ready to send: a message type, a request id and a payload.
|
|
222
|
+
|
|
223
|
+
`checksum` is passed to :meth:`encode` rather than remembered here, because the
|
|
224
|
+
decision belongs to the :class:`Worker` that made the handshake agreement.
|
|
225
|
+
"""
|
|
226
|
+
|
|
227
|
+
__slots__ = ("message_type", "request_id", "payload", "flags")
|
|
228
|
+
|
|
229
|
+
def __init__(
|
|
230
|
+
self,
|
|
231
|
+
message_type: int,
|
|
232
|
+
request_id: int,
|
|
233
|
+
payload: bytes = b"",
|
|
234
|
+
flags: int = 0,
|
|
235
|
+
) -> None:
|
|
236
|
+
self.message_type = message_type
|
|
237
|
+
self.request_id = request_id
|
|
238
|
+
self.payload = payload
|
|
239
|
+
self.flags = flags
|
|
240
|
+
|
|
241
|
+
def encode(self, checksum: bool = True) -> bytes:
|
|
242
|
+
"""Serialize to header + payload.
|
|
243
|
+
|
|
244
|
+
With the checksum off the field is written as zero and ``FLAG_NO_CHECKSUM`` is
|
|
245
|
+
raised, which is what makes the zero mean "not a checksum" rather than "the
|
|
246
|
+
checksum of an empty payload".
|
|
247
|
+
"""
|
|
248
|
+
flags = self.flags if checksum else self.flags | FLAG_NO_CHECKSUM
|
|
249
|
+
return struct.pack(
|
|
250
|
+
"<IBBHII",
|
|
251
|
+
len(self.payload),
|
|
252
|
+
PROTOCOL_VERSION,
|
|
253
|
+
self.message_type,
|
|
254
|
+
flags,
|
|
255
|
+
self.request_id,
|
|
256
|
+
crc32c(self.payload) if checksum else 0,
|
|
257
|
+
) + self.payload
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def json_bytes(value: Any) -> bytes:
|
|
261
|
+
"""JSON-encode a payload the way the protocol expects it.
|
|
262
|
+
|
|
263
|
+
`separators` carries no spaces because the golden fixtures in both SDKs are written
|
|
264
|
+
against that spelling, and a payload is compared byte for byte by those tests.
|
|
265
|
+
"""
|
|
266
|
+
return json.dumps(value, separators=(",", ":")).encode("utf-8")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def decode_json(payload: bytes, what: str) -> dict[str, Any]:
|
|
270
|
+
"""Decode a JSON payload, turning a bad payload into a protocol error."""
|
|
271
|
+
if not payload:
|
|
272
|
+
return {}
|
|
273
|
+
try:
|
|
274
|
+
decoded = json.loads(payload)
|
|
275
|
+
except ValueError as error:
|
|
276
|
+
raise ProtocolError(f"{what} was not valid JSON: {error}") from None
|
|
277
|
+
if not isinstance(decoded, dict):
|
|
278
|
+
raise ProtocolError(f"{what} must be a JSON object")
|
|
279
|
+
return decoded
|
volucer_worker/py.typed
ADDED
|
File without changes
|