sw-serverless 10.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sw_serverless/__init__.py +28 -0
- sw_serverless/_adapter.py +180 -0
- sw_serverless/_hpack.py +186 -0
- sw_serverless/_http2.py +253 -0
- sw_serverless/_huffman.py +37 -0
- sw_serverless/_runner.py +565 -0
- sw_serverless/_types.py +124 -0
- sw_serverless/_wire.py +242 -0
- sw_serverless-10.2.3.dist-info/METADATA +58 -0
- sw_serverless-10.2.3.dist-info/RECORD +12 -0
- sw_serverless-10.2.3.dist-info/WHEEL +5 -0
- sw_serverless-10.2.3.dist-info/top_level.txt +1 -0
sw_serverless/_runner.py
ADDED
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
"""Running an adapter: the stdin handshake, the gRPC stream to the host, and every frame on it.
|
|
2
|
+
|
|
3
|
+
The process is started by the host, which writes one JSON line on stdin — where to dial and a
|
|
4
|
+
one-time token — and keeps stdin open: when it closes, the host is gone and the adapter exits.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import asyncio
|
|
8
|
+
import contextvars
|
|
9
|
+
import inspect
|
|
10
|
+
import itertools
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import traceback
|
|
18
|
+
|
|
19
|
+
from . import _adapter, _types, _wire
|
|
20
|
+
from ._http2 import GrpcStream
|
|
21
|
+
|
|
22
|
+
SDK_VERSION = "10.2.3"
|
|
23
|
+
SDK_LANGUAGE = "python"
|
|
24
|
+
PROTOCOL = 2
|
|
25
|
+
ATTACH = "/sw.serverless.v1.AdapterHost/Attach"
|
|
26
|
+
DESCRIBE_FLAG = "--describe"
|
|
27
|
+
DESCRIBE_VERSION = 1
|
|
28
|
+
MEMORY_LIMIT_VARIABLE = "SW_SERVERLESS_MEMORY_LIMIT_BYTES"
|
|
29
|
+
|
|
30
|
+
# The memory ceiling this process runs under once apply_memory_limit has set it; None otherwise.
|
|
31
|
+
_memory_limit = None
|
|
32
|
+
|
|
33
|
+
# ILogger levels, which the host's log pipeline reads: Trace 0 .. Critical 5.
|
|
34
|
+
_LEVELS = ((logging.CRITICAL, 5), (logging.ERROR, 4), (logging.WARNING, 3), (logging.INFO, 2), (logging.DEBUG, 1))
|
|
35
|
+
|
|
36
|
+
_context = contextvars.ContextVar("sw_context", default=None)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class AdapterError(Exception):
|
|
40
|
+
"""An error with a type of the adapter's choosing, which the host records as given."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, message, *, type=None, detail=None):
|
|
43
|
+
super().__init__(message)
|
|
44
|
+
self.type = type
|
|
45
|
+
self.detail = detail
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Context:
|
|
49
|
+
"""What a call can reach beyond its argument: its session, the host's state and events, logs."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, runner, session_id=None, command=None, cancelled=None):
|
|
52
|
+
self._runner = runner
|
|
53
|
+
self.session_id = session_id
|
|
54
|
+
self.command = command
|
|
55
|
+
self.cancelled = cancelled or asyncio.Event()
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def adapter_id(self):
|
|
59
|
+
return self._runner.handshake.get("adapterId", "")
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def instance_key(self):
|
|
63
|
+
return self._runner.handshake.get("instanceKey", "")
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def stopping(self):
|
|
67
|
+
"""Set once the host has asked the adapter to stop."""
|
|
68
|
+
return self._runner.stopping
|
|
69
|
+
|
|
70
|
+
def value_of(self, name, default=None):
|
|
71
|
+
return _adapter.value_of(name, default)
|
|
72
|
+
|
|
73
|
+
async def publish(self, payload, *, dedupe_key="", content_type="", headers=None, endpoint=""):
|
|
74
|
+
"""Hands an event to the host and waits until it is persisted. Acknowledge the source only
|
|
75
|
+
after this returns: that is what makes delivery at-least-once. Returns the host's reference."""
|
|
76
|
+
ack = await self._runner.request("event", {
|
|
77
|
+
"payload": _types.encode(payload), "dedupe_key": dedupe_key, "content_type": content_type,
|
|
78
|
+
"headers": headers or {}, "endpoint": endpoint}, "event_ack")
|
|
79
|
+
if not ack.get("accepted"):
|
|
80
|
+
error = ack.get("error") or {}
|
|
81
|
+
raise AdapterError(error.get("message") or "the host did not accept the event", type=error.get("type"))
|
|
82
|
+
return ack.get("reference", "")
|
|
83
|
+
|
|
84
|
+
async def get_state(self, name):
|
|
85
|
+
result = await self._state(0, name)
|
|
86
|
+
return result.get("value", "") if result.get("found") else None
|
|
87
|
+
|
|
88
|
+
async def set_state(self, name, value):
|
|
89
|
+
await self._state(1, name, value)
|
|
90
|
+
|
|
91
|
+
async def delete_state(self, name):
|
|
92
|
+
await self._state(2, name)
|
|
93
|
+
|
|
94
|
+
async def _state(self, op, name, value=""):
|
|
95
|
+
result = await self._runner.request("state", {"op": op, "name": name, "value": value}, "state_result")
|
|
96
|
+
if result.get("error"):
|
|
97
|
+
raise AdapterError(result["error"].get("message", "state request failed"), type=result["error"].get("type"))
|
|
98
|
+
return result
|
|
99
|
+
|
|
100
|
+
def metric(self, name, value, tags=None):
|
|
101
|
+
self._runner.telemetry("metric", {"name": name, "value": float(value), "tags": tags or {}})
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def context():
|
|
105
|
+
"""The current call's :class:`Context`, or the adapter's own outside a call."""
|
|
106
|
+
current = _context.get()
|
|
107
|
+
if current is None:
|
|
108
|
+
raise RuntimeError("there is no adapter context outside a running adapter")
|
|
109
|
+
return current
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class _HostLogHandler(logging.Handler):
|
|
113
|
+
"""Python logging, forwarded to the host as log frames once the stream is open."""
|
|
114
|
+
|
|
115
|
+
def __init__(self, runner):
|
|
116
|
+
super().__init__()
|
|
117
|
+
self._runner = runner
|
|
118
|
+
|
|
119
|
+
def emit(self, record):
|
|
120
|
+
level = next((sw for py, sw in _LEVELS if record.levelno >= py), 0)
|
|
121
|
+
if level < self._runner.minimum_level:
|
|
122
|
+
return
|
|
123
|
+
try:
|
|
124
|
+
message = record.getMessage()
|
|
125
|
+
except Exception:
|
|
126
|
+
message = str(record.msg)
|
|
127
|
+
exception = "".join(traceback.format_exception(*record.exc_info)) if record.exc_info else ""
|
|
128
|
+
self._runner.telemetry("log", {
|
|
129
|
+
"level": level, "message": message, "exception": exception,
|
|
130
|
+
"properties": {"logger": record.name}, "timestamp_unix_ms": int(record.created * 1000)})
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class Runner:
|
|
134
|
+
def __init__(self, adapter):
|
|
135
|
+
self.adapter = adapter
|
|
136
|
+
self.cls = type(adapter)
|
|
137
|
+
self.commands = _adapter.commands_of(adapter)
|
|
138
|
+
self.handshake = {}
|
|
139
|
+
self.stream = None
|
|
140
|
+
self.loop = None
|
|
141
|
+
self.stopping = None
|
|
142
|
+
self.minimum_level = 1
|
|
143
|
+
self._ids = itertools.count(1)
|
|
144
|
+
self._pending = {}
|
|
145
|
+
self._running = {}
|
|
146
|
+
self._telemetry = None
|
|
147
|
+
self._ready = None
|
|
148
|
+
self._starting = None
|
|
149
|
+
|
|
150
|
+
# ------------------------------------------------------------------ description
|
|
151
|
+
|
|
152
|
+
def hello(self):
|
|
153
|
+
return {
|
|
154
|
+
"token": self.handshake.get("token") or "",
|
|
155
|
+
"adapter_id": self.handshake.get("adapterId") or "",
|
|
156
|
+
"instance_key": self.handshake.get("instanceKey") or "",
|
|
157
|
+
"protocol_version": PROTOCOL,
|
|
158
|
+
"sdk_version": SDK_VERSION,
|
|
159
|
+
"sdk_language": SDK_LANGUAGE,
|
|
160
|
+
"capabilities": self.capabilities(),
|
|
161
|
+
"commands": [self._command_info(c) for c in self.commands.values()],
|
|
162
|
+
"settings": [{"name": s.name, "description": s.description or "", "required": s.required,
|
|
163
|
+
"secret": s.secret, "default_value": s.default or "", "type": s.type}
|
|
164
|
+
for s in _adapter.declared_settings()],
|
|
165
|
+
"kinds": _adapter.kinds_of(self.cls),
|
|
166
|
+
"contracts": _adapter.contracts_of(self.cls),
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
@staticmethod
|
|
170
|
+
def _command_info(command):
|
|
171
|
+
info = command.info()
|
|
172
|
+
return {"name": info["name"], "parameter_type": info["parameter_type"],
|
|
173
|
+
"returns_value": info["returns_value"], "description": info["description"] or "",
|
|
174
|
+
"input_schema": json.dumps(info["input_schema"]) if info["input_schema"] is not None else "",
|
|
175
|
+
"output_schema": json.dumps(info["output_schema"]) if info["output_schema"] is not None else ""}
|
|
176
|
+
|
|
177
|
+
def capabilities(self):
|
|
178
|
+
caps = []
|
|
179
|
+
if _adapter.is_resident(self.cls):
|
|
180
|
+
caps.append("resident")
|
|
181
|
+
if callable(getattr(self.adapter, "reset", None)):
|
|
182
|
+
caps.append("resettable")
|
|
183
|
+
caps.append("cancel")
|
|
184
|
+
caps.extend("command:" + name for name in self.commands)
|
|
185
|
+
return caps
|
|
186
|
+
|
|
187
|
+
# ------------------------------------------------------------------ running
|
|
188
|
+
|
|
189
|
+
async def run(self, handshake_line):
|
|
190
|
+
self.handshake = json.loads(handshake_line)
|
|
191
|
+
if int(self.handshake.get("protocol") or 2) < PROTOCOL:
|
|
192
|
+
raise RuntimeError(f"the host speaks protocol {self.handshake.get('protocol')}; this SDK speaks {PROTOCOL}")
|
|
193
|
+
socket_path = self.handshake.get("socket")
|
|
194
|
+
if not socket_path:
|
|
195
|
+
raise RuntimeError("the handshake names no socket; this SDK runs on Linux and macOS hosts")
|
|
196
|
+
|
|
197
|
+
self.loop = asyncio.get_running_loop()
|
|
198
|
+
self.stopping = asyncio.Event()
|
|
199
|
+
self._ready = asyncio.Event()
|
|
200
|
+
self._telemetry = asyncio.Queue(maxsize=10000)
|
|
201
|
+
_context.set(Context(self))
|
|
202
|
+
|
|
203
|
+
self.stream = await GrpcStream.open(socket_path, ATTACH)
|
|
204
|
+
await self.send({"hello": self.hello()})
|
|
205
|
+
|
|
206
|
+
handler = _HostLogHandler(self)
|
|
207
|
+
logging.getLogger().addHandler(handler)
|
|
208
|
+
if logging.getLogger().level > logging.INFO or logging.getLogger().level == logging.NOTSET:
|
|
209
|
+
logging.getLogger().setLevel(logging.INFO)
|
|
210
|
+
|
|
211
|
+
watcher = threading.Thread(target=self._watch_parent, daemon=True)
|
|
212
|
+
watcher.start()
|
|
213
|
+
telemetry = self.loop.create_task(self._pump_telemetry())
|
|
214
|
+
try:
|
|
215
|
+
await self._read_loop()
|
|
216
|
+
finally:
|
|
217
|
+
self.stopping.set()
|
|
218
|
+
logging.getLogger().removeHandler(handler)
|
|
219
|
+
telemetry.cancel()
|
|
220
|
+
await self.stream.aclose()
|
|
221
|
+
|
|
222
|
+
def _watch_parent(self):
|
|
223
|
+
# Stdin stays open while the host lives; end of file means it's gone.
|
|
224
|
+
try:
|
|
225
|
+
while sys.stdin.readline():
|
|
226
|
+
pass
|
|
227
|
+
except Exception:
|
|
228
|
+
pass
|
|
229
|
+
if self.loop and not self.loop.is_closed():
|
|
230
|
+
self.loop.call_soon_threadsafe(self.stopping.set)
|
|
231
|
+
|
|
232
|
+
async def _read_loop(self):
|
|
233
|
+
stop = self.loop.create_task(self.stopping.wait())
|
|
234
|
+
try:
|
|
235
|
+
while True:
|
|
236
|
+
receive = self.loop.create_task(self.stream.receive())
|
|
237
|
+
done, _ = await asyncio.wait({receive, stop}, return_when=asyncio.FIRST_COMPLETED)
|
|
238
|
+
if receive not in done:
|
|
239
|
+
receive.cancel()
|
|
240
|
+
return
|
|
241
|
+
data = receive.result()
|
|
242
|
+
if data is None:
|
|
243
|
+
return
|
|
244
|
+
frame = _wire.decode("HostFrame", data)
|
|
245
|
+
if await self._on_frame(frame):
|
|
246
|
+
return
|
|
247
|
+
finally:
|
|
248
|
+
stop.cancel()
|
|
249
|
+
|
|
250
|
+
async def _on_frame(self, frame):
|
|
251
|
+
"""True when the adapter should stop."""
|
|
252
|
+
frame_id = frame.get("id", 0)
|
|
253
|
+
if "ready" in frame:
|
|
254
|
+
ready = frame["ready"]
|
|
255
|
+
_adapter._startup_values.clear()
|
|
256
|
+
_adapter._startup_values.update(ready.get("startup_values", {}))
|
|
257
|
+
# Started beside the read loop, not inside it: a start that publishes an event or reads its
|
|
258
|
+
# state waits for the host's answer, which only the read loop can take in. Commands wait
|
|
259
|
+
# for _ready; if start fails, the adapter stops.
|
|
260
|
+
self._starting = self.loop.create_task(self._start())
|
|
261
|
+
elif "invoke" in frame:
|
|
262
|
+
self._start_invoke(frame_id, frame["invoke"])
|
|
263
|
+
elif "cancel" in frame:
|
|
264
|
+
running = self._running.get(frame_id)
|
|
265
|
+
if running:
|
|
266
|
+
running[1].set()
|
|
267
|
+
running[0].cancel()
|
|
268
|
+
elif "ping" in frame:
|
|
269
|
+
self.loop.create_task(self._pong(frame_id))
|
|
270
|
+
elif "reset" in frame:
|
|
271
|
+
self.loop.create_task(self._reset(frame_id, frame["reset"].get("session_id", "")))
|
|
272
|
+
elif "set_log_level" in frame:
|
|
273
|
+
self.minimum_level = frame["set_log_level"].get("level", 0)
|
|
274
|
+
elif "shutdown" in frame:
|
|
275
|
+
await self._shutdown(frame["shutdown"])
|
|
276
|
+
return True
|
|
277
|
+
else:
|
|
278
|
+
for answer in ("state_result", "event_ack"):
|
|
279
|
+
if answer in frame:
|
|
280
|
+
waiter = self._pending.pop(frame_id, None)
|
|
281
|
+
if waiter and not waiter.done():
|
|
282
|
+
waiter.set_result(frame[answer])
|
|
283
|
+
return False
|
|
284
|
+
|
|
285
|
+
async def _start(self):
|
|
286
|
+
start = getattr(self.adapter, "start", None)
|
|
287
|
+
try:
|
|
288
|
+
if callable(start):
|
|
289
|
+
await _call(start)
|
|
290
|
+
self._ready.set()
|
|
291
|
+
except Exception:
|
|
292
|
+
logging.getLogger("sw_serverless").exception("the adapter failed to start")
|
|
293
|
+
traceback.print_exc()
|
|
294
|
+
self.stopping.set()
|
|
295
|
+
|
|
296
|
+
# ------------------------------------------------------------------ commands
|
|
297
|
+
|
|
298
|
+
def _start_invoke(self, frame_id, invoke):
|
|
299
|
+
cancelled = asyncio.Event()
|
|
300
|
+
task = self.loop.create_task(self._invoke(frame_id, invoke, cancelled))
|
|
301
|
+
self._running[frame_id] = (task, cancelled)
|
|
302
|
+
task.add_done_callback(lambda _: self._running.pop(frame_id, None))
|
|
303
|
+
|
|
304
|
+
async def _invoke(self, frame_id, invoke, cancelled):
|
|
305
|
+
name = invoke.get("command", "")
|
|
306
|
+
try:
|
|
307
|
+
await self._ready.wait()
|
|
308
|
+
command = self.commands.get(name)
|
|
309
|
+
if command is None:
|
|
310
|
+
raise AdapterError(f"the adapter has no command named {name}", type="MissingMethodException")
|
|
311
|
+
_adapter._call_values.set(invoke.get("properties") or {})
|
|
312
|
+
_context.set(Context(self, invoke.get("session_id") or str(frame_id), name, cancelled))
|
|
313
|
+
args = [_types.decode(command.argument_type, invoke.get("payload", b""))] if command.takes_argument else []
|
|
314
|
+
coro = _call(command.method, *args)
|
|
315
|
+
timeout = invoke.get("timeout_seconds") or 0
|
|
316
|
+
result = await (asyncio.wait_for(coro, timeout) if timeout > 0 else coro)
|
|
317
|
+
payload = _types.encode(result) if command.returns_value else b""
|
|
318
|
+
await self.send({"id": frame_id, "invoke_result": {"payload": payload}})
|
|
319
|
+
except asyncio.CancelledError:
|
|
320
|
+
await self._fail(frame_id, "OperationCanceledException", "the call was cancelled", "")
|
|
321
|
+
except MemoryError as ex:
|
|
322
|
+
message = str(ex)
|
|
323
|
+
if _memory_limit:
|
|
324
|
+
message = (f"the adapter ran out of memory: it runs under a memory limit of "
|
|
325
|
+
f"{_memory_limit // (1024 * 1024)} MB" + (f" ({message})" if message else ""))
|
|
326
|
+
await self._fail(frame_id, _qualified(type(ex)), message, traceback.format_exc())
|
|
327
|
+
except Exception as ex:
|
|
328
|
+
await self._fail(frame_id, getattr(ex, "type", None) or _qualified(type(ex)), str(ex),
|
|
329
|
+
getattr(ex, "detail", None) or traceback.format_exc())
|
|
330
|
+
|
|
331
|
+
async def _fail(self, frame_id, error_type, message, detail):
|
|
332
|
+
try:
|
|
333
|
+
await self.send({"id": frame_id, "invoke_result": {"error": {
|
|
334
|
+
"type": error_type, "message": message, "detail": detail}}})
|
|
335
|
+
except Exception:
|
|
336
|
+
pass
|
|
337
|
+
|
|
338
|
+
async def _pong(self, frame_id):
|
|
339
|
+
pong = {"connected": True, "state": "Running", "in_flight": len(self._running)}
|
|
340
|
+
status = getattr(self.adapter, "status", None)
|
|
341
|
+
if callable(status):
|
|
342
|
+
try:
|
|
343
|
+
reported = await _call(status) or {}
|
|
344
|
+
pong.update({k: v for k, v in reported.items() if k in
|
|
345
|
+
("connected", "state", "in_flight", "last_error", "last_message_unix_ms", "details")})
|
|
346
|
+
except Exception as ex:
|
|
347
|
+
pong.update(connected=False, state="StatusFailed", last_error=str(ex))
|
|
348
|
+
await self.send({"id": frame_id, "pong": pong})
|
|
349
|
+
|
|
350
|
+
async def _reset(self, frame_id, session_id):
|
|
351
|
+
result = {}
|
|
352
|
+
reset = getattr(self.adapter, "reset", None)
|
|
353
|
+
if callable(reset):
|
|
354
|
+
try:
|
|
355
|
+
await _call(reset, session_id)
|
|
356
|
+
except Exception as ex:
|
|
357
|
+
result["error"] = {"type": _qualified(type(ex)), "message": str(ex), "detail": traceback.format_exc()}
|
|
358
|
+
await self.send({"id": frame_id, "invoke_result": result})
|
|
359
|
+
|
|
360
|
+
async def _shutdown(self, shutdown):
|
|
361
|
+
# Ask it to stop first, let commands already running answer, and only then go: a drain
|
|
362
|
+
# promised those answers.
|
|
363
|
+
deadline = time.monotonic() + (30 if shutdown.get("drain") else 5)
|
|
364
|
+
# A start still running finishes first: a stop that overtook it would leave half of what
|
|
365
|
+
# start set up in place.
|
|
366
|
+
if self._starting is not None and not self._starting.done():
|
|
367
|
+
await asyncio.wait({self._starting}, timeout=max(0.1, deadline - time.monotonic()))
|
|
368
|
+
stop = getattr(self.adapter, "stop", None)
|
|
369
|
+
if callable(stop):
|
|
370
|
+
try:
|
|
371
|
+
await asyncio.wait_for(_call(stop), max(0.1, deadline - time.monotonic()))
|
|
372
|
+
except Exception:
|
|
373
|
+
logging.getLogger("sw_serverless").warning("stop() failed", exc_info=True)
|
|
374
|
+
running = [task for task, _ in list(self._running.values())]
|
|
375
|
+
if running:
|
|
376
|
+
await asyncio.wait(running, timeout=max(0.1, deadline - time.monotonic()))
|
|
377
|
+
await self._flush_telemetry()
|
|
378
|
+
|
|
379
|
+
# ------------------------------------------------------------------ outbound
|
|
380
|
+
|
|
381
|
+
async def send(self, frame):
|
|
382
|
+
await self.stream.send(_wire.encode("AdapterFrame", frame))
|
|
383
|
+
|
|
384
|
+
async def request(self, kind, body, answer):
|
|
385
|
+
"""Sends a frame of the adapter's own and waits for the host's answer to it."""
|
|
386
|
+
frame_id = next(self._ids)
|
|
387
|
+
waiter = self.loop.create_future()
|
|
388
|
+
self._pending[frame_id] = waiter
|
|
389
|
+
await self.send({"id": frame_id, kind: body})
|
|
390
|
+
stopping = self.loop.create_task(self.stopping.wait())
|
|
391
|
+
try:
|
|
392
|
+
done, _ = await asyncio.wait({waiter, stopping}, return_when=asyncio.FIRST_COMPLETED)
|
|
393
|
+
if waiter not in done:
|
|
394
|
+
raise AdapterError("the adapter is stopping", type="OperationCanceledException")
|
|
395
|
+
return waiter.result()
|
|
396
|
+
finally:
|
|
397
|
+
stopping.cancel()
|
|
398
|
+
self._pending.pop(frame_id, None)
|
|
399
|
+
|
|
400
|
+
def telemetry(self, kind, body):
|
|
401
|
+
"""Logs and metrics: queued, and dropped rather than allowed to hold anything up."""
|
|
402
|
+
if self.loop is None or self._telemetry is None:
|
|
403
|
+
return
|
|
404
|
+
|
|
405
|
+
def put():
|
|
406
|
+
try:
|
|
407
|
+
self._telemetry.put_nowait({kind: body})
|
|
408
|
+
except asyncio.QueueFull:
|
|
409
|
+
pass
|
|
410
|
+
|
|
411
|
+
try:
|
|
412
|
+
if _in_loop(self.loop):
|
|
413
|
+
put()
|
|
414
|
+
else:
|
|
415
|
+
self.loop.call_soon_threadsafe(put)
|
|
416
|
+
except RuntimeError:
|
|
417
|
+
pass # the loop has closed
|
|
418
|
+
|
|
419
|
+
async def _pump_telemetry(self):
|
|
420
|
+
while True:
|
|
421
|
+
frame = await self._telemetry.get()
|
|
422
|
+
try:
|
|
423
|
+
await self.send(frame)
|
|
424
|
+
except Exception:
|
|
425
|
+
return
|
|
426
|
+
|
|
427
|
+
async def _flush_telemetry(self):
|
|
428
|
+
while not self._telemetry.empty():
|
|
429
|
+
try:
|
|
430
|
+
await self.send(self._telemetry.get_nowait())
|
|
431
|
+
except Exception:
|
|
432
|
+
return
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _in_loop(loop):
|
|
436
|
+
try:
|
|
437
|
+
return asyncio.get_running_loop() is loop
|
|
438
|
+
except RuntimeError:
|
|
439
|
+
return False
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _qualified(cls):
|
|
443
|
+
module = cls.__module__
|
|
444
|
+
return cls.__qualname__ if module in ("builtins", "__main__") else f"{module}.{cls.__qualname__}"
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
async def _call(fn, *args):
|
|
448
|
+
"""Calls a hook or command, sync or async. Sync ones run on a worker thread, with the call's
|
|
449
|
+
context, so a blocking adapter doesn't stop the host's pings being answered."""
|
|
450
|
+
if inspect.iscoroutinefunction(fn):
|
|
451
|
+
return await fn(*args)
|
|
452
|
+
result = await asyncio.to_thread(fn, *args)
|
|
453
|
+
if inspect.isawaitable(result):
|
|
454
|
+
return await result
|
|
455
|
+
return result
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
# ---------------------------------------------------------------------------- entry point
|
|
459
|
+
|
|
460
|
+
def describe(adapter):
|
|
461
|
+
"""What ``--describe`` prints: the adapter's settings, commands, kinds and contracts."""
|
|
462
|
+
cls = adapter if isinstance(adapter, type) else type(adapter)
|
|
463
|
+
warnings = []
|
|
464
|
+
instance = adapter
|
|
465
|
+
if isinstance(adapter, type):
|
|
466
|
+
try:
|
|
467
|
+
instance = adapter()
|
|
468
|
+
except Exception as ex:
|
|
469
|
+
warnings.append(f"the adapter could not be built without its settings ({ex}); "
|
|
470
|
+
"settings it declares when built are missing")
|
|
471
|
+
instance = adapter
|
|
472
|
+
commands = []
|
|
473
|
+
for command in _adapter.commands_of(instance).values():
|
|
474
|
+
info = command.info()
|
|
475
|
+
commands.append({"name": info["name"], "description": info["description"],
|
|
476
|
+
"inputSchema": info["input_schema"], "outputSchema": info["output_schema"],
|
|
477
|
+
"returnsValue": info["returns_value"]})
|
|
478
|
+
return {
|
|
479
|
+
"describeVersion": DESCRIBE_VERSION,
|
|
480
|
+
"sdkLanguage": SDK_LANGUAGE,
|
|
481
|
+
"sdkVersion": SDK_VERSION,
|
|
482
|
+
"lifecycle": "resident" if _adapter.is_resident(cls) else "classic",
|
|
483
|
+
"protocol": {"min": PROTOCOL, "max": PROTOCOL},
|
|
484
|
+
"settings": [s.describe() for s in _adapter.declared_settings()],
|
|
485
|
+
"commands": commands,
|
|
486
|
+
"kinds": _adapter.kinds_of(cls),
|
|
487
|
+
"contracts": _adapter.contracts_of(cls),
|
|
488
|
+
"warnings": warnings,
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def apply_memory_limit(environ=None, platform=None, resource_module=None):
|
|
493
|
+
"""Applies the memory ceiling the host hands a Python adapter in SW_SERVERLESS_MEMORY_LIMIT_BYTES.
|
|
494
|
+
|
|
495
|
+
Python has no heap ceiling of its own, so on Linux it becomes RLIMIT_DATA, soft and hard: an
|
|
496
|
+
allocation past it raises MemoryError inside the adapter rather than growing until the host's
|
|
497
|
+
watchdog or the kernel kills it. RLIMIT_DATA rather than RLIMIT_AS: address space counts what
|
|
498
|
+
is only reserved — 64 MB per malloc arena, a stack per thread — so a few worker threads put an
|
|
499
|
+
adapter using 30 MB at over 600 MB of it. Elsewhere (macOS, Windows) nothing is set, and the
|
|
500
|
+
host's watchdog is the ceiling. With no variable, nothing is ever set.
|
|
501
|
+
|
|
502
|
+
Returns the ceiling applied in bytes, or None.
|
|
503
|
+
"""
|
|
504
|
+
global _memory_limit
|
|
505
|
+
environ = os.environ if environ is None else environ
|
|
506
|
+
platform = sys.platform if platform is None else platform
|
|
507
|
+
raw = (environ.get(MEMORY_LIMIT_VARIABLE) or "").strip()
|
|
508
|
+
if not raw:
|
|
509
|
+
return None
|
|
510
|
+
try:
|
|
511
|
+
limit = int(raw)
|
|
512
|
+
except ValueError:
|
|
513
|
+
print(f"{MEMORY_LIMIT_VARIABLE}={raw!r} is not a number of bytes; no memory limit applied", file=sys.stderr)
|
|
514
|
+
return None
|
|
515
|
+
if limit <= 0 or not platform.startswith("linux"):
|
|
516
|
+
return None
|
|
517
|
+
try:
|
|
518
|
+
if resource_module is None:
|
|
519
|
+
import resource as resource_module
|
|
520
|
+
_, hard = resource_module.getrlimit(resource_module.RLIMIT_DATA)
|
|
521
|
+
# A process can lower its hard limit but not raise it.
|
|
522
|
+
if hard != resource_module.RLIM_INFINITY and 0 <= hard < limit:
|
|
523
|
+
limit = hard
|
|
524
|
+
resource_module.setrlimit(resource_module.RLIMIT_DATA, (limit, limit))
|
|
525
|
+
except Exception as ex:
|
|
526
|
+
print(f"the memory limit of {limit} bytes could not be applied: {ex}", file=sys.stderr)
|
|
527
|
+
return None
|
|
528
|
+
_memory_limit = limit
|
|
529
|
+
return limit
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _check_kinds(adapter):
|
|
533
|
+
check = getattr(adapter, "__sw_check__", None)
|
|
534
|
+
if callable(check):
|
|
535
|
+
check()
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def run(adapter):
|
|
539
|
+
"""Runs the adapter: describes it for ``--describe``, otherwise serves the host that started it.
|
|
540
|
+
|
|
541
|
+
``adapter`` is an instance, or a class built with no arguments.
|
|
542
|
+
"""
|
|
543
|
+
if DESCRIBE_FLAG in sys.argv[1:]:
|
|
544
|
+
sys.stdout.write(json.dumps(describe(adapter), indent=2) + "\n")
|
|
545
|
+
sys.stdout.flush()
|
|
546
|
+
return
|
|
547
|
+
|
|
548
|
+
apply_memory_limit()
|
|
549
|
+
instance = adapter() if isinstance(adapter, type) else adapter
|
|
550
|
+
_check_kinds(instance)
|
|
551
|
+
|
|
552
|
+
line = sys.stdin.readline()
|
|
553
|
+
if not line:
|
|
554
|
+
print("stdin closed before the handshake arrived; the host is gone", file=sys.stderr)
|
|
555
|
+
sys.exit(1)
|
|
556
|
+
try:
|
|
557
|
+
asyncio.run(Runner(instance).run(line))
|
|
558
|
+
except KeyboardInterrupt:
|
|
559
|
+
pass
|
|
560
|
+
except Exception:
|
|
561
|
+
traceback.print_exc()
|
|
562
|
+
sys.exit(1)
|
|
563
|
+
# Worker threads running blocking commands must not keep a stopped adapter alive.
|
|
564
|
+
sys.stdout.flush()
|
|
565
|
+
os._exit(0)
|
sw_serverless/_types.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""How command arguments and results cross the wire, and the JSON Schema that describes them.
|
|
2
|
+
|
|
3
|
+
The encoding is the protocol's, the same in every language: a string is its raw UTF-8 text, bytes
|
|
4
|
+
are passed as they are, nothing is an empty payload, and anything else is JSON. A class can take
|
|
5
|
+
charge of its own JSON with ``to_wire``/``from_wire`` and describe it with ``__sw_schema__`` — the
|
|
6
|
+
types of a contract package do, to keep the property names the contract fixes.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import dataclasses
|
|
11
|
+
import inspect
|
|
12
|
+
import json
|
|
13
|
+
import typing
|
|
14
|
+
|
|
15
|
+
_EMPTY = inspect.Parameter.empty
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _origin(tp):
|
|
19
|
+
return typing.get_origin(tp) or tp
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _strip_optional(tp):
|
|
23
|
+
if typing.get_origin(tp) is typing.Union:
|
|
24
|
+
args = [a for a in typing.get_args(tp) if a is not type(None)]
|
|
25
|
+
if len(args) == 1:
|
|
26
|
+
return args[0]
|
|
27
|
+
return tp
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def decode(tp, payload):
|
|
31
|
+
"""The argument a command receives, from the payload it was sent."""
|
|
32
|
+
tp = _strip_optional(tp)
|
|
33
|
+
if tp in (_EMPTY, typing.Any, None):
|
|
34
|
+
if not payload:
|
|
35
|
+
return None
|
|
36
|
+
text = payload.decode("utf-8")
|
|
37
|
+
try:
|
|
38
|
+
return json.loads(text)
|
|
39
|
+
except ValueError:
|
|
40
|
+
return text
|
|
41
|
+
if tp is bytes:
|
|
42
|
+
return bytes(payload)
|
|
43
|
+
text = payload.decode("utf-8")
|
|
44
|
+
if tp is str:
|
|
45
|
+
return text
|
|
46
|
+
if not text:
|
|
47
|
+
return None
|
|
48
|
+
value = json.loads(text)
|
|
49
|
+
return from_json(tp, value)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def from_json(tp, value):
|
|
53
|
+
tp = _strip_optional(tp)
|
|
54
|
+
if value is None:
|
|
55
|
+
return None
|
|
56
|
+
if hasattr(tp, "from_wire"):
|
|
57
|
+
return tp.from_wire(value)
|
|
58
|
+
if dataclasses.is_dataclass(tp):
|
|
59
|
+
hints = typing.get_type_hints(tp)
|
|
60
|
+
names = {f.name for f in dataclasses.fields(tp)}
|
|
61
|
+
return tp(**{k: from_json(hints.get(k, typing.Any), v) for k, v in value.items() if k in names})
|
|
62
|
+
origin = _origin(tp)
|
|
63
|
+
args = typing.get_args(tp)
|
|
64
|
+
if origin in (list, tuple, set) and args:
|
|
65
|
+
return origin(from_json(args[0], v) for v in value)
|
|
66
|
+
if origin is dict and len(args) == 2:
|
|
67
|
+
return {k: from_json(args[1], v) for k, v in value.items()}
|
|
68
|
+
if tp is bytes and isinstance(value, str):
|
|
69
|
+
return base64.b64decode(value)
|
|
70
|
+
return value
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def encode(value):
|
|
74
|
+
"""The payload a command's result becomes."""
|
|
75
|
+
if value is None:
|
|
76
|
+
return b""
|
|
77
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
78
|
+
return bytes(value)
|
|
79
|
+
if isinstance(value, str):
|
|
80
|
+
return value.encode("utf-8")
|
|
81
|
+
return json.dumps(to_json(value), separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def to_json(value):
|
|
85
|
+
if hasattr(value, "to_wire"):
|
|
86
|
+
return value.to_wire()
|
|
87
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
88
|
+
return {f.name: to_json(getattr(value, f.name)) for f in dataclasses.fields(value)}
|
|
89
|
+
if isinstance(value, dict):
|
|
90
|
+
return {str(k): to_json(v) for k, v in value.items()}
|
|
91
|
+
if isinstance(value, (list, tuple, set, frozenset)):
|
|
92
|
+
return [to_json(v) for v in value]
|
|
93
|
+
if isinstance(value, (bytes, bytearray)):
|
|
94
|
+
return base64.b64encode(bytes(value)).decode("ascii")
|
|
95
|
+
return value
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def schema(tp):
|
|
99
|
+
"""A JSON Schema for a command's argument or result type; None when it has none."""
|
|
100
|
+
tp = _strip_optional(tp)
|
|
101
|
+
if tp in (_EMPTY, None, type(None)):
|
|
102
|
+
return None
|
|
103
|
+
if hasattr(tp, "__sw_schema__"):
|
|
104
|
+
return tp.__sw_schema__()
|
|
105
|
+
simple = {str: {"type": "string"}, int: {"type": "integer"}, float: {"type": "number"},
|
|
106
|
+
bool: {"type": "boolean"}, bytes: {"type": "string", "contentEncoding": "binary"}}
|
|
107
|
+
if tp in simple:
|
|
108
|
+
return dict(simple[tp])
|
|
109
|
+
if dataclasses.is_dataclass(tp):
|
|
110
|
+
hints = typing.get_type_hints(tp)
|
|
111
|
+
properties = {f.name: schema(hints.get(f.name, typing.Any)) or {} for f in dataclasses.fields(tp)}
|
|
112
|
+
required = [f.name for f in dataclasses.fields(tp)
|
|
113
|
+
if f.default is dataclasses.MISSING and f.default_factory is dataclasses.MISSING]
|
|
114
|
+
result = {"type": "object", "properties": properties}
|
|
115
|
+
if required:
|
|
116
|
+
result["required"] = required
|
|
117
|
+
return result
|
|
118
|
+
origin = _origin(tp)
|
|
119
|
+
args = typing.get_args(tp)
|
|
120
|
+
if origin in (list, tuple, set):
|
|
121
|
+
return {"type": "array", "items": (schema(args[0]) or {}) if args else {}}
|
|
122
|
+
if origin is dict:
|
|
123
|
+
return {"type": "object", "additionalProperties": (schema(args[1]) or {}) if len(args) == 2 else {}}
|
|
124
|
+
return {}
|