superkernel-adapter 0.9.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- superkernel_adapter/__init__.py +25 -0
- superkernel_adapter/kernel.py +646 -0
- superkernel_adapter/launcher.py +40 -0
- superkernel_adapter/logfile.py +28 -0
- superkernel_adapter/schema.py +179 -0
- superkernel_adapter/state.py +217 -0
- superkernel_adapter-0.9.7.dist-info/METADATA +132 -0
- superkernel_adapter-0.9.7.dist-info/RECORD +12 -0
- superkernel_adapter-0.9.7.dist-info/WHEEL +5 -0
- superkernel_adapter-0.9.7.dist-info/entry_points.txt +2 -0
- superkernel_adapter-0.9.7.dist-info/licenses/LICENSE +201 -0
- superkernel_adapter-0.9.7.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""SuperKernel Adapter: shared local Jupyter integration for remote compute."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version("superkernel-adapter")
|
|
9
|
+
except PackageNotFoundError:
|
|
10
|
+
__version__ = "0.9.7"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def __getattr__(name: str):
|
|
14
|
+
"""Load the local ipykernel only when a client explicitly requests it."""
|
|
15
|
+
if name == "SuperKernel":
|
|
16
|
+
from superkernel_adapter.kernel import SuperKernel
|
|
17
|
+
|
|
18
|
+
return SuperKernel
|
|
19
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"SuperKernel",
|
|
24
|
+
"__version__",
|
|
25
|
+
]
|
|
@@ -0,0 +1,646 @@
|
|
|
1
|
+
"""Jupyter kernel runtime for the kernel-cloud SSE execution API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import sys
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
import traceback
|
|
14
|
+
import uuid
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, ClassVar, Literal
|
|
18
|
+
|
|
19
|
+
import requests
|
|
20
|
+
from ipykernel.kernelapp import IPKernelApp
|
|
21
|
+
from ipykernel.kernelbase import Kernel
|
|
22
|
+
from requests import Response
|
|
23
|
+
from requests.exceptions import (
|
|
24
|
+
ChunkedEncodingError,
|
|
25
|
+
HTTPError,
|
|
26
|
+
ReadTimeout,
|
|
27
|
+
RequestException,
|
|
28
|
+
Timeout,
|
|
29
|
+
)
|
|
30
|
+
from requests.exceptions import (
|
|
31
|
+
ConnectionError as RequestsConnectionError,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
from superkernel_adapter import __version__
|
|
35
|
+
from superkernel_adapter.logfile import DEFAULT_LOG_FILE, append_log
|
|
36
|
+
from superkernel_adapter.schema import (
|
|
37
|
+
CodeExecutionRequest,
|
|
38
|
+
CodeExecutionResumeRequest,
|
|
39
|
+
ControlRequest,
|
|
40
|
+
ControlResponse,
|
|
41
|
+
)
|
|
42
|
+
from superkernel_adapter.state import (
|
|
43
|
+
_delete_runtime_status,
|
|
44
|
+
_normalize_gateway_url,
|
|
45
|
+
_read_auth_token,
|
|
46
|
+
_write_runtime_status,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class StaleSuperKernelError(RuntimeError):
|
|
51
|
+
"""Signal that this saved local registration cannot be used anymore."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class UnresolvedExecutionError(RuntimeError):
|
|
55
|
+
"""Refuse different code while a previous execution needs reconciliation."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
AdmissionState = Literal["idle", "waiting", "admitting", "admitted"]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SuperKernel(Kernel):
|
|
62
|
+
"""Forward notebook execution to one gateway-provisioned Super Kernel."""
|
|
63
|
+
|
|
64
|
+
implementation = "superkernel_adapter"
|
|
65
|
+
implementation_version = __version__
|
|
66
|
+
language = "python"
|
|
67
|
+
language_version = f"{sys.version_info.major}.{sys.version_info.minor}"
|
|
68
|
+
language_info: ClassVar[dict[str, str]] = {
|
|
69
|
+
"name": "python",
|
|
70
|
+
"mimetype": "text/x-python",
|
|
71
|
+
"file_extension": ".py",
|
|
72
|
+
}
|
|
73
|
+
banner = "SuperKernel Adapter — local Jupyter kernel with remote execution"
|
|
74
|
+
boot: ClassVar[dict[str, Any]] = {}
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def _session_id_from_connection_file(connection_file: str) -> str:
|
|
78
|
+
"""Use Jupyter's runtime connection-file ID as the notebook identity."""
|
|
79
|
+
filename = Path(connection_file).name
|
|
80
|
+
if not filename.endswith(".json"):
|
|
81
|
+
raise RuntimeError("Jupyter connection file is unavailable")
|
|
82
|
+
session_id = filename[:-5]
|
|
83
|
+
session_id = session_id.removeprefix("kernel-")
|
|
84
|
+
if not re.fullmatch(r"[A-Za-z0-9._-]{1,128}", session_id):
|
|
85
|
+
raise RuntimeError("Jupyter connection file has an invalid kernel ID")
|
|
86
|
+
return session_id
|
|
87
|
+
|
|
88
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
89
|
+
super().__init__(**kwargs)
|
|
90
|
+
config = SuperKernel.boot
|
|
91
|
+
self.gateway_url = _normalize_gateway_url(str(config["gateway_url"]))
|
|
92
|
+
self.endpoint = f"{self.gateway_url}/kernel/execute"
|
|
93
|
+
self.control_endpoint = f"{self.gateway_url}/kernel/control"
|
|
94
|
+
self.remote_kernel_id = str(config["remote_kernel_id"])
|
|
95
|
+
self.auth_file = str(config["auth_file"])
|
|
96
|
+
self.timeout = float(os.getenv("SUPERKERNEL_EXECUTION_TIMEOUT", "300"))
|
|
97
|
+
connection_file = str(
|
|
98
|
+
config.get("connection_file")
|
|
99
|
+
or getattr(self.parent, "connection_file", "")
|
|
100
|
+
or getattr(IPKernelApp.instance(), "connection_file", "")
|
|
101
|
+
)
|
|
102
|
+
self.session_id = self._session_id_from_connection_file(connection_file)
|
|
103
|
+
self._execution_lock = threading.Lock()
|
|
104
|
+
self._pending_code_digest: str | None = None
|
|
105
|
+
self._pending_execution_id: str | None = None
|
|
106
|
+
self._pending_reply_id: str | None = None
|
|
107
|
+
self._pending_cursor_id: str | None = None
|
|
108
|
+
self._admission_state: AdmissionState = "idle"
|
|
109
|
+
self._output_runner_id: str | None = None
|
|
110
|
+
self._output_stream_name: str | None = None
|
|
111
|
+
self._output_ends_with_newline = True
|
|
112
|
+
self._runner_virtual_newlines: set[tuple[str, str]] = set()
|
|
113
|
+
log_file = config.get("log_file") or os.getenv("SUPERKERNEL_LOG_FILE")
|
|
114
|
+
self._log_path = Path(log_file).expanduser() if log_file else DEFAULT_LOG_FILE
|
|
115
|
+
self._runtime_status = "idle"
|
|
116
|
+
self._last_execution_at: float | None = None
|
|
117
|
+
self._silent_execution = False
|
|
118
|
+
self._runtime_status_lock = threading.RLock()
|
|
119
|
+
self._runtime_status_stop = threading.Event()
|
|
120
|
+
self._publish_runtime_status()
|
|
121
|
+
self._runtime_status_thread = threading.Thread(
|
|
122
|
+
target=self._runtime_status_heartbeat,
|
|
123
|
+
name=f"superkernel_adapter-status-{self.session_id}",
|
|
124
|
+
daemon=True,
|
|
125
|
+
)
|
|
126
|
+
self._runtime_status_thread.start()
|
|
127
|
+
|
|
128
|
+
for message_type in ("comm_open", "comm_msg", "comm_close"):
|
|
129
|
+
self.shell_handlers[message_type] = self._ignore_comm_message
|
|
130
|
+
self.control_handlers[message_type] = self._ignore_comm_message
|
|
131
|
+
|
|
132
|
+
def _ignore_comm_message(self, _stream: Any, _idents: Any, _msg: Any) -> None:
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
def _write_stream(self, name: str, text: str) -> None:
|
|
136
|
+
if getattr(self, "_silent_execution", False):
|
|
137
|
+
return
|
|
138
|
+
self.send_response(self.iopub_socket, "stream", {"name": name, "text": text})
|
|
139
|
+
|
|
140
|
+
def _write_runner_stream(self, runner_id: str, name: str, text: str) -> None:
|
|
141
|
+
"""Keep interleaved runner chunks readable without buffering them."""
|
|
142
|
+
if not text:
|
|
143
|
+
return
|
|
144
|
+
runner_stream = (runner_id, name)
|
|
145
|
+
if text.startswith("\n") and runner_stream in self._runner_virtual_newlines:
|
|
146
|
+
self._runner_virtual_newlines.remove(runner_stream)
|
|
147
|
+
text = text[1:]
|
|
148
|
+
if not text:
|
|
149
|
+
return
|
|
150
|
+
if (
|
|
151
|
+
self._output_runner_id is not None
|
|
152
|
+
and runner_id != self._output_runner_id
|
|
153
|
+
and not self._output_ends_with_newline
|
|
154
|
+
and not text.startswith("\n")
|
|
155
|
+
):
|
|
156
|
+
previous_stream = self._output_stream_name or name
|
|
157
|
+
self._write_stream(previous_stream, "\n")
|
|
158
|
+
self._runner_virtual_newlines.add((self._output_runner_id, previous_stream))
|
|
159
|
+
self._write_stream(name, text)
|
|
160
|
+
self._output_runner_id = runner_id
|
|
161
|
+
self._output_stream_name = name
|
|
162
|
+
self._output_ends_with_newline = text.endswith("\n")
|
|
163
|
+
|
|
164
|
+
def _reset_runner_stream(self) -> None:
|
|
165
|
+
self._output_runner_id = None
|
|
166
|
+
self._output_stream_name = None
|
|
167
|
+
self._output_ends_with_newline = True
|
|
168
|
+
self._runner_virtual_newlines.clear()
|
|
169
|
+
|
|
170
|
+
def _send_display_data(self, data: dict[str, Any]) -> None:
|
|
171
|
+
if getattr(self, "_silent_execution", False):
|
|
172
|
+
return
|
|
173
|
+
self.send_response(
|
|
174
|
+
self.iopub_socket,
|
|
175
|
+
"display_data",
|
|
176
|
+
{"data": data, "metadata": {}},
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
def _log(self, message: str) -> None:
|
|
180
|
+
try:
|
|
181
|
+
append_log(message, self._log_path)
|
|
182
|
+
except Exception: # noqa: BLE001, S110 - logging must never break execution
|
|
183
|
+
pass
|
|
184
|
+
|
|
185
|
+
def _publish_runtime_status(self) -> None:
|
|
186
|
+
try:
|
|
187
|
+
with self._runtime_status_lock:
|
|
188
|
+
if self._runtime_status_stop.is_set():
|
|
189
|
+
return
|
|
190
|
+
_write_runtime_status(
|
|
191
|
+
self.gateway_url,
|
|
192
|
+
self.remote_kernel_id,
|
|
193
|
+
self.session_id,
|
|
194
|
+
self._runtime_status,
|
|
195
|
+
self._last_execution_at,
|
|
196
|
+
)
|
|
197
|
+
except Exception as exc: # noqa: BLE001 - status must not break the kernel
|
|
198
|
+
self._log(
|
|
199
|
+
f"Local runtime status update failed: {type(exc).__name__}: {exc}"
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
def _set_runtime_status(self, status: str) -> None:
|
|
203
|
+
with self._runtime_status_lock:
|
|
204
|
+
self._runtime_status = status
|
|
205
|
+
if status == "busy":
|
|
206
|
+
self._last_execution_at = time.time()
|
|
207
|
+
self._publish_runtime_status()
|
|
208
|
+
|
|
209
|
+
def _runtime_status_heartbeat(self) -> None:
|
|
210
|
+
while not self._runtime_status_stop.wait(5.0):
|
|
211
|
+
self._publish_runtime_status()
|
|
212
|
+
|
|
213
|
+
def _execution_id(self, code: str) -> str:
|
|
214
|
+
digest = hashlib.sha256(code.encode("utf-8")).hexdigest()
|
|
215
|
+
if self._pending_code_digest == digest and self._pending_execution_id:
|
|
216
|
+
return self._pending_execution_id
|
|
217
|
+
if self._pending_execution_id:
|
|
218
|
+
raise UnresolvedExecutionError(
|
|
219
|
+
"A previous execution is unresolved. Re-run its original code to "
|
|
220
|
+
"reconnect, or restart the kernel before submitting different code."
|
|
221
|
+
)
|
|
222
|
+
self._pending_code_digest = digest
|
|
223
|
+
self._pending_execution_id = f"exec_{uuid.uuid4().hex}"
|
|
224
|
+
self._pending_reply_id = None
|
|
225
|
+
self._pending_cursor_id = None
|
|
226
|
+
return self._pending_execution_id
|
|
227
|
+
|
|
228
|
+
def _clear_pending_execution(self) -> None:
|
|
229
|
+
self._pending_code_digest = None
|
|
230
|
+
self._pending_execution_id = None
|
|
231
|
+
self._pending_reply_id = None
|
|
232
|
+
self._pending_cursor_id = None
|
|
233
|
+
self._admission_state = "idle"
|
|
234
|
+
|
|
235
|
+
@staticmethod
|
|
236
|
+
def _iter_sse(response: Response) -> Iterator[dict[str, str]]:
|
|
237
|
+
"""Parse one HTTP response into SSE frames."""
|
|
238
|
+
response.encoding = "utf-8"
|
|
239
|
+
fields: dict[str, str] = {}
|
|
240
|
+
data_lines: list[str] = []
|
|
241
|
+
for line in response.iter_lines(decode_unicode=True, chunk_size=1):
|
|
242
|
+
if line == "":
|
|
243
|
+
if fields or data_lines:
|
|
244
|
+
fields["data"] = "\n".join(data_lines)
|
|
245
|
+
yield fields
|
|
246
|
+
fields = {}
|
|
247
|
+
data_lines = []
|
|
248
|
+
continue
|
|
249
|
+
if not line or line.startswith(":"):
|
|
250
|
+
continue
|
|
251
|
+
name, separator, value = line.partition(":")
|
|
252
|
+
if not separator:
|
|
253
|
+
value = ""
|
|
254
|
+
elif value.startswith(" "):
|
|
255
|
+
value = value[1:]
|
|
256
|
+
if name == "data":
|
|
257
|
+
data_lines.append(value)
|
|
258
|
+
else:
|
|
259
|
+
fields[name] = value
|
|
260
|
+
if fields or data_lines:
|
|
261
|
+
fields["data"] = "\n".join(data_lines)
|
|
262
|
+
yield fields
|
|
263
|
+
|
|
264
|
+
def _handle_sse_frame(self, frame: dict[str, str]) -> tuple[str | None, bool]:
|
|
265
|
+
"""Render one SSE frame and return terminal status/error state."""
|
|
266
|
+
event_type = frame.get("event", "message")
|
|
267
|
+
try:
|
|
268
|
+
payload = json.loads(frame.get("data", "{}"))
|
|
269
|
+
except json.JSONDecodeError as exc:
|
|
270
|
+
raise RuntimeError(f"Gateway sent invalid SSE JSON: {exc}") from exc
|
|
271
|
+
|
|
272
|
+
if event_type == "runner_execution_started":
|
|
273
|
+
return None, False
|
|
274
|
+
|
|
275
|
+
if event_type == "runner_execution_finished":
|
|
276
|
+
succeeded = str(payload.get("status") or "") == "success"
|
|
277
|
+
# The runner atomically publishes a cluster-wide interrupt before
|
|
278
|
+
# this failure event. Return to Jupyter immediately instead of
|
|
279
|
+
# waiting for every interrupted rank to report its final state.
|
|
280
|
+
return (None, False) if succeeded else ("failed", True)
|
|
281
|
+
|
|
282
|
+
if event_type in {"result", "stream"}:
|
|
283
|
+
data = str(payload.get("data") or "")
|
|
284
|
+
if data:
|
|
285
|
+
stream = str(payload.get("stream") or "stdout")
|
|
286
|
+
self._write_runner_stream(
|
|
287
|
+
str(payload.get("runner_id") or ""),
|
|
288
|
+
"stderr" if stream == "stderr" else "stdout",
|
|
289
|
+
data if event_type == "stream" else data.rstrip("\n") + "\n",
|
|
290
|
+
)
|
|
291
|
+
return None, str(payload.get("status") or "success") != "success"
|
|
292
|
+
|
|
293
|
+
if event_type == "execution_finished":
|
|
294
|
+
if payload.get("status") == "kernel_restarted":
|
|
295
|
+
self._write_stream(
|
|
296
|
+
"stderr",
|
|
297
|
+
"[Super Kernel] Cluster restarted or recovered. "
|
|
298
|
+
"In-memory notebook state was reset; workspace files were retained.\n",
|
|
299
|
+
)
|
|
300
|
+
succeeded = str(payload.get("status") or "") == "success"
|
|
301
|
+
return "completed" if succeeded else "failed", not succeeded
|
|
302
|
+
|
|
303
|
+
if event_type == "error":
|
|
304
|
+
return "failed", True
|
|
305
|
+
|
|
306
|
+
raise RuntimeError(f"Gateway sent unsupported SSE event: {event_type}")
|
|
307
|
+
|
|
308
|
+
def _stream_execute(self, request: CodeExecutionRequest) -> bool:
|
|
309
|
+
"""Admit locally queued code once, then resume its stable reply stream."""
|
|
310
|
+
deadline = time.monotonic() + self.timeout
|
|
311
|
+
reply_id = self._pending_reply_id
|
|
312
|
+
last_event_id = self._pending_cursor_id
|
|
313
|
+
self._admission_state = "admitted" if reply_id is not None else "waiting"
|
|
314
|
+
execution_failed = False
|
|
315
|
+
|
|
316
|
+
while True:
|
|
317
|
+
remaining = deadline - time.monotonic()
|
|
318
|
+
if remaining <= 0:
|
|
319
|
+
raise Timeout(
|
|
320
|
+
"Execution did not reach a terminal status before the timeout; "
|
|
321
|
+
"rerun the same cell to reconnect"
|
|
322
|
+
)
|
|
323
|
+
headers = {
|
|
324
|
+
"Accept": "text/event-stream",
|
|
325
|
+
"Content-Type": "application/json",
|
|
326
|
+
}
|
|
327
|
+
access_token = _read_auth_token(self.auth_file, self.gateway_url)
|
|
328
|
+
headers["Authorization"] = f"Bearer {access_token}"
|
|
329
|
+
if last_event_id is not None:
|
|
330
|
+
headers["Last-Event-ID"] = str(last_event_id)
|
|
331
|
+
body = (
|
|
332
|
+
CodeExecutionResumeRequest(
|
|
333
|
+
kernel_id=self.remote_kernel_id,
|
|
334
|
+
reply_id=reply_id,
|
|
335
|
+
).model_dump(mode="json")
|
|
336
|
+
if reply_id is not None
|
|
337
|
+
else request.model_dump(mode="json")
|
|
338
|
+
)
|
|
339
|
+
self._log(
|
|
340
|
+
f"POST {self.endpoint} remote_kernel_id={self.remote_kernel_id} "
|
|
341
|
+
f"execution_id={request.execution_id} reply_id={reply_id} "
|
|
342
|
+
f"reconnect_after={last_event_id} "
|
|
343
|
+
f"code_length={len(request.code)}"
|
|
344
|
+
)
|
|
345
|
+
try:
|
|
346
|
+
if reply_id is None:
|
|
347
|
+
# The Gateway may admit this request before its response
|
|
348
|
+
# reaches us, so interruption is remote-best-effort while
|
|
349
|
+
# the initial request is in flight.
|
|
350
|
+
self._admission_state = "admitting"
|
|
351
|
+
with requests.post(
|
|
352
|
+
self.endpoint,
|
|
353
|
+
headers=headers,
|
|
354
|
+
json=body,
|
|
355
|
+
stream=True,
|
|
356
|
+
timeout=(10.0, min(60.0, max(1.0, remaining))),
|
|
357
|
+
) as response:
|
|
358
|
+
if response.status_code == 409:
|
|
359
|
+
try:
|
|
360
|
+
detail = str(response.json().get("detail") or "")
|
|
361
|
+
except (TypeError, ValueError):
|
|
362
|
+
detail = ""
|
|
363
|
+
if detail == "kernel_busy" and reply_id is None:
|
|
364
|
+
self._admission_state = "waiting"
|
|
365
|
+
retry_after = response.headers.get("Retry-After", "1")
|
|
366
|
+
try:
|
|
367
|
+
delay = min(5.0, max(0.05, float(retry_after)))
|
|
368
|
+
except ValueError:
|
|
369
|
+
delay = 1.0
|
|
370
|
+
time.sleep(min(delay, max(0.01, remaining)))
|
|
371
|
+
continue
|
|
372
|
+
if response.status_code in {408, 425, 429, 502, 503, 504}:
|
|
373
|
+
time.sleep(min(1.0, max(0.01, remaining)))
|
|
374
|
+
continue
|
|
375
|
+
response.raise_for_status()
|
|
376
|
+
response_reply_id = response.headers.get("SuperKernel-Reply-ID")
|
|
377
|
+
if not response_reply_id:
|
|
378
|
+
raise RuntimeError(
|
|
379
|
+
"Gateway omitted the execution reply identifier"
|
|
380
|
+
)
|
|
381
|
+
if reply_id is not None and response_reply_id != reply_id:
|
|
382
|
+
raise RuntimeError(
|
|
383
|
+
"Gateway changed the execution reply identifier"
|
|
384
|
+
)
|
|
385
|
+
reply_id = response_reply_id
|
|
386
|
+
self._pending_reply_id = reply_id
|
|
387
|
+
self._admission_state = "admitted"
|
|
388
|
+
content_type = response.headers.get("content-type", "")
|
|
389
|
+
if not content_type.lower().startswith("text/event-stream"):
|
|
390
|
+
raise RuntimeError(
|
|
391
|
+
f"Gateway returned {content_type or 'no content type'}; "
|
|
392
|
+
"expected text/event-stream"
|
|
393
|
+
)
|
|
394
|
+
for frame in self._iter_sse(response):
|
|
395
|
+
raw_event_id = frame.get("id")
|
|
396
|
+
if raw_event_id is not None:
|
|
397
|
+
if not raw_event_id.strip():
|
|
398
|
+
raise RuntimeError(
|
|
399
|
+
"Gateway sent an invalid SSE event ID"
|
|
400
|
+
)
|
|
401
|
+
last_event_id = raw_event_id
|
|
402
|
+
self._pending_cursor_id = last_event_id
|
|
403
|
+
status, frame_failed = self._handle_sse_frame(frame)
|
|
404
|
+
execution_failed = execution_failed or frame_failed
|
|
405
|
+
if status is not None:
|
|
406
|
+
# A terminal SSE frame is the execution boundary. Do not
|
|
407
|
+
# wait for a reverse proxy to close its HTTP connection.
|
|
408
|
+
return status == "completed" and not execution_failed
|
|
409
|
+
except HTTPError:
|
|
410
|
+
raise
|
|
411
|
+
except (ChunkedEncodingError, ReadTimeout, RequestsConnectionError):
|
|
412
|
+
if time.monotonic() >= deadline:
|
|
413
|
+
raise
|
|
414
|
+
self._log(
|
|
415
|
+
f"SSE reconnect execution_id={request.execution_id} "
|
|
416
|
+
f"last_event_id={last_event_id}"
|
|
417
|
+
)
|
|
418
|
+
time.sleep(0.25)
|
|
419
|
+
|
|
420
|
+
def _post_control(
|
|
421
|
+
self,
|
|
422
|
+
action: str,
|
|
423
|
+
execution_id: str | None = None,
|
|
424
|
+
) -> ControlResponse:
|
|
425
|
+
payload = ControlRequest(
|
|
426
|
+
action=action,
|
|
427
|
+
kernel_id=self.remote_kernel_id,
|
|
428
|
+
session_id=self.session_id,
|
|
429
|
+
execution_id=execution_id,
|
|
430
|
+
)
|
|
431
|
+
self._log(
|
|
432
|
+
f"POST {self.control_endpoint} remote_kernel_id={self.remote_kernel_id} "
|
|
433
|
+
f"action={action} execution_id={execution_id}"
|
|
434
|
+
)
|
|
435
|
+
headers = {"Content-Type": "application/json"}
|
|
436
|
+
access_token = _read_auth_token(self.auth_file, self.gateway_url)
|
|
437
|
+
headers["Authorization"] = f"Bearer {access_token}"
|
|
438
|
+
response = requests.post(
|
|
439
|
+
self.control_endpoint,
|
|
440
|
+
headers=headers,
|
|
441
|
+
json=payload.model_dump(mode="json"),
|
|
442
|
+
timeout=(10.0, min(30.0, self.timeout)),
|
|
443
|
+
)
|
|
444
|
+
response.raise_for_status()
|
|
445
|
+
return ControlResponse.model_validate(response.json())
|
|
446
|
+
|
|
447
|
+
def _interrupt_pending_execution(self, execution_id: str) -> None:
|
|
448
|
+
"""Cancel a local wait or best-effort interrupt possible remote work."""
|
|
449
|
+
if self._admission_state == "waiting":
|
|
450
|
+
self._log(f"Local admission wait cancelled execution_id={execution_id}")
|
|
451
|
+
return
|
|
452
|
+
try:
|
|
453
|
+
self._post_control("interrupt", execution_id)
|
|
454
|
+
except Exception as exc: # noqa: BLE001 - interruption is best effort
|
|
455
|
+
self._log(f"Super Kernel interrupt failed: {type(exc).__name__}: {exc}")
|
|
456
|
+
|
|
457
|
+
def do_execute(
|
|
458
|
+
self,
|
|
459
|
+
code: str,
|
|
460
|
+
silent: bool,
|
|
461
|
+
store_history: bool = True,
|
|
462
|
+
user_expressions: dict[str, Any] | None = None,
|
|
463
|
+
allow_stdin: bool = False,
|
|
464
|
+
) -> dict[str, Any]:
|
|
465
|
+
# Keep the remote stream strictly aligned with Jupyter's local kernel.
|
|
466
|
+
# This also protects direct/concurrent callers outside the shell loop.
|
|
467
|
+
with self._execution_lock:
|
|
468
|
+
self._silent_execution = silent
|
|
469
|
+
try:
|
|
470
|
+
return self._do_execute(
|
|
471
|
+
code,
|
|
472
|
+
silent,
|
|
473
|
+
store_history,
|
|
474
|
+
user_expressions,
|
|
475
|
+
allow_stdin,
|
|
476
|
+
)
|
|
477
|
+
finally:
|
|
478
|
+
self._silent_execution = False
|
|
479
|
+
|
|
480
|
+
def _do_execute(
|
|
481
|
+
self,
|
|
482
|
+
code: str,
|
|
483
|
+
silent: bool,
|
|
484
|
+
store_history: bool = True,
|
|
485
|
+
user_expressions: dict[str, Any] | None = None,
|
|
486
|
+
allow_stdin: bool = False,
|
|
487
|
+
) -> dict[str, Any]:
|
|
488
|
+
if not code.strip():
|
|
489
|
+
return self._reply("ok")
|
|
490
|
+
try:
|
|
491
|
+
request = CodeExecutionRequest(
|
|
492
|
+
kernel_id=self.remote_kernel_id,
|
|
493
|
+
execution_id="validation",
|
|
494
|
+
session_id=self.session_id,
|
|
495
|
+
execution_count=self.execution_count,
|
|
496
|
+
code=code,
|
|
497
|
+
)
|
|
498
|
+
execution_id = self._execution_id(code)
|
|
499
|
+
request.execution_id = execution_id
|
|
500
|
+
self._set_runtime_status("busy")
|
|
501
|
+
self._reset_runner_stream()
|
|
502
|
+
succeeded = self._stream_execute(request)
|
|
503
|
+
except KeyboardInterrupt:
|
|
504
|
+
if self._pending_execution_id:
|
|
505
|
+
self._interrupt_pending_execution(self._pending_execution_id)
|
|
506
|
+
self._write_stream("stderr", "[Super Kernel] Execution interrupted.\n")
|
|
507
|
+
self._clear_pending_execution()
|
|
508
|
+
self._set_runtime_status("idle")
|
|
509
|
+
return self._reply("abort")
|
|
510
|
+
except RequestException as exc:
|
|
511
|
+
self._log(f"Gateway request failed: {type(exc).__name__}: {exc}")
|
|
512
|
+
if (
|
|
513
|
+
isinstance(exc, HTTPError)
|
|
514
|
+
and exc.response is not None
|
|
515
|
+
and exc.response.status_code == 409
|
|
516
|
+
):
|
|
517
|
+
self._clear_pending_execution()
|
|
518
|
+
self._write_stream(
|
|
519
|
+
"stderr",
|
|
520
|
+
"[Super Kernel] Previous execution is unavailable; its outcome is unknown. "
|
|
521
|
+
"No code was replayed. Re-run explicitly when the cluster is ready.\n",
|
|
522
|
+
)
|
|
523
|
+
self._write_stream("stderr", "[Super Kernel] Error.\n")
|
|
524
|
+
self._set_runtime_status("error")
|
|
525
|
+
if (
|
|
526
|
+
isinstance(exc, HTTPError)
|
|
527
|
+
and exc.response is not None
|
|
528
|
+
and exc.response.status_code in {401, 404}
|
|
529
|
+
):
|
|
530
|
+
return self._reply(
|
|
531
|
+
"error",
|
|
532
|
+
StaleSuperKernelError("Saved Super Kernel is unavailable"),
|
|
533
|
+
)
|
|
534
|
+
return self._reply("error", RuntimeError("Error"))
|
|
535
|
+
except UnresolvedExecutionError as exc:
|
|
536
|
+
self._write_stream("stderr", f"[Super Kernel] {exc}\n")
|
|
537
|
+
self._set_runtime_status("error")
|
|
538
|
+
return self._reply("error", exc)
|
|
539
|
+
except Exception as exc: # noqa: BLE001 - kernel protocol boundary
|
|
540
|
+
self._log(
|
|
541
|
+
f"Execution failed: {type(exc).__name__}: {exc}\n"
|
|
542
|
+
f"{traceback.format_exc()}"
|
|
543
|
+
)
|
|
544
|
+
self._write_stream("stderr", "[Super Kernel] Error.\n")
|
|
545
|
+
self._set_runtime_status("error")
|
|
546
|
+
return self._reply("error", RuntimeError("Error"))
|
|
547
|
+
|
|
548
|
+
self._clear_pending_execution()
|
|
549
|
+
if succeeded:
|
|
550
|
+
self._set_runtime_status("idle")
|
|
551
|
+
return self._reply("ok")
|
|
552
|
+
self._write_stream("stderr", "[Super Kernel] Error.\n")
|
|
553
|
+
self._set_runtime_status("error")
|
|
554
|
+
error = RuntimeError("Error")
|
|
555
|
+
return self._reply("error", error)
|
|
556
|
+
|
|
557
|
+
def _reply(self, status: str, error: Exception | None = None) -> dict[str, Any]:
|
|
558
|
+
reply: dict[str, Any] = {
|
|
559
|
+
"status": status,
|
|
560
|
+
"execution_count": self.execution_count,
|
|
561
|
+
"payload": [],
|
|
562
|
+
"user_expressions": {},
|
|
563
|
+
}
|
|
564
|
+
if status == "error" and error is not None:
|
|
565
|
+
reply.update(
|
|
566
|
+
{
|
|
567
|
+
"ename": type(error).__name__,
|
|
568
|
+
"evalue": str(error),
|
|
569
|
+
"traceback": [],
|
|
570
|
+
}
|
|
571
|
+
)
|
|
572
|
+
return reply
|
|
573
|
+
|
|
574
|
+
def do_shutdown(self, restart: bool = False) -> dict[str, Any]:
|
|
575
|
+
"""Reset remote session state for restart, then stop the local process."""
|
|
576
|
+
if restart:
|
|
577
|
+
try:
|
|
578
|
+
self._post_control("restart")
|
|
579
|
+
except Exception as exc: # noqa: BLE001 - shutdown must complete
|
|
580
|
+
self._log(f"Super Kernel restart failed: {type(exc).__name__}: {exc}")
|
|
581
|
+
self._clear_pending_execution()
|
|
582
|
+
try:
|
|
583
|
+
with self._runtime_status_lock:
|
|
584
|
+
self._runtime_status_stop.set()
|
|
585
|
+
_delete_runtime_status(
|
|
586
|
+
self.gateway_url,
|
|
587
|
+
self.remote_kernel_id,
|
|
588
|
+
self.session_id,
|
|
589
|
+
)
|
|
590
|
+
except Exception as exc: # noqa: BLE001 - shutdown must complete
|
|
591
|
+
self._log(
|
|
592
|
+
f"Local runtime status cleanup failed: {type(exc).__name__}: {exc}"
|
|
593
|
+
)
|
|
594
|
+
return {"status": "ok", "restart": restart}
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def _build_arg_parser() -> argparse.ArgumentParser:
|
|
598
|
+
parser = argparse.ArgumentParser(
|
|
599
|
+
prog="superkernel",
|
|
600
|
+
description="Launch a local Jupyter kernel bound to one Super Kernel ID.",
|
|
601
|
+
)
|
|
602
|
+
parser.add_argument(
|
|
603
|
+
"--version", action="version", version=f"SuperKernel Adapter {__version__}"
|
|
604
|
+
)
|
|
605
|
+
parser.add_argument("--gateway-url", required=True)
|
|
606
|
+
parser.add_argument("--remote-kernel-id", required=True)
|
|
607
|
+
parser.add_argument(
|
|
608
|
+
"--auth-file",
|
|
609
|
+
required=True,
|
|
610
|
+
help="Private UI-managed auth.json file",
|
|
611
|
+
)
|
|
612
|
+
return parser
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _connection_file_argument(argv: list[str]) -> str:
|
|
616
|
+
"""Return the runtime connection file selected by Jupyter."""
|
|
617
|
+
for index, argument in enumerate(argv):
|
|
618
|
+
if argument in {"-f", "--IPKernelApp.connection_file"}:
|
|
619
|
+
if index + 1 >= len(argv):
|
|
620
|
+
break
|
|
621
|
+
return argv[index + 1]
|
|
622
|
+
for prefix in ("--f=", "--IPKernelApp.connection_file="):
|
|
623
|
+
if argument.startswith(prefix):
|
|
624
|
+
return argument[len(prefix) :]
|
|
625
|
+
raise SystemExit("Jupyter connection file is required")
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def main(argv: list[str] | None = None) -> None:
|
|
629
|
+
args, remaining = _build_arg_parser().parse_known_args(
|
|
630
|
+
argv if argv is not None else sys.argv[1:]
|
|
631
|
+
)
|
|
632
|
+
try:
|
|
633
|
+
_read_auth_token(args.auth_file, args.gateway_url)
|
|
634
|
+
except RuntimeError as exc:
|
|
635
|
+
raise SystemExit(str(exc)) from exc
|
|
636
|
+
SuperKernel.boot = {
|
|
637
|
+
"gateway_url": args.gateway_url,
|
|
638
|
+
"remote_kernel_id": args.remote_kernel_id,
|
|
639
|
+
"auth_file": args.auth_file,
|
|
640
|
+
"connection_file": _connection_file_argument(remaining),
|
|
641
|
+
}
|
|
642
|
+
IPKernelApp.launch_instance(kernel_class=SuperKernel, argv=remaining)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
if __name__ == "__main__":
|
|
646
|
+
main()
|