conkernelclient 0.0.16__tar.gz → 0.0.17__tar.gz

{conkernelclient-0.0.16/conkernelclient.egg-info → conkernelclient-0.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: conkernelclient
-Version: 0.0.16
+Version: 0.0.17
 Summary: Concurrent-safe Jupyter KernelClient
 Author-email: Jeremy Howard <github@jhoward.fastmail.fm>
 License: Apache-2.0
@@ -13,9 +13,14 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: jupyter_client
-Requires-Dist: fastcore
+Requires-Dist: fastcore>=2.1.17
+Requires-Dist: jupywire>=0.0.1
 Provides-Extra: dev
 Requires-Dist: ipykernel; extra == "dev"
+Requires-Dist: ipymini>=0.1.17; extra == "dev"
+Requires-Dist: ipyfuncs>=0.0.1; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-timeout; extra == "dev"
 Dynamic: license-file
 
 # conkernelclient

{conkernelclient-0.0.16 → conkernelclient-0.0.17}/conkernelclient/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.0.16"
+__version__ = "0.0.17"
 from .core import *
 from .ops import *

{conkernelclient-0.0.16 → conkernelclient-0.0.17}/conkernelclient/_modidx.py

@@ -14,6 +14,8 @@ d = { 'settings': { 'branch': 'main',
                                                                                              'conkernelclient/core.py'),
                                       'conkernelclient.core.ConKernelClient._fail_pending': ( 'core.html#conkernelclient._fail_pending',
                                                                                               'conkernelclient/core.py'),
+                                      'conkernelclient.core.ConKernelClient.astop_channels': ( 'core.html#conkernelclient.astop_channels',
+                                                                                               'conkernelclient/core.py'),
                                       'conkernelclient.core.ConKernelClient.execute': ( 'core.html#conkernelclient.execute',
                                                                                         'conkernelclient/core.py'),
                                       'conkernelclient.core.ConKernelClient.start_channels': ( 'core.html#conkernelclient.start_channels',
@@ -22,6 +24,8 @@ d = { 'settings': { 'branch': 'main',
                                                                                            'conkernelclient/core.py'),
                                       'conkernelclient.core.ConKernelClient.stop_channels': ( 'core.html#conkernelclient.stop_channels',
                                                                                               'conkernelclient/core.py'),
+                                      'conkernelclient.core.ConKernelClient.wait_for_ready': ( 'core.html#conkernelclient.wait_for_ready',
+                                                                                               'conkernelclient/core.py'),
                                       'conkernelclient.core.ConKernelManager': ('core.html#conkernelmanager', 'conkernelclient/core.py'),
                                       'conkernelclient.core.ConKernelManager._transport_encryption_default': ( 'core.html#conkernelmanager._transport_encryption_default',
                                                                                                                'conkernelclient/core.py'),
@@ -29,7 +33,9 @@ d = { 'settings': { 'branch': 'main',
                                       'conkernelclient.core._send': ('core.html#_send', 'conkernelclient/core.py'),
                                       'conkernelclient.core.apply_session_patch': ( 'core.html#apply_session_patch',
                                                                                     'conkernelclient/core.py')},
-            'conkernelclient.ops': { 'conkernelclient.ops.ConKernelClient.cmd': ('ops.html#conkernelclient.cmd', 'conkernelclient/ops.py'),
+            'conkernelclient.ops': { 'conkernelclient.ops.ConKernelClient._pre_ipy': ( 'ops.html#conkernelclient._pre_ipy',
+                                                                                       'conkernelclient/ops.py'),
+                                     'conkernelclient.ops.ConKernelClient.cmd': ('ops.html#conkernelclient.cmd', 'conkernelclient/ops.py'),
                                      'conkernelclient.ops.ConKernelClient.control_request': ( 'ops.html#conkernelclient.control_request',
                                                                                               'conkernelclient/ops.py'),
                                      'conkernelclient.ops.ConKernelClient.ctl': ('ops.html#conkernelclient.ctl', 'conkernelclient/ops.py'),
@@ -50,6 +56,8 @@ d = { 'settings': { 'branch': 'main',
                                                                                           'conkernelclient/ops.py'),
                                      'conkernelclient.ops.ConKernelClient.iopub_flush': ( 'ops.html#conkernelclient.iopub_flush',
                                                                                           'conkernelclient/ops.py'),
+                                     'conkernelclient.ops.ConKernelClient.reply': ( 'ops.html#conkernelclient.reply',
+                                                                                    'conkernelclient/ops.py'),
                                      'conkernelclient.ops.ConKernelClient.shell_request': ( 'ops.html#conkernelclient.shell_request',
                                                                                             'conkernelclient/ops.py'),
                                      'conkernelclient.ops.ConKernelClient.user_exprs': ( 'ops.html#conkernelclient.user_exprs',
@@ -66,7 +74,6 @@ d = { 'settings': { 'branch': 'main',
                                      'conkernelclient.ops.iopub_msgs': ('ops.html#iopub_msgs', 'conkernelclient/ops.py'),
                                      'conkernelclient.ops.iopub_streams': ('ops.html#iopub_streams', 'conkernelclient/ops.py'),
                                      'conkernelclient.ops.iter_timeout': ('ops.html#iter_timeout', 'conkernelclient/ops.py'),
-                                     'conkernelclient.ops.nb_outputs': ('ops.html#nb_outputs', 'conkernelclient/ops.py'),
                                      'conkernelclient.ops.parent_id': ('ops.html#parent_id', 'conkernelclient/ops.py'),
                                      'conkernelclient.ops.parse_expr': ('ops.html#parse_expr', 'conkernelclient/ops.py'),
                                      'conkernelclient.ops.reconnect': ('ops.html#reconnect', 'conkernelclient/ops.py'),

{conkernelclient-0.0.16 → conkernelclient-0.0.17}/conkernelclient/core.py

@@ -12,6 +12,9 @@ from jupyter_client import AsyncKernelClient, AsyncKernelManager
 from jupyter_client.session import Session
 from zmq.error import ZMQError
 from traitlets import Type, default
+from fastcore.utils import patch
+from jupywire.ops import EvalOps
+from queue import Empty
 import asyncio, zmq.asyncio, time, logging
 
 # %% ../nbs/00_core.ipynb #737a0fc1
@@ -45,7 +48,7 @@ def apply_session_patch():
     Session.send = _send
 
 # %% ../nbs/00_core.ipynb #d6a5fa6a
-class ConKernelClient(AsyncKernelClient):
+class ConKernelClient(EvalOps, AsyncKernelClient):
     def __init__(self, *args, **kwargs):
         apply_session_patch()
         # jupyter_client's `get_connection_info` returns curve keys as str, but the client traits are `Bytes`
@@ -91,7 +94,6 @@ class ConKernelClient(AsyncKernelClient):
                     self._fail_pending(exc, skip=mid)
         self._shell_reader_task = asyncio.create_task(_reader())
         await _ready.wait()
-        await asyncio.sleep(0.2)
         return self
 
     def stop_channels(self):
@@ -101,7 +103,6 @@ class ConKernelClient(AsyncKernelClient):
         if (tk := getattr(self, '_shell_reader_task', None)):
             tk.cancel()
             self._shell_reader_task = None
-        time.sleep(0.2)
 
     async def _async_recv_reply(self, msg_id, timeout=None, channel="shell"):
         if channel == "control": return await self._async_get_control_msg(timeout=timeout)
@@ -136,6 +137,52 @@ class ConKernelClient(AsyncKernelClient):
         except (AssertionError, OSError, ZMQError) as e: raise DeadKernelError(f"Kernel socket closed: {e}") from e
         await asyncio.sleep(0.01)
 
+# %% ../nbs/00_core.ipynb #3771c654
+@patch
+async def wait_for_ready(self:ConKernelClient, timeout=None):
+    "Wait for the kernel to be ready: deterministic via `iopub_welcome` (JEP 65) when the kernel sends one, else jupyter_client's probe loop"
+    deadline = time.monotonic()+timeout if timeout is not None else None
+    def _left():
+        if deadline is None: return 1.0
+        t = deadline-time.monotonic()
+        if t <= 0: raise RuntimeError(f"Kernel didn't respond in {timeout} seconds")
+        return min(t, 1.0)
+    async def _get(ch):
+        while True:
+            try: return await ch.get_msg(timeout=_left())
+            except Empty:
+                if not await self._async_is_alive(): raise RuntimeError("Kernel died before replying to kernel_info")
+    msg = None
+    while msg is None:
+        self.kernel_info()
+        try: msg = await self.iopub_channel.get_msg(timeout=_left())
+        except Empty:
+            if not await self._async_is_alive(): raise RuntimeError("Kernel died before replying to kernel_info")
+    if msg['msg_type']=='iopub_welcome':
+        mid = self.kernel_info()
+        while (reply := await _get(self.shell_channel))['parent_header'].get('msg_id') != mid: pass
+        self._handle_kernel_info_reply(reply)
+        while not (msg['msg_type']=='status' and msg['parent_header'].get('msg_id')==mid and msg['content']['execution_state']=='idle'):
+            msg = await _get(self.iopub_channel)
+    else:
+        while (reply := await _get(self.shell_channel))['msg_type'] != 'kernel_info_reply': pass
+        self._handle_kernel_info_reply(reply)
+        while True:
+            try: await self.iopub_channel.get_msg(timeout=0.2)
+            except Empty: break
+
+# %% ../nbs/00_core.ipynb #967c10a1
+@patch
+async def astop_channels(self:ConKernelClient):
+    "Stop channels and cancel the background shell-reply reader, awaiting its exit"
+    self._fail_pending(RuntimeError("Shell channels stopped before reply"))
+    if (tk := getattr(self, '_shell_reader_task', None)):
+        tk.cancel()
+        try: await tk
+        except asyncio.CancelledError: pass
+        self._shell_reader_task = None
+    super(ConKernelClient, self).stop_channels()
+
 # %% ../nbs/00_core.ipynb #b828c222
 class ConKernelManager(AsyncKernelManager):
     client_class,client_factory = ConKernelClient,Type(ConKernelClient)

{conkernelclient-0.0.16 → conkernelclient-0.0.17}/conkernelclient/ops.py

@@ -5,12 +5,14 @@ Docs: https://AnswerDotAI.github.io/conkernelclient/ops.html.md"""
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_ops.ipynb.
 
 # %% auto #0
-__all__ = ['default_timeout', 'output_types', 'parent_id', 'iter_timeout', 'nb_outputs', 'iopub_msgs', 'iopub_streams',
-           'EvalError', 'parse_expr', 'run_kernel', 'reconnect']
+__all__ = ['default_timeout', 'output_types', 'parent_id', 'iter_timeout', 'iopub_msgs', 'iopub_streams', 'EvalError',
+           'parse_expr', 'run_kernel', 'reconnect']
 
 # %% ../nbs/01_ops.ipynb #23936f1c
 from fastcore.utils import *
-from .core import ConKernelClient, ConKernelManager
+from fastcore.ansi import strip_ansi
+from fastcore.nbio import msgs2outs, preferred_out
+from .core import ConKernelClient, ConKernelManager, DeadKernelError
 from contextlib import asynccontextmanager
 from ast import literal_eval
 from queue import Empty
@@ -52,15 +54,6 @@ async def iopub_flush(self:ConKernelClient, timeout=0.1):
 # %% ../nbs/01_ops.ipynb #add288fe
 output_types = {'stream', 'execute_result', 'display_data', 'error'}
 
-def nb_outputs(msgs):
-    "Convert iopub `msgs` to nbformat-style output dicts, dropping non-output messages"
-    res = []
-    for m in msgs:
-        if (mt := m['msg_type']) not in output_types: continue
-        d = dict(output_type=mt, **m['content'])
-        d.pop('transient', None)
-        res.append(d)
-    return res
 
 # %% ../nbs/01_ops.ipynb #ebdbd6eb
 def iopub_msgs(msgs, msg_type=None):
@@ -92,7 +85,7 @@ async def exec_ok(self:ConKernelClient, code, timeout=default_timeout, **kw):
 async def exec_outs(self:ConKernelClient, code, timeout=default_timeout, **kw):
     "Execute `code` and return just its nbformat-style outputs"
     reply, outputs = await self.exec_drain(code, timeout=timeout, **kw)
-    return nb_outputs(outputs)
+    return msgs2outs(outputs)
 
 # %% ../nbs/01_ops.ipynb #77ab9da4
 class EvalError(Exception):
@@ -117,6 +110,26 @@ async def eval_expr(self:ConKernelClient, expr:str, code:str='', timeout=default
     if r.get('status') != 'ok': raise EvalError(f"{r.get('ename')}: {r.get('evalue')}")
     return parse_expr(r['data']['text/plain'])
 
+# %% ../nbs/01_ops.ipynb #d9bbaa8d
+@patch
+def reply(self:ConKernelClient,
+    code, # A string of code in the kernel's language.
+    user_expressions:NoneType=None, # A dict mapping names to expressions to be evaluated in the user's dict
+    allow_stdin:NoneType=None, # Flag for whether the kernel can send stdin requests to frontends.
+    cts_typ:str='code', timeout:int=None, msg_id:NoneType=None, priority:bool=False, **kw
+):
+    if priority: assert self.priority, 'no priority subshell configured'
+    return self.execute(code, user_expressions=user_expressions, allow_stdin=allow_stdin, reply=True,
+        cts_typ=cts_typ, timeout=timeout or getattr(self, 'default_timeout', default_timeout), msg_id=msg_id, subsh_id=self.priority if priority else None, **kw)
+
+# %% ../nbs/01_ops.ipynb #5de02acb
+from jupywire.ops import EvalOps, EvalException, try_eval
+
+@patch
+def _pre_ipy(self:ConKernelClient):
+    if not self.shell_channel.is_alive(): raise DeadKernelError()
+
+
 # %% ../nbs/01_ops.ipynb #aafc1ab4
 @patch
 def shell_request(

{conkernelclient-0.0.16 → conkernelclient-0.0.17/conkernelclient.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: conkernelclient
-Version: 0.0.16
+Version: 0.0.17
 Summary: Concurrent-safe Jupyter KernelClient
 Author-email: Jeremy Howard <github@jhoward.fastmail.fm>
 License: Apache-2.0
@@ -13,9 +13,14 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: jupyter_client
-Requires-Dist: fastcore
+Requires-Dist: fastcore>=2.1.17
+Requires-Dist: jupywire>=0.0.1
 Provides-Extra: dev
 Requires-Dist: ipykernel; extra == "dev"
+Requires-Dist: ipymini>=0.1.17; extra == "dev"
+Requires-Dist: ipyfuncs>=0.0.1; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-timeout; extra == "dev"
 Dynamic: license-file
 
 # conkernelclient

{conkernelclient-0.0.16 → conkernelclient-0.0.17}/conkernelclient.egg-info/SOURCES.txt

@@ -11,4 +11,5 @@ conkernelclient.egg-info/SOURCES.txt
 conkernelclient.egg-info/dependency_links.txt
 conkernelclient.egg-info/entry_points.txt
 conkernelclient.egg-info/requires.txt
-conkernelclient.egg-info/top_level.txt
+conkernelclient.egg-info/top_level.txt
+tests/test_races.py

conkernelclient-0.0.17/conkernelclient.egg-info/requires.txt

@@ -0,0 +1,10 @@
+jupyter_client
+fastcore>=2.1.17
+jupywire>=0.0.1
+
+[dev]
+ipykernel
+ipymini>=0.1.17
+ipyfuncs>=0.0.1
+pytest
+pytest-timeout

{conkernelclient-0.0.16 → conkernelclient-0.0.17}/pyproject.toml

@@ -17,12 +17,17 @@ classifiers = [
 ]
 dependencies = [
   'jupyter_client',
-  'fastcore',
+  'fastcore>=2.1.17',
+  'jupywire>=0.0.1',
 ]
 
 [project.optional-dependencies]
 dev = [
   "ipykernel",
+  "ipymini>=0.1.17",
+  "ipyfuncs>=0.0.1",
+  "pytest",
+  "pytest-timeout",
 ]
 
 [project.urls]
@@ -44,3 +49,6 @@ allowed_cell_metadata_keys = ['solveit_ai']
 
 [tool.chkstyle]
 skip_paths = ["_modidx.py"]
+
+[tool.pytest.ini_options]
+timeout = 120

conkernelclient-0.0.17/tests/test_races.py

@@ -0,0 +1,76 @@
+"""Race-condition regression tests: adversarial timing on the shell send path.
+
+`LockedSendChannel` serializes shell sends under a lock with a random 10-15ms sleep,
+widening the window between the sync zmq send path and the async receive side. The burst
+test then keeps many replies pending at once, so the background reader races every send.
+The parked-recv test is the minimal repro of the edge-triggered FD bug the Session patch
+fixes: a recv parked in poll(None) must still wake after a sync send consumes the FD edge.
+"""
+import asyncio, functools, random, threading, time
+from queue import Empty
+
+from jupyter_client import AsyncKernelManager
+from jupyter_client.channels import AsyncZMQSocketChannel
+from traitlets import Type
+
+from conkernelclient import ConKernelClient, ConKernelManager
+from conkernelclient.core import apply_session_patch
+from conkernelclient.ops import run_kernel
+
+
+def locked_sleep(f):
+    @functools.wraps(f)
+    def _f(self, *args, **kwargs):
+        with self.lock:
+            time.sleep(random.uniform(0.01, 0.015))
+            return f(self, *args, **kwargs)
+    return _f
+
+class LockedSendChannel(AsyncZMQSocketChannel):
+    lock = threading.RLock()
+    @locked_sleep
+    def send(self, msg): return super().send(msg)
+
+class LockedClient(ConKernelClient):
+    shell_channel_class = LockedSendChannel
+
+class LockedKernelManager(ConKernelManager):
+    client_class,client_factory = LockedClient,Type(LockedClient)
+
+
+def test_burst_replies_under_send_jitter():
+    "20 pending replies at once, every send jittered: the reader must route each reply to its waiter."
+    async def _run():
+        async with run_kernel(kernel_name="ipymini", manager_cls=LockedKernelManager) as (km, kc):
+            reps = [kc.execute(f"x{i} = {i}; x{i}", reply=True, timeout=30) for i in range(20)]
+            for i, r in enumerate(await asyncio.gather(*reps)): assert r["content"]["status"] == "ok", (i, r["content"])
+    asyncio.run(_run())
+
+
+def test_parked_recv_survives_sync_send():
+    "A recv parked in poll(None) must wake with the right reply after sync sends while the loop was busy."
+    async def _run():
+        apply_session_patch()
+        km = AsyncKernelManager()
+        await km.start_kernel()
+        kc = km.client()
+        kc.start_channels()
+        await kc.wait_for_ready()
+        try:
+            # Drain replies to any extra kernel_info requests from a slow start,
+            # so the parked recv below can only receive the execute reply.
+            try:
+                while True: await kc.get_shell_msg(timeout=0.3)
+            except Empty: pass
+
+            fut = asyncio.create_task(kc.get_shell_msg(timeout=None))
+            await asyncio.sleep(0.1)
+            mid = kc.execute("1+1")  # loop busy below, so the reply buffers and sets the FD edge
+            time.sleep(0.4)
+            kc.execute("2+2")  # sync send consumes the edge via process_commands
+            reply = await asyncio.wait_for(fut, 3)  # TimeoutError here is the regression
+            assert reply["parent_header"]["msg_id"] == mid
+        finally:
+            kc.stop_channels()
+            await km.shutdown_kernel(now=True)
+    asyncio.run(_run())

conkernelclient-0.0.16/conkernelclient.egg-info/requires.txt

@@ -1,5 +0,0 @@
-jupyter_client
-fastcore
-
-[dev]
-ipykernel