sferriol-python 0.9.4__tar.gz → 0.10.1__tar.gz

{sferriol_python-0.9.4 → sferriol_python-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sferriol-python
-Version: 0.9.4
+Version: 0.10.1
 Summary: Python utilities
 Author-email: Sylvain Ferriol <sylvain.ferriol@cnrs.fr>
 License: 

{sferriol_python-0.9.4 → sferriol_python-0.10.1}/sferriol/python/__init__.py

@@ -4,6 +4,7 @@ import time
 import types
 from typing import Any, Callable
 
+import sferriol.python.asyncio_ as asyncio
 import sferriol.python.os_ as os
 
 

sferriol_python-0.9.4/sferriol/python/asyncio.py → sferriol_python-0.10.1/sferriol/python/asyncio_/__init__.py

@@ -1,6 +1,8 @@
-import asyncio as _asyncio
+import asyncio
 import signal
 
+from sferriol.python.asyncio_.stream import *
+
 
 async def run_shell_cmd(cmd: str) -> (int, str, str):
     """Execute shell command
@@ -8,8 +10,8 @@ async def run_shell_cmd(cmd: str) -> (int, str, str):
     Returns:
       Tuple (returned code, stdout, stderr)
     """
-    proc = await _asyncio.create_subprocess_shell(
-        cmd, stdout=_asyncio.subprocess.PIPE, stderr=_asyncio.subprocess.PIPE)
+    proc = await asyncio.create_subprocess_shell(
+        cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
     stdout, stderr = await proc.communicate()
     return proc.returncode, stdout.decode().strip(), stderr.decode().strip()
 
@@ -18,7 +20,7 @@ async def wait_sigint_signal() -> None:
     """
     Wait until the SIGINT signal is received. SIGINT is the Interrupt from keyboard (CTRL + C).
     """
-    evt = _asyncio.Event()
-    loop = _asyncio.get_running_loop()
+    evt = asyncio.Event()
+    loop = asyncio.get_running_loop()
     loop.add_signal_handler(signal.SIGINT, lambda: evt.set())
     await evt.wait()

sferriol_python-0.10.1/sferriol/python/asyncio_/stream.py

@@ -0,0 +1,146 @@
+import asyncio
+from collections import deque
+from dataclasses import dataclass
+from typing import Any, Deque
+
+__all__ = ["Buffered_Stream"]
+
+
+class Stream_Closed(Exception):
+    """Stream is closed."""
+
+
+class Stream_Overrun(Exception):
+    """Reader fell too far behind, and some data has been overwritten."""
+
+
+@dataclass(frozen=True)
+class _Item:
+    sequence: int
+    value: Any
+
+
+class Buffered_Stream:
+    """Asynchronous stream for publishing and consuming a sequence of items.
+    Supports multiple independent subscribers with configurable sliding windows."""
+
+    def __init__(self, *, buffer_size: int) -> None:
+        if buffer_size <= 0:
+            raise ValueError("buffer_size must be > 0")
+
+        self.buffer_size = buffer_size
+
+        self._buffer: Deque[_Item] = deque(maxlen=buffer_size)
+        self._next_sequence = 0
+
+        self._condition = asyncio.Condition()
+        self._closed = False
+
+    async def publish(self, value: Any) -> None:
+        """Publish a new item."""
+        async with self._condition:
+            if self._closed:
+                raise StreamClosed
+
+            self._buffer.append(
+                _Item(
+                    sequence=self._next_sequence,
+                    value=value,
+                ))
+            self._next_sequence += 1
+
+            self._condition.notify_all()
+
+    async def close(self) -> None:
+        async with self._condition:
+            self._closed = True
+            self._condition.notify_all()
+
+    def subscribe(self, *, window_size: int, step: int):
+        """Subscribe to the stream with a sliding window of `window_size` items advancing by `step` items.
+        Returns:
+          Buffered_Stream_Reader
+        """
+        if window_size <= 0:
+            raise ValueError("window_size must be > 0")
+
+        if step <= 0:
+            raise ValueError("step must be > 0")
+
+        if window_size > self.buffer_size:
+            raise ValueError("window_size cannot be greater than buffer_size")
+
+        return Buffered_Stream_Reader(
+            stream=self,
+            window_size=window_size,
+            step=step,
+        )
+
+    async def _get_window(self, *, start_sequence: int,
+                          window_size: int) -> list[Any]:
+
+        end_sequence = start_sequence + window_size
+
+        async with self._condition:
+
+            def ready() -> bool:
+                if self._closed:
+                    return True
+
+                if not self._buffer:
+                    return False
+
+                latest = self._buffer[-1].sequence
+
+                return latest >= end_sequence - 1
+
+            await self._condition.wait_for(ready)
+
+            if not self._buffer:
+                raise StreamClosed
+
+            oldest = self._buffer[0].sequence
+            latest = self._buffer[-1].sequence
+
+            if start_sequence < oldest:
+                raise StreamOverrun(
+                    f"reader needs sequence {start_sequence}, "
+                    f"but oldest available sequence is {oldest}")
+
+            if end_sequence - 1 > latest:
+                raise StreamClosed
+
+            items = list(self._buffer)
+
+            return [
+                item.value for item in items
+                if start_sequence <= item.sequence < end_sequence
+            ]
+
+
+class Buffered_Stream_Reader:
+
+    def __init__(self, *, stream: Buffered_Stream, window_size: int,
+                 step: int) -> None:
+        self._stream = stream
+        self._next_sequence = stream._next_sequence
+        self.window_size = window_size
+        self.step = step
+
+    async def get(self) -> list[Any]:
+        """Attend la prochaine fenêtre et la retourne."""
+        start = self._next_sequence
+
+        window = await self._stream._get_window(
+            start_sequence=start,
+            window_size=self.window_size,
+        )
+
+        if len(window) != self.window_size:
+            raise RuntimeError(
+                f"internal error: expected {self.window_size} items, "
+                f"got {len(window)}")
+
+        self._next_sequence += self.step
+
+        return window

sferriol_python-0.10.1/sferriol/python/test/test__asyncio.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+#
+#  Copyright 2024 sferriol <s.ferriol@ip2i.in2p3.fr>
+import asyncio
+import multiprocessing
+import os
+import signal
+import tempfile
+import time
+import unittest
+
+from sferriol.python.asyncio_ import *
+
+
+class Test__Buffered_Stream(unittest.IsolatedAsyncioTestCase):
+
+    async def _producer(self, stream, n_values):
+        for i in range(n_values):
+            await stream.publish(i)
+            await asyncio.sleep(0)
+        await stream.close()
+
+    async def _consumer(self, name, stream, window_size, step):
+        self.consumer_windows[name] = list()
+        reader = stream.subscribe(
+            window_size=window_size,
+            step=step,
+        )
+        stop = False
+        while not stop:
+            try:
+                window = await reader.get()
+                self.consumer_windows[name].append(window)
+            except:
+                stop = True
+
+    async def test(self):
+        np_stream = Buffered_Stream(buffer_size=70)
+        self.consumer_windows = dict()
+        await asyncio.gather(
+            self._consumer("overlap", np_stream, window_size=5, step=2),
+            self._consumer("no_overlap", np_stream, window_size=5, step=5),
+            self._producer(np_stream, n_values=20),
+        )
+        self.assertEqual(len(self.consumer_windows["overlap"]), 8)
+        self.assertEqual(len(self.consumer_windows["no_overlap"]), 4)
+
+
+class Test__Run_In_Shell(unittest.IsolatedAsyncioTestCase):
+
+    async def test__stdout(self):
+        ret, stdout, stderr = await run_shell_cmd("echo toto")
+        self.assertEqual(ret, 0)
+        self.assertEqual(stdout, "toto")
+        self.assertEqual(stderr, "")
+
+    async def test__stderr(self):
+        ret, stdout, stderr = await run_shell_cmd("man toto")
+        self.assertNotEqual(ret, 0)
+        self.assertEqual(stdout, "")
+        self.assertNotEqual(stderr, "")
+
+
+def _worker():
+    import asyncio
+    asyncio.run(wait_sigint_signal())
+
+
+class Test__Wait_Sigint_Signal(unittest.IsolatedAsyncioTestCase):
+
+    async def test(self):
+        p = multiprocessing.Process(target=_worker)
+        p.start()
+        await asyncio.sleep(1)
+        os.kill(p.pid, signal.SIGINT)
+        await asyncio.to_thread(p.join, 1)
+        self.assertFalse(p.is_alive())
+        self.assertEqual(p.exitcode, 0)

{sferriol_python-0.9.4 → sferriol_python-0.10.1}/sferriol_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sferriol-python
-Version: 0.9.4
+Version: 0.10.1
 Summary: Python utilities
 Author-email: Sylvain Ferriol <sylvain.ferriol@cnrs.fr>
 License: 

{sferriol_python-0.9.4 → sferriol_python-0.10.1}/sferriol_python.egg-info/SOURCES.txt

@@ -2,13 +2,14 @@ Licence_CeCILL-B_V1-en.txt
 README.md
 pyproject.toml
 sferriol/python/__init__.py
-sferriol/python/asyncio.py
 sferriol/python/env.py
 sferriol/python/json.py
 sferriol/python/net.py
 sferriol/python/object.py
 sferriol/python/os_.py
 sferriol/python/pkg.py
+sferriol/python/asyncio_/__init__.py
+sferriol/python/asyncio_/stream.py
 sferriol/python/dictionary/__init__.py
 sferriol/python/test/__init__.py
 sferriol/python/test/test__asyncio.py

sferriol_python-0.9.4/sferriol/python/test/test__asyncio.py

@@ -1,44 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-#  Copyright 2024 sferriol <s.ferriol@ip2i.in2p3.fr>
-import asyncio
-import multiprocessing
-import os
-import signal
-import tempfile
-import time
-import unittest
-
-from sferriol.python.asyncio import *
-
-
-class Test__Run_In_Shell(unittest.IsolatedAsyncioTestCase):
-
-    async def test__stdout(self):
-        ret, stdout, stderr = await run_shell_cmd("echo toto")
-        self.assertEqual(ret, 0)
-        self.assertEqual(stdout, "toto")
-        self.assertEqual(stderr, "")
-
-    async def test__stderr(self):
-        ret, stdout, stderr = await run_shell_cmd("man toto")
-        self.assertNotEqual(ret, 0)
-        self.assertEqual(stdout, "")
-        self.assertNotEqual(stderr, "")
-
-
-def _worker():
-    import asyncio
-    asyncio.run(wait_sigint_signal())
-
-
-class Test__Wait_Sigint_Signal(unittest.IsolatedAsyncioTestCase):
-
-    async def test(self):
-        p = multiprocessing.Process(target=_worker)
-        p.start()
-        await asyncio.sleep(1)
-        os.kill(p.pid, signal.SIGINT)
-        await asyncio.to_thread(p.join, 1)
-        self.assertFalse(p.is_alive())
-        self.assertEqual(p.exitcode, 0)