serverish 1.2.2__tar.gz → 1.4.0__tar.gz

{serverish-1.2.2 → serverish-1.4.0}/PKG-INFO

@@ -1,13 +1,11 @@
 Metadata-Version: 2.1
 Name: serverish
-Version: 1.2.2
+Version: 1.4.0
 Summary: helpers for server alike projects
 Home-page: https://github.com/AkondLab/serverish
 Author: Mikołaj Kałuszyński
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -32,7 +30,7 @@ Description-Content-Type: text/markdown
 See [`doc` directory](doc/) for more documentation.
 
 ## Optional (extras) dependencies
-Following extras are available:
+The following extras are available:
 * `messenger` - for using NATS based Messenger. Will install `nats-py` package.
 * `dns` - for using `aiodns` DNS resolver for connection status diagnostics, it's extracted as an extra dependency because it's not available on all platforms.
 
@@ -49,6 +47,7 @@ this will install `nats-py` package.
 - Pytest fixtures for NATS on CI
 
 ## Changes
+* 1.4 Contains improvements to the network problems recovery and connection tracking.
 * 1.1 Switches to `param` 2.* nad `py-nats` 1.7.*. Also, publisher may reraise exceptions or ignore them.
 * 1.0  Changes NATS subscriptions to pull only
 * 0.12 Extends messges metdata of `nats` section, Handles reconnection of subscriptions, 

{serverish-1.2.2 → serverish-1.4.0}/README.md

@@ -10,7 +10,7 @@
 See [`doc` directory](doc/) for more documentation.
 
 ## Optional (extras) dependencies
-Following extras are available:
+The following extras are available:
 * `messenger` - for using NATS based Messenger. Will install `nats-py` package.
 * `dns` - for using `aiodns` DNS resolver for connection status diagnostics, it's extracted as an extra dependency because it's not available on all platforms.
 
@@ -27,6 +27,7 @@ this will install `nats-py` package.
 - Pytest fixtures for NATS on CI
 
 ## Changes
+* 1.4 Contains improvements to the network problems recovery and connection tracking.
 * 1.1 Switches to `param` 2.* nad `py-nats` 1.7.*. Also, publisher may reraise exceptions or ignore them.
 * 1.0  Changes NATS subscriptions to pull only
 * 0.12 Extends messges metdata of `nats` section, Handles reconnection of subscriptions, 

{serverish-1.2.2 → serverish-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "serverish"
-version = "1.2.2"
+version = "1.4.0"
 description = "helpers for server alike projects"
 authors = ["Mikołaj Kałuszyński", "MMME team"]
 readme = "README.md"
@@ -8,7 +8,7 @@ repository = "https://github.com/AkondLab/serverish"
 include = ["serverish/schema/*"]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.10"
 aiodns = {version="^3.2.0", optional=true}
 param = "^2.1.0"
 nats-py = { version = "^2.7.2", optional= true}

{serverish-1.2.2 → serverish-1.4.0}/serverish/base/exceptions.py

@@ -7,8 +7,12 @@ class MessengerNotConnected(Exception):
 class MessengerReaderStopped(Exception):
     pass
 
+class MessengerReaderAlreadyOpen(RuntimeError):
+    pass
+
 class MessengerRequestNoResponse(Exception):
     pass
+
 class MessengerRequestNoResponders(MessengerRequestNoResponse):
     pass
 
@@ -21,7 +25,7 @@ class MessengerRequestJetStreamSubject(MessengerRequestNoResponse):
 class MessengerRequestCanceled(MessengerRequestNoResponse):
     pass
 
-class MessengerRequestTimeout(MessengerRequestNoResponse):
+class MessengerRequestTimeout(MessengerRequestNoResponse, TimeoutError):
     pass
 
 class MessengerRequestNoResultYet(MessengerRequestNoResponse):

serverish-1.4.0/serverish/base/iterators.py

@@ -0,0 +1,97 @@
+import asyncio
+from typing import List, Dict, Any, Tuple, Set
+
+
+class AsyncRangeIter:
+    """This class represent async range iterator (instead of sync).
+
+    WARNING: Noticed that end is included (not like in standard iterator).
+    Example 1: start=1, end=4, will return [1, 2, 3, 4]
+    Example 2: async for n in AsyncRangeIter(start=1, end=3): ...
+    """
+
+    def __init__(self, start: int, end: int) -> None:
+        self.start = start
+        self.end = end
+
+    def __aiter__(self) -> Any:
+        self.current = self.start
+        return self
+
+    async def __anext__(self) -> int:
+        if self.current <= self.end:
+            value = self.current
+            self.current += 1
+            await asyncio.sleep(0)
+            return value
+        else:
+            raise StopAsyncIteration
+
+
+class AsyncListIter:
+    """This class represent async list iterator (instead of sync).
+
+    Example of use: async for n in AsyncListIter(some_list): ...
+    """
+    def __init__(self, iterable: List | Tuple | Set | Any):
+        self.iterable = iterable
+        self.index: int = 0
+
+    def __aiter__(self) -> Any:
+        self.index = 0
+        return self
+
+    async def __anext__(self):
+        if self.index < len(self.iterable):
+            value = self.iterable[self.index]
+            self.index += 1
+            await asyncio.sleep(0)
+            return value
+        else:
+            raise StopAsyncIteration
+
+
+class AsyncEnumerateIter:
+    """This class represent async enumerate iterator (instead of sync).
+
+    Example of use: async for current_index, value in AsyncEnumerateIter(some_iterable): ...
+    """
+
+    def __init__(self, iterable: List | Tuple | Set | Any) -> None:
+        self.iterable = iterable
+
+    def __aiter__(self):
+        self.index = 0
+        return self
+
+    async def __anext__(self):
+        if self.index < len(self.iterable):
+            value = self.iterable[self.index]
+            current_index = self.index
+            self.index += 1
+            await asyncio.sleep(0)
+            return current_index, value
+        else:
+            raise StopAsyncIteration
+
+
+class AsyncDictItemsIter:
+    """This class represent async dict items iterator (instead of sync).
+
+    Example of use: async for key, value in AsyncEnumerateIter(some_dict): ...
+    """
+
+    def __init__(self, data_dict: Dict) -> None:
+        self.data_dict = data_dict
+
+    def __aiter__(self) -> Any:
+        self.iterator = iter(self.data_dict.items())
+        return self
+
+    async def __anext__(self) -> Tuple:
+        try:
+            n, m = next(self.iterator)
+            await asyncio.sleep(0)
+            return n, m
+        except StopIteration:
+            raise StopAsyncIteration

{serverish-1.2.2 → serverish-1.4.0}/serverish/connection/connection_jets.py

@@ -89,9 +89,14 @@ class ConnectionJetStream(ConnectionNATS):
 
     async def nats_reconnected_cb(self):
         await super().nats_reconnected_cb()
-        self.setup_jetstream()
-        await self.update_statuses()
-        logging.info(f'Jeststream reconnect status: {self.format_status()}')
+        try:
+            self.setup_jetstream()
+            await self.update_statuses()
+            logging.info(f'JetStream reconnect status: {self.format_status()}')
+        except Exception as e:
+            logging.error(f'Error setting up JetStream on reconnection cb: {e}')
+            # Update status to reflect the error
+            await self.update_statuses()
 
     async def ensure_subject_in_stream(self, stream: str, subject: str,
                                        create_stram_if_needed: bool = False,

{serverish-1.2.2 → serverish-1.4.0}/serverish/messenger/msg_reader.py

@@ -17,7 +17,8 @@ from nats.js import JetStreamContext
 from nats.js.api import DeliverPolicy, ConsumerConfig
 
 from serverish.base import wait_for_psce
-from serverish.base.exceptions import MessengerReaderStopped
+from serverish.base.exceptions import (MessengerReaderStopped, MessengerNotConnected,
+                                       MessengerReaderAlreadyOpen, MessengerRequestTimeout)
 from serverish.base.fifoset import FifoSet
 from serverish.messenger import Messenger
 from serverish.messenger.messenger import MsgDriver
@@ -154,14 +155,24 @@ class MsgReader(MsgDriver):
                         await bmsg.ack()
                     except Exception as e:
                         log.warning(self.fmt(f"Error acking message: {e}"))
-                    data, meta = self.reader.messenger.unpack_nats_msg(bmsg)
+                        # Not essential - we can continue without acking
+
+                    try:
+                        data, meta = self.reader.messenger.unpack_nats_msg(bmsg)
+                    except Exception as e:
+                        log.warning(self.fmt(f"Error unpacking message: {e}"))
+                        # Mesaage malformed - non recoverable error (for this message)
+                        raise self.ErrorException('pop', e)
+
                     try:
                         if self.reader.last_seq is not None and meta['nats']['seq'] <= self.reader.last_seq: # dupes can happen on reopen
                             log.info(self.fmt(f"Skipping duplicated message seq={meta['nats']['seq']}, last_seq={self.reader.last_seq}"))
                             raise self.ContinueException('pop')
                     except self.ContinueException:
+                        # Duplicate removed from queue - recovered from a problem
                         raise
                     except Exception as e:
+                        # Error checking sequence, can be ignored
                         log.warning(self.fmt(f"Error checking seq: {e}"))
                     meta['receive_mode'] = 'pull'
                     try: # nonessential
@@ -170,36 +181,52 @@ class MsgReader(MsgDriver):
                         if len(self.reader.messages) == 0:
                             self.reader._emptied.set()
                     except Exception as e:
-                        log.warning(self.fmt(f"Error unpacking message: {e}"))
+                        # Error in logging, can be ignored
+                        log.warning(self.fmt(f"Error posprocessing message: {e}"))
                     # log.info(self.fmt(f"Returning message seq={meta['nats']['seq']}, last_seq={self.reader.last_seq}"))
                     raise  self.ReturnException('pop', data, meta)
+
             @async_shield
             async def ensure_open(self) -> None:
                 if not self.reader.is_open:
                     if self.reader._expect_beeing_open:
                         log.warning(self.fmt("The reader is not open, but once it has been, trying to reopen"))
-                    await self.reader.open()
+                    await self.reader.open()  # exceptions handled by the async_shield decorator
                     if self.reader.is_open:
                         self.reader._expect_beeing_open = True
+                        # Recover from a problem
                         raise self.ContinueException('open')
                     else:
-                        raise Exception('Opened, but still close')
+                        raise Exception('Opened, but still close') # exception handled by the async_shield decorator
+
             @async_shield
             async def ensure_not_stopped(self) -> None:
                 if self.reader._stop.is_set():
-                    await self.reader.close()
+                    try:
+                        await self.reader.close()
+                    except Exception as e:
+                        # Error in closing can be ignored we are finishing iteration anyway
+                        log.warning(self.fmt(f"Error closing reader: {e}"))
                     raise self.EndIterationException('stop')
+
             @async_shield
             async def ensure_consumer(self) -> None:
                 if self.error is not None:
                     try:
                         ci = await self.reader.pull_subscription.consumer_info()
-                        print(ci)
+                        # Consumer exists and is accessible
+                        log.debug(self.fmt(f"Consumer check OK: {ci.name}"))
                     except nats.js.errors.NotFoundError:
                         log.warning(self.fmt("Consumer has gone, trying to recreate it"))
                         await self.reader._reopen()
                         log.info(self.fmt(f"Consumer re-opened"))
                         raise self.ContinueException('reopen')
+                    except Exception as e:
+                        log.warning(self.fmt(f"Error checking consumer, will try to recreate anyway: {e}"))
+                        await self.reader._reopen()
+                        log.info(self.fmt(f"Consumer looks like re-opened after error"))
+                        raise self.ContinueException('reopen')
+
             @async_shield
             async def read_batch(self) -> None:
                 if len(self.reader.messages) == 0:
@@ -242,6 +269,15 @@ class MsgReader(MsgDriver):
         # The loop which tries hard to get some data
         while True:
             try:
+                # If the step method:
+                #  - retuns data                       - it will raiseReturnException
+                #  - corrected something               - it will raise ContinueException
+                #  - wants stop iteration              - it will raise EndIterationException
+                #  - encounters an unrecoverable error - it will raise ErrorException
+                #  - did not do anything               - it will return and next step will be called
+                # Those mthod should not raise any other exceptions, if they do, it is a bug,
+                # this is enforced by @async_shield (conversion of nay exception to ErrorException)
+
                 # 0. Do we have some data to return already?
                 await st.pop_msg()
                 # 1. Check if iteration have been stopped externally
@@ -281,11 +317,12 @@ class MsgReader(MsgDriver):
                         log.warning(st.fmt(f"read_next error, (retry in {wait_time:.1f}s): {e.error}"))
                         await asyncio.sleep(wait_time)
                     case _:  # should not be reached
-                        log.error(st.fmt(f"Invalid on_connection_close value {self.error_behavior}"))
-                        exit(-1) # this is not i/o error but programming error
+                        log.error(st.fmt(f"Invalid on_connection_close value {self.error_behavior}. "
+                                         f"Raport this as a bug to the serverish maintainers!"))
+                        exit(-107) # this is not i/o error but programming error
             except Exception as e:# should never happen
-                log.error(st.fmt(f"unhandled exception {e}"))
-                exit(-1) # this is not i/o error but programming error
+                log.error(st.fmt(f"Unhandled exception {e}. Raport this as a bug to the serverish maintainers!"))
+                exit(-108) # this is not i/o error but programming error
             st.n += 1
         # end while
 
@@ -406,7 +443,7 @@ class MsgReader(MsgDriver):
 
     async def open(self) -> None:
         if self.pull_subscription is not None:
-            raise RuntimeError("Reader already open, do not reuse MsgReader instances")
+            raise MessengerReaderAlreadyOpen("Reader already open, do not reuse MsgReader instances")
 
         log.debug(f"Opening {self}")
         js = self.connection.js
@@ -433,25 +470,52 @@ class MsgReader(MsgDriver):
     async def _create_pull_subscribtion(self, consumer_conf: ConsumerConfig):
         # Durable consumer is probably not needed (at least problematic)
         # consumer_conf.durable_name = self.name if consumer_conf.durable_name is None else consumer_conf.durable_name
-        ret = await self.connection.js.pull_subscribe(self.subject,
-                                                       durable=consumer_conf.durable_name,
-                                                       config=consumer_conf)
+        if self.connection is None or self.connection.js is None:
+            raise MessengerNotConnected('Cannot create pull subscription, not connected to NATS')
+        try:
+            ret = await self.connection.js.pull_subscribe(self.subject,
+                                                           durable=consumer_conf.durable_name,
+                                                           config=consumer_conf)
+        except TimeoutError as e:
+            raise MessengerRequestTimeout(e)
         ci = await ret.consumer_info()
         return ret
 
     async def _reopen(self) -> None:
-        try:
-            await self.pull_subscription.unsubscribe()
-        except Exception:
-            pass
-        consumer_conf = await self._create_consumer_cfg()
-        # set policy for 'new messages' for BY_START_SEQUENCE if any message was received
-        if self.last_seq is not None:
-            log.info(f"Reopening {self}, from seq={self.last_seq + 1}")
-            consumer_conf.deliver_policy = DeliverPolicy.BY_START_SEQUENCE
-            consumer_conf.opt_start_time = None
-            consumer_conf.opt_start_seq = self.last_seq + 1
-        self.pull_subscription = await self._create_pull_subscribtion(consumer_conf=consumer_conf)
+        """Reopens the pull subscription with retry logic and better error handling"""
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                # Clean up existing subscription
+                if self.pull_subscription is not None:
+                    try:
+                        await self.pull_subscription.unsubscribe()
+                    except Exception as e:
+                        log.debug(f"Error unsubscribing during reopen attempt {attempt + 1}: {e}")
+                
+                # Create new consumer configuration
+                consumer_conf = await self._create_consumer_cfg()
+                
+                # Set policy for 'new messages' if any message was received
+                if self.last_seq is not None:
+                    log.info(f"Reopening {self}, from seq={self.last_seq + 1}")
+                    consumer_conf.deliver_policy = DeliverPolicy.BY_START_SEQUENCE
+                    consumer_conf.opt_start_time = None
+                    consumer_conf.opt_start_seq = self.last_seq + 1
+                
+                # Create new subscription
+                self.pull_subscription = await self._create_pull_subscribtion(consumer_conf=consumer_conf)
+                log.info(f"Successfully reopened {self} after {attempt + 1} attempts")
+                return  # Success!
+                
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    log.error(f"Failed to reopen {self} after {max_retries} attempts: {e}")
+                    raise
+                
+                wait_time = (attempt + 1) * 2  # Exponential backoff: 2s, 4s, 6s
+                log.warning(f"Reopen attempt {attempt + 1} failed, retrying in {wait_time}s: {e}")
+                await asyncio.sleep(wait_time)
 
 
 
@@ -479,7 +543,7 @@ class MsgReader(MsgDriver):
         return consumer_conf
 
     async def close(self) -> None:
-        Messenger().connection.remove_reconnect_cb(self.on_nats_reconnect)
+        self.connection.remove_reconnect_cb(self.on_nats_reconnect)
         await super().close()
         await self._close_pull_subscription()
         await self._close_push_subscription()

{serverish-1.2.2 → serverish-1.4.0}/serverish/messenger/msg_rpc_resp.py

@@ -58,7 +58,7 @@ class MsgRpcResponder(MsgDriver):
             rpc.set_response(data={'c': c})
 
         responder = MsgRpcResponder(subject='subject')
-        responder.open()
+        await responder.open()
         try:
             await responder.register_function(callback)
             # ... wait for incoming messages
@@ -135,7 +135,7 @@ def get_rpcresponder(subject: str) -> 'MsgRpcResponder':
             rpc.set_response(data={'c': c})
 
         responder = MsgRpcResponder(subject='subject')
-        responder.open()
+        await responder.open()
         try:
             await responder.register_function(callback)
             # ... wait for incoming messages

{serverish-1.2.2 → serverish-1.4.0}/serverish/messenger/msg_single_read.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import logging
 import asyncio
 from asyncio import Event
+from time import time
 
 import nats.errors
 import param
@@ -13,6 +14,7 @@ from nats.js.api import DeliverPolicy, ConsumerConfig
 from serverish.messenger import Messenger
 from serverish.messenger.messenger import MsgDriver
 from serverish.messenger.msg_reader import MsgReader
+from serverish.base.exceptions import MessengerReaderAlreadyOpen
 
 log = logging.getLogger(__name__.rsplit('.')[-1])
 
@@ -37,7 +39,27 @@ class MsgSingleReader(MsgReader):
             tuple: data, meta
 
         """
-        await self.open()
+        interval = 0.1
+        max_delay = 5.0
+        # abolute utc timeout for waiting for data (now (utc) + wait):
+        timeout = time() + wait if wait is not None else None
+
+        while True:
+            try:
+                await self.open()
+                break
+            except MessengerReaderAlreadyOpen:
+                break
+            except Exception as e: # pragma: no cover
+                now = time()
+                if timeout is None or now > timeout:
+                    log.error(f"Failed to open single reader for subject {self.subject}: {e}")
+                    raise e
+                log.warning(f"Failed to open single reader for subject {self.subject}: {e}, retrying in {interval} seconds")
+                remaining = timeout - time() if timeout is not None else None
+                await asyncio.sleep(min(interval, remaining, max_delay) if remaining is not None else min(interval, max_delay))
+                interval *= 2
+
         try:
             data, meta = await asyncio.wait_for(self.read_next(), timeout=wait)
         finally: