serverish 1.2.0__tar.gz → 1.2.1__tar.gz

{serverish-1.2.0 → serverish-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: serverish
-Version: 1.2.0
+Version: 1.2.1
 Summary: helpers for server alike projects
 Home-page: https://github.com/AkondLab/serverish
 Author: Mikołaj Kałuszyński

{serverish-1.2.0 → serverish-1.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "serverish"
-version = "1.2.0"
+version = "1.2.1"
 description = "helpers for server alike projects"
 authors = ["Mikołaj Kałuszyński", "MMME team"]
 readme = "README.md"

{serverish-1.2.0 → serverish-1.2.1}/serverish/messenger/messenger.py

@@ -338,7 +338,7 @@ class Messenger(Singleton):
         Args:
             subject (str): subject to subscribe to
             deliver_policy (str): deliver policy, one of 'all', 'last', 'new', 'by_start_time', will be passed to consumer config
-            opt_start_time (datetime): start time for 'by_start_time' deliver policy, will be passed to consumer config
+            opt_start_time (datetime): start time for 'by_start_time' deliver policy, will be passed to consumer config, must carry TZ info
             kwargs: additional arguments to pass to the reader and underlying NATS consumer config
 
         Returns:

{serverish-1.2.0 → serverish-1.2.1}/serverish/messenger/msg_reader.py

@@ -4,10 +4,12 @@ import logging
 import asyncio
 from asyncio import Event
 from collections import deque
-from datetime import datetime
+from datetime import datetime, timezone
 import time
 from dataclasses import dataclass, field
 import functools
+from uuid import uuid4
+
 
 import nats.errors
 import param
@@ -153,6 +155,14 @@ class MsgReader(MsgDriver):
                     except Exception as e:
                         log.warning(self.fmt(f"Error acking message: {e}"))
                     data, meta = self.reader.messenger.unpack_nats_msg(bmsg)
+                    try:
+                        if self.reader.last_seq is not None and meta['nats']['seq'] <= self.reader.last_seq: # dupes can happen on reopen
+                            log.info(self.fmt(f"Skipping duplicated message seq={meta['nats']['seq']}, last_seq={self.reader.last_seq}"))
+                            raise self.ContinueException('pop')
+                    except self.ContinueException:
+                        raise
+                    except Exception as e:
+                        log.warning(self.fmt(f"Error checking seq: {e}"))
                     meta['receive_mode'] = 'pull'
                     try: # nonessential
                         self.reader.messenger.log_msg_trace(data, meta, f"SUB PULL iteration from {self.reader.subject}")
@@ -161,6 +171,7 @@ class MsgReader(MsgDriver):
                             self.reader._emptied.set()
                     except Exception as e:
                         log.warning(self.fmt(f"Error unpacking message: {e}"))
+                    # log.info(self.fmt(f"Returning message seq={meta['nats']['seq']}, last_seq={self.reader.last_seq}"))
                     raise  self.ReturnException('pop', data, meta)
             @async_shield
             async def ensure_open(self) -> None:
@@ -192,12 +203,29 @@ class MsgReader(MsgDriver):
             @async_shield
             async def read_batch(self) -> None:
                 if len(self.reader.messages) == 0:
-                    timeout = min(0.1 + self.n/10.0, 5.0)
+                    timeout = 100.0 # min(0.1 + self.n/10.0, 5.0)
                     # batch = 1 if self.error is not None else 100 # recover slowly
-                    log.debug(self.fmt(f"Pulling {self.reader.batch} messages with timeout {timeout}s"))
+                    log.info(self.fmt(f"Pulling {self.reader.batch} messages with timeout {timeout}s"))
                     new_msgs = await self.reader.fetch_available(batch=self.reader.batch, timeout=timeout)
-                    log.debug(self.fmt(f"Pulled {len(new_msgs)} messages"))
+                    log.info(self.fmt(f"Pulled {len(new_msgs)} messages"))
+
+                    # If no messages were available immediately, switch to blocking mode
+                    # to wait for at least one message more efficiently
+                    if len(new_msgs) == 0:
+                        log.info(self.fmt(f"No messages available immediately, waiting for at least one message with timeout {timeout:0.1f}s"))
+
+                        # Use the regular fetch operation from nats (which blocks)
+                        try:
+                            new_msgs = await self.reader.pull_subscription.fetch(1, timeout=timeout)
+                            log.info(self.fmt(f"Received {len(new_msgs)} message while blocking waiting"))
+                        except asyncio.TimeoutError:
+                            log.info(self.fmt(f"No meesge before timeout"))
+                        except Exception as e:
+                            log.warning(self.fmt(f"Error waiting for messages: {e}"))
+                    # fi : no messages available immediately
+
                     self.reader.messages.extend(new_msgs)
+
                     raise self.ContinueException('read')
 
             def fmt(self, msg: str) -> str:
@@ -263,7 +291,7 @@ class MsgReader(MsgDriver):
 
 
     async def fetch_available(self, batch=10, timeout = 0.1):
-        """Fetch only immediately available messages without blocking
+        """[NATS fix] Fetch only immediately available messages without blocking
 
         Path on lack of functionality of JetStreamContext.PullSubscription.fetch
         By default uses only very short timeout to account for network latency
@@ -309,7 +337,7 @@ class MsgReader(MsgDriver):
                 break
 
             try:
-                msg = await pull_subscription._sub.next_msg(timeout=deadline)
+                msg = await self.next_msg(timeout=0.0) # nonblocking here
                 status = JetStreamContext.is_status_msg(msg)
                 if not status:
                     msgs.append(msg)
@@ -318,9 +346,64 @@ class MsgReader(MsgDriver):
                     break
             except asyncio.TimeoutError:
                 break
+            except asyncio.QueueEmpty:
+                log.info(f"{self} Queue is empty")
+                break
 
         return msgs
 
+    async def next_msg(self, timeout: Optional[float] = None) -> Msg:
+        """ [NATS fix] Fetch the next message from the subscription
+        :params timeout: Time in seconds to wait for next message before timing out.
+                        supports 0 for non-blocking and None for blocking indefinitely.
+        :raises nats.errors.TimeoutError:
+
+        next_msg can be used to retrieve the next message from a stream of messages using
+        await syntax, this only works when not passing a callback on `subscribe`::
+
+            sub = await nc.subscribe('hello')
+            msg = await sub.next_msg(timeout=1)
+
+        """
+        ps = self.pull_subscription._sub
+        if ps._conn.is_closed:
+            raise errors.ConnectionClosedError
+
+        if ps._cb:
+            raise errors.Error(
+                'nats: next_msg cannot be used in async subscriptions'
+            )
+
+        task_name = str(uuid4())
+        try:
+            if timeout == 0.0:
+                future = asyncio.create_task(
+                    ps._pending_queue.get_nowait()
+                )
+            else:
+                future = asyncio.create_task(
+                    asyncio.wait_for(ps._pending_queue.get(), timeout)
+                )
+            ps._pending_next_msgs_calls[task_name] = future
+            msg = await future
+        except asyncio.TimeoutError:
+            if ps._conn.is_closed:
+                raise errors.ConnectionClosedError
+            raise errors.TimeoutError
+        except asyncio.CancelledError:
+            if ps._conn.is_closed:
+                raise errors.ConnectionClosedError
+            raise
+        else:
+            ps._pending_size -= len(msg.data)
+            # For sync subscriptions we will consider a message
+            # to be done once it has been consumed by the client
+            # regardless of whether it has been processed.
+            ps._pending_queue.task_done()
+            return msg
+        finally:
+            ps._pending_next_msgs_calls.pop(task_name, None)
+
     async def open(self) -> None:
         if self.pull_subscription is not None:
             raise RuntimeError("Reader already open, do not reuse MsgReader instances")
@@ -364,6 +447,7 @@ class MsgReader(MsgDriver):
         consumer_conf = await self._create_consumer_cfg()
         # set policy for 'new messages' for BY_START_SEQUENCE if any message was received
         if self.last_seq is not None:
+            log.info(f"Reopening {self}, from seq={self.last_seq + 1}")
             consumer_conf.deliver_policy = DeliverPolicy.BY_START_SEQUENCE
             consumer_conf.opt_start_time = None
             consumer_conf.opt_start_seq = self.last_seq + 1
@@ -371,52 +455,6 @@ class MsgReader(MsgDriver):
 
 
 
-    async def _transform_to_pull_sub(self):
-        log.info(f"Attempt to transform {self.subject} MeassageReader mode to PULL (from seq: {self.last_seq})")
-        try:
-            await self.push_subscription.unsubscribe()
-        except (nats.errors.ConnectionClosedError, AttributeError):  # no connection, no reason to break
-            pass
-        self.push_subscription = None
-        self._emptied.clear()
-        consumer_conf = await self._create_consumer_cfg()
-        # set policy for 'new messages' for push subscription
-        if self.last_seq is not None:
-            consumer_conf.deliver_policy = DeliverPolicy.BY_START_SEQUENCE
-            consumer_conf.opt_start_time = None
-            consumer_conf.opt_start_seq = self.last_seq + 1
-        else:
-            consumer_conf.deliver_policy = DeliverPolicy.NEW
-            consumer_conf.opt_start_time = None
-            consumer_conf.opt_start_seq = None
-        self.pull_subscription = await self._create_pull_subscribtion(consumer_conf=consumer_conf)
-        log.info(f"{self}:  PULL subscription set up "
-                 f"(policy={consumer_conf.deliver_policy}, seq={consumer_conf.opt_start_seq})")
-
-
-
-    async def _transform_to_push_sub(self):
-        """After fetching existing messages, switch to push based consumer"""
-        log.info(f"Attempt to transform {self.subject} MeassageReader mode to PUSH for future messages")
-        pull = self.pull_subscription
-        if pull is None: # no transition needed or not possible
-            log.debug(f"No pull - no transform {self}")
-            return
-
-        consumer_conf = await self._create_consumer_cfg()
-        # set policy for 'new messages' for push subscription
-        consumer_conf.deliver_policy = DeliverPolicy.NEW
-        consumer_conf.opt_start_time = None
-        consumer_conf.opt_start_seq = None
-        js = self.connection.js
-
-        push: JetStreamContext.PushSubscription = await js.subscribe(
-            self.subject, config=consumer_conf
-        )
-        self.push_subscription = push
-        log.info(f"{self}:  PUSH subscription set up")
-        # self._push.set()
-
     async def _create_consumer_cfg(self) -> ConsumerConfig:
         cfg = self.consumer_cfg.copy()
         # Convert the delivery policy from a string to the appropriate DeliverPolicy enum
@@ -427,8 +465,14 @@ class MsgReader(MsgDriver):
             if isinstance(self.opt_start_time, str):
                 cfg['opt_start_time'] = self.opt_start_time
             else:
-                # TODO: Check if it is proper format for NATS!
-                cfg['opt_start_time'] = self.opt_start_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+                # Check if timezone is set, if not warn
+                if self.opt_start_time.tzinfo is None:
+                    log.warning(f"opt_start_time should have timezone information, converting to UTC: "
+                                f"{self.opt_start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')}"
+                                f"=>{self.opt_start_time.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')}"
+                                f" , use e.g. `datetime.now(tz=timezone.utc)`")
+
+                cfg['opt_start_time'] = self.opt_start_time.astimezone(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
 
         # Create the pull consumer configuration
         consumer_conf = ConsumerConfig(**cfg)