serverish 2.1.0__tar.gz → 2.2.1__tar.gz

{serverish-2.1.0 → serverish-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: serverish
-Version: 2.1.0
+Version: 2.2.1
 Summary: helpers for server alike projects
 License-File: LICENSE
 Author: Mikołaj Kałuszyński
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: dns
 Provides-Extra: messenger
-Requires-Dist: aiodns (>=3.2.0,<4.0.0) ; extra == "dns"
+Requires-Dist: aiodns (>=3.2,<5) ; extra == "dns"
 Requires-Dist: jsonschema (>=4.20.0,<5.0.0)
 Requires-Dist: nats-py (>=2.7.2,<3.0.0) ; extra == "messenger"
 Requires-Dist: param (>=2.1.0,<3.0.0)

{serverish-2.1.0 → serverish-2.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "serverish"
-version = "2.1.0"
+version = "2.2.1"
 description = "helpers for server alike projects"
 authors = ["Mikołaj Kałuszyński", "MMME team"]
 readme = "README.md"
@@ -9,7 +9,7 @@ include = ["serverish/schema/*"]
 
 [tool.poetry.dependencies]
 python = "^3.10"
-aiodns = {version="^3.2.0", optional=true}
+aiodns = {version=">=3.2,<5", optional=true}
 param = "^2.1.0"
 nats-py = { version = "^2.7.2", optional= true}
 jsonschema = "^4.20.0"
@@ -20,12 +20,12 @@ dns = ["aiodns"]
 
 
 [tool.poetry.group.dev.dependencies]
-pytest = ">=8.2,<9"
-pytest-asyncio = ">=0.24,<1"
+pytest = ">=9.1,<10"
+pytest-asyncio = "^1.4"
 nats-py = "^2.6.0"
 jupyter = "*"
 rich = "^13.5.2"
-aiodns = "^3.1.1"
+aiodns = ">=3.1.1,<5"
 testcontainers = {extras = ["nats"], version = "^4.14.2"}
 pytest-timeout = "^2.4.0"
 

{serverish-2.1.0 → serverish-2.2.1}/serverish/base/exceptions.py

@@ -1,13 +1,24 @@
-class MessengerCannotConnect(Exception):
+class ServerishError(Exception):
+    """Common base for all serverish exceptions
+
+    Catch this to handle any error originating from the serverish library,
+    regardless of the subsystem (messenger, connection, KV). Concrete
+    exceptions additionally inherit fitting standard bases (ValueError,
+    TimeoutError, KeyError...), so idiomatic generic handlers keep working.
+    """
+    pass
+
+
+class MessengerCannotConnect(ServerishError):
     pass
 
-class MessengerNotConnected(Exception):
+class MessengerNotConnected(ServerishError):
     pass
 
-class MessengerReaderStopped(Exception):
+class MessengerReaderStopped(ServerishError):
     pass
 
-class MessengerReaderConfigError(ValueError):
+class MessengerReaderConfigError(ServerishError, ValueError):
     """Raised when a MsgReader is misconfigured (e.g. missing required start marker).
 
     This is a fatal error — the same configuration will never succeed against
@@ -16,10 +27,10 @@ class MessengerReaderConfigError(ValueError):
     """
     pass
 
-class MessengerReaderAlreadyOpen(RuntimeError):
+class MessengerReaderAlreadyOpen(ServerishError, RuntimeError):
     pass
 
-class MessengerRequestNoResponse(Exception):
+class MessengerRequestNoResponse(ServerishError):
     pass
 
 class MessengerRequestNoResponders(MessengerRequestNoResponse):
@@ -41,7 +52,19 @@ class MessengerRequestNoResultYet(MessengerRequestNoResponse):
     pass
 
 
-class MessengerKvError(Exception):
+class MessengerPublishAckTimeout(ServerishError, TimeoutError):
+    """Raised when JetStream did not confirm a publish within the ack timeout.
+
+    The message MAY have been delivered and stored — only the acknowledgement
+    did not arrive (or was not processed) in time. A starved client event loop
+    is the most common cause, not an actual delivery failure. Publishes carry
+    a ``Nats-Msg-Id`` header, so JetStream deduplicates retries of the same
+    message within the stream's duplicate window.
+    """
+    pass
+
+
+class MessengerKvError(ServerishError):
     pass
 
 
@@ -61,4 +84,4 @@ class MessengerKvMalformed(MessengerKvError):
     An entry written by a non-serverish client fails loudly instead of being
     silently returned as empty.
     """
-    pass
+    pass

{serverish-2.1.0 → serverish-2.2.1}/serverish/messenger/messenger.py

@@ -24,7 +24,7 @@ from nats.aio.msg import Msg
 from serverish.base import dt_utcnow_array, dt_from_array, Task, create_task, dt_ensure_array
 from serverish.base.collector import Collector
 from serverish.connection.connection_jets import ConnectionJetStream
-from serverish.base.idmanger import gen_id
+from serverish.base.idmanger import gen_id, gen_uid
 from serverish.base.manageable import Manageable
 from serverish.messenger.msgvalidator import MsgValidator
 from serverish.base.singleton import Singleton
@@ -51,6 +51,13 @@ class Messenger(Singleton):
     def __init__(self, name: str = None, parent: Collector = None, **kwargs) -> None:
         self.validator = MsgValidator()
         self.opener_task: Task | None = None
+        # Random per-process token, part of the Nats-Msg-Id dedup header.
+        # meta.id (gen_id) is only process-locally unique - two processes (or one
+        # process restarted within the stream's duplicate window) both emit
+        # 'msg-1', 'msg-2', ..., and JetStream would silently drop the later
+        # publishes as duplicates. The instance prefix makes the dedup identity
+        # global while retries still reuse the exact same header.
+        self.instance_id: str = gen_uid('mi', 12)
         super().__init__(name, parent, **kwargs)
 
     @property

{serverish-2.1.0 → serverish-2.2.1}/serverish/messenger/msg_publisher.py

@@ -7,7 +7,7 @@ import nats.errors
 import nats.js
 import param
 
-from serverish.base import MessengerNotConnected
+from serverish.base import MessengerNotConnected, MessengerPublishAckTimeout
 from serverish.messenger import Messenger
 from serverish.messenger.messenger import MsgDriver, log
 
@@ -18,8 +18,19 @@ class MsgPublisher(MsgDriver):
     Use this class if you want to publish data to a messenger subject.
     Check for specialist publishers for common use cases.
 
+    Every publish carries a ``Nats-Msg-Id`` header equal to the message
+    ``meta.id``, so JetStream deduplicates repeated deliveries of the same
+    message within the stream's duplicate window. This makes retrying after
+    an ack timeout safe: if the original publish did reach the stream (a
+    common case when the client event loop is starved and the ack is simply
+    processed too late), the retry is discarded by the server.
+
     Parameters:
         raise_on_publish_error (bool): Raise on publish error, default `True` re-raises underlying exceptions
+        ack_timeout_retries (int): How many times to retry a publish for which no JetStream
+            ack arrived in time (default 2). Retries are deduplicated server-side via
+            ``Nats-Msg-Id``. When retries are exhausted, `MessengerPublishAckTimeout` is
+            raised (subject to `raise_on_publish_error`).
 
     Health monitoring:
         The publisher tracks publish statistics accessible via `health_status` property:
@@ -30,6 +41,9 @@ class MsgPublisher(MsgDriver):
     """
 
     raise_on_publish_error = param.Boolean(default=True, doc="Raise on publish error")
+    ack_timeout_retries = param.Integer(default=2, bounds=(0, None),
+                                        doc="Retries when JetStream ack does not arrive in time; "
+                                            "safe thanks to Nats-Msg-Id deduplication")
 
     def __init__(self, **kwargs) -> None:
         # Health monitoring fields
@@ -60,8 +74,14 @@ class MsgPublisher(MsgDriver):
             log.error(f"Message {msg['meta']['id']} validation error: {e}")
             raise e
         self.messenger.log_msg_trace(msg['data'], msg['meta'], f"PUB to {self.subject}")
+        headers = dict(kwargs.pop('headers', None) or {})
+        # Server-side dedup id: makes ack-timeout retries safe. Must be globally
+        # unique, hence the per-process instance prefix - meta.id alone is only
+        # process-locally unique and would collide across processes/restarts,
+        # silently dropping messages within the stream's duplicate window.
+        headers.setdefault('Nats-Msg-Id', f"{self.messenger.instance_id}:{msg['meta']['id']}")
         try:
-            await self.connection.js.publish(self.subject, bdata, **kwargs)
+            await self._publish_with_ack_retries(bdata, headers, msg['meta']['id'], **kwargs)
             # Track successful publish
             self._publish_count += 1
             self._last_publish_time = time.monotonic()
@@ -90,6 +110,33 @@ class MsgPublisher(MsgDriver):
                 msg['meta']['status'] = str(e)
         return msg
 
+    async def _publish_with_ack_retries(self, bdata: bytes, headers: dict, msg_id: str, **kwargs) -> None:
+        """Publishes to JetStream, retrying when the ack does not arrive in time
+
+        A missing ack does not mean the message was not delivered - it may
+        well be stored while the ack was processed too late (e.g. starved
+        client event loop). Retries are deduplicated server-side via the
+        Nats-Msg-Id header, so this never produces duplicates.
+
+        Raises:
+            MessengerPublishAckTimeout: no ack after all retries
+        """
+        for attempt in range(self.ack_timeout_retries + 1):
+            try:
+                await self.connection.js.publish(self.subject, bdata, headers=headers, **kwargs)
+                return
+            except nats.errors.TimeoutError as e:
+                if attempt < self.ack_timeout_retries:
+                    log.warning(f"No JetStream ack for message {msg_id} on '{self.subject}' "
+                                f"(attempt {attempt + 1}/{self.ack_timeout_retries + 1}), retrying - "
+                                f"deduplicated by Nats-Msg-Id")
+                    continue
+                raise MessengerPublishAckTimeout(
+                    f"No JetStream ack for message {msg_id} on '{self.subject}' "
+                    f"after {self.ack_timeout_retries + 1} attempts. The message MAY have been "
+                    f"delivered - a missing ack often means a starved client event loop, "
+                    f"not a delivery failure") from e
+
     @property
     def health_status(self) -> dict:
         """Returns current health status of the publisher for monitoring

{serverish-2.1.0 → serverish-2.2.1}/serverish/messenger/msg_reader.py

@@ -86,6 +86,7 @@ class MsgReader(MsgDriver):
         }
         if consumer_cfg is not None:
             consumer_cfg_defaults.update(consumer_cfg)
+        # pull request size == read-ahead buffer bound; becomes BatchPolicy (docs/design/POLICIES.md)
         self.batch = 100
         self.messages = deque()
         self.pull_subscription: JetStreamContext.PullSubscription | None = None
@@ -435,12 +436,16 @@ class MsgReader(MsgDriver):
         Uses no_wait=True protocol:
         - Server sends available messages immediately
         - Server sends 404 status if no messages exist
-        - We wait for server response (network latency) but not for new messages to arrive
+
+        End-of-data is detected deterministically from the `num_pending`
+        counter the server stamps on every delivered message - the call
+        returns as soon as the server confirms there is nothing more (or the
+        batch is full), with no fixed waiting anywhere on the happy path.
 
         Args:
             batch: Maximum number of messages to fetch
-            timeout: Network latency budget (default 2s for local/fast networks)
-                     NOT a "wait for messages" timeout - server won't wait
+            timeout: Network-failure backstop (default 2s). Never consumed on
+                     the happy path - NOT a "wait for messages" timeout
 
         Returns:
             List of messages that were immediately available on server
@@ -477,78 +482,93 @@ class MsgReader(MsgDriver):
             pull_subscription._deliver,
         )
 
-        # Wait for server response (messages and/or 404 status)
-        # The server will respond quickly because no_wait=True
-        # We just need to account for network latency
+        # Wait for server response (messages and/or closing status).
+        #
+        # Deterministic end-of-data: every JetStream delivery is stamped with
+        # `num_pending` — the server-side count of messages still remaining
+        # for this consumer *after* that message (correct also for filtered
+        # and last_per_subject consumers). The loop therefore ends without
+        # any arbitrary grace period:
+        #   - a message arrives with num_pending == 0 → nothing left, return,
+        #   - batch filled while num_pending > 0 → return, the caller pulls again,
+        #   - 404 (empty subject) / 408 (partial-batch closure) → server-confirmed end.
+        # `timeout` is a network-failure backstop only; no happy path consumes
+        # it, so the call is as fast as the server and the network allow.
         start_time = time.monotonic()
-        got_404_status = False
-        got_any_message = False
-
-        while not got_404_status:
-            deadline = timeout - (time.monotonic() - start_time)
-            if deadline <= 0:
-                # Timeout - if we got some messages, that's OK (server might not send 404 after exact batch)
-                # If we got nothing, this is a network problem
-                if not got_any_message and len(msgs) == 0:
-                    log.warning(f"{self} Timeout after {timeout}s waiting for server response (network issue?)")
-                else:
-                    log.debug(f"{self} Timeout after getting {len(msgs)} messages (no 404 received)")
-                break
-
-            try:
-                # Wait for server to respond (messages or 404)
-                msg = await asyncio.wait_for(queue.get(), timeout=deadline)
-                pull_subscription._sub._pending_size -= len(msg.data)
-                queue.task_done()
 
-                status = JetStreamContext.is_status_msg(msg)
-
-                if status == "404":
-                    # Server says: no (more) messages available
-                    # This is the definitive "done" signal
-                    log.debug(f"{self} Server confirmed no more messages (404 status)")
-                    got_404_status = True
-                    break
-                elif status == "408":
-                    # Request Timeout - server gave us what it had but couldn't fill batch
-                    # This is effectively "no more messages available right now"
-                    log.debug(f"{self} Server returned partial batch (408 status - request timeout)")
-                    got_404_status = True  # Treat same as 404 - no more messages available
-                    break
-                elif status:
-                    # Other status message (shouldn't happen with pull, but handle it)
-                    log.debug(f"{self} Received status: {status}")
-                    continue
-                else:
-                    # Real message
-                    msgs.append(msg)
-                    got_any_message = True
-                    needed -= 1
+        def log_backstop():
+            if msgs:
+                log.warning(f"{self} No end-of-data signal within {timeout}s backstop, "
+                            f"returning {len(msgs)} messages (no num_pending metadata? network issue?)")
+            else:
+                log.warning(f"{self} Timeout after {timeout}s waiting for server response (network issue?)")
 
-                    # If we got all requested messages, give server very short time to send 404
-                    # but don't wait long - likely there are more messages
-                    if needed == 0:
-                        # Very short grace period for 404 status (100ms)
-                        # If 404 doesn't arrive, we'll just fetch again on next iteration
-                        remaining_time = timeout - (time.monotonic() - start_time)
-                        if remaining_time > 0.1:
-                            deadline = 0.1
-                            timeout = time.monotonic() - start_time + 0.1  # Update timeout to exit on next deadline check
+        while True:
+            remaining = timeout - (time.monotonic() - start_time)
+            if remaining <= 0:
+                log_backstop()
+                break
 
+            try:
+                msg = await asyncio.wait_for(queue.get(), timeout=remaining)
             except asyncio.TimeoutError:
-                # Timeout after getting messages is OK - server might not send 404 if we got exact batch
-                if got_any_message or len(msgs) > 0:
-                    log.debug(f"{self} Got {len(msgs)} messages, no 404 status (normal for exact batch)")
-                else:
-                    # No messages at all - this could be empty subject or network problem
-                    log.debug(f"{self} No messages or 404 within {timeout}s")
+                log_backstop()
                 break
             except Exception as e:
                 log.warning(f"{self} Error in fetch_available: {e}")
                 break
+            pull_subscription._sub._pending_size -= len(msg.data)
+            queue.task_done()
+
+            status = JetStreamContext.is_status_msg(msg)
+            if status == "404":
+                # Server says: no messages available at all
+                log.debug(f"{self} Server confirmed no messages (404 status)")
+                break
+            elif status == "408":
+                # Request Timeout - server closed the request after a partial batch
+                log.debug(f"{self} Server closed partial batch (408 status)")
+                break
+            elif status:
+                # Other status message (shouldn't happen with pull, but handle it)
+                log.debug(f"{self} Received status: {status}")
+                continue
+
+            # Real message
+            msgs.append(msg)
+            needed -= 1
+
+            num_pending = self._num_pending(msg)
+            if num_pending == 0:
+                # Server-confirmed end of data - no reason to wait for the
+                # closing status (which may trail by a network round trip,
+                # or never come for an exact-batch response)
+                log.debug(f"{self} Server reports num_pending=0, returning {len(msgs)} messages")
+                break
+            if needed == 0:
+                # Batch full and the server has more - return immediately,
+                # the caller decides whether to pull again
+                log.debug(f"{self} Batch of {len(msgs)} filled, "
+                          f"server reports {num_pending} message(s) still pending")
+                break
+            # num_pending unknown (no JS metadata): keep reading until a
+            # closing status arrives or the backstop hits (degraded mode)
 
         return msgs
 
+    @staticmethod
+    def _num_pending(msg: Msg) -> int | None:
+        """Returns server-side `num_pending` from JetStream message metadata
+
+        `num_pending` is the number of messages still remaining for this
+        consumer after the given message. Returns None if the metadata is
+        unavailable (non-JetStream message).
+        """
+        try:
+            return msg.metadata.num_pending
+        except Exception:
+            return None
+
     async def next_msg(self, timeout: Optional[float] = None) -> Msg:
         """ [NATS fix] Fetch the next message from the subscription
         :params timeout: Time in seconds to wait for next message before timing out.