log-foundry 0.10.2.dev37__tar.gz → 0.10.2.dev39__tar.gz

{log_foundry-0.10.2.dev37 → log_foundry-0.10.2.dev39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.10.2.dev37
+Version: 0.10.2.dev39
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE

{log_foundry-0.10.2.dev37 → log_foundry-0.10.2.dev39}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 ]
 
 # Optional features. Install with: pip install log-foundry[aws]
-version = "0.10.2.dev37"
+version = "0.10.2.dev39"
 
 [project.optional-dependencies]
 aws = ["boto3>=1.43.61"]              # SQSSink, SNSSink, KinesisSink, FirehoseSink

{log_foundry-0.10.2.dev37 → log_foundry-0.10.2.dev39}/src/log_foundry/decorator.py

@@ -350,6 +350,21 @@ def _offer_orphan_signal(sink: Sink) -> None:
     leave the drain thread serving a full backoff across ``Worker.shutdown``'s join, which is the
     global pause SPEC-027 exists to remove.
 
+    Ownership alone is not the whole predicate either, and this site is the one place in the
+    module where neither of the two categories fits (SPEC-035 FR-001). :func:`_live_worker` was
+    wrong: ``retired`` latches on **entry** to ``Worker.shutdown``, so for the whole of the drain
+    the skip stopped applying and an orphan log handed the sink a fresh unset event — precisely
+    the one the drain thread was about to wait on. Measured, a 20 s backoff against
+    ``shutdown(timeout=3)`` was then still outstanding when the shutdown expired at 3.01 s with
+    the sink left open — the wait was never cut, not merely slow. But bare ``_worker.sink is
+    sink`` is wrong in the opposite direction: it skips for a worker whose shutdown has
+    **finished**, leaving a sink still being written to holding a set event that can never clear,
+    which is SPEC-033 FR-004's tight retry loop and is covered by a test that spec shipped. The
+    predicate is therefore the ownership term **conjoined with** ``Worker.draining`` — the
+    moment as well as the identity, since without the identity an orphan log to sink Y would be
+    skipped merely because a live worker is draining into sink X. That property's docstring
+    carries the rest of the reasoning.
+
     A fresh event replaces one that is already set, because an ``Event`` never clears and
     ``sinks/_retry.wait`` returns immediately on a set one — a sink handed the shutdown's event
     would have every subsequent backoff collapsed to zero, which against a rate-limited
@@ -368,8 +383,8 @@ def _offer_orphan_signal(sink: Sink) -> None:
       None.
     """
     global _orphan_stop
-    worker = _live_worker()
-    if worker is not None and worker.sink is sink:
+    worker = _worker
+    if worker is not None and worker.sink is sink and worker.draining:
         return
     if _orphan_stop.is_set():
         _orphan_stop = threading.Event()
@@ -506,7 +521,10 @@ def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) ->
     ``sinks/base.py`` requires ``close()`` to tolerate a concurrent ``emit`` (SPEC-028 FR-001).
     This inherits that contract rather than weakening it.
 
-    **Who performs the swap and who owns the old sink's close are two questions.** The first is
+    **Who performs the swap, who owns the old sink's close, and who owns the new one when the
+    swap is declined are three questions** — the third added by SPEC-035 FR-003 and answered by
+    :func:`_adopt_declined_swap`, off ``Worker.swap_sink``'s return value rather than a predicate
+    here, because only the worker knows whether it got as far as reassigning. The first is
     liveness — a retired worker performs nothing, since :meth:`Worker.swap_sink` returns early
     once shut down, so routing the swap to it loses the handoff entirely. The second is
     ownership, exactly as in :func:`_close_orphan_sink`: a worker that *holds* ``old`` has either
@@ -559,16 +577,65 @@ def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) ->
                 closer = _lifecycle.close_detached(old)
     if worker is not None:
         try:
-            worker.swap_sink(new_sink, timeout)
+            adopted = worker.swap_sink(new_sink, timeout)
         except Exception as exc:
             _diag.absorbed(
                 "swapping the log sink", exc, "events may still be delivered to the previous sink"
             )
+            return
+        if not adopted:
+            _adopt_declined_swap(new_sink)
         return
     if closer is not None:
         closer.join(timeout)
 
 
+def _adopt_declined_swap(new_sink: Sink) -> None:
+    """Takes ownership of a sink a worker refused mid-swap (SPEC-035 FR-003).
+
+    ``Worker.swap_sink`` re-checks retirement after its first ``flush()`` and returns early once
+    ``_shutdown_done`` latched, but :func:`_swap_sink` had already cleared ``_orphan_sink`` on the
+    strength of a worker being live a few instructions earlier. The new sink then sat in the
+    config, installed nowhere and recorded nowhere: measured, ``config.sink is B`` was ``True``,
+    ``B`` was never closed, and ``health()`` read entirely clean — for a sink whose ``close()``
+    *is* its delivery, a ``KafkaSink`` flushing its producer, a silently lost buffer.
+
+    Only the new sink is re-homed. ``Worker.swap_sink`` declines before reassigning anything, so
+    whatever the worker held it still holds, and the worker closes it through its own
+    ``_close_if_owed``; closing it here would be the double-close :func:`_close_orphan_sink`
+    exists to avoid. The one case with no "old" sink at all is a retired worker that already
+    holds ``new_sink`` — the entry check tests ``_shutdown_done`` before identity, so it declines
+    a swap that was already satisfied — and there the same ownership guard makes the worker
+    perform the single close.
+
+    **A sink already recorded as closed is refused re-arming**, the guard
+    :func:`_note_orphan_emit` carries and for its reason. Without it, an orphan log arming
+    ``_orphan_sink`` while this thread is inside ``swap_sink``'s first drain, followed by a
+    ``shutdown()`` that closes it, lets this re-arm a closed sink for a second ``close()`` at
+    exit — reproduced, and ``Sink.close`` promises no idempotency.
+
+    ``incomplete_swaps`` is deliberately not moved. It counts an unconfirmed *drain*, and this
+    swap had no drain to confirm — the worker declined before reassigning anything — so counting
+    it would stop telling an operator whether events were misrouted or a close was merely slow.
+
+    Args:
+      new_sink: The sink the worker refused to adopt.
+
+    Returns:
+      None.
+
+    Raises:
+      None.
+    """
+    global _orphan_sink
+    with _worker_lock:
+        _offer_orphan_signal(new_sink)
+        if new_sink is _orphan_sink or new_sink is _orphan_closed_sink:
+            return
+        _register_exit_handler()
+        _orphan_sink = new_sink
+
+
 def _flush_worker(timeout: float | None = 5.0) -> bool:
     """Drains the process worker without retiring it, backing ``flush()`` (SPEC-013 FR-003).
 

{log_foundry-0.10.2.dev37 → log_foundry-0.10.2.dev39}/src/log_foundry/worker.py

@@ -226,6 +226,7 @@ class Worker:
         self._queue: queue.Queue[object] = queue.Queue(maxsize=max_queue)
         self._stop = threading.Event()
         self._drain_finished = threading.Event()
+        self._drain_settled = threading.Event()
         self._shutdown_done = False
         self._sink_closed = False
         self._lock = threading.Lock()
@@ -351,6 +352,54 @@ class Worker:
         """
         return self._shutdown_done
 
+    @property
+    def draining(self) -> bool:
+        """Whether this worker's own ``_stop`` is still the sink's route to a cut-short backoff.
+
+        Neither ``retired`` nor a bare ``_worker is not None`` answers this, and the two specs
+        that got it wrong disagreed in opposite directions (SPEC-035 FR-001). SPEC-033 requires a
+        sink still being written to *after* a shutdown to be handed a **fresh** event, because
+        ``_stop`` is set forever and an ``Event`` never clears, so every later backoff would
+        collapse to zero. SPEC-035 requires a sink whose drain is **in flight** to keep the event
+        that drain is about to wait on, or the shutdown serves a full backoff and expires. Both
+        are the same sink held by the same worker; only the moment differs, so the predicate has
+        to be the moment.
+
+        An abandoned drain counts as not draining. The thread is wedged and the shutdown has
+        already given up on it (``_close_orphan_sink`` and SPEC-027 FR-004 leave the sink open
+        for that reason), so nothing further will cut its backoff — where SPEC-033's tight retry
+        loop would go on costing the still-running application every emit.
+
+        It reads **one** event, ``_drain_settled``, set both where the loop stops and where a
+        shutdown gives up on it — not a conjunction of the two facts. A reader that tested them
+        separately would be correct only at the instant it looked: :meth:`shutdown`'s idempotent
+        path evaluates this and *then* waits, so an abandonment landing in between has to release
+        that wait, and only an event already being waited on can do that. Measured before the
+        change, with the first caller expiring while the second was inside its wait: a second
+        ``shutdown(timeout=20)`` returned after 20.01 s, and with ``timeout=None`` the process
+        never exited at all.
+
+        ``_drain_finished`` is deliberately left alone rather than widened: :meth:`flush` and
+        :meth:`shutdown`'s sentinel gate both read it as "the loop stopped reading the queue",
+        which an abandoned — and still running — drain has not done. The two events stay
+        distinguishable without a third flag, which is why no ``_drain_abandoned`` is kept:
+        abandoned is ``_drain_settled`` set with ``_drain_finished`` unset, and the operator-facing
+        form of the same fact is ``stopped_reason == "ShutdownTimeout"``.
+
+        Read without the lock, as ``retired`` is: the event is set once and never cleared, so a
+        racing reader sees one of two answers and both are momentarily true.
+
+        Args:
+          None.
+
+        Returns:
+          Whether the drain loop is still running and has not been abandoned.
+
+        Raises:
+          None.
+        """
+        return not self._drain_settled.is_set()
+
     def health(self) -> Health:
         """Snapshots the delivery counters (SPEC-017 FR-005, SPEC-019 FR-003).
 
@@ -476,7 +525,7 @@ class Worker:
             return False
         return marker.delivered
 
-    def swap_sink(self, new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> None:
+    def swap_sink(self, new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> bool:
         """Retargets delivery at a new sink, draining and closing the previous one (FR-003).
 
         This is what makes a late ``configure(sink=...)`` mean what it says. The sink was
@@ -515,33 +564,41 @@ class Worker:
             hold ``configure()`` for a multiple of the budget. ``None`` waits indefinitely.
 
         Returns:
-          None. The outcome is reported through ``health().incomplete_swaps`` and one stderr
-          line rather than a return value, because the caller is ``configure()``, which has
-          never had one and whose callers do not check.
+          Whether this worker now holds ``new_sink``, and therefore owns its close. ``False``
+          means **declined**, which happens only when ``_shutdown_done`` latched — and the
+          caller must then own the handoff itself, because a declined swap leaves the new sink
+          installed nowhere (SPEC-035 FR-003). ``True`` covers the sink already being this
+          worker's, which owes nothing further. The *quality* of the swap is still reported
+          through ``health().incomplete_swaps`` and one stderr line rather than here: an
+          unconfirmed drain is a swap that happened, so it returns ``True``, and conflating the
+          two would make ``_swap_sink`` re-home a sink this worker is delivering to.
 
         Raises:
           None on a sink fault. A close that fails is announced, as everywhere else
           (SPEC-025 FR-004).
         """
         with self._lock:
-            if self._shutdown_done or self.sink is new_sink:
-                return
+            if self._shutdown_done:
+                return False
+            if self.sink is new_sink:
+                return True
         deadline = None if timeout is None else time.monotonic() + timeout
         drained = self.flush(timeout)
         with self._lock:
             if self._shutdown_done:
-                return
+                return False
             old = self.sink
             if old is new_sink:
-                return
+                return True
             self.sink = new_sink
         self._offer_stop_signal()
         remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
         if not (drained and self.flush(remaining)):
             self._record_incomplete_swap(timeout)
-            return
+            return True
         left = None if deadline is None else max(0.0, deadline - time.monotonic())
         self._close_swapped_out(old, left)
+        return True
 
     def _record_incomplete_swap(self, timeout: float | None) -> None:
         """Counts a swap whose drain could not be confirmed, then announces it (FR-003).
@@ -663,6 +720,19 @@ class Worker:
         liveness test would queue a sentinel through that whole window. The flag is set before
         that call, so this one does not.
 
+        **The idempotent path waits for the drain it found running** (SPEC-035 FR-004).
+        ``_shutdown_done`` latches on *entry*, so a second caller used to take that branch and
+        return in under a millisecond over a drain that had barely started. The shape is rarely
+        two user calls: it is a ``shutdown()`` on one thread and ``atexit`` on the main thread,
+        and measured with a 2 s-emit sink and three traced calls it delivered **nothing**, never
+        closed the sink, and left the process gone in 0.39 s with nothing on stderr. The
+        wait is on ``_drain_settled`` and bounded by *this* call's own budget, so
+        ``shutdown(timeout=0)`` still returns promptly and a caller never inherits the other
+        call's deadline. It is skipped where :attr:`draining` is already false, and **released**
+        where the drain is abandoned after the wait began — the first caller sets the same event
+        when its join expires, which is what stops a wedged thread costing the second caller its
+        whole budget in front of an exit that has to happen.
+
         :meth:`_release_waiters` runs on the way out for the sibling case the ordering cannot
         reach: a ``flush()`` that passed its liveness check microseconds before the thread
         finished can still queue a marker nothing will answer, and with ``timeout=None`` that
@@ -691,6 +761,10 @@ class Worker:
             first = not self._shutdown_done
             self._shutdown_done = True
         if not first:
+            if self.draining:
+                self._drain_settled.wait(
+                    None if deadline is None else max(0.0, deadline - time.monotonic())
+                )
             self._close_if_owed()
             self._join_closers(None if deadline is None else max(0.0, deadline - time.monotonic()))
             return
@@ -706,6 +780,7 @@ class Worker:
             with self._lock:
                 if self.stopped_reason is None:
                     self.stopped_reason = "ShutdownTimeout"
+            self._drain_settled.set()
             _diag.lost(
                 "item",
                 queued,
@@ -879,6 +954,7 @@ class Worker:
                 self._drain(pending)
             finally:
                 self._drain_finished.set()
+                self._drain_settled.set()
         except BaseException as exc:
             self._terminal_failure(exc, len(pending))
         finally: