log-foundry 0.10.2.dev102__tar.gz → 0.10.2.dev106__tar.gz

{log_foundry-0.10.2.dev102 → log_foundry-0.10.2.dev106}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.10.2.dev102
+Version: 0.10.2.dev106
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE

{log_foundry-0.10.2.dev102 → log_foundry-0.10.2.dev106}/pyproject.toml

@@ -74,7 +74,7 @@ keywords = [
 # vulnerability-reporting channel. The repository is still named `log-forge` — the ORIGINAL name,
 # which PyPI rejected for the distribution — so these URLs deliberately do not match the package
 # name. See the note on `name` above before "correcting" them.
-version = "0.10.2.dev102"
+version = "0.10.2.dev106"
 
 [project.urls]
 Homepage = "https://github.com/agriffi10/log-forge"

{log_foundry-0.10.2.dev102 → log_foundry-0.10.2.dev106}/src/log_foundry/_lifecycle.py

@@ -368,6 +368,56 @@ nested with either of the other two, so there is no cycle. It is held only acros
 membership test or a single mutation, never across a ``close()``.
 """
 
+_orphan_closing = 0
+"""How many orphan closes are running right now, written under ``_state._lock`` (SPEC-050 FR-002).
+
+The orphan path's half of the C3 residue :meth:`~log_foundry.worker.Worker._close_if_owed` fixes
+for the worker: :func:`_close_orphan_sink` empties ``_orphan_owed`` under that lock and *then*
+closes, so a second caller found nothing owed and returned instantly — and an ``atexit`` call that
+returns while a background thread is still inside a close-is-delivery sink's ``close()`` exits
+through it and kills it.
+
+**A count and a gate, not one close's event.** A single slot was tried and is wrong, because the
+orphan close is not once-only: ``_orphan_owed`` is repopulated by :func:`_note_orphan_emit` and
+:func:`_adopt_declined_swap`, so a second close overwrote the first's event and its own completion
+then cleared the slot — a bystander arriving after that read nothing, waited for nothing, and the
+interpreter exited through the *first* close, still running. Measured against a one-second close:
+the bystander waited 1.005 s alone and 0.000 s with a second close completing in between, losing
+the first sink's whole buffer. It is the correction SPEC-045 made to the owed-close *record*,
+arriving here for the same reason.
+
+The predicate a bystander needs is "no orphan close is in flight", so that is what is published:
+:data:`_orphan_idle` is cleared while the count is non-zero and set when it returns to zero. One
+``Event``, built once at module scope, which is also what keeps it where ``_fork``'s repair walk
+can replace it.
+
+**A leaked count is permanent, so the increment is inside the ``try`` and the decrement is
+clamped.** Two ways it leaked, both reproduced: a ``KeyboardInterrupt`` delivered at a bytecode
+boundary between the increment and the ``try`` — measured leaking once in a few hundred iterations
+under a real ``SIGINT`` storm — and a ``fork()`` from *inside* the inline close, where the child's
+handler zeroes the count and the forking thread's own ``finally`` then takes it to ``-1``, which
+``if not _orphan_closing`` never satisfies again. Either leaves the gate clear forever and every
+later caller paying the whole grace, in a process that survives rather than exits. The count is
+therefore taken and released under one ``try``, guarded by a flag set in the same critical section
+as the increment, and floored at zero.
+
+**The gate is process-wide, not per-sink**, so a worker-path ``shutdown()`` can pay the grace for
+an orphan close it has nothing to do with — measured at 2.007 s wall, 0.000 s CPU, with the live
+sink still drained and closed. That is the price of not exiting through a running close, and it is
+bounded by the grace either way.
+"""
+
+_orphan_idle = threading.Event()
+"""Set exactly when :data:`_orphan_closing` is zero — what a bystander waits on (SPEC-050 FR-002).
+
+Starts set: a process with no orphan close in flight must not make the first bystander wait.
+Not in :data:`_FORK_SKIP` — an ``Event`` is what ``_fork``'s repair walk exists to replace. What
+the walk cannot know is that a child is idle whatever the parent was doing, since the threads that
+would have finished those closes did not survive the fork; :func:`_clear_closing_after_fork` says
+so.
+"""
+_orphan_idle.set()
+
 
 def _closing(sink: Sink) -> bool:
     """Whether a release of this sink is in flight on some thread right now (FR-003).
@@ -402,6 +452,12 @@ def _clear_closing_after_fork() -> None:
     the placement is free rather than load-bearing: the registry holds ``int`` ids and no handler
     reads it.
 
+    :data:`_orphan_closing` is zeroed here for the same reason and by the same argument
+    (SPEC-050 FR-002). It counts closes running on threads that did not survive the fork, so a
+    child inheriting a non-zero count would make its next bystander wait out the whole closer
+    grace for closes that can never finish. The fork walk replaces the ``Event`` but not the count
+    that keeps it clear, which is the distinction this handler exists for.
+
     Args:
       None.
 
@@ -411,8 +467,12 @@ def _clear_closing_after_fork() -> None:
     Raises:
       None.
     """
+    global _orphan_closing
     with _closing_now_lock:
         _closing_now.clear()
+    with _state._lock:
+        _orphan_closing = 0
+        _orphan_idle.set()
 
 
 _FORK_SKIP = ("_owned",)
@@ -1451,7 +1511,62 @@ def _inline_close_choice(owed: list[Sink]) -> Sink:
     return owed[-1]
 
 
-def _close_orphan_sink() -> None:
+def _bystander_grace(deadline: float | None) -> float:
+    """Returns how long a caller may wait on a close another caller is performing (SPEC-050 FR-002).
+
+    ``join_closers``'s arithmetic, and :func:`~log_foundry.worker._closer_grace` is its twin on the
+    worker path: capped at :data:`DEFAULT_CLOSER_GRACE` because this is an exit waiting on a close
+    it does not own, and carved from the caller's own budget so a ``shutdown(timeout=0)`` does not
+    inherit somebody else's. The flat cap this replaced made the two paths disagree — the worker
+    half returned in under half a second on a ``timeout=0`` call while this one took the whole two
+    seconds.
+
+    Args:
+      deadline: The calling ``shutdown``'s monotonic deadline, or ``None`` for an unbounded caller,
+        which takes the cap rather than waiting indefinitely.
+
+    Returns:
+      Seconds to wait, never negative and never above the cap.
+
+    Raises:
+      None.
+    """
+    if deadline is None:
+        return DEFAULT_CLOSER_GRACE
+    return max(0.0, min(DEFAULT_CLOSER_GRACE, deadline - monotonic()))
+
+
+def discharge_owed(sink: Sink) -> None:
+    """Records that a sink's owed close is being performed elsewhere (SPEC-050 FR-004).
+
+    The orphan record and :attr:`~log_foundry.worker.Worker._unclosed_swaps` can name the **same**
+    sink: an unconfirmed swap strands it in the worker's record, and an orphan emit that resolved
+    it before the swap and resumed after the re-arm slot had moved on puts it back in
+    ``_orphan_owed``. Both then close it — measured ``A.closes == 2`` with a preemption point at
+    ``_ensure_sink``, which is SPEC-044 FR-004's shape at a record that did not exist then.
+
+    Called with ``_state._lock`` held, by the worker taking its own record under it, so the take
+    and the discharge are one critical section: split, a concurrent ``_close_orphan_sink`` can read
+    the sink out of ``_orphan_owed`` in the gap and close it alongside.
+
+    ``_orphan_closed_sink`` is latched as well as the entry removed, for the reason
+    :func:`_get_worker` latches it: removal stops *this* close being performed twice, and the latch
+    stops a later orphan emit re-arming a sink whose close is already under way.
+
+    Args:
+      sink: The sink whose close the caller is about to perform.
+
+    Returns:
+      None.
+
+    Raises:
+      None.
+    """
+    _state._orphan_owed.pop(id(sink), None)
+    _state._orphan_closed_sink = sink
+
+
+def _close_orphan_sink(deadline: float | None = None) -> None:
     """Closes a sink only the orphan path ever wrote to, once (SPEC-031 FR-006).
 
     A process that never opens a span builds no worker, so nothing owned the sink's close and
@@ -1459,6 +1574,33 @@ def _close_orphan_sink() -> None:
     on a synchronous one the flush and the resource were lost, and ``health()`` read all-clear
     because every field it carries describes a worker that does not exist.
 
+    **A caller that finds nothing owed waits for the one that is closing** (SPEC-050 FR-002).
+    The record is emptied under ``_state._lock`` before any close begins, so a second caller took
+    the early return below while the first was still inside an unbounded ``close()`` — and where
+    that first caller is a background thread and the second is ``atexit``, the interpreter exits
+    through a running close and kills it. For a sink whose ``close()`` *is* the delivery that is
+    total loss of its buffer.
+
+    **A separate caller cannot wait on itself, and the guard for that is the `if owed:` on the
+    write.** (A sink whose own ``close()`` calls ``shutdown()`` re-enters on the closing thread and
+    does wait out its grace — bounded, no deadlock, and pathological usage rather than a case this
+    guards.)
+    :data:`_orphan_closing` is installed only where ``owed`` is non-empty, so a caller that takes
+    the work never reaches the read below and a bystander never installs anything. Capturing
+    ``waiting`` before the write is *not* what makes this safe — both happen under
+    ``_state._lock``, so reading the global there and reading the captured value are the same
+    read, and mutating one into the other is an equivalent mutant. The conditional is the live
+    guard: made unconditional, a caller with nothing owed leaves a permanently unset event behind
+    it, and every call after the next one pays the whole grace on an event nothing will set —
+    measured at the *third* successive orphan ``shutdown()``, which is why a test doing two of
+    them cannot see it. The local exists so a reader can tell which of the two a caller is
+    without re-deriving it.
+
+    The wait is :func:`_bystander_grace` — capped at :data:`DEFAULT_CLOSER_GRACE` and carved from
+    the caller's own deadline, the same arithmetic the worker path uses, so a
+    ``shutdown(timeout=0)`` does not inherit another caller's budget on one path and not the other.
+    It took a flat cap first, which made the two paths disagree by two seconds on that call.
+
     A worker that owns *this* sink closes it instead, and this returns — that is what makes a
     mixed process exactly one ``close()`` in either order. It also inherits that worker's reasons
     for *not* closing: an expired :meth:`Worker.shutdown` leaves the sink open because the drain
@@ -1516,7 +1658,8 @@ def _close_orphan_sink() -> None:
     choice is between closing inline and never closing at all.
 
     Args:
-      None.
+      deadline: The calling ``shutdown``'s monotonic deadline, or ``None``. It bounds only
+        the wait for another caller's close, never a close performed here.
 
     Returns:
       None.
@@ -1526,18 +1669,25 @@ def _close_orphan_sink() -> None:
         traceback carrying the message arch §6 keeps out of anything the library says about
         itself. ``Exception``, never ``BaseException`` (SPEC-025 FR-004).
     """
-    with _state._lock:
-        owed: list[Sink] = [
-            sink for sink in _state._orphan_owed.values() if not _state.worker_owns(sink)
-        ]
-        for sink in owed:
-            del _state._orphan_owed[id(sink)]
-            _state._orphan_closed_sink = sink
-    if not owed:
-        return
-    inline = _inline_close_choice(owed)
+    global _orphan_closing
+    took = False
     started: list[threading.Thread] = []
     try:
+        with _state._lock:
+            owed: list[Sink] = [
+                sink for sink in _state._orphan_owed.values() if not _state.worker_owns(sink)
+            ]
+            for sink in owed:
+                del _state._orphan_owed[id(sink)]
+                _state._orphan_closed_sink = sink
+            if owed:
+                _orphan_closing += 1
+                _orphan_idle.clear()
+                took = True
+        if not owed:
+            _orphan_idle.wait(_bystander_grace(deadline))
+            return
+        inline = _inline_close_choice(owed)
         for sink in owed:
             if sink is inline:
                 continue
@@ -1559,6 +1709,11 @@ def _close_orphan_sink() -> None:
     finally:
         for closer in started:
             closer.join()
+        if took:
+            with _state._lock:
+                _orphan_closing = max(0, _orphan_closing - 1)
+                if not _orphan_closing:
+                    _orphan_idle.set()
 def _shutdown_worker(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
     """Drains and closes the process worker, or closes an orphan-only sink, backing ``shutdown()``.
 
@@ -1625,10 +1780,10 @@ def _shutdown_worker(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
             _state._shutdown_running += 1
     if worker is not None:
         worker.shutdown(timeout)
-        _close_orphan_sink()
+        _close_orphan_sink(deadline)
         return
     try:
-        _close_orphan_sink()
+        _close_orphan_sink(deadline)
     finally:
         with _state._lock:
             late_worker = _state._late_worker
@@ -1699,6 +1854,15 @@ def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) ->
     but the library does not *rely* on that — it cannot enforce what a third-party sink does — so
     it performs one close (SPEC-032).
 
+    **A sink this branch closes is dropped from the worker's owed-swap record** (SPEC-050
+    FR-004). That record is what makes :meth:`~log_foundry.worker.Worker._close_if_owed`'s close
+    of a stranded sink once-only, and this is the one route by which a sink in it can acquire a
+    different closer: the re-arm guard next to this line is a single slot, so a second
+    unconfirmed swap overwrites the first sink's protection and an orphan emit could put it back
+    in the owed record. Defensive rather than reproduced — no reachable sequence for it was found
+    — and one call, taken under the worker's own lock beneath this one, the same nesting
+    :meth:`~log_foundry.worker.Worker.swap_sink` already performs.
+
     The latch is keyed on ``worker.sink``, **not** on the orphan record. The sink
     ``Worker.swap_sink`` is about to close is the one the worker holds, and in the reproduced case
     the record is ``None`` — the worker cleared it when it was built — so keying on the record
@@ -1738,6 +1902,8 @@ def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) ->
             for stale in _state.take_orphan_owed():
                 if stale is not new_sink and stale is not worker.sink:
                     _state._orphan_closed_sink = stale
+                    with worker._lock:
+                        worker._discard_owed_swap(stale)
                     closer = release(stale, detached=True)
                     if closer is not None:
                         closers.append(closer)

{log_foundry-0.10.2.dev102 → log_foundry-0.10.2.dev106}/src/log_foundry/decorator.py

@@ -393,15 +393,19 @@ def _note_orphan_loss() -> None:
         _orphan_lost += 1
 
 
-def _note_in_span_loss() -> None:
-    """Counts one event lost while being built inside a span (SPEC-036 FR-003).
+def _note_in_span_loss(count: int = 1) -> None:
+    """Counts events lost while being built or handed over inside a span (SPEC-036 FR-003).
 
     Separate from :func:`_note_orphan_loss` because the two aggregate different failure
-    populations: this path cannot fail at ``emit``, so a non-zero count here always means the
-    data, never the destination.
+    populations: this path cannot fail at a sink's ``emit``, which is ``failed_batches``. It has
+    **two** causes, not one (SPEC-050 FR-003): a value that could not be built into an event, and
+    a process that could not give the library a thread to deliver through at all — where
+    ``Worker.__init__`` cannot start the drain thread, so the span's whole buffer is lost with no
+    worker in existence to record anything. The count is what the span was holding, so the second
+    cause moves it by more than one.
 
     Args:
-      None.
+      count: How many events were lost, defaulting to the single event the build path loses.
 
     Returns:
       None.
@@ -411,7 +415,7 @@ def _note_in_span_loss() -> None:
     """
     global _in_span_lost
     with _loss_lock:
-        _in_span_lost += 1
+        _in_span_lost += count
 
 
 def _read_losses() -> tuple[int, int]:
@@ -459,6 +463,15 @@ def _flush(span: Span) -> None:
     eight threads, 20,000 spans each). The **append** window this does not close is a different
     one, needs a per-span lock, and is declined in ``architecture.md`` §13.
 
+    **The worker is resolved before the buffer is detached** (SPEC-050 FR-003), which is what
+    lets :func:`_end` count what was lost. Resolving second, the events were already in a local
+    when :func:`~log_foundry._lifecycle._get_worker` raised, so they died with the exception and
+    ``len(span.events)`` read zero on exactly the path the count is for. Nothing between the
+    detach and the hand-off can fail: :meth:`~log_foundry.worker.Worker.submit` is total on every
+    path, so the events cannot be stranded in the local by this order. It also **narrows** the
+    SPEC-036 FR-004 window rather than widening it — the gap between detaching and submitting
+    falls from the whole of ``_get_worker()`` to a lock release.
+
     Args:
       span: The finished span whose buffered events are submitted.
 
@@ -468,9 +481,10 @@ def _flush(span: Span) -> None:
     Raises:
       Exception: Whatever creating the worker or submitting raises; :func:`_end` is the guard.
     """
+    worker = _lifecycle._get_worker()
     with _sweep_lock:
         events, span.events = span.events, []
-    _lifecycle._get_worker().submit(events)
+    worker.submit(events)
 
 
 type _SpanScope = tuple[
@@ -546,6 +560,14 @@ def _end(
     would fall through that function's own guard to ``set(())`` and wipe the whole stack,
     detaching the parent of an untraced nested call and splitting its trace.
 
+    **The absorbed close is counted, not merely announced** (SPEC-050 FR-003). It used to write
+    one stderr line and move no field, so a process that could not start a drain thread lost every
+    span's events under ``Health(stopped_reason=None, in_span_lost=0)`` — all zeros over total
+    loss, which is the reading SPEC-036 FR-003 added the counter to remove. The count is
+    ``len(span.events)``, which :func:`_flush` leaves intact until there is a worker to take them,
+    and it is recorded **before** the announcement for the reason every other site here is:
+    stderr may be wedged, and the counter must not be lost to it.
+
     Args:
       span: The span to close, or ``None`` if none was opened.
       token: The span-stack token to release, or ``None``.
@@ -565,6 +587,7 @@ def _end(
         try:
             _close_span(span, status, error)
         except Exception as exc:
+            _note_in_span_loss(len(span.events))
             _diag.absorbed("closing a span", exc, "the span's events were lost")
     if token is not None:
         context.pop_span(token)

{log_foundry-0.10.2.dev102 → log_foundry-0.10.2.dev106}/src/log_foundry/sinks/file.py

@@ -8,6 +8,7 @@ import threading
 import time
 from typing import TextIO
 
+from log_foundry import _diag
 from log_foundry.sinks.base import SinkDeliveryError
 
 __all__ = ["FileSink", "RotatingFileSink"]
@@ -267,6 +268,7 @@ class RotatingFileSink:
         self._stream: TextIO = open(path, "a", encoding=self._encoding)
         self._size = os.path.getsize(path) if os.path.exists(path) else 0
         self._next_rollover = self._schedule_next()
+        self._rotation_failing = False
         self._closed = False
         self._lock = threading.Lock()
 
@@ -278,6 +280,15 @@ class RotatingFileSink:
         ``_should_rotate`` check and the write that follows it must see the same stream, or a
         rotation between them sends the line to a closed handle.
 
+        **The batch is flushed before a rotation is attempted** (SPEC-048 FR-006). ``_rotate``
+        begins by closing the stream, which flushes it, while this loop otherwise flushes once at
+        the end — so under the canonical rotation failure, a full or read-only filesystem, it is
+        that flush that raises and the batch's buffered lines are gone before any rename is tried.
+        Flushing here means every event of the batch is on disk before the rotation can fail, at
+        every one of ``_rotate``'s raise sites rather than only at the renames. A flush that
+        raises *here* is deliberately not absorbed: nothing was written, so it is the
+        genuinely-total failure the worker's retry exists for.
+
         Args:
           batch: The events to write.
 
@@ -285,7 +296,9 @@ class RotatingFileSink:
           None.
 
         Raises:
-          OSError: If a write, flush or rotation fails.
+          OSError: If a write or a flush fails. A failed *rotation* no longer raises; see
+            :meth:`_rotate_or_continue`.
+          SinkDeliveryError: If the sink is closed.
         """
         if not batch:
             return
@@ -298,7 +311,8 @@ class RotatingFileSink:
                 line = json.dumps(event) + "\n"
                 data = len(line.encode(self._encoding))
                 if self._should_rotate(data):
-                    self._rotate()
+                    self._stream.flush()
+                    self._rotate_or_continue()
                 self._stream.write(line)
                 self._size += data
             self._stream.flush()
@@ -425,6 +439,68 @@ class RotatingFileSink:
             return True
         return self._next_rollover is not None and time.monotonic() >= self._next_rollover
 
+    def _rotate_or_continue(self) -> None:
+        """Rotates, or absorbs the failure and carries on writing to the un-rotated file.
+
+        A rotation that raised used to cost the batch twice. ``_rotate`` closes the active stream
+        first, so the events already written in this batch were on disk and the ``OSError``
+        propagated out of ``emit`` — the worker then re-sent the whole batch and wrote them again.
+        Measured: an 8-event batch failing after 3 were written put 11 lines on disk, 3 of them
+        duplicates. A persistent failure was worse: the sink kept a **closed** stream and every
+        later batch raised a raw ``PermissionError``, which is not a ``SinkDeliveryError`` and has
+        no ``losses()`` behind it.
+
+        Absorbing it costs nothing and duplicates nothing. The active file simply exceeds
+        ``max_bytes`` until a rotation succeeds, which is what happens anyway when rotation is
+        impossible — the trade SPEC-027 FR-004 already took, that a leaked resource beats a
+        corrupt write.
+
+        ``_next_rollover`` is re-armed as well as ``_size``: ``_rotate`` sets it on its last line,
+        so an absorbed failure would otherwise leave a **time** trigger permanently in the past.
+
+        **The re-arm does not damp the size trigger, and the diagnostic is what carries that.**
+        ``_size`` is re-seeded from a file that is now over ``max_bytes``, so ``_should_rotate``'s
+        size branch stays true and every subsequent event attempts a rotation again — measured at
+        598 attempts over 600 events. The attempts are cheap and lose nothing, but an unthrottled
+        stderr write per event is not: ``PostgresSink._reconnect_if_broken`` records the same rule
+        for the same reason, that a diagnostic which floods is one an operator stops reading. So
+        the failure is announced **once per outage** and the flag clears on the next successful
+        rotation. The remaining per-event attempt is recorded in ``architecture.md`` §12 rather
+        than fixed here, because damping it means deferring a rotation the caller asked for.
+
+        **The reopen can itself raise, and that is not absorbed.** It is the same
+        ``open(self._path, "a")`` call ``_rotate`` ends with, so whatever defeats it there —
+        a read-only mount, ``EMFILE``, a directory that lost write permission — defeats it here.
+        At that point this sink has no stream and cannot continue, so there is nothing to absorb
+        *into*; the ``OSError`` reaches ``emit`` and the worker retries the batch, duplicating the
+        prefix the pre-rotation flush had already written. That residue is recorded rather than
+        fixed: it is unchanged from before SPEC-048, the surviving events are on disk rather than
+        lost, and inventing a half-open state to avoid a duplicate would trade a visible
+        duplication for a silent loss.
+
+        Args:
+          None.
+
+        Returns:
+          None.
+
+        Raises:
+          OSError: If the *reopen* fails, per the paragraph above. A failed **rotation** is
+            absorbed and announced through ``_diag``; the batch continues, nothing is dropped, no
+            counter moves, and this class still has no ``losses()``.
+        """
+        try:
+            self._rotate()
+        except OSError as err:
+            self._stream = open(self._path, "a", encoding=self._encoding)
+            self._size = os.path.getsize(self._path) if os.path.exists(self._path) else 0
+            self._next_rollover = self._schedule_next()
+            if not self._rotation_failing:
+                self._rotation_failing = True
+                _diag.absorbed("rotating RotatingFileSink", err)
+            return
+        self._rotation_failing = False
+
     def _rotate(self) -> None:
         """Closes the active file, shifts and prunes backups, then opens a fresh active file.
 

{log_foundry-0.10.2.dev102 → log_foundry-0.10.2.dev106}/src/log_foundry/sinks/firehose.py

@@ -203,6 +203,22 @@ class FirehoseSink:
         this loop re-sends the records the destination flagged, and the canonical reason it flags
         them is throttling, which an immediate re-send makes worse.
 
+        **A client exception costs this chunk, never the batch** (SPEC-048 FR-002). It used to
+        propagate out of :meth:`emit`, so a failure on chunk N after chunks 1..N-1 had landed made
+        the worker re-send the whole batch and duplicate everything already delivered — the exit
+        drain, one large batch by construction, is exactly that shape. The guard sits around the
+        client call *inside* this loop rather than around ``_send`` in ``emit``, so only the
+        records still outstanding at the failing attempt are charged and the ones already accepted
+        still count toward the return.
+
+        A client exception is treated as **provable non-delivery** for the chunk, so a wholly
+        failed batch still raises and ``unknown`` is untouched: SPEC-018's "unadjudicable" is a
+        property of a *response*, and an exception is not a response. The cost is written down —
+        a read timeout means the request went out and the reply was lost, so a re-send may
+        duplicate — and taken because an unreachable endpoint is the common case and is exactly
+        what the worker's retry exists for, while suppressing the raise would lose every event of
+        every batch for a whole outage, silently.
+
         Args:
           records: One chunk's request entries.
 
@@ -214,13 +230,27 @@ class FirehoseSink:
           SPEC-018 settled must never be re-sent.
 
         Raises:
-          Exception: Whatever the client raises.
+          None. ``Exception`` is caught rather than ``BaseException``, so a ``KeyboardInterrupt``
+            or ``SystemExit`` still reaches the caller (SPEC-025 FR-004).
         """
         sent = len(records)
         for attempt in range(self.max_retries + 1):
-            response = self.client.put_record_batch(
-                DeliveryStreamName=self.delivery_stream, Records=records
-            )
+            try:
+                response = self.client.put_record_batch(
+                    DeliveryStreamName=self.delivery_stream, Records=records
+                )
+            except Exception as err:
+                if attempt < self.max_retries:
+                    wait(_BACKOFF_BASE * (2**attempt), self.log_foundry_stop_signal)
+                    continue
+                with self._counter_lock:
+                    self.failed += len(records)
+                _diag.lost(
+                    "record",
+                    len(records),
+                    f"FirehoseSink, {self.max_retries + 1} attempt(s), {type(err).__name__}",
+                )
+                return sent - len(records)
             if not response.get("FailedPutCount"):
                 return sent
             results = usable_results(response.get("RequestResponses"))