log-foundry 0.5.1.dev7__tar.gz → 0.5.1.dev9__tar.gz

{log_foundry-0.5.1.dev7 → log_foundry-0.5.1.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.5.1.dev7
+Version: 0.5.1.dev9
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE
@@ -395,6 +395,9 @@ A few conventions hold across every sink below:
   open for you to manage.
 - **Never crashes the app.** A failing sink is retried with backoff and then counted (`.failed`,
   `.dropped_oversized`, …) rather than raised — a broken destination degrades logging, nothing more.
+  The one deliberate exception is a `MultiSink` whose children *all* failed: it re-raises so the
+  worker's retry engages, since nothing was delivered and there are no duplicates to risk. That
+  still doesn't reach your code — the worker is what catches it.
 
 #### Built-in, zero-dependency
 
@@ -612,7 +615,22 @@ background worker via a fast, non-blocking submit — your function returns with
 the sink. The worker batches events (by count and time), emits them on its own thread, retries
 a failing sink with backoff, and applies backpressure so a slow or down sink can never block or
 back-pressure the app: when its bounded queue is full it drops the newest submissions and counts
-them (`worker.dropped`) rather than stalling.
+them rather than stalling.
+
+Those losses are deliberate, so the library gives you a way to notice them. `log_foundry.health()`
+returns a snapshot of the worker's counters:
+
+```python
+h = log_foundry.health()
+if h.dropped or h.failed_batches:
+    ...  # logs were silently lost — worth an alert
+```
+
+`dropped` counts submissions discarded because the queue filled; `failed_batches` counts batches
+abandoned after the retry budget was spent. Overflow also warns on stderr — on the first drop and
+every thousandth after it, since overflow is a high-rate condition and a line per drop would be its
+own outage. A process that has never logged has no worker, and asking after its health does not
+create one.
 
 Because delivery is asynchronous, drain before the process exits. There are two drains, and
 which one you want depends on whether the process is about to end:

{log_foundry-0.5.1.dev7 → log_foundry-0.5.1.dev9}/README.md

@@ -359,6 +359,9 @@ A few conventions hold across every sink below:
   open for you to manage.
 - **Never crashes the app.** A failing sink is retried with backoff and then counted (`.failed`,
   `.dropped_oversized`, …) rather than raised — a broken destination degrades logging, nothing more.
+  The one deliberate exception is a `MultiSink` whose children *all* failed: it re-raises so the
+  worker's retry engages, since nothing was delivered and there are no duplicates to risk. That
+  still doesn't reach your code — the worker is what catches it.
 
 #### Built-in, zero-dependency
 
@@ -576,7 +579,22 @@ background worker via a fast, non-blocking submit — your function returns with
 the sink. The worker batches events (by count and time), emits them on its own thread, retries
 a failing sink with backoff, and applies backpressure so a slow or down sink can never block or
 back-pressure the app: when its bounded queue is full it drops the newest submissions and counts
-them (`worker.dropped`) rather than stalling.
+them rather than stalling.
+
+Those losses are deliberate, so the library gives you a way to notice them. `log_foundry.health()`
+returns a snapshot of the worker's counters:
+
+```python
+h = log_foundry.health()
+if h.dropped or h.failed_batches:
+    ...  # logs were silently lost — worth an alert
+```
+
+`dropped` counts submissions discarded because the queue filled; `failed_batches` counts batches
+abandoned after the retry budget was spent. Overflow also warns on stderr — on the first drop and
+every thousandth after it, since overflow is a high-rate condition and a line per drop would be its
+own outage. A process that has never logged has no worker, and asking after its health does not
+create one.
 
 Because delivery is asynchronous, drain before the process exits. There are two drains, and
 which one you want depends on whether the process is about to end:

{log_foundry-0.5.1.dev7 → log_foundry-0.5.1.dev9}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 ]
 
 # Optional features. Install with: pip install log-foundry[aws]
-version = "0.5.1.dev7"
+version = "0.5.1.dev9"
 
 [project.optional-dependencies]
 aws = ["boto3>=1.34"]              # SQSSink, SNSSink, KinesisSink, FirehoseSink

{log_foundry-0.5.1.dev7 → log_foundry-0.5.1.dev9}/src/log_foundry/__init__.py

@@ -18,6 +18,7 @@ from log_foundry.context import (
     current_traceparent,
 )
 from log_foundry.decorator import continue_trace, trace
+from log_foundry.worker import Health
 
 try:
     # Distribution name ("log-foundry") differs from the import name ("log_foundry").
@@ -49,6 +50,27 @@ def flush(timeout: float | None = 5.0) -> bool:
     return _flush_worker(timeout)
 
 
+def health() -> Health:
+    """Snapshot the background worker's delivery counters (SPEC-017 FR-005). Never raises.
+
+    Returns ``queued`` / ``dropped`` / ``failed_batches``. A non-zero ``dropped`` means the queue
+    filled and submissions were discarded to keep your code non-blocking; a non-zero
+    ``failed_batches`` means a sink stayed broken through the whole retry budget. Both are
+    losses the library absorbs on purpose, and this is how you notice them::
+
+        h = log_foundry.health()
+        if h.dropped or h.failed_batches:
+            ...  # raise an alert; logs were silently lost
+
+    A process that has never logged has no worker, and asking after its health does not create
+    one — the snapshot is simply zeroed. Valid after :func:`shutdown`, which leaves the final
+    counters readable.
+    """
+    from log_foundry.decorator import _worker_health
+
+    return _worker_health()
+
+
 def shutdown() -> None:
     """Flush buffered events and close the sink, blocking until drained. Idempotent.
 
@@ -80,5 +102,7 @@ __all__ = [
     "current_baggage_header",
     "flush",
     "shutdown",
+    "health",
+    "Health",
     "__version__",
 ]

{log_foundry-0.5.1.dev7 → log_foundry-0.5.1.dev9}/src/log_foundry/decorator.py

@@ -40,7 +40,7 @@ from log_foundry.ids import (
     parse_traceparent,
 )
 from log_foundry.model import Span, backfill_baggage, end_event, start_event
-from log_foundry.worker import Worker
+from log_foundry.worker import Health, Worker
 
 __all__ = ["trace", "continue_trace"]
 
@@ -235,6 +235,19 @@ def _flush_worker(timeout: float | None = 5.0) -> bool:
         return False
 
 
+def _worker_health() -> Health:
+    """Snapshot the process worker's counters, or zeros if none was ever created.
+
+    Backs :func:`log_foundry.health` (SPEC-017 FR-005). Like :func:`_flush_worker` this
+    deliberately does *not* call :func:`_get_worker`: starting a thread and registering an
+    ``atexit`` drain in order to report three zeros would be pure cost.
+    """
+    worker = _worker
+    if worker is None:
+        return Health(queued=0, dropped=0, failed_batches=0)
+    return worker.health()
+
+
 def _flush(span: Span) -> None:
     """Hand the finished span's events to the background worker — non-blocking (FR-001).
 

{log_foundry-0.5.1.dev7 → log_foundry-0.5.1.dev9}/src/log_foundry/sinks/multi.py

@@ -6,6 +6,12 @@ whose ``emit`` (or ``close``) raises is isolated — the failure is counted on `
 logged to stderr, and its siblings still run — so one broken destination never fails the whole
 fan-out or the worker's retry. This mirrors the worker's own survive-a-sink-failure isolation
 boundary (arch §9; best-practices §7 sanctions the broad catch here).
+
+The one exception is **total** failure: if every child raised, ``emit`` re-raises rather than
+reporting success, so the worker's retry engages (SPEC-017 FR-004). Swallowing there meant a
+``MultiSink`` whose destinations were all down reported healthy on every batch and the loss was
+invisible. ``close()`` keeps the unconditional isolate-and-continue behaviour — a failed close
+has nothing to retry.
 """
 
 from __future__ import annotations
@@ -29,16 +35,35 @@ class MultiSink:
         self.failed = 0
 
     def emit(self, batch: list[dict[str, object]]) -> None:
-        """Forward ``batch`` to every child in construction order, isolating failures (FR-002)."""
+        """Forward ``batch`` to every child in construction order, isolating failures (FR-002).
+
+        When **every** child failed, re-raise the first child's exception so the worker's bounded
+        retry sees the total loss (SPEC-017 FR-004). Partial success stays isolated: a retry there
+        would re-deliver the batch to the children that already took it, and duplicates are worse
+        than the one failure already counted on ``failed`` and written to stderr. Total failure
+        delivered nothing, so it has no duplicates to create and is the one case worth retrying.
+        """
+        first_error: Exception | None = None
+        delivered = 0
         for sink in self._sinks:
             try:
                 sink.emit(batch)
             except Exception as err:  # isolation boundary: one child must not fail the rest
                 self.failed += 1
+                if first_error is None:
+                    first_error = err
                 sys.stderr.write(
                     f"log-foundry: MultiSink child {type(sink).__name__}.emit "
                     f"failed and was skipped: {err!r}\n"
                 )
+            else:
+                delivered += 1
+        if delivered == 0 and first_error is not None:
+            # Raised outside any ``except`` block, so ``__context__`` is untouched and the
+            # exception propagates by identity with its original traceback. The ``is not None``
+            # is also what keeps an empty ``MultiSink`` a no-op rather than a raise — otherwise a
+            # misconfigured fan-out with no children would retry every batch to exhaustion.
+            raise first_error
 
     def close(self) -> None:
         """Close every child, isolating a failing child so the rest still close (FR-002)."""

{log_foundry-0.5.1.dev7 → log_foundry-0.5.1.dev9}/src/log_foundry/worker.py

@@ -23,16 +23,36 @@ import queue
 import sys
 import threading
 import time
-from typing import cast
+from typing import NamedTuple, cast
 
 from log_foundry.sinks.base import Sink
 
-__all__ = ["Worker"]
+__all__ = ["Health", "Worker"]
 
 # Sentinel enqueued by shutdown() to wake a worker blocked in queue.get() so it stops promptly
 # instead of waiting out the flush_interval. It is never emitted.
 _SHUTDOWN = object()
 
+# Queue overflow is a high-rate condition by nature: a line per dropped submission would be its
+# own outage. Warn on the first drop, then every this-many-th (SPEC-017 FR-005).
+_DROP_WARN_EVERY = 1000
+
+
+class Health(NamedTuple):
+    """A point-in-time snapshot of the worker's delivery counters (SPEC-017 FR-005).
+
+    Attributes:
+        queued: Submissions currently buffered. Approximate by nature — it is read without
+            stopping the world, and briefly counts the internal flush/shutdown markers
+            alongside real submissions.
+        dropped: Submissions discarded because the queue was full (backpressure).
+        failed_batches: Batches abandoned after the retry budget was spent.
+    """
+
+    queued: int
+    dropped: int
+    failed_batches: int
+
 
 class _FlushMarker:
     """A drain request travelling the queue in FIFO order (SPEC-013 FR-002).
@@ -87,13 +107,35 @@ class Worker:
 
         Enqueues via ``put_nowait`` and returns immediately without touching the sink. When the
         queue is full, drops this submission (drop-newest) and counts it in ``dropped`` rather
-        than blocking the caller (FR-001, FR-004).
+        than blocking the caller (FR-001, FR-004), warning on a throttle (SPEC-017 FR-005).
         """
         try:
             self._queue.put_nowait(events)
         except queue.Full:
             with self._lock:
                 self.dropped += 1
+                total = self.dropped  # read under the lock: every value is produced exactly once
+            # Written outside the lock deliberately. stderr can block on a slow reader, and
+            # ``_lock`` also guards flush()/shutdown()'s once-only flag — holding it across a
+            # blocking write would let a wedged console stall the drain path, not just other
+            # submitters. Lines may therefore interleave out of order under concurrency; the
+            # counts they carry are still exact.
+            if total == 1 or total % _DROP_WARN_EVERY == 0:
+                sys.stderr.write(
+                    f"log-foundry: log queue full, dropped {total} submission(s) so far\n"
+                )
+
+    def health(self) -> Health:
+        """Snapshot the delivery counters (SPEC-017 FR-005). Never raises.
+
+        Valid after :meth:`shutdown` — the counters are plain integers that outlive the thread,
+        and the final drain consumes the queue, so ``queued`` reads 0 rather than a stale marker.
+        """
+        with self._lock:
+            dropped, failed_batches = self.dropped, self.failed_batches
+        return Health(
+            queued=self._queue.qsize(), dropped=dropped, failed_batches=failed_batches
+        )
 
     def flush(self, timeout: float | None = 5.0) -> bool:
         """Drain everything submitted before this call through the sink, without stopping.
@@ -221,7 +263,10 @@ class Worker:
                 return
             except Exception:  # noqa: BLE001 — any sink failure must not kill the worker thread
                 if attempt >= self.max_retries:
-                    self.failed_batches += 1
+                    # Under the lock so a concurrent health() sees a coherent snapshot rather
+                    # than a half-updated pair. No deadlock: shutdown() releases before join().
+                    with self._lock:
+                        self.failed_batches += 1
                     sys.stderr.write(
                         f"log-foundry: abandoned a batch of {len(batch)} event(s) after "
                         f"{self.max_retries + 1} failed emit attempts\n"