log-foundry 0.10.2.dev11__tar.gz → 0.10.2.dev13__tar.gz

{log_foundry-0.10.2.dev11 → log_foundry-0.10.2.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.10.2.dev11
+Version: 0.10.2.dev13
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE

{log_foundry-0.10.2.dev11 → log_foundry-0.10.2.dev13}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 ]
 
 # Optional features. Install with: pip install log-foundry[aws]
-version = "0.10.2.dev11"
+version = "0.10.2.dev13"
 
 [project.optional-dependencies]
 aws = ["boto3>=1.43.61"]              # SQSSink, SNSSink, KinesisSink, FirehoseSink

log_foundry-0.10.2.dev13/src/log_foundry/_diag.py

@@ -0,0 +1,215 @@
+"""The library's own diagnostic channel — one line on stderr, for what it lost or absorbed.
+
+This library *is* the logger, so it cannot report its own faults through itself: an event
+describing a broken sink would be handed to that same broken sink. Stderr is the channel of last
+resort, and **every** line the library writes about itself goes through here, so the wording and
+the safety rules are decided once rather than remembered at twenty-odd call sites — which is how
+twelve of them came to disagree with the other eight (SPEC-029).
+
+Three rules, all load-bearing:
+
+* **The exception's type, never its message.** A message can carry a value from the event that
+  provoked it, and arch §6 keeps user data out of anything the library emits about itself — the
+  same rule SPEC-019 applies to ``Health.stopped_reason`` and ``sanitize`` applies by refusing
+  ``repr(value)``. ``PostgresSink`` is the sharpest case: ``_row`` binds the whole
+  ``json.dumps(event)`` as a statement parameter, and a psycopg error repr routinely reprints the
+  failing statement *and its parameters* — so a naive diagnostic for a failed insert would reprint
+  the event, PII included, into a stream nobody was asked to secure. A type name tells a
+  ``ConnectionError`` from a ``TypeError``, which is what the reader needs. Where a bare type is
+  genuinely not diagnosable — an ``OSError`` alone does not say "refused" from "host unknown" —
+  the caller passes a ``detail`` built from values the *library* controls: an ``errno`` (see
+  :func:`errno_of`), an HTTP status, an attempt count, a sink's class name. Never the exception's
+  text. Any ``detail`` is escaped and bounded regardless, so a value that reached one could not
+  forge a second line in an operator's console.
+* **No stream fault escapes.** These writers are called from ``except`` and ``finally`` blocks
+  whose entire purpose is to stop an exception reaching the caller, and from the worker thread
+  where an escaping exception ends delivery for good. A failure here — a closed stderr at
+  interpreter shutdown, a stream that rejects the write, ``sys.stderr`` set to ``None`` — must not
+  become the failure those guards existed to prevent. A ``BaseException`` still passes through, as
+  everywhere else in this library: a ``KeyboardInterrupt`` landing mid-write is the operator's
+  intent, not a stream fault to swallow.
+* **Record first, announce second.** A counter moves *before* its line is attempted, at every call
+  site. The write is best-effort by construction; the counter that ``health()`` reports is not, and
+  must not be able to ride on it.
+
+Stderr stays the channel: routing through the ``logging`` module would make the library's own
+failure reports depend on machinery that may be the thing failing, and ``LoggingSink`` already
+exists for the opposite direction. These lines are for humans — the structured stream is the sink.
+
+**This module must import nothing from its own package.** ``decorator``, ``api``, ``worker`` and
+every sink reach it with ``from log_foundry import _diag`` at module scope, which executes while
+the package is still partially initialised and resolves only because ``_diag`` is a leaf — it needs
+``sys`` and nothing else. Give it an intra-package import and every module that is imported *first*
+in a fresh interpreter starts failing.
+"""
+
+from __future__ import annotations
+
+import sys
+
+__all__ = ["absorbed", "errno_of", "lost", "rejected"]
+
+# Bound on any caller-supplied detail string, applied after escaping so it bounds what is
+# *written*. Generous enough for an attempt count plus a sink name plus an errno; far short of a
+# payload.
+_MAX_DETAIL = 200
+
+# Bound on how much of a rejected inbound value is echoed (SPEC-014). Tighter than _MAX_DETAIL:
+# the value is attacker-controllable, and its shape is the diagnosis, not its contents.
+_MAX_REJECTED_ECHO = 64
+
+_ESCAPES = {"\t": "\\t", "\n": "\\n", "\r": "\\r"}
+
+
+def absorbed(where: str, exc: BaseException, detail: str = "") -> None:
+    """Report a failure the library swallowed rather than propagated.
+
+    Never raises on a stream fault; a ``BaseException`` from the write still propagates.
+
+    Args:
+        where: What was being attempted, as a participle phrase that reads after "while" —
+            ``"closing a span"``, ``"emitting an orphan log"``. A literal, never a runtime value.
+        exc: The absorbed exception. Only ``type(exc).__name__`` is written.
+        detail: Optional consequence for the reader, appended after a semicolon — most usefully
+            what was lost, since an absorbed failure is invisible apart from this line. Escaped
+            and truncated to ``_MAX_DETAIL``; must not be derived from ``exc``'s text.
+    """
+    try:
+        sys.stderr.write(
+            f"log-foundry: absorbed a failure while {_escape(where)} "
+            f"({_escape(type(exc).__name__)}){_suffix(detail)}\n"
+        )
+    except Exception:
+        # The channel of last resort has no fallback of its own. Reporting is best-effort by
+        # construction: losing the line is bad, raising from the guard that exists to protect
+        # the caller's exception would be worse (arch §4).
+        pass
+
+
+def lost(what: str, count: int, detail: str = "") -> None:
+    """Report a counted loss — events, messages, batches or rows that will not be delivered.
+
+    Never raises on a stream fault; a ``BaseException`` from the write still propagates.
+
+    Args:
+        what: Singular noun for the unit lost — ``"event"``, ``"message"``, ``"row"``. Rendered
+            with the corpus-wide ``(s)`` suffix, so it must be a literal, never a runtime value.
+        count: How many this line reports as lost — normally the increment the caller's counter
+            has just taken. A **throttled** site instead passes its running total, because a line
+            written on every thousandth drop that said "lost 1" would read as one loss rather than
+            a thousand; such a site says so in ``detail``. Recorded before this call either way.
+        detail: Optional circumstances — the sink's class name, the attempt count, an ``errno``,
+            an exception *type*. Escaped and truncated to ``_MAX_DETAIL``.
+    """
+    try:
+        sys.stderr.write(f"log-foundry: lost {count} {_escape(what)}(s){_suffix(detail)}\n")
+    except Exception:
+        pass  # best-effort, as above; the counter this line describes is already recorded.
+
+
+def rejected(reason: str, value: object) -> None:
+    """Report an inbound trace context the library refused to adopt (SPEC-014).
+
+    The offending value is echoed as a **bounded ``repr``** — the one place in the library where a
+    ``repr`` is correct, because the input is an inbound *header* rather than an exception, and its
+    exact shape is what makes a rejection diagnosable. Unbounded it would be a log-injection
+    surface: the value is attacker-controllable.
+
+    The ``repr`` is escaped afterwards even though ``repr`` of a ``str`` is already printable
+    throughout, which makes this a no-op for every value the call sites actually pass. It is not a
+    no-op in general: ``repr`` runs ``__repr__``, which is user code free to *return* a raw newline
+    no matter what the built-in reprs would have done. Escaping the result is the difference
+    between "the built-ins happen to escape" and "this cannot forge a line".
+
+    Never raises on a stream fault; a ``BaseException`` from the write still propagates.
+
+    Args:
+        reason: Why it was refused — ``"unparseable traceparent"``, ``"invalid trace_id"``. A
+            literal, never a runtime value: unlike the echo, it is not bounded.
+        value: The refused value, echoed as a ``repr`` bounded to ``_MAX_REJECTED_ECHO``.
+    """
+    try:
+        shown = _escape(repr(value))
+        if len(shown) > _MAX_REJECTED_ECHO:
+            shown = shown[:_MAX_REJECTED_ECHO] + "…"
+        sys.stderr.write(
+            f"log-foundry: ignoring inbound trace context ({_escape(reason)}): {shown}\n"
+        )
+    except Exception:
+        pass  # best-effort, as above.
+
+
+def errno_of(exc: BaseException) -> str:
+    """Return ``"errno=N"`` for an exception carrying one, else ``""``.
+
+    The library-controlled detail that makes an ``OSError`` line actionable: "connection refused"
+    and "host unknown" are the same type name and are told apart only by the code. An integer from
+    the OS is not caller data, which is what makes it safe to write where the message is not.
+
+    ``urllib``'s ``URLError`` carries the underlying socket error on ``.reason`` rather than
+    setting its own ``errno``, so that is consulted too. Total: attribute access on an arbitrary
+    exception can run a property that raises, and a helper for a diagnostic must not become one.
+
+    The value is rendered through ``int()`` rather than interpolated as found. ``isinstance(x,
+    int)`` admits any *subclass*, and a driver's error code is routinely one — an ``IntEnum`` or a
+    bespoke class whose ``__str__`` returns whatever its author chose, which the f-string would
+    then have written verbatim. That is the leak this helper exists to be the alternative to.
+    """
+    try:
+        code = getattr(exc, "errno", None)
+        if code is None:
+            code = getattr(getattr(exc, "reason", None), "errno", None)
+        return f"errno={int(code)}" if isinstance(code, int) else ""
+    except Exception:
+        return ""
+
+
+def _suffix(detail: str) -> str:
+    """Render a detail as a bounded, escaped ``"; ..."`` suffix, or ``""`` when there is none.
+
+    Escaping precedes truncation so the bound applies to what is written, and so truncation can
+    only ever remove characters from an already-safe string rather than split an escape into
+    something that isn't one.
+
+    Guarded in its own right, and returning ``""`` rather than propagating: this runs inside the
+    caller's f-string, so a detail that cannot be rendered would otherwise take the *whole* line
+    with it — including the count, which is the part an operator cannot reconstruct.
+    """
+    if not detail:
+        return ""
+    try:
+        shown = _escape(detail)
+        if len(shown) > _MAX_DETAIL:
+            shown = shown[:_MAX_DETAIL] + "…"
+        return f"; {shown}"
+    except Exception:
+        return ""
+
+
+def _escape(text: str) -> str:
+    """Render every non-printable character visibly, so nothing in ``text`` can forge a line.
+
+    ``str.isprintable()`` is the test rather than a table over ``range(0x20)``: it is ``False`` for
+    the C0 block *and* for DEL, the C1 block (U+009B is CSI, which a terminal reads as the start of
+    an escape sequence), U+0085, U+2028, U+2029, and the bidi format characters. Python's own
+    ``splitlines()`` breaks on three of those a C0 table misses, so a log shipper or a reader doing
+    the obvious thing would see a forged ``log-foundry:`` line that a newline count says is not
+    there. Space is printable, so ordinary text is untouched.
+
+    The whole-string fast path keeps the overwhelmingly common case — a sink name, an attempt
+    count, an exception type — to a single C-level call.
+    """
+    if text.isprintable():
+        return text
+    return "".join(char if char.isprintable() else _escaped(char) for char in text)
+
+
+def _escaped(char: str) -> str:
+    """Escape one non-printable character, following Python's own ``repr`` conventions."""
+    escape = _ESCAPES.get(char)
+    if escape is not None:
+        return escape
+    code = ord(char)
+    if code <= 0xFF:
+        return f"\\x{code:02x}"
+    return f"\\u{code:04x}" if code <= 0xFFFF else f"\\U{code:08x}"

{log_foundry-0.10.2.dev11 → log_foundry-0.10.2.dev13}/src/log_foundry/decorator.py

@@ -24,7 +24,6 @@ from __future__ import annotations
 import asyncio
 import atexit
 import functools
-import sys
 import threading
 from collections.abc import Callable
 from time import monotonic
@@ -47,9 +46,6 @@ if TYPE_CHECKING:
 
 __all__ = ["continue_trace", "trace"]
 
-# Bound on how much of a rejected inbound value is echoed into a stderr warning.
-_MAX_REJECTED_ECHO = 64
-
 # One background worker per process (SPEC-004), created lazily from the configured sink on the
 # first flush. The double-checked lock makes concurrent first-flushes create exactly one.
 _worker: Worker | None = None
@@ -86,19 +82,6 @@ def _open_span(name: str, defaults: dict[str, object] | None) -> Span:
     return span
 
 
-def _warn_rejected(reason: str, value: object) -> None:
-    """Report a rejected inbound context on stderr, as ``worker`` and ``SQSSink`` do.
-
-    The offending value is echoed only as a **bounded ``repr``**. Unbounded is a log-injection
-    surface — the value is attacker-controllable, and `repr` additionally escapes newlines and
-    control characters so it cannot forge a second log line in an operator's console.
-    """
-    shown = repr(value)
-    if len(shown) > _MAX_REJECTED_ECHO:
-        shown = shown[:_MAX_REJECTED_ECHO] + "…"
-    sys.stderr.write(f"log-foundry: ignoring inbound trace context ({reason}): {shown}\n")
-
-
 def continue_trace(
     traceparent: str | None = None,
     *,
@@ -144,19 +127,19 @@ def continue_trace(
     if traceparent is not None:
         if trace_id is not None or parent_span_id is not None:
             # A programming error, not bad input: the caller supplied the same thing twice.
-            _warn_rejected("both traceparent and explicit ids given; traceparent wins", traceparent)
+            _diag.rejected("both traceparent and explicit ids given; traceparent wins", traceparent)
         parsed = parse_traceparent(traceparent)
         if parsed is None:
-            _warn_rejected("unparseable traceparent", traceparent)
+            _diag.rejected("unparseable traceparent", traceparent)
         else:
             adopted = parsed
     elif trace_id is not None:
         if not is_valid_trace_id(trace_id):
-            _warn_rejected("invalid trace_id", trace_id)
+            _diag.rejected("invalid trace_id", trace_id)
         elif parent_span_id is not None and not is_valid_span_id(parent_span_id):
             # Drop just the parent and join as another root rather than reject the whole
             # context: being in the right trace without a parent beats being in a fresh one.
-            _warn_rejected("invalid parent_span_id; joining as a root", parent_span_id)
+            _diag.rejected("invalid parent_span_id; joining as a root", parent_span_id)
             adopted = (trace_id, None)
         else:
             # parent_span_id may legitimately be omitted — a consumer that knows the trace but
@@ -175,7 +158,7 @@ def continue_trace(
         if parsed_baggage is None:
             # Deliberately independent of the trace context above: losing correlating fields is
             # bad, and losing the trace join because one field was malformed is worse.
-            _warn_rejected("unusable baggage header", baggage)
+            _diag.rejected("unusable baggage header", baggage)
         else:
             context.set_baggage(**parsed_baggage)
 

{log_foundry-0.10.2.dev11 → log_foundry-0.10.2.dev13}/src/log_foundry/worker.py

@@ -20,7 +20,6 @@ nothing about spans or context (the same dumbness that makes sinks swappable).
 from __future__ import annotations
 
 import queue
-import sys
 import threading
 import time
 from typing import TYPE_CHECKING, NamedTuple, cast
@@ -162,15 +161,7 @@ class Worker:
             # submitters. Lines may therefore interleave out of order under concurrency; the
             # counts they carry are still exact.
             if total == 1 or total % _DROP_WARN_EVERY == 0:
-                try:
-                    sys.stderr.write(
-                        f"log-foundry: log queue full, dropped {total} submission(s) so far\n"
-                    )
-                except Exception:  # submit() runs on the *caller's* thread, so an
-                    # unwritable stderr (closed fd, broken pipe, daemonized process) would raise
-                    # straight into the app. A diagnostic about dropped logs must never itself be
-                    # the reason a decorated function fails. The counter is already recorded.
-                    pass
+                _diag.lost("submission", total, "log queue full; count is cumulative")
 
     def health(self) -> Health:
         """Snapshot the delivery counters (SPEC-017 FR-005, SPEC-019 FR-003). Never raises.
@@ -323,36 +314,36 @@ class Worker:
 
         Recording precedes announcing: stderr may be closed or wedged, and unlike the overflow
         warning this line is written exactly once and cannot be re-emitted later, so the record
-        must not be able to ride on it. The exception's *type* is reported and its message is
-        not — a sink's exception text can carry event data, and arch §6 keeps caller data out of
-        places it was not asked for (the same rule behind ``sanitize``'s type-name placeholder).
+        must not be able to ride on it. The exception's *type* is reported and its message is not
+        — the rule ``_diag`` now applies to every line the library writes (SPEC-029), and the
+        reason this site had it first.
 
-        The count reports what was *in hand* and what was still *queued behind it* (SPEC-021
-        FR-002). Held alone under-reads the loss: nothing will drain the queue either, so an
-        operator reading "1 undrained event-list(s)" could conclude far less was lost than was.
+        The announcement is an :func:`~log_foundry._diag.absorbed`, not a fourth kind of line: the
+        thread's death *is* an exception this method caught and did not propagate, and what it
+        cost belongs in the detail. The count reports what was *in hand* and what was still
+        *queued behind it* (SPEC-021 FR-002). Held alone under-reads the loss: nothing will drain
+        the queue either, so an operator reading "1 undrained event-list(s)" could conclude far
+        less was lost than was.
 
         The queued figure is "items", not "event-lists", and says so: like ``Health.queued`` it is
         read without stopping the world, so it counts any internal flush/shutdown marker sitting
         alongside real submissions, and a producer thread can add to the queue between the death
         and the read. It is a floor on what was lost, which is the useful direction.
         """
-        name = type(exc).__name__
         with self._lock:
-            self.stopped_reason = name
+            self.stopped_reason = type(exc).__name__
         try:
             # In its own guard, and after the record: ``qsize()`` is not guaranteed on every
             # platform's queue, and a diagnostic must not be the reason the diagnosis is lost.
             queued: object = self._queue.qsize()
         except Exception:
             queued = "?"
-        try:
-            sys.stderr.write(
-                f"log-foundry: worker thread stopped on {name}; {undrained} undrained "
-                f"event-list(s) held and {queued} queued item(s) undelivered, nothing further "
-                f"will be delivered\n"
-            )
-        except Exception:  # best-effort: the record above is what an operator reads.
-            pass
+        _diag.absorbed(
+            "draining the log queue",
+            exc,
+            f"worker thread stopped; {undrained} undrained event-list(s) held and {queued} "
+            f"queued item(s) undelivered, nothing further will be delivered",
+        )
 
     def _drain(self, pending: list[list[dict[str, object]]]) -> None:
         """The drain loop proper. ``pending`` is owned by :meth:`_run`, which reports its size.
@@ -477,9 +468,12 @@ class Worker:
                     # than a half-updated pair. No deadlock: shutdown() releases before join().
                     with self._lock:
                         self.failed_batches += 1
-                    sys.stderr.write(
-                        f"log-foundry: abandoned a batch of {len(batch)} event(s) after "
-                        f"{retries + 1} failed emit attempts\n"
+                    # Through ``_diag`` so the write is guarded: unguarded, a broken stderr raised
+                    # out of here, through ``_drain``, into ``_run``'s handler, and the drain
+                    # thread died for good — a diagnostic about one lost batch costing every batch
+                    # after it (SPEC-029 FR-003).
+                    _diag.lost(
+                        "event", len(batch), f"batch abandoned after {retries + 1} emit attempts"
                     )
                     return
                 # Backoff between attempts; _stop.wait returns at once during shutdown, so a

log_foundry-0.10.2.dev11/src/log_foundry/_diag.py

@@ -1,59 +0,0 @@
-"""The library's own diagnostic channel — one line on stderr, for failures it absorbed.
-
-This library *is* the logger, so it cannot report its own faults through itself: an event
-describing a broken sink would be handed to that same broken sink. Stderr is the channel of last
-resort, and every site that swallows an exception to keep architecture §4's promise (logging never
-breaks the application) announces it through here, so the wording and the safety rules are decided
-once rather than per call site.
-
-Two rules, both load-bearing:
-
-* **The exception's type, never its message.** A message can carry a value from the event that
-  provoked it, and arch §6 keeps user data out of anything the library emits about itself — the
-  same rule SPEC-019 applies to ``Health.stopped_reason``. A type name is enough to tell a
-  ``ConnectionError`` from a ``TypeError``, which is what the reader needs.
-* **No stream fault escapes.** It is called from ``except`` and ``finally`` blocks whose entire
-  purpose is to stop an exception reaching the caller, so a failure here — a closed stderr at
-  interpreter shutdown, a stream that rejects the write, ``sys.stderr`` set to ``None`` — must not
-  become the exception those guards existed to prevent. A ``BaseException`` still passes through,
-  as everywhere else in this library: a ``KeyboardInterrupt`` landing mid-write is the operator's
-  intent, not a stream fault to swallow.
-
-**This module must import nothing from its own package.** ``decorator``, ``api`` and ``worker``
-all reach it with ``from log_foundry import _diag`` at module scope, which executes while the
-package is still partially initialised and resolves only because ``_diag`` is a leaf — it needs
-``sys`` and nothing else. Give it an intra-package import and every module that is imported
-*first* in a fresh interpreter starts failing.
-
-SPEC-029 takes ownership of this module and moves the ``repr(exception)`` sink sites onto it.
-"""
-
-from __future__ import annotations
-
-import sys
-
-__all__ = ["absorbed"]
-
-
-def absorbed(where: str, exc: BaseException, detail: str = "") -> None:
-    """Report a failure the library swallowed rather than propagated.
-
-    Never raises on a stream fault; a ``BaseException`` from the write still propagates.
-
-    Args:
-        where: What was being attempted, as a participle phrase that reads after "while" —
-            ``"closing a span"``, ``"emitting an orphan log"``.
-        exc: The absorbed exception. Only ``type(exc).__name__`` is written.
-        detail: Optional consequence for the reader, appended after a semicolon — most usefully
-            what was lost, since an absorbed failure is invisible apart from this line.
-    """
-    try:
-        suffix = f"; {detail}" if detail else ""
-        sys.stderr.write(
-            f"log-foundry: absorbed a failure while {where} ({type(exc).__name__}){suffix}\n"
-        )
-    except Exception:
-        # The channel of last resort has no fallback of its own. Reporting is best-effort by
-        # construction: losing the line is bad, raising from the guard that exists to protect
-        # the caller's exception would be worse (arch §4).
-        pass