log-foundry 0.5.1.dev5__tar.gz → 0.5.1.dev7__tar.gz

{log_foundry-0.5.1.dev5 → log_foundry-0.5.1.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.5.1.dev5
+Version: 0.5.1.dev7
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE

{log_foundry-0.5.1.dev5 → log_foundry-0.5.1.dev7}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 ]
 
 # Optional features. Install with: pip install log-foundry[aws]
-version = "0.5.1.dev5"
+version = "0.5.1.dev7"
 
 [project.optional-dependencies]
 aws = ["boto3>=1.34"]              # SQSSink, SNSSink, KinesisSink, FirehoseSink

{log_foundry-0.5.1.dev5 → log_foundry-0.5.1.dev7}/src/log_foundry/config.py

@@ -19,18 +19,33 @@ if TYPE_CHECKING:
 
 @dataclass
 class Config:
-    """Process-wide settings stamped onto every event / consulted by the pipeline."""
+    """Process-wide settings stamped onto every event / consulted by the pipeline.
+
+    The four ``max_*`` ceilings bound every event payload (SPEC-017 FR-002). Defaults are set so
+    that the overwhelming majority of events are untouched; they exist to stop *one* pathological
+    value getting a whole event rejected by a sink's hard limit.
+    """
 
     service: str = "unknown"
     version: str = "0.0.0"
     env: str = "dev"
     sink: Sink | None = None
     defaults: dict[str, object] = field(default_factory=dict)
+    max_value_bytes: int = 8192  # per str value, UTF-8 bytes
+    max_stack_bytes: int = 32768  # error.stack only — legitimately long, and worth keeping
+    max_keys: int = 256  # per mapping / sequence
+    max_depth: int = 8  # nesting levels
 
 
 _config = Config()  # module-level singleton; the whole library reads through get_config()
 
 
+def _require_positive(name: str, value: int | None) -> None:
+    """Reject a non-positive ceiling. A ceiling of zero would empty every event it touched."""
+    if value is not None and value < 1:
+        raise ValueError(f"{name} must be >= 1, got {value}")
+
+
 def configure(
     *,
     service: str | None = None,
@@ -38,13 +53,26 @@ def configure(
     env: str | None = None,
     sink: Sink | None = None,
     defaults: dict[str, object] | None = None,
+    max_value_bytes: int | None = None,
+    max_stack_bytes: int | None = None,
+    max_keys: int | None = None,
+    max_depth: int | None = None,
 ) -> None:
     """Patch the global config. Call once at startup.
 
     Only the arguments you pass are applied, so repeated calls compose rather than reset.
     If no sink has ever been set, defaults to :class:`~log_foundry.sinks.stdout.StdoutSink`
     (the zero-dependency dev default, arch §8) once that phase lands.
+
+    The four ``max_*`` ceilings bound event payloads (SPEC-017 FR-006); each must be >= 1.
     """
+    # Validate every ceiling *before* assigning anything: a rejected call must leave the config
+    # exactly as it found it, not half-applied with `service` set and the ceiling rejected.
+    _require_positive("max_value_bytes", max_value_bytes)
+    _require_positive("max_stack_bytes", max_stack_bytes)
+    _require_positive("max_keys", max_keys)
+    _require_positive("max_depth", max_depth)
+
     if service is not None:
         _config.service = service
     if version is not None:
@@ -55,6 +83,14 @@ def configure(
         _config.sink = sink
     if defaults is not None:
         _config.defaults = dict(defaults)
+    if max_value_bytes is not None:
+        _config.max_value_bytes = max_value_bytes
+    if max_stack_bytes is not None:
+        _config.max_stack_bytes = max_stack_bytes
+    if max_keys is not None:
+        _config.max_keys = max_keys
+    if max_depth is not None:
+        _config.max_depth = max_depth
 
     _ensure_sink()
 

{log_foundry-0.5.1.dev5 → log_foundry-0.5.1.dev7}/src/log_foundry/model.py

@@ -12,6 +12,12 @@ import time
 import traceback
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+from log_foundry.sanitize import sanitize_fields, truncate_str, truncate_tail
+
+if TYPE_CHECKING:
+    from log_foundry.config import Config
 
 __all__ = ["Span", "build_event", "start_event", "end_event", "backfill_baggage"]
 
@@ -20,6 +26,11 @@ __all__ = ["Span", "build_event", "start_event", "end_event", "backfill_baggage"
 _START_MESSAGE = "span.start"
 _END_MESSAGE = "span.end"
 
+# Marks an event that a ceiling clipped (SPEC-017 FR-002). Set **only ever to True**, never to
+# False — that single invariant is what makes "OR with whatever an earlier stage set" and
+# "absent, not false, on a clean event" both fall out with no read-modify-write anywhere.
+_TRUNCATED = "truncated"
+
 
 @dataclass
 class Span:
@@ -57,16 +68,22 @@ def build_event(
 
     Field precedence, lowest to highest: config ``defaults`` → ``span.defaults`` → ``baggage``
     → per-call ``fields`` (arch §5.1). Later sources win on a key conflict.
+
+    The merged mapping is coerced and size-bounded here (SPEC-017 FR-001/FR-002), which is what
+    makes every sink's bare ``json.dumps`` safe without any of them changing. ``message`` is
+    bounded too: it is a base field, but unlike the other eleven it is caller-supplied free text.
     """
     from log_foundry.config import get_config
     from log_foundry.ids import new_log_id
 
     cfg = get_config()
     merged: dict[str, object] = {**cfg.defaults, **span.defaults, **baggage, **fields}
-    return {
+    safe, clipped = sanitize_fields(merged, cfg=cfg)
+    bounded_message, message_clipped = truncate_str(message, cfg.max_value_bytes)
+    event: dict[str, object] = {
         "timestamp": _iso_now(),
         "level": level,
-        "message": message,
+        "message": bounded_message,
         "trace_id": span.trace_id,
         "span_id": span.span_id,
         "parent_span_id": span.parent_span_id,
@@ -75,8 +92,12 @@ def build_event(
         "service": cfg.service,
         "version": cfg.version,
         "env": cfg.env,
-        "fields": merged,
     }
+    # Before ``fields`` so it reads ahead of the payload blob in a rendered log line.
+    if clipped or message_clipped:
+        event[_TRUNCATED] = True
+    event["fields"] = safe
+    return event
 
 
 def start_event(span: Span) -> dict[str, object]:
@@ -84,6 +105,57 @@ def start_event(span: Span) -> dict[str, object]:
     return build_event(span, "INFO", _START_MESSAGE, fields={}, baggage={})
 
 
+def _exception_message(exc: BaseException) -> str:
+    """``str(exc)`` — ``""`` for an exception raised with no arguments (SPEC-017 FR-003).
+
+    Guarded because a user exception may define a ``__str__`` that itself raises. This runs
+    inside the decorator's ``except`` block, so an exception escaping here would *replace* the
+    one the user's code raised, demoting theirs to ``__context__`` — logging breaking the app,
+    which is the thing this spec exists to stop.
+    """
+    try:
+        return str(exc)
+    except Exception:  # noqa: BLE001 — see above; a hostile __str__ must not escape.
+        return "<unprintable message>"
+
+
+def _exception_stack(exc: BaseException) -> str:
+    """The formatted traceback.
+
+    ``traceback.format_exception`` already renders a failing ``__str__`` as
+    ``<exception str() failed>`` rather than propagating, so this guard is for the rarer case of
+    a frame that cannot be rendered at all.
+    """
+    try:
+        return "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
+    except Exception:  # noqa: BLE001 — same reasoning as _exception_message.
+        return f"<unformattable traceback: {type(exc).__name__}>"
+
+
+def _error_fields(exc: BaseException, *, cfg: Config) -> tuple[dict[str, object], bool]:
+    """Build the bounded ``error`` sub-document; report whether a ceiling fired.
+
+    ``type`` keeps the bare class name consumers already index; ``module`` is added alongside it
+    so two same-named exception classes from different packages stay distinguishable, rather
+    than qualifying ``type`` in place and breaking every existing query (SPEC-017 FR-003).
+
+    ``stack`` gets its own, larger ceiling: a traceback is legitimately long and is the most
+    valuable thing on the event. It is clipped from the *head*, because ``format_exception``
+    puts the exception and the innermost frames last.
+    """
+    type_name, t1 = truncate_str(type(exc).__name__, cfg.max_value_bytes)
+    module, t2 = truncate_str(str(getattr(type(exc), "__module__", "")), cfg.max_value_bytes)
+    message, t3 = truncate_str(_exception_message(exc), cfg.max_value_bytes)
+    stack, t4 = truncate_tail(_exception_stack(exc), cfg.max_stack_bytes)
+    error: dict[str, object] = {
+        "type": type_name,
+        "module": module,
+        "message": message,
+        "stack": stack,
+    }
+    return error, (t1 or t2 or t3 or t4)
+
+
 def end_event(
     span: Span,
     status: str,
@@ -92,17 +164,20 @@ def end_event(
     """Build the span-end boundary event.
 
     Adds ``duration_ms`` (from a monotonic delta), ``status`` (``"ok"``/``"error"``), and on
-    failure a nested ``error`` with the exception type and formatted stack (arch §6).
+    failure a nested ``error`` with the exception type, module, message and formatted stack
+    (arch §6, SPEC-017 FR-003).
     """
+    from log_foundry.config import get_config
+
     level = "INFO" if status == "ok" else "ERROR"
     event = build_event(span, level, _END_MESSAGE, fields={}, baggage={})
     event["duration_ms"] = (time.monotonic() - span.start_ts) * 1000.0
     event["status"] = status
     if exc is not None:
-        event["error"] = {
-            "type": type(exc).__name__,
-            "stack": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
-        }
+        error, clipped = _error_fields(exc, cfg=get_config())
+        event["error"] = error
+        if clipped:
+            event[_TRUNCATED] = True  # only ever True — this *is* the OR with build_event's
     return event
 
 
@@ -123,13 +198,24 @@ def backfill_baggage(span: Span, baggage: dict[str, object]) -> None:
 
     The baggage is a parameter rather than read from ``context``: this module does not know where
     the current span lives (arch §6).
+
+    These values bypass :func:`build_event`'s pass entirely — ``set_baggage`` accepts arbitrary
+    objects — so they are coerced here, **once** above the loop rather than per event: the same
+    mapping is merged into every boundary event, and ``fields`` is already sanitized. The merge
+    can push a mapping past ``max_keys``; that is deliberate, because re-capping here would drop
+    the correlation keys SPEC-015 shipped to add (SPEC-017 FR-001).
     """
     if not baggage:
         return
+    from log_foundry.config import get_config
+
+    safe, clipped = sanitize_fields(baggage, cfg=get_config())
     for event in span.events:
         # Matched on the message constants, not a position — that ``span.start`` is index 0 is an
         # implementation detail of when it happens to be appended.
         if event.get("message") in (_START_MESSAGE, _END_MESSAGE):
             fields = event.get("fields")
             if isinstance(fields, dict):
-                event["fields"] = {**fields, **baggage}
+                event["fields"] = {**fields, **safe}
+                if clipped:
+                    event[_TRUNCATED] = True

log_foundry-0.5.1.dev7/src/log_foundry/sanitize.py

@@ -0,0 +1,281 @@
+"""Coercion and size-bounding for event values (SPEC-017 FR-001, FR-002).
+
+An event dict must never contain a value ``json.dumps`` would reject, and no value may be
+unbounded. Both guarantees are established **once**, where the event is assembled, rather than
+in each sink: one pass per event instead of one per destination (which matters under
+``MultiSink``), and the guarantee then holds for the non-JSON sinks — ``postgres``, ``mongo``,
+``sqlite`` — for free. Every bare ``json.dumps`` in ``sinks/`` is correct by consequence.
+
+Three rules worth knowing before changing anything here:
+
+* **Total by contract.** Nothing in this module raises. It runs on the caller's own stack (a
+  level call with no active span emits synchronously, ``api.py``), so an exception escaping
+  here is precisely the failure SPEC-017 exists to remove.
+* **The ``truncated`` marker means a *ceiling* fired** — ``max_value_bytes``, ``max_stack_bytes``,
+  ``max_keys`` or ``max_depth``. It is *not* set by :data:`_CIRCULAR` or an unserializable
+  placeholder, which are coercion outcomes, not clipping.
+* **``max_depth`` is what bounds the recursion**, not cycle detection. A cycle shallower than the
+  depth limit terminates there regardless; the ancestor tracking exists only so the value reads
+  ``<circular>`` rather than ``<depth limit>``. Don't "harden" it into a safety mechanism.
+
+The unserializable fallback is a type-name placeholder rather than ``repr(value)`` on purpose.
+Architecture §6 refuses to auto-capture argument and return values so the library cannot leak
+secrets or PII, and ``repr()`` of an arbitrary object routinely prints attribute values — a
+credential held on a client object would land in the log. The placeholder identifies what was
+dropped without disclosing it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+from datetime import date, datetime, time
+from decimal import Decimal
+from enum import Enum
+from typing import TYPE_CHECKING, Any
+from uuid import UUID
+
+if TYPE_CHECKING:
+    # Type-only import: this module then has *no* runtime dependency on any other package
+    # module, so it can never take part in an import cycle. Same idiom as ``config.Sink``.
+    from log_foundry.config import Config
+
+__all__ = ["TRUNCATION_MARKER", "coerce", "sanitize_fields", "truncate_str", "truncate_tail"]
+
+TRUNCATION_MARKER = "…[truncated]"
+_MARKER_BYTES = len(TRUNCATION_MARKER.encode("utf-8"))
+
+_CIRCULAR = "<circular>"
+_DEPTH_LIMIT = "<depth limit>"
+
+# Exact-type membership, deliberately not ``isinstance``. ``IntEnum``/``StrEnum`` members *are*
+# ``int``/``str`` instances, so an isinstance check would pass the enum member itself through and
+# hand a sink an ``Enum`` where a plain value was promised. Exact typing lets them fall to the
+# ``Enum`` branch and degrade to ``.value``. It is also one hash lookup rather than an
+# ``ABCMeta.__instancecheck__``.
+_PLAIN_SCALARS: frozenset[type] = frozenset({int, float, bool})
+
+# ``str``/``bytes``/``bytearray`` are Sequences, and so — less obviously — is ``memoryview``.
+# Without this guard ``memoryview(b"x")`` would render as ``[120]`` instead of ``"x"``.
+_TEXTLIKE: tuple[type, ...] = (str, bytes, bytearray, memoryview)
+
+
+def _measured(value: str) -> bytes:
+    """UTF-8 bytes of ``value``, tolerating lone surrogates.
+
+    A ``str`` carrying an unpaired surrogate (anything that went through ``surrogateescape``,
+    e.g. ``os.fsdecode`` of an undecodable filename) raises ``UnicodeEncodeError`` on a bare
+    ``.encode("utf-8")`` — inside a function contracted never to raise. ``errors="replace"``
+    is the same tolerance ``context`` applies when measuring an inbound baggage header.
+    """
+    return value.encode("utf-8", errors="replace")
+
+
+def truncate_str(value: str, max_bytes: int) -> tuple[str, bool]:
+    """Clip ``value`` to ``max_bytes`` UTF-8 bytes, keeping the head.
+
+    The returned string — marker included — never exceeds ``max_bytes``, so a caller sizing
+    against a hard downstream limit can rely on the ceiling being a ceiling. Cuts on a character
+    boundary, so the result always decodes cleanly even when the budget falls mid-sequence.
+
+    Returns ``(value, was_truncated)``.
+    """
+    raw = _measured(value)
+    if len(raw) <= max_bytes:
+        return value, False
+    budget = max_bytes - _MARKER_BYTES
+    if budget <= 0:
+        # No room for anything but the marker — and a marker alone is still the honest answer.
+        return TRUNCATION_MARKER, True
+    # ``errors="ignore"`` drops a partial trailing sequence rather than emitting U+FFFD, which
+    # is what makes the cut land on a character boundary.
+    return raw[:budget].decode("utf-8", errors="ignore") + TRUNCATION_MARKER, True
+
+
+def truncate_tail(value: str, max_bytes: int) -> tuple[str, bool]:
+    """Clip ``value`` to ``max_bytes`` UTF-8 bytes, keeping the **tail**.
+
+    For ``error.stack``: ``traceback.format_exception`` puts the exception type, its message and
+    the innermost frames *last*, so the head of an over-long traceback is the least useful part
+    of it. The marker is prepended, and the total stays within ``max_bytes`` as above.
+
+    Returns ``(value, was_truncated)``.
+    """
+    raw = _measured(value)
+    if len(raw) <= max_bytes:
+        return value, False
+    budget = max_bytes - _MARKER_BYTES
+    if budget <= 0:
+        return TRUNCATION_MARKER, True
+    # ``raw[-budget:]`` would return the *whole* string at ``budget == 0``, which is why the
+    # guard above is not merely defensive: ``max_stack_bytes`` may legally be smaller than the
+    # marker, and silently truncating nothing would be worse than truncating everything.
+    return TRUNCATION_MARKER + raw[-budget:].decode("utf-8", errors="ignore"), True
+
+
+class _Coercer:
+    """One event's coercion pass: the ceilings, the truncation flag, and the ancestor chain.
+
+    A pass object rather than a pure function because the ceilings fire *deep* in the recursion
+    — a 300-key mapping eight levels down still has to set ``truncated`` on the top-level event —
+    and a recursive function returning a bare value has nowhere to report that. One instance per
+    event, ``__slots__``-ed; against the timestamp, UUID and two dicts ``build_event`` already
+    allocates per event, it is noise.
+    """
+
+    __slots__ = ("_cfg", "_parents", "truncated")
+
+    def __init__(self, cfg: Config) -> None:
+        self._cfg = cfg
+        self.truncated = False
+        # ``id()``s of the containers currently being descended through. Ancestors only, never a
+        # global "visited" set: an ancestor is held alive by the recursion for the whole descent,
+        # so its ``id()`` cannot be recycled underneath us, and two siblings referencing the same
+        # object are not a cycle and must both render.
+        self._parents: list[int] = []
+
+    def value(self, value: object, depth: int) -> object:
+        """Coerce one node. Total — never raises, whatever the value does."""
+        try:
+            return self._dispatch(value, depth)
+        except Exception:  # noqa: BLE001 — a hostile __iter__/__str__/__eq__ must not reach the
+            return self._placeholder(value)  # caller's stack; this is the whole point (FR-001).
+
+    def _dispatch(self, value: object, depth: int) -> object:
+        if depth >= self._cfg.max_depth:
+            self.truncated = True
+            return _DEPTH_LIMIT
+        if value is None:
+            return None
+
+        # Exact types first: the common case, one hash lookup, and it is what lets IntEnum and
+        # StrEnum fall through to the Enum branch below instead of passing through as themselves.
+        kind = type(value)
+        if kind is str:
+            return self.text(value)  # type: ignore[arg-type]
+        if kind in _PLAIN_SCALARS:
+            return value
+        if kind is dict:
+            return self.mapping(value, depth)  # type: ignore[arg-type]
+        if kind is list:
+            return self.members(value, depth)  # type: ignore[arg-type]
+
+        if isinstance(value, Enum):
+            member = value.value
+            if type(member) in _PLAIN_SCALARS or member is None:
+                return member
+            if isinstance(member, str):
+                return self.text(member)
+            # A structured ``.value`` (a tuple payload, say) has no plain form; ``str(value)``
+            # renders the member as ``Class.NAME``, which identifies it without a ``repr``.
+            return self.text(str(value))
+        if isinstance(value, bool):  # bool before int: it is an int subclass.
+            return value
+        if isinstance(value, (int, float)):  # int/float *subclasses* land here.
+            return value
+        if isinstance(value, (datetime, date, time)):
+            return self.text(value.isoformat())
+        if isinstance(value, UUID):
+            return self.text(str(value))
+        if isinstance(value, Decimal):
+            # A string, not a float: ``Decimal("0.1")`` must not become ``0.1000000000000000055``.
+            return self.text(str(value))
+        if isinstance(value, (bytes, bytearray, memoryview)):
+            return self.text(bytes(value).decode("utf-8", errors="replace"))
+        if isinstance(value, str):  # ``str`` subclass.
+            return self.text(str(value))
+        if isinstance(value, Mapping):
+            return self.mapping(value, depth)
+        if isinstance(value, (set, frozenset)):  # sets are not Sequences.
+            return self.members(value, depth)
+        if isinstance(value, Sequence) and not isinstance(value, _TEXTLIKE):
+            return self.members(value, depth)
+        return self._placeholder(value)
+
+    def mapping(self, value: Mapping[Any, object], depth: int) -> object:
+        """Coerce a mapping, capping it at ``max_keys`` and guarding against a cycle.
+
+        The key type is ``Any`` rather than ``object`` because ``Mapping`` is invariant in its
+        key: a ``Mapping[str, object]`` (which is what ``build_event`` passes) is not a
+        ``Mapping[object, object]``. :meth:`key` coerces whatever actually arrives.
+        """
+        ident = id(value)
+        if ident in self._parents:
+            return _CIRCULAR
+        self._parents.append(ident)
+        try:
+            out: dict[str, object] = {}
+            for key, item in value.items():
+                if len(out) >= self._cfg.max_keys:
+                    self.truncated = True
+                    break
+                out[self.key(key)] = self.value(item, depth + 1)
+            return out
+        finally:
+            self._parents.pop()
+
+    def members(self, value: Iterable[object], depth: int) -> object:
+        """Coerce an iterable's members into a list, capping length at ``max_keys``."""
+        ident = id(value)
+        if ident in self._parents:
+            return _CIRCULAR
+        self._parents.append(ident)
+        try:
+            out: list[object] = []
+            for item in value:
+                if len(out) >= self._cfg.max_keys:
+                    self.truncated = True
+                    break
+                out.append(self.value(item, depth + 1))
+            return out
+        finally:
+            self._parents.pop()
+
+    def key(self, key: object) -> str:
+        """Coerce a mapping key to a bounded ``str`` — JSON object keys are always strings."""
+        text = key if isinstance(key, str) else str(key)
+        return self.text(text)
+
+    def text(self, value: str) -> str:
+        """Apply ``max_value_bytes``, recording whether it fired."""
+        clipped, was_truncated = truncate_str(value, self._cfg.max_value_bytes)
+        if was_truncated:
+            self.truncated = True
+        return clipped
+
+    def _placeholder(self, value: object) -> str:
+        """Name the type that could not be coerced, without disclosing the value."""
+        try:
+            name = type(value).__name__
+        except Exception:  # noqa: BLE001 — a pathological metaclass is still not our problem.
+            name = "?"
+        return f"<unserializable: {name}>"
+
+
+def coerce(value: object, *, cfg: Config) -> object:
+    """Return a JSON-serializable, size-bounded equivalent of ``value``. Never raises.
+
+    Use :func:`sanitize_fields` for a whole event mapping — it reports whether a ceiling fired,
+    which this cannot.
+    """
+    return _Coercer(cfg).value(value, 0)
+
+
+def sanitize_fields(
+    fields: Mapping[str, object], *, cfg: Config
+) -> tuple[dict[str, object], bool]:
+    """Coerce and bound a whole ``fields`` mapping.
+
+    Returns ``(fields, any_ceiling_was_applied)``. The flag drives the event's ``truncated``
+    marker, so it must reflect ceilings that fired arbitrarily deep in the structure — hence the
+    accumulator rather than a pure recursive function.
+    """
+    coercer = _Coercer(cfg)
+    try:
+        result = coercer.mapping(fields, 0)
+    except Exception:  # noqa: BLE001 — belt and braces: `_Coercer.value` is already total, but
+        return {}, True  # a hostile top-level mapping must not reach the caller either.
+    if not isinstance(result, dict):
+        # ``fields`` was itself circular — impossible from ``build_event``, which always passes a
+        # freshly merged dict, but ``sanitize_fields`` is public and must stay total.
+        return {}, True
+    return result, coercer.truncated