log-foundry 0.10.2.dev55__tar.gz → 0.10.2.dev56__tar.gz

{log_foundry-0.10.2.dev55 → log_foundry-0.10.2.dev56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.10.2.dev55
+Version: 0.10.2.dev56
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE

{log_foundry-0.10.2.dev55 → log_foundry-0.10.2.dev56}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 ]
 
 # Optional features. Install with: pip install log-foundry[aws]
-version = "0.10.2.dev55"
+version = "0.10.2.dev56"
 
 [project.optional-dependencies]
 aws = ["boto3>=1.43.61"]              # SQSSink, SNSSink, KinesisSink, FirehoseSink

{log_foundry-0.10.2.dev55 → log_foundry-0.10.2.dev56}/src/log_foundry/_fork.py

@@ -330,7 +330,43 @@ def _fresh_primitive(value: Any, memo: dict[int, Any], keepalive: list[Any]) ->
     return fresh
 
 
-def _reinit_primitives() -> None:
+_DISCARD_HOOK = "discard_buffered_after_fork"
+"""The optional sink member a forked child asks to strand the parent's pending bytes.
+
+Probed by name, as ``losses()`` and ``log_foundry_stop_signal`` are, so no existing sink stops
+satisfying ``Sink`` by not having it. ``sinks/base.py`` states the contract; this is the only
+place the name is read.
+"""
+
+
+def _offers_discard(holder: object) -> bool:
+    """Whether this object carries FR-004's buffer hook, without letting the question raise.
+
+    Every other read the walk makes is guarded individually, and this one has to be too: an
+    owned ``__getattr__`` raising anything but ``AttributeError`` would abort the walk, and
+    what is lost then is not a buffer but the **lock repair** for everything the walk had not
+    reached yet — the hang this module exists to remove, arrived at through the probe for a
+    different hazard. Nothing owned defines ``__getattr__`` today.
+
+    Callability is part of the question, so an attribute of that name which is not a method is
+    simply not the hook.
+
+    Args:
+      holder: Any object the walk has entered.
+
+    Returns:
+      Whether it offers the hook.
+
+    Raises:
+      None.
+    """
+    try:
+        return callable(getattr(holder, _DISCARD_HOOK, None))
+    except Exception:
+        return False
+
+
+def _reinit_primitives() -> list[Any]:
     """Replaces every lock and event this package owns, wherever the walk reaches one.
 
     An inherited ``Lock`` stays locked with no owner — measured, ``acquire(timeout=1)`` returns
@@ -345,11 +381,17 @@ def _reinit_primitives() -> None:
     events measured 202 ms, against 0.45 ms idle. That is accepted rather than bounded: a cap
     would be a lock this cannot promise to find, and the alternative to finding it is a hang.
 
+    **The sinks carrying FR-004's buffer hook are collected on this same walk** rather than by
+    a second one, because the traversal is the expensive part and its cost is proportional to
+    caller data — paying 202 ms twice on a process that forks is a worse trade than one function
+    reporting what it passed. Collected, not called: a hook may take a lock, and the contract is
+    that every lock is the child's own before any of them runs (FR-001 AC-2).
+
     Args:
       None.
 
     Returns:
-      None.
+      The owned instances carrying :data:`_DISCARD_HOOK`, in the order the walk reached them.
 
     Raises:
       None.
@@ -357,6 +399,7 @@ def _reinit_primitives() -> None:
     memo: dict[int, Any] = {}
     keepalive: list[Any] = []
     seen: set[int] = set()
+    buffered: list[Any] = []
     stack: list[Any] = [
         module
         for name, module in list(sys.modules.items())
@@ -369,14 +412,49 @@ def _reinit_primitives() -> None:
         seen.add(id(holder))
         if _is_container(holder):
             stack.extend(child for child in _container_children(holder) if _is_traversable(child))
-        if not isinstance(holder, types.ModuleType | type) and not _is_owned(holder):
-            continue
+        if not isinstance(holder, types.ModuleType | type):
+            if not _is_owned(holder):
+                continue
+            if _offers_discard(holder):
+                buffered.append(holder)
         for name, value in _namespace_items(holder):
             fresh = _fresh_primitive(value, memo, keepalive)
             if fresh is not None:
                 _assign(holder, name, fresh)
             elif _is_traversable(value):
                 stack.append(value)
+    return buffered
+
+
+def _discard_buffers(holders: list[Any]) -> None:
+    """Asks each sink that owns a buffered stream to strand what it inherited (FR-004).
+
+    A fork landing inside ``emit`` — after the write loop, before the flush — leaves the child
+    holding the parent's unflushed bytes, which both processes then write: measured, the event
+    at the fork point appeared on disk twice. Without a ``before`` handler there is nowhere to
+    empty the buffer from (FR-001), so the child throws its copy away instead.
+
+    Args:
+      holders: What :func:`_reinit_primitives` collected on its walk.
+
+    Returns:
+      None.
+
+    Raises:
+      None. One sink's failure is absorbed separately from the rest, so a hook that cannot
+        strand its buffer costs that sink a duplicated batch rather than costing every other
+        sink its repair — and the consequence is named, because an absorbed failure is
+        invisible apart from that line.
+    """
+    for holder in holders:
+        try:
+            getattr(holder, _DISCARD_HOOK)()
+        except Exception as exc:
+            _diag.absorbed(
+                "discarding an inherited buffer after a fork",
+                exc,
+                f"{type(holder).__name__} may write the parent's pending bytes again",
+            )
 
 
 def register_child_handler(fn: Callable[[], None]) -> None:
@@ -414,11 +492,12 @@ def _reinit_after_fork() -> None:
     """Repairs the library in a child that has just returned from ``fork``.
 
     **The order of work here is the contract** (FR-001 AC-2): locks and events first, then the
-    registered handlers. A lock re-initialised *after* a handler that takes it is a handler
-    that hangs, and it hangs on the child's only thread with nothing to interrupt it. Work that
-    must happen before *any* handler belongs inline between the two steps rather than registered
-    — the buffer discard FR-004 adds is the case, and registering it would put it after
-    ``decorator``'s rebuild, which has started a live drain thread by then.
+    buffer discard, then the registered handlers. A lock re-initialised *after* a handler that
+    takes it is a handler that hangs, and it hangs on the child's only thread with nothing to
+    interrupt it. Work that must happen before *any* handler stays inline between the two steps
+    rather than being registered — the discard is the case, and registering it would put it
+    after ``decorator``'s rebuild, which has started a live drain thread by then, emitting into
+    the very sink whose buffer is still the parent's.
 
     Args:
       None.
@@ -436,10 +515,12 @@ def _reinit_after_fork() -> None:
         registers another simply causes it to run — and :func:`register_child_handler` is the
         only writer, appending only.
     """
+    buffered: list[Any] = []
     try:
-        _reinit_primitives()
+        buffered = _reinit_primitives()
     except Exception as exc:
         _diag.absorbed("repairing the library after a fork", exc, "this child may block or lose")
+    _discard_buffers(buffered)
     for handler in _child_handlers:
         try:
             handler()

{log_foundry-0.10.2.dev55 → log_foundry-0.10.2.dev56}/src/log_foundry/sinks/base.py

@@ -89,6 +89,33 @@ class Sink(Protocol):
     already used, with a ``threading.Event``. Reproduced. A prefixed name cannot collide by
     accident, and the cost is one verbose attribute on the sinks that want interruptibility.
 
+    A fifth is optional in the same way and exists for one event only:
+    ``discard_buffered_after_fork() -> None``. A sink that owns a **buffered** stream — one it
+    opened itself, rather than the process's ``sys.stdout`` — inherits the parent's unflushed
+    bytes in a forked child, and both processes then write them (SPEC-039 FR-004). Defining this
+    method is how a sink says it can strand what it inherited; a sink holding no buffer of its
+    own needs nothing here. It is *called* rather than assigned, which is why it carries no
+    ``log_foundry_`` prefix: the collision the fourth member's name guards against is an
+    attribute the library writes onto an object it does not own.
+
+    **Which sinks are actually asked is narrower than "whichever define it", and the boundary is
+    the same one every fork repair has.** The child's repair walks this package's own objects —
+    a sink whose class is defined here or **inherits** one that is, ``Sink`` included, and the
+    **owned** sinks such an object holds, ``MultiSink``'s children among them. A third-party
+    sink satisfying this Protocol *structurally*, which is how every shipped sink satisfies it,
+    is outside that walk exactly as its locks are (SPEC-039 FR-005): its hook is never called,
+    and a fork mid-batch there can still duplicate — including when a wrapper this library owns
+    is holding it. Inheriting from ``Sink`` or from a shipped sink is what brings it inside. A
+    **wrapper** sink the walk enters need do nothing itself, because the owned sinks it holds
+    are reached directly rather than through it.
+
+    **It runs in a child that has not yet returned from ``fork``, on that child's only thread,
+    and must not block.** Nothing else is running there, so there is no holder a lock could be
+    waiting for — and no watchdog to end a wait taken anyway, since a pending alarm does not
+    survive the fork and this runs before any application code can arm one. It may raise: the
+    library absorbs and announces that, and the cost is a child that may write the parent's
+    pending bytes a second time.
+
     "Safe to call during an emit" is a concurrency requirement once :meth:`emit` is (SPEC-028
     FR-003). The shipped sinks keep their loss counters under a **dedicated** lock, separate
     from whatever guards their transport: an increment is a read-modify-write that Python does

{log_foundry-0.10.2.dev55 → log_foundry-0.10.2.dev56}/src/log_foundry/sinks/file.py

@@ -20,6 +20,46 @@ _WHEN_SECONDS = {
 }
 
 
+def _reopen_discarding(stream: TextIO, path: str, encoding: str) -> TextIO:
+    """Strands an inherited stream's pending bytes and returns a fresh one on the same path.
+
+    ``os.dup2`` points the inherited descriptor at ``os.devnull``, so the buffer this process
+    inherited can only ever reach the null device — whether it is flushed deliberately, by the
+    interpreter at exit, or by the garbage collector when the old object is dropped. The two
+    steps are written in this order for readability and **not** because the order is what makes
+    it safe: the replacement takes a different descriptor either way, since the inherited one is
+    still occupied, and nothing else is running to flush anything in between. Reopening rather
+    than reusing the descriptor is what gives the child a stream of its own, and it picks up the
+    currently active file if the parent rotated.
+
+    ``dup2`` leaves the redirected descriptor **inheritable** across an ``exec``, where the one
+    ``open`` produced carried ``O_CLOEXEC``. That is a behaviour change and it is harmless: the
+    descriptor names the null device, so what an exec'd process inherits is a handle on nothing.
+
+    Args:
+      stream: The inherited stream, still holding whatever the parent had not flushed.
+      path: The file to reopen. **Append mode**, never write mode: the child shares this file
+        with the parent and with whatever was written before either existed, so truncating here
+        would destroy a log to protect it — strictly worse than the duplication this prevents.
+      encoding: The text encoding to open it in, carried across so a child does not start
+        writing a differently-encoded second half into the parent's file.
+
+    Returns:
+      The replacement stream.
+
+    Raises:
+      OSError: If the descriptor cannot be redirected or the path cannot be reopened.
+      ValueError: If the stream has no usable descriptor, which means there is no inherited
+        buffer this can strand and the caller must not carry on as though there were.
+    """
+    devnull = os.open(os.devnull, os.O_WRONLY)
+    try:
+        os.dup2(devnull, stream.fileno())
+    finally:
+        os.close(devnull)
+    return open(path, "a", encoding=encoding)
+
+
 class FileSink:
     """A :class:`~log_foundry.sinks.base.Sink` that appends events as NDJSON to one file.
 
@@ -84,6 +124,35 @@ class FileSink:
                 self._stream.write(json.dumps(event) + "\n")
             self._stream.flush()
 
+    def discard_buffered_after_fork(self) -> None:
+        """Throws away the parent's unflushed bytes in a forked child (SPEC-039 FR-004).
+
+        ``emit`` writes a whole batch into a **buffered** stream and flushes once at the end, so
+        a fork landing inside it leaves both processes holding the same pending bytes and both
+        writing them: measured, the event at the fork point appeared on disk twice. Without a
+        ``before`` handler there is nowhere to empty the buffer from (FR-001), so the child
+        strands it instead — the parent's copy is untouched and still reaches disk exactly once.
+
+        No lock is taken, for the reason ``decorator._rebuild_worker_after_fork`` gives: there is
+        one thread here by construction, and the lock was re-initialised moments earlier, so
+        taking it could only wait on a holder that cannot exist. A hook that blocks here blocks a
+        child that has not yet returned from ``fork``, where no watchdog can reach it.
+
+        Args:
+          None.
+
+        Returns:
+          None.
+
+        Raises:
+          OSError: If the descriptor cannot be redirected or the file cannot be reopened.
+          ValueError: If the stream has no usable descriptor. ``_fork`` absorbs and announces
+            either, since a child that cannot strand its buffer still has working locks.
+        """
+        if self._closed:
+            return
+        self._stream = _reopen_discarding(self._stream, self._path, self._encoding)
+
     def close(self) -> None:
         """Flushes and closes the file handle, with a second call a no-op (FR-001).
 
@@ -221,6 +290,34 @@ class RotatingFileSink:
                 self._size += data
             self._stream.flush()
 
+    def discard_buffered_after_fork(self) -> None:
+        """Throws away the parent's unflushed bytes in a forked child (SPEC-039 FR-004).
+
+        Identical to :meth:`FileSink.discard_buffered_after_fork` and measured on this class
+        too, because the window is the same one: a whole batch written into a buffered stream
+        and flushed once at the end.
+
+        ``_size`` is deliberately left as the parent set it. It counts bytes this sink believes
+        are in the active file, and the moment two processes append to one path that is an
+        approximation whichever way it is computed (FR-005 AC-1) — re-reading the file's size
+        here would claim a precision a shared handle cannot support, and the only consequence of
+        the stale count is a rotation that fires marginally early.
+
+        Args:
+          None.
+
+        Returns:
+          None.
+
+        Raises:
+          OSError: If the descriptor cannot be redirected or the file cannot be reopened.
+          ValueError: If the stream has no usable descriptor. ``_fork`` absorbs and announces
+            either, since a child that cannot strand its buffer still has working locks.
+        """
+        if self._closed:
+            return
+        self._stream = _reopen_discarding(self._stream, self._path, self._encoding)
+
     def close(self) -> None:
         """Flushes and closes the active handle, with a second call a no-op (FR-002).