log-foundry 0.2.1.dev3__py3-none-any.whl → 0.2.1.dev4__py3-none-any.whl

log_foundry/__init__.py

@@ -1,8 +1,8 @@
 """log_foundry — consistent, structured logs per decorated function call.
 
 Public façade (module-function shape). Exposes configuration, the ``@trace`` decorator, the
-``debug/info/warning/error/critical`` emitters, ``set_baggage``, and ``shutdown`` (graceful
-drain of the background worker).
+``debug/info/warning/error/critical`` emitters, ``set_baggage``, ``flush`` (drain and keep
+logging) and ``shutdown`` (drain, close the sink, and stop).
 """
 
 from importlib.metadata import PackageNotFoundError
@@ -19,11 +19,38 @@ except PackageNotFoundError:  # running from a source tree that isn't installed
     __version__ = "0.0.0"
 
 
+def flush(timeout: float | None = 5.0) -> bool:
+    """Drain buffered events through the sink without closing it.
+
+    Every event submitted before this call has been passed to ``sink.emit`` when it returns
+    ``True``. (Events submitted concurrently by another thread may or may not be included —
+    the caller cannot have meant those.) Unlike :func:`shutdown` the background worker stays
+    alive and the sink stays open, so logging continues normally afterwards.
+
+    This is the drain for a process that is *frozen* rather than exited — an AWS Lambda
+    handler must drain before it returns, but will be invoked again on the same warm
+    container. Call it in a ``finally``: the invocation worth logging is the one that failed.
+
+    ``timeout=None`` waits indefinitely, which is unsafe in any environment with an execution
+    deadline — it converts "some logs were lost" into "the invocation timed out".
+
+    Returns ``False`` if the drain did not complete within ``timeout``, or if the worker has
+    already been shut down. Never raises.
+    """
+    from log_foundry.decorator import _flush_worker
+
+    return _flush_worker(timeout)
+
+
 def shutdown() -> None:
     """Flush buffered events and close the sink, blocking until drained. Idempotent.
 
     Also registered via ``atexit``; call it explicitly before a fast process exit when you
     want to be certain the tail of the queue reached the sink (SPEC-004 FR-005).
+
+    This is **terminal** — the worker does not come back. Do not call it per-invocation in a
+    serverless handler: the first invocation on a warm container would log and every later one
+    would silently log nothing. Use :func:`flush` there, which drains and keeps the worker.
     """
     from log_foundry.decorator import _shutdown_worker
 
@@ -40,6 +67,7 @@ __all__ = [
     "error",
     "critical",
     "set_baggage",
+    "flush",
     "shutdown",
     "__version__",
 ]

log_foundry/decorator.py

@@ -80,6 +80,25 @@ def _shutdown_worker() -> None:
         _worker.shutdown()
 
 
+def _flush_worker(timeout: float | None = 5.0) -> bool:
+    """Drain the process worker without retiring it. Backs ``flush()`` (SPEC-013 FR-003).
+
+    Returns ``True`` when no worker exists: a process that never logged has nothing to drain,
+    and building one here — with the thread and ``atexit`` registration :func:`_get_worker`
+    brings — in order to flush nothing would be pure cost. So this deliberately does *not* call
+    :func:`_get_worker`.
+    """
+    worker = _worker
+    if worker is None:
+        return True
+    try:
+        return worker.flush(timeout)
+    except Exception:  # noqa: BLE001 — a flush is the call most likely to be made in a
+        # `finally`, so the library must never be the reason a caller's function fails. Any
+        # failure is reported by the return value instead (FR-003).
+        return False
+
+
 def _flush(span: Span) -> None:
     """Hand the finished span's events to the background worker — non-blocking (FR-001).
 

log_foundry/worker.py

@@ -8,6 +8,11 @@ bounded and overflow is dropped-newest with a counter (arch §9). Emit failures
 with backoff; a graceful :meth:`shutdown` drains the queue, emits the tail, and closes the
 sink so buffered events survive process exit.
 
+Two drains, deliberately distinct (SPEC-013): :meth:`shutdown` is terminal — it stops the thread
+and closes the sink, and the worker never comes back — while :meth:`flush` drains on demand and
+leaves everything running. A process that is frozen rather than exited (a serverless handler
+between invocations) needs the second, because it will be asked to log again.
+
 This module owns *delivery mechanics only*; it receives already-built event dicts and knows
 nothing about spans or context (the same dumbness that makes sinks swappable).
 """
@@ -29,6 +34,25 @@ __all__ = ["Worker"]
 _SHUTDOWN = object()
 
 
+class _FlushMarker:
+    """A drain request travelling the queue in FIFO order (SPEC-013 FR-002).
+
+    The mechanism follows from the queue being FIFO: everything submitted *before* ``flush()``
+    was called is necessarily ahead of the marker, so by the time the worker dequeues it those
+    events are either already emitted or sitting in ``pending``. The worker emits ``pending``,
+    then sets ``event``. No lock, no inspection of queue internals, and no coordination with the
+    batching triggers.
+
+    Like ``_SHUTDOWN`` it is never emitted — but unlike ``_SHUTDOWN`` it carries state, so it is
+    a class rather than a bare sentinel object.
+    """
+
+    __slots__ = ("event",)
+
+    def __init__(self) -> None:
+        self.event = threading.Event()
+
+
 class Worker:
     """Owns a bounded queue + daemon thread that batches and flushes events to ``sink``."""
 
@@ -71,6 +95,36 @@ class Worker:
             with self._lock:
                 self.dropped += 1
 
+    def flush(self, timeout: float | None = 5.0) -> bool:
+        """Drain everything submitted before this call through the sink, without stopping.
+
+        Returns ``True`` once the worker has emitted them, ``False`` on timeout or when the
+        worker has already been shut down. Unlike :meth:`shutdown` the thread keeps running, the
+        sink is **not** closed, and the once-only shutdown flag is untouched, so logging
+        continues normally afterwards (SPEC-013 FR-002).
+        """
+        with self._lock:
+            if self._shutdown_done:
+                # Nothing will ever consume a marker now, so report the failure immediately
+                # rather than make the caller wait out `timeout` for a drain that cannot happen.
+                # A caller with an execution deadline would pay that wait for nothing (FR-003).
+                return False
+        if not self._thread.is_alive():
+            return False
+        marker = _FlushMarker()
+        # One deadline shared by the put and the wait: a caller asked for a bound on the whole
+        # call, not on each half of it, so the two cannot add up to 2 * timeout.
+        deadline = None if timeout is None else time.monotonic() + timeout
+        try:
+            # A *blocking* put, never put_nowait: on a full queue put_nowait would skip the
+            # flush and return as though it had succeeded, which is the one outcome a flush must
+            # never produce silently. A put that times out is reported as False.
+            self._queue.put(marker, timeout=timeout)
+        except queue.Full:
+            return False
+        remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
+        return marker.event.wait(remaining)
+
     def shutdown(self) -> None:
         """Stop the thread, drain + emit everything queued, then ``close()`` the sink.
 
@@ -101,6 +155,22 @@ class Worker:
                 item = self._queue.get(timeout=timeout)
             except queue.Empty:
                 item = None
+            if isinstance(item, _FlushMarker):
+                # Drain on demand: emit immediately, ignoring both the batch_size and the
+                # flush_interval trigger — a caller who asked for a flush is not interested in
+                # the batching policy. Note this branch *returns to the top of the loop*: the
+                # marker must never fall through to the append below, where it would be treated
+                # as a list of events and handed to sink.emit, killing this thread.
+                try:
+                    if pending:
+                        self._emit(pending)
+                        pending = []
+                finally:
+                    # Signal even if the emit died, so a waiter is released rather than left to
+                    # wait out its timeout on a thread that is no longer running.
+                    last_flush = time.monotonic()
+                    item.event.set()
+                continue
             if item is not None and item is not _SHUTDOWN:
                 pending.append(cast("list[dict[str, object]]", item))
             now = time.monotonic()
@@ -116,15 +186,24 @@ class Worker:
 
     def _final_drain(self, pending: list[list[dict[str, object]]]) -> None:
         """On stop, pull anything still queued and emit the tail as one final batch."""
+        markers: list[_FlushMarker] = []
         while True:
             try:
                 item = self._queue.get_nowait()
             except queue.Empty:
                 break
+            if isinstance(item, _FlushMarker):
+                # This guard is a second copy of _run's and needs the same exclusion. Markers
+                # are answered *after* the final emit below, so a flush() that raced shutdown()
+                # still returns True — and truthfully: its events really did reach the sink.
+                markers.append(item)
+                continue
             if item is not None and item is not _SHUTDOWN:
                 pending.append(cast("list[dict[str, object]]", item))
         if pending:
             self._emit(pending)
+        for marker in markers:
+            marker.event.set()
 
     def _emit(self, event_lists: list[list[dict[str, object]]]) -> None:
         """Flatten queued per-span event-lists into one batch and emit, retrying with backoff.

{log_foundry-0.2.1.dev3.dist-info → log_foundry-0.2.1.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.2.1.dev3
+Version: 0.2.1.dev4
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE
@@ -206,6 +206,12 @@ never crashes the worker or the app. If the bounded queue fills completely, new
 dropped (newest-first) and counted rather than blocking your code. On `shutdown()` the worker
 stops, sweeps anything still queued into one final batch, emits it, and closes the sink.
 
+Both triggers can be pre-empted: `flush()` puts a marker in the queue and the worker emits the
+pending pile the moment it reaches it, ignoring the count and time triggers. Because the queue
+is FIFO, everything submitted before the call is necessarily ahead of that marker — which is
+exactly why the guarantee is "events submitted before this call", and why concurrent
+submissions from other threads may or may not be included.
+
 ## Usage
 
 ### `configure(...)`
@@ -494,17 +500,70 @@ a failing sink with backoff, and applies backpressure so a slow or down sink can
 back-pressure the app: when its bounded queue is full it drops the newest submissions and counts
 them (`worker.dropped`) rather than stalling.
 
-Because delivery is asynchronous, drain before the process exits:
+Because delivery is asynchronous, drain before the process exits. There are two drains, and
+which one you want depends on whether the process is about to end:
 
 ```python
 import log_foundry as lf
 
-lf.shutdown()   # flush buffered events and close the sink; blocks until drained
+lf.flush()      # drain to the sink and keep going; returns True when everything landed
+lf.shutdown()   # drain, close the sink, and stop for good; blocks until drained
 ```
 
+| | `flush()` | `shutdown()` |
+|---|---|---|
+| Drains buffered events | yes | yes |
+| Closes the sink | no | yes |
+| Worker survives | yes | **no** — it never comes back |
+| Repeatable | yes | idempotent, but only the first call does anything |
+| Use it | before returning from a handler, or at a checkpoint | once, as the process exits |
+
 `shutdown()` is also registered via `atexit`, so a normal exit flushes automatically — call it
-explicitly when you need to be certain the tail reached the sink before a fast exit (e.g. at the
-end of a short script or an AWS Lambda handler). It is idempotent.
+explicitly when you need to be certain the tail reached the sink before a fast exit, e.g. at the
+end of a short script. It is idempotent.
+
+`flush(timeout=5.0)` returns `True` when every event submitted before the call has been passed
+to the sink, and `False` if that did not happen within `timeout` (or the worker was already shut
+down). It never raises — a logging call must not be the reason your function fails. Passing
+`timeout=None` waits indefinitely, which is unsafe anywhere with an execution deadline.
+
+#### Serverless / short-lived processes
+
+In AWS Lambda (and anything else that freezes rather than exits) the rules are different, and
+getting them wrong is silent:
+
+- **Flush before the handler returns.** Lambda freezes the execution environment the instant
+  your handler returns, so the worker's interval-based flush stops mid-interval and whatever is
+  still queued is lost when the container is eventually reaped. `atexit` does not save you —
+  a frozen environment is killed without running exit handlers, so `flush()` is the *only*
+  guaranteed drain there.
+- **Put it in a `finally`.** A flush written as the last line of the handler body is precisely
+  the line that does not run when the handler raises, and the invocation whose logs are most
+  worth having is the one that failed.
+- **Never call `shutdown()` per invocation.** It is terminal: the worker does not come back, so
+  the first invocation on a warm container would log and every later one would silently log
+  nothing. That failure reads as "works locally, broken in production".
+
+```python
+import log_foundry as lf
+from log_foundry.sinks.sqs import SQSSink
+
+lf.configure(service="billing-api", env="prod", sink=SQSSink(queue_url=QUEUE_URL))
+
+@lf.trace
+def handler(event, context):
+    lf.info("received", records=len(event["Records"]))
+    try:
+        return do_work(event)
+    finally:
+        lf.flush()      # in `finally`: the failed invocation is the one worth logging.
+                        # NEVER shutdown() here — the worker does not come back, and every
+                        # later invocation on this warm container would log nothing.
+```
+
+Note that each invocation is its own trace: trace context does not cross a process boundary
+today, so N invocations produce N `trace_id`s. Correlate them with a shared
+[`set_baggage`](#logging-inside-a-span) field until cross-process continuation ships.
 
 ## Event schema
 

{log_foundry-0.2.1.dev3.dist-info → log_foundry-0.2.1.dev4.dist-info}/RECORD

@@ -1,9 +1,9 @@
-log_foundry/__init__.py,sha256=omYgsLnsjBuqEuD9J3gzSCUuiY9md_eh79333klUus0,1384
+log_foundry/__init__.py,sha256=bNIgcgWwtQc2BpmxdObbkRcyMOYOmTA6y5hlzKW5JCc,2860
 log_foundry/api.py,sha256=X4q1EnJCfqtfUglCoZkpUN4uHRs3lXCLBRB6hU8WI8k,3813
 log_foundry/config.py,sha256=vv0WkDffEmbSw3PjOiHBH0OHmyaqWcCUHXyRJsMBSiU,2700
 log_foundry/console.py,sha256=etACHsfxwx_lUF3bej-Hzj63VU_5hxa9g2_Ecpc-0ZI,1199
 log_foundry/context.py,sha256=MnqBFngMB0joOU3ybpXwmK1TwWqsHwAG5Tdye5B2768,2168
-log_foundry/decorator.py,sha256=0a_O7Cwsv1a8IxB_QXrWoCr_ZiDjoQ_i4E6q52SUSDk,5983
+log_foundry/decorator.py,sha256=Fe-p6BGEVARTjqYNi8JwTvs7StnYr2bQRGiV4O3HrIM,6832
 log_foundry/ids.py,sha256=yKJPdCEepG9ZgVP3c-CQyvraHyKRzRUBhUp2-1A8-8g,912
 log_foundry/model.py,sha256=yL8RRBkHcPvTut5XfVNnm4vztN2YkRIAfFaC6i-U8qY,3591
 log_foundry/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -44,8 +44,8 @@ log_foundry/sinks/stdout.py,sha256=X_hVTUAAsppx1k8znuBveOZZwj3FqcTwLH185_JWJUU,1
 log_foundry/sinks/syslog.py,sha256=CAQ957jBIlAzrLyrOmtzspqcnAFBfYHGrIN8I35DGMw,3381
 log_foundry/sinks/transform.py,sha256=eeZpaMxc7urrZ07Vc4GQ-sIq2mTYhoA_dI4Ur4Mmb4s,1613
 log_foundry/sinks/util.py,sha256=SCYFxW9Jm-bIVsyOKir5JWWbwP7u5VFnhNmncMIkwBM,2759
-log_foundry/worker.py,sha256=9sX6-Zqq4bChlAAM26nDcV3CZGPkgaKHzolIvTeWRjk,6689
-log_foundry-0.2.1.dev3.dist-info/METADATA,sha256=0tkkYzwkQ4xQ6bEQ5R0iBtkxDAqdq-NMcAeMcbDSSX8,31265
-log_foundry-0.2.1.dev3.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
-log_foundry-0.2.1.dev3.dist-info/licenses/LICENSE,sha256=HaCckyCX8P6TR8gY5xyEsFi050Hs2kl-JqaO9Vi0Sw0,1072
-log_foundry-0.2.1.dev3.dist-info/RECORD,,
+log_foundry/worker.py,sha256=bbBwMwecVOBu1k6rbSLPwM0zzxN-ebGdy5X2Mo5Q14s,11014
+log_foundry-0.2.1.dev4.dist-info/METADATA,sha256=BUNLTnO5yKWnzM3KI1tNZyfUMVYHng9kdcazs8Gz7w4,34456
+log_foundry-0.2.1.dev4.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
+log_foundry-0.2.1.dev4.dist-info/licenses/LICENSE,sha256=HaCckyCX8P6TR8gY5xyEsFi050Hs2kl-JqaO9Vi0Sw0,1072
+log_foundry-0.2.1.dev4.dist-info/RECORD,,