log-foundry 0.10.2.dev25__tar.gz → 0.10.2.dev27__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/PKG-INFO +73 -5
  2. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/README.md +72 -4
  3. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/pyproject.toml +1 -1
  4. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/__init__.py +23 -5
  5. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/config.py +52 -1
  6. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/decorator.py +34 -1
  7. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/base.py +23 -1
  8. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/callback.py +5 -0
  9. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/datadog.py +4 -0
  10. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/elasticsearch.py +4 -0
  11. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/filtering.py +4 -0
  12. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/firehose.py +4 -0
  13. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/honeycomb.py +4 -0
  14. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/http.py +7 -0
  15. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/kafka.py +42 -3
  16. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/kinesis.py +4 -0
  17. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/logging_sink.py +5 -0
  18. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/logstash.py +8 -0
  19. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/loki.py +4 -0
  20. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/multi.py +7 -0
  21. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/pubsub.py +53 -5
  22. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/redis.py +35 -6
  23. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sentry.py +8 -2
  24. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sns.py +4 -0
  25. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/splunk.py +4 -0
  26. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sqs.py +4 -0
  27. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/stdout.py +12 -0
  28. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/syslog.py +6 -0
  29. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/transform.py +4 -0
  30. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/util.py +8 -0
  31. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/worker.py +222 -4
  32. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/LICENSE +0 -0
  33. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/_diag.py +0 -0
  34. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/api.py +0 -0
  35. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/console.py +0 -0
  36. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/context.py +0 -0
  37. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/ids.py +0 -0
  38. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/model.py +0 -0
  39. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/py.typed +0 -0
  40. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sanitize.py +0 -0
  41. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/__init__.py +0 -0
  42. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_batch.py +0 -0
  43. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_chunk.py +0 -0
  44. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_retry.py +0 -0
  45. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_socket.py +0 -0
  46. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_time.py +0 -0
  47. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/clickhouse.py +0 -0
  48. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/eventhubs.py +0 -0
  49. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/file.py +0 -0
  50. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/mongodb.py +0 -0
  51. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/nats.py +0 -0
  52. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/newrelic.py +0 -0
  53. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/postgres.py +0 -0
  54. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/rabbitmq.py +0 -0
  55. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sqlite.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev25
3
+ Version: 0.10.2.dev27
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -234,6 +234,34 @@ lf.configure(
234
234
  If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
235
235
  works with zero configuration.
236
236
 
237
+ **Passing `sink=` after logging has started swaps the live destination.** The background worker
238
+ captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
239
+ what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
240
+ what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
241
+ it, and points the worker at the new one:
242
+
243
+ ```python
244
+ lf.configure(sink=StdoutSink())
245
+ do_some_work() # these events go to stdout
246
+
247
+ lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
248
+ do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
249
+ ```
250
+
251
+ The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
252
+ you asked for that sink — but the previous sink is left **open** rather than closed, because the
253
+ drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
254
+ the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
255
+ do not hand it back to a later call.
256
+
257
+ The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
258
+ a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
259
+ an unreachable broker is the case to watch. Configure the sink before the first log where you can;
260
+ that path has no worker to retarget and nothing to close.
261
+
262
+ `configure()` is still a startup call. It is not thread-safe, and a span finishing on another
263
+ thread mid-swap may land on either sink.
264
+
237
265
  ### `@trace`
238
266
 
239
267
  Decorate any **synchronous** function. Usable bare or with arguments:
@@ -668,7 +696,7 @@ in `CallbackSink`.
668
696
 
669
697
  #### Writing your own sink
670
698
 
671
- `Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
699
+ `Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
672
700
  fails and one about *when* it is called. They are not stylistic — the library's whole
673
701
  loss-reporting apparatus is built on them:
674
702
 
@@ -688,6 +716,15 @@ loss-reporting apparatus is built on them:
688
716
  - **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
689
717
  partial success re-delivers the records that already arrived, and duplicates downstream are worse
690
718
  than a counted loss.
719
+ - **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
720
+ released transport has delivered nothing, so it is the first rule again by another route — and
721
+ it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
722
+ wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
723
+ one appended a delivery future nothing would resolve, and one transparently *reconnected*,
724
+ leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
725
+ If your `close()` releases nothing — you open a fresh connection per request, or the client is
726
+ the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
727
+ invented. `emit([])` stays a no-op either way.
691
728
 
692
729
  A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
693
730
  never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
@@ -704,12 +741,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
704
741
  class MySink:
705
742
  def __init__(self) -> None:
706
743
  self._dropped = self._failed = 0
744
+ self._closed = False
707
745
  self._lock = threading.Lock() # transport state
708
746
  self._counter_lock = threading.Lock() # counters only, never held across I/O
709
747
 
710
748
  def emit(self, batch: list[dict[str, object]]) -> None:
749
+ if not batch:
750
+ return
711
751
  delivered = 0
712
752
  with self._lock: # your connection, socket or stream
753
+ if self._closed: # refuse: nothing here can deliver it
754
+ raise SinkDeliveryError(
755
+ f"MySink delivered none of {len(batch)} event(s): the sink is closed"
756
+ )
713
757
  for chunk in self._chunks(batch):
714
758
  if self._send(chunk): # your own bounded retry
715
759
  delivered += len(chunk)
@@ -725,6 +769,9 @@ class MySink:
725
769
 
726
770
  def close(self) -> None:
727
771
  with self._lock: # never release under an active writer
772
+ if self._closed: # idempotent: atexit races your own cleanup
773
+ return
774
+ self._closed = True # set the flag, *then* release
728
775
  ...
729
776
  ```
730
777
 
@@ -746,7 +793,11 @@ returns a snapshot of the worker's counters:
746
793
 
747
794
  ```python
748
795
  h = log_foundry.health()
749
- if h.dropped or h.failed_batches or h.stopped_reason or (h.sink and (h.sink.dropped or h.sink.failed)):
796
+ if (
797
+ h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
798
+ or (h.sink and (h.sink.dropped or h.sink.failed))
799
+ or (h.retired and h.submitted_after_shutdown)
800
+ ):
750
801
  ... # logs were silently lost — worth an alert
751
802
  ```
752
803
 
@@ -759,6 +810,8 @@ They tell you different things, and they want different responses:
759
810
  | `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
760
811
  | `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
761
812
  | `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
813
+ | `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
814
+ | `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
762
815
 
763
816
  `h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
764
817
  the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
@@ -780,8 +833,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
780
833
  thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
781
834
  pointing at the wrong fix.
782
835
 
836
+ `retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
837
+ logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
838
+ means every log line since the shutdown has gone nowhere. That state used to read as perfectly
839
+ healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
840
+
783
841
  Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
784
- fields over time — a fourth (`stopped_reason`) in `v0.7.0` and a fifth (`sink`) not yet in a tagged
842
+ fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
843
+ seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
785
844
  release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
786
845
  field keeps its position when a new one is appended, so attribute and index access stay stable.
787
846
 
@@ -859,6 +918,13 @@ getting them wrong is silent:
859
918
  the first invocation on a warm container would log and every later one would silently log
860
919
  nothing. That failure reads as "works locally, broken in production".
861
920
 
921
+ It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
922
+ `health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
923
+ pair, and only that pair, is this mistake. The first such submission also writes one stderr
924
+ line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
925
+ `stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
926
+ drain where the repeatable one belonged.
927
+
862
928
  ```python
863
929
  import log_foundry as lf
864
930
  from log_foundry.sinks.sqs import SQSSink
@@ -876,9 +942,11 @@ def handler(event, context):
876
942
  # would log nothing.
877
943
  drained = lf.flush()
878
944
  h = lf.health()
879
- if not drained or h.failed_batches or h.dropped or h.stopped_reason:
945
+ if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
880
946
  # `drained` covers this invocation's tail; the counters cover anything the worker
881
947
  # lost earlier — a batch its own interval trigger already gave up on, for instance.
948
+ # `h.retired` catches the mistake above: inside a handler it can only mean something
949
+ # called shutdown(), and from here on this container logs nothing.
882
950
  # Emitting this through your platform's own logger keeps it outside the pipeline
883
951
  # that just failed.
884
952
  print(f"log-foundry: undelivered logs ({drained=}, {h=})")
@@ -198,6 +198,34 @@ lf.configure(
198
198
  If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
199
199
  works with zero configuration.
200
200
 
201
+ **Passing `sink=` after logging has started swaps the live destination.** The background worker
202
+ captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
203
+ what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
204
+ what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
205
+ it, and points the worker at the new one:
206
+
207
+ ```python
208
+ lf.configure(sink=StdoutSink())
209
+ do_some_work() # these events go to stdout
210
+
211
+ lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
212
+ do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
213
+ ```
214
+
215
+ The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
216
+ you asked for that sink — but the previous sink is left **open** rather than closed, because the
217
+ drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
218
+ the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
219
+ do not hand it back to a later call.
220
+
221
+ The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
222
+ a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
223
+ an unreachable broker is the case to watch. Configure the sink before the first log where you can;
224
+ that path has no worker to retarget and nothing to close.
225
+
226
+ `configure()` is still a startup call. It is not thread-safe, and a span finishing on another
227
+ thread mid-swap may land on either sink.
228
+
201
229
  ### `@trace`
202
230
 
203
231
  Decorate any **synchronous** function. Usable bare or with arguments:
@@ -632,7 +660,7 @@ in `CallbackSink`.
632
660
 
633
661
  #### Writing your own sink
634
662
 
635
- `Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
663
+ `Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
636
664
  fails and one about *when* it is called. They are not stylistic — the library's whole
637
665
  loss-reporting apparatus is built on them:
638
666
 
@@ -652,6 +680,15 @@ loss-reporting apparatus is built on them:
652
680
  - **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
653
681
  partial success re-delivers the records that already arrived, and duplicates downstream are worse
654
682
  than a counted loss.
683
+ - **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
684
+ released transport has delivered nothing, so it is the first rule again by another route — and
685
+ it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
686
+ wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
687
+ one appended a delivery future nothing would resolve, and one transparently *reconnected*,
688
+ leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
689
+ If your `close()` releases nothing — you open a fresh connection per request, or the client is
690
+ the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
691
+ invented. `emit([])` stays a no-op either way.
655
692
 
656
693
  A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
657
694
  never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
@@ -668,12 +705,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
668
705
  class MySink:
669
706
  def __init__(self) -> None:
670
707
  self._dropped = self._failed = 0
708
+ self._closed = False
671
709
  self._lock = threading.Lock() # transport state
672
710
  self._counter_lock = threading.Lock() # counters only, never held across I/O
673
711
 
674
712
  def emit(self, batch: list[dict[str, object]]) -> None:
713
+ if not batch:
714
+ return
675
715
  delivered = 0
676
716
  with self._lock: # your connection, socket or stream
717
+ if self._closed: # refuse: nothing here can deliver it
718
+ raise SinkDeliveryError(
719
+ f"MySink delivered none of {len(batch)} event(s): the sink is closed"
720
+ )
677
721
  for chunk in self._chunks(batch):
678
722
  if self._send(chunk): # your own bounded retry
679
723
  delivered += len(chunk)
@@ -689,6 +733,9 @@ class MySink:
689
733
 
690
734
  def close(self) -> None:
691
735
  with self._lock: # never release under an active writer
736
+ if self._closed: # idempotent: atexit races your own cleanup
737
+ return
738
+ self._closed = True # set the flag, *then* release
692
739
  ...
693
740
  ```
694
741
 
@@ -710,7 +757,11 @@ returns a snapshot of the worker's counters:
710
757
 
711
758
  ```python
712
759
  h = log_foundry.health()
713
- if h.dropped or h.failed_batches or h.stopped_reason or (h.sink and (h.sink.dropped or h.sink.failed)):
760
+ if (
761
+ h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
762
+ or (h.sink and (h.sink.dropped or h.sink.failed))
763
+ or (h.retired and h.submitted_after_shutdown)
764
+ ):
714
765
  ... # logs were silently lost — worth an alert
715
766
  ```
716
767
 
@@ -723,6 +774,8 @@ They tell you different things, and they want different responses:
723
774
  | `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
724
775
  | `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
725
776
  | `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
777
+ | `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
778
+ | `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
726
779
 
727
780
  `h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
728
781
  the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
@@ -744,8 +797,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
744
797
  thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
745
798
  pointing at the wrong fix.
746
799
 
800
+ `retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
801
+ logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
802
+ means every log line since the shutdown has gone nowhere. That state used to read as perfectly
803
+ healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
804
+
747
805
  Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
748
- fields over time — a fourth (`stopped_reason`) in `v0.7.0` and a fifth (`sink`) not yet in a tagged
806
+ fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
807
+ seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
749
808
  release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
750
809
  field keeps its position when a new one is appended, so attribute and index access stay stable.
751
810
 
@@ -823,6 +882,13 @@ getting them wrong is silent:
823
882
  the first invocation on a warm container would log and every later one would silently log
824
883
  nothing. That failure reads as "works locally, broken in production".
825
884
 
885
+ It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
886
+ `health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
887
+ pair, and only that pair, is this mistake. The first such submission also writes one stderr
888
+ line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
889
+ `stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
890
+ drain where the repeatable one belonged.
891
+
826
892
  ```python
827
893
  import log_foundry as lf
828
894
  from log_foundry.sinks.sqs import SQSSink
@@ -840,9 +906,11 @@ def handler(event, context):
840
906
  # would log nothing.
841
907
  drained = lf.flush()
842
908
  h = lf.health()
843
- if not drained or h.failed_batches or h.dropped or h.stopped_reason:
909
+ if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
844
910
  # `drained` covers this invocation's tail; the counters cover anything the worker
845
911
  # lost earlier — a batch its own interval trigger already gave up on, for instance.
912
+ # `h.retired` catches the mistake above: inside a handler it can only mean something
913
+ # called shutdown(), and from here on this container logs nothing.
846
914
  # Emitting this through your platform's own logger keeps it outside the pipeline
847
915
  # that just failed.
848
916
  print(f"log-foundry: undelivered logs ({drained=}, {h=})")
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev25"
23
+ version = "0.10.2.dev27"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -58,22 +58,35 @@ def health() -> Health:
58
58
  code non-blocking, and a non-zero ``failed_batches`` means a sink stayed broken through the
59
59
  whole retry budget — both are losses the library absorbs on purpose, and this is how you
60
60
  notice them. A non-``None`` ``stopped_reason`` is worse than either: the background thread
61
- died on that exception type, so nothing further will be delivered at all (SPEC-019)::
61
+ died on that exception type, so nothing further will be delivered at all (SPEC-019). A
62
+ ``retired`` worker still being handed events is the same total loss arrived at from the
63
+ other direction — someone called :func:`shutdown` in a process that logs again (SPEC-030)::
62
64
 
63
65
  h = log_foundry.health()
64
- if h.dropped or h.failed_batches or h.stopped_reason or (
66
+ if h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps or (
65
67
  h.sink and (h.sink.dropped or h.sink.failed)
66
- ):
68
+ ) or (h.retired and h.submitted_after_shutdown):
67
69
  ... # raise an alert; logs were silently lost
68
70
 
71
+ ``retired`` alone is not a fault — a process that shuts down and then stops logging is
72
+ doing the right thing, which is why it is paired with the count rather than alerted on.
73
+
69
74
  Args:
70
75
  None.
71
76
 
72
77
  Returns:
73
- The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason`` and
74
- ``sink``. The last is the configured sink's own
78
+ The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason``, ``sink``,
79
+ ``retired``, ``submitted_after_shutdown`` and ``incomplete_swaps``. ``retired`` says
80
+ :func:`shutdown` was called, and ``submitted_after_shutdown`` counts events accepted
81
+ afterwards, which are queued where nothing will drain them — non-zero together, that is
82
+ the ``shutdown()``-per-invocation mistake, and the remedy is :func:`flush`.
83
+ ``incomplete_swaps`` counts late ``configure(sink=...)`` calls whose drain of the
84
+ previous sink could not be confirmed, leaving that sink open. ``sink`` is the configured
85
+ sink's own
75
86
  :class:`~log_foundry.sinks.base.SinkLosses` — loss the sink absorbed rather than the
76
87
  worker (SPEC-026) — and is ``None`` when no worker exists or the sink reports nothing.
88
+ Note ``sink`` describes whichever sink is live now, so a swap takes the previous sink's
89
+ absorbed losses out of the snapshot with it.
77
90
  Its ``dropped`` is not the worker's: the worker's is backpressure at the queue, the
78
91
  sink's is an event that never reached the wire, and the stderr line names which. Its
79
92
  ``failed`` is an upper bound on loss rather than a count of it, since a sink that raises
@@ -98,6 +111,11 @@ def shutdown(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
98
111
  registered via ``atexit``, so call it explicitly only when you want to be certain the tail
99
112
  of the queue reached the sink before a fast process exit (SPEC-004 FR-005).
100
113
 
114
+ Logging afterwards is **accepted and undeliverable**: those events are queued and nothing
115
+ will drain them. It is not silent any more — :func:`health` then reports ``retired`` with a
116
+ non-zero ``submitted_after_shutdown``, and the first such submission writes one stderr line
117
+ (SPEC-030). That pair is the reading that catches the mistake above.
118
+
101
119
  Args:
102
120
  timeout: Seconds bounding the wait for the background thread (SPEC-027 FR-004). ``None``
103
121
  waits indefinitely, which is what this did unconditionally before and is still
@@ -76,11 +76,31 @@ def configure(
76
76
  the zero-dependency dev default (arch §8). Every ceiling is validated before anything is
77
77
  assigned, so a rejected call leaves the config exactly as it found it.
78
78
 
79
+ A ``sink=`` passed after logging has already started is the one argument that needs more
80
+ than an assignment, because the background worker captured its sink when it was built
81
+ (arch §7). It **swaps the live delivery target**: everything submitted so far is drained to
82
+ the previous sink, that sink is closed, and subsequent events go to the new one. The drains
83
+ are bounded, and a swap whose drain could not be confirmed leaves the previous sink open and
84
+ records ``health().incomplete_swaps`` (SPEC-030 FR-003) — so "repeated calls compose rather
85
+ than reset" holds for the sink too, at the cost of one bounded wait. Passing the sink that
86
+ is already live is a no-op: no drain, no close. The previous sink is closed and must not be
87
+ handed back to a later call.
88
+
89
+ The *closing* of the previous sink is not bounded, because ``Sink.close`` takes no timeout —
90
+ a destination that blocks in ``close()`` blocks this call. It is the same gap
91
+ ``architecture.md`` §13 already records for ``shutdown()``, and it has the same fix, which
92
+ is a change to the sink contract rather than to this function.
93
+
94
+ This is still a startup call. It is not thread-safe, and a span finishing on another thread
95
+ during a swap may land on either sink.
96
+
79
97
  Args:
80
98
  service: The service name stamped onto every event.
81
99
  version: The service version stamped onto every event.
82
100
  env: The deployment environment stamped onto every event.
83
- sink: The destination every event is delivered to.
101
+ sink: The destination every event is delivered to. Passed after the first log, it swaps
102
+ the live target as described above rather than only updating what ``get_config()``
103
+ reports.
84
104
  defaults: Fields merged into every event at the lowest precedence.
85
105
  max_value_bytes: Per-value ceiling, in UTF-8 bytes or rendered digits.
86
106
  max_stack_bytes: Ceiling for ``error.stack`` alone.
@@ -119,6 +139,37 @@ def configure(
119
139
 
120
140
  _ensure_sink()
121
141
 
142
+ if sink is not None:
143
+ _swap_live_sink(sink)
144
+
145
+
146
+ def _swap_live_sink(sink: Sink) -> None:
147
+ """Points an already-running worker at a newly configured sink (SPEC-030 FR-003).
148
+
149
+ The import is local for the reason ``_ensure_sink``'s is: ``decorator`` imports this module
150
+ at module scope, so reaching back the other way at import time would be a cycle. It also
151
+ keeps the dependency to the one call that needs it — ``configure()`` without a ``sink=``
152
+ never touches the worker at all.
153
+
154
+ The budget is passed explicitly rather than left to the parameter default, so that the
155
+ bound this call actually applies is resolved when it runs. A default argument is bound at
156
+ definition time, which would leave the end-to-end bound untestable without reaching past
157
+ the function under test.
158
+
159
+ Args:
160
+ sink: The sink just written to the config.
161
+
162
+ Returns:
163
+ None.
164
+
165
+ Raises:
166
+ None.
167
+ """
168
+ from log_foundry.decorator import _swap_sink
169
+ from log_foundry.worker import DEFAULT_SWAP_TIMEOUT
170
+
171
+ _swap_sink(sink, DEFAULT_SWAP_TIMEOUT)
172
+
122
173
 
123
174
  def get_config() -> Config:
124
175
  """Returns the current global config singleton.
@@ -20,11 +20,13 @@ from log_foundry.ids import (
20
20
  parse_traceparent,
21
21
  )
22
22
  from log_foundry.model import Span, backfill_baggage, end_event, start_event
23
- from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, Health, Worker
23
+ from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_SWAP_TIMEOUT, Health, Worker
24
24
 
25
25
  if TYPE_CHECKING:
26
26
  import contextvars
27
27
 
28
+ from log_foundry.sinks.base import Sink
29
+
28
30
  __all__ = ["continue_trace", "trace"]
29
31
 
30
32
  _worker: Worker | None = None
@@ -231,6 +233,37 @@ def _shutdown_worker(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
231
233
  _worker.shutdown(timeout)
232
234
 
233
235
 
236
+ def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> None:
237
+ """Retargets the process worker at a new sink, backing a late ``configure(sink=...)``.
238
+
239
+ Like :func:`_flush_worker` this deliberately does not call :func:`_get_worker`: a process
240
+ that has not logged has captured no sink, so there is nothing to swap and building a thread
241
+ to prove it would be pure cost — that is also the case where the old behaviour was already
242
+ correct (SPEC-030 FR-003).
243
+
244
+ Args:
245
+ new_sink: The sink already written to the config, to be made the live delivery target.
246
+ timeout: Seconds bounding the drain of the previous sink.
247
+
248
+ Returns:
249
+ None.
250
+
251
+ Raises:
252
+ None. This runs inside ``configure()``, which has never raised for anything but a
253
+ rejected ceiling, and a sink swap that fails must not become the reason an application
254
+ cannot start.
255
+ """
256
+ worker = _worker
257
+ if worker is None:
258
+ return
259
+ try:
260
+ worker.swap_sink(new_sink, timeout)
261
+ except Exception as exc:
262
+ _diag.absorbed(
263
+ "swapping the log sink", exc, "events may still be delivered to the previous sink"
264
+ )
265
+
266
+
234
267
  def _flush_worker(timeout: float | None = 5.0) -> bool:
235
268
  """Drains the process worker without retiring it, backing ``flush()`` (SPEC-013 FR-003).
236
269
 
@@ -94,9 +94,25 @@ class Sink(Protocol):
94
94
  re-delivered wholesale by the worker's retry, and duplicates downstream are worse than
95
95
  the counted loss (SPEC-017 FR-004, SPEC-018). Report that through ``losses()`` instead.
96
96
 
97
+ **After ``close()`` has released or invalidated something, raise rather than absorb**
98
+ (SPEC-032 FR-002). This is the same obligation applied to the sink's own lifecycle
99
+ rather than the destination's: an absorbed batch is one the worker believes, so the
100
+ retry never engages, ``failed_batches`` stays at zero and ``flush()`` returns True while
101
+ the events are gone — reached not through a destination that is down but through a
102
+ transport the library itself has already let go. Three shipped sinks failed this rule
103
+ for four specs, in three different ways: a produce into a batch nothing would flush
104
+ again, a future nothing would resolve, and a client that quietly reconnected.
105
+
106
+ The converse is equally binding: a sink holding nothing to release **keeps accepting**.
107
+ ``close()`` on a sink that opens a fresh connection per request, or whose client belongs
108
+ to the caller, has invalidated nothing, so a later batch still delivers and refusing it
109
+ would be loss the library invented rather than loss it reported. Which of the two
110
+ applies is a property of the sink, so each shipped sink records its answer in its class
111
+ docstring and a test holds it to it.
112
+
97
113
  Args:
98
114
  batch: The events to ship. ``emit([])`` is a no-op and never raises, since an empty
99
- batch has not failed to deliver.
115
+ batch has not failed to deliver — closed or not.
100
116
 
101
117
  Returns:
102
118
  None.
@@ -116,6 +132,12 @@ class Sink(Protocol):
116
132
  succeeding. Taking the same lock ``emit`` takes satisfies this; so does an idempotent
117
133
  guard checked under that lock.
118
134
 
135
+ The flag this sets is the one a later :meth:`emit` reads, so the two are one decision
136
+ rather than two (SPEC-032 FR-002). Set it *before* releasing anything: an emit arriving
137
+ during a release must be refused rather than handed a half-released transport. Make the
138
+ release idempotent for the same reason a second ``close()`` is expected at all —
139
+ ``atexit`` racing a caller's own cleanup is the documented case.
140
+
119
141
  Args:
120
142
  None.
121
143
 
@@ -17,6 +17,11 @@ class CallbackSink:
17
17
  function, without writing a ``Sink`` implementation. Attributes are internal; the observable
18
18
  contract is that ``emit`` hands the batch to the callable unchanged and ``close`` invokes
19
19
  the close hook once when one was supplied.
20
+
21
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
22
+ (SPEC-032 FR-003). Both decisions belong to the callable: this class holds nothing, and what
23
+ a hook releases is not knowable from here — a callable needing either guarantee must provide
24
+ it, exactly as a hand-written ``Sink`` implementation would.
20
25
  """
21
26
 
22
27
  def __init__(
@@ -16,6 +16,10 @@ class DatadogSink(HTTPSink):
16
16
 
17
17
  The batch goes as a JSON array to the region-specific intake with a ``DD-API-KEY`` header,
18
18
  each entry enriched with ``ddsource``, ``service`` and ``ddtags``.
19
+
20
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
21
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
22
+ is no transport held and ``close()`` releases nothing.
19
23
  """
20
24
 
21
25
  def __init__(
@@ -25,6 +25,10 @@ class ElasticsearchSink(HTTPSink):
25
25
  dropped_unadjudicated: Events whose outcome a ``_bulk`` response did not describe, because
26
26
  its ``items`` array did not line up with the batch sent. Abandoned rather than retried,
27
27
  for SPEC-018's reason: the request succeeded, so re-sending would duplicate what landed.
28
+
29
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
30
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
31
+ is no transport held and ``close()`` releases nothing.
28
32
  """
29
33
 
30
34
  def __init__(self, url: str, *, index: str, auth: str | tuple[str, str] | None = None,
@@ -24,6 +24,10 @@ class FilteringSink:
24
24
  This is a static, emit-time filter in front of an inner sink, not the reserved
25
25
  tail-sampling ``should_send`` seam (arch §10) — that stays deferred and owns rate policy at
26
26
  span-decision time, while this only reshapes an already-built batch on its way to a sink.
27
+
28
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
29
+ (SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
30
+ belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
27
31
  """
28
32
 
29
33
  def __init__(
@@ -36,6 +36,10 @@ class FirehoseSink:
36
36
  The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
37
37
  lock. ``boto3`` clients are documented thread-safe, this one is built once in
38
38
  ``__init__``, and the sink rebinds nothing after construction.
39
+
40
+ It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
41
+ because the client is the caller's to release or the SDK's to reap, so a batch emitted
42
+ afterwards still reaches the delivery stream.
39
43
  """
40
44
 
41
45
  MAX_RECORDS = 500
@@ -14,6 +14,10 @@ class HoneycombSink(HTTPSink):
14
14
 
15
15
  The request goes to ``/1/batch/<dataset>`` with an ``X-Honeycomb-Team`` header, in
16
16
  Honeycomb's ``[{"data": <event>}, ...]`` batch shape.
17
+
18
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
19
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
20
+ is no transport held and ``close()`` releases nothing.
17
21
  """
18
22
 
19
23
  def __init__(
@@ -74,6 +74,13 @@ class HTTPSink:
74
74
  failed: Requests abandoned past the retry bound.
75
75
  dropped_oversized: Events dropped for exceeding a destination's hard size limit, used by
76
76
  subclasses that enforce one; the generic core imposes no universal limit.
77
+
78
+ The two lifecycle decisions this class records, inherited by every platform subclass. It
79
+ takes **no** transport lock (SPEC-028 FR-002): there is no transport to guard, since
80
+ ``urllib`` opens a fresh connection per request and nothing is rebound after construction.
81
+ And it **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` releases nothing —
82
+ a batch emitted afterwards still reaches the endpoint, and refusing it would be loss the
83
+ library invented rather than loss it reported.
77
84
  """
78
85
 
79
86
  def __init__(