log-foundry 0.10.2.dev25__tar.gz → 0.10.2.dev26__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/PKG-INFO +53 -4
  2. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/README.md +52 -3
  3. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/pyproject.toml +1 -1
  4. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/__init__.py +23 -5
  5. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/config.py +52 -1
  6. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/decorator.py +34 -1
  7. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/worker.py +222 -4
  8. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/LICENSE +0 -0
  9. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/_diag.py +0 -0
  10. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/api.py +0 -0
  11. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/console.py +0 -0
  12. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/context.py +0 -0
  13. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/ids.py +0 -0
  14. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/model.py +0 -0
  15. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/py.typed +0 -0
  16. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sanitize.py +0 -0
  17. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/__init__.py +0 -0
  18. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_batch.py +0 -0
  19. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_chunk.py +0 -0
  20. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_retry.py +0 -0
  21. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_socket.py +0 -0
  22. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_time.py +0 -0
  23. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/base.py +0 -0
  24. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/callback.py +0 -0
  25. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/clickhouse.py +0 -0
  26. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/datadog.py +0 -0
  27. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/elasticsearch.py +0 -0
  28. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/eventhubs.py +0 -0
  29. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/file.py +0 -0
  30. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/filtering.py +0 -0
  31. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/firehose.py +0 -0
  32. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/honeycomb.py +0 -0
  33. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/http.py +0 -0
  34. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/kafka.py +0 -0
  35. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/kinesis.py +0 -0
  36. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/logging_sink.py +0 -0
  37. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/logstash.py +0 -0
  38. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/loki.py +0 -0
  39. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/mongodb.py +0 -0
  40. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/multi.py +0 -0
  41. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/nats.py +0 -0
  42. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/newrelic.py +0 -0
  43. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/postgres.py +0 -0
  44. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/pubsub.py +0 -0
  45. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/rabbitmq.py +0 -0
  46. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/redis.py +0 -0
  47. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sentry.py +0 -0
  48. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sns.py +0 -0
  49. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/splunk.py +0 -0
  50. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sqlite.py +0 -0
  51. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sqs.py +0 -0
  52. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/stdout.py +0 -0
  53. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/syslog.py +0 -0
  54. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/transform.py +0 -0
  55. {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/util.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev25
3
+ Version: 0.10.2.dev26
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -234,6 +234,34 @@ lf.configure(
234
234
  If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
235
235
  works with zero configuration.
236
236
 
237
+ **Passing `sink=` after logging has started swaps the live destination.** The background worker
238
+ captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
239
+ what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
240
+ what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
241
+ it, and points the worker at the new one:
242
+
243
+ ```python
244
+ lf.configure(sink=StdoutSink())
245
+ do_some_work() # these events go to stdout
246
+
247
+ lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
248
+ do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
249
+ ```
250
+
251
+ The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
252
+ you asked for that sink — but the previous sink is left **open** rather than closed, because the
253
+ drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
254
+ the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
255
+ do not hand it back to a later call.
256
+
257
+ The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
258
+ a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
259
+ an unreachable broker is the case to watch. Configure the sink before the first log where you can;
260
+ that path has no worker to retarget and nothing to close.
261
+
262
+ `configure()` is still a startup call. It is not thread-safe, and a span finishing on another
263
+ thread mid-swap may land on either sink.
264
+
237
265
  ### `@trace`
238
266
 
239
267
  Decorate any **synchronous** function. Usable bare or with arguments:
@@ -746,7 +774,11 @@ returns a snapshot of the worker's counters:
746
774
 
747
775
  ```python
748
776
  h = log_foundry.health()
749
- if h.dropped or h.failed_batches or h.stopped_reason or (h.sink and (h.sink.dropped or h.sink.failed)):
777
+ if (
778
+ h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
779
+ or (h.sink and (h.sink.dropped or h.sink.failed))
780
+ or (h.retired and h.submitted_after_shutdown)
781
+ ):
750
782
  ... # logs were silently lost — worth an alert
751
783
  ```
752
784
 
@@ -759,6 +791,8 @@ They tell you different things, and they want different responses:
759
791
  | `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
760
792
  | `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
761
793
  | `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
794
+ | `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
795
+ | `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
762
796
 
763
797
  `h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
764
798
  the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
@@ -780,8 +814,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
780
814
  thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
781
815
  pointing at the wrong fix.
782
816
 
817
+ `retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
818
+ logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
819
+ means every log line since the shutdown has gone nowhere. That state used to read as perfectly
820
+ healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
821
+
783
822
  Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
784
- fields over time — a fourth (`stopped_reason`) in `v0.7.0` and a fifth (`sink`) not yet in a tagged
823
+ fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
824
+ seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
785
825
  release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
786
826
  field keeps its position when a new one is appended, so attribute and index access stay stable.
787
827
 
@@ -859,6 +899,13 @@ getting them wrong is silent:
859
899
  the first invocation on a warm container would log and every later one would silently log
860
900
  nothing. That failure reads as "works locally, broken in production".
861
901
 
902
+ It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
903
+ `health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
904
+ pair, and only that pair, is this mistake. The first such submission also writes one stderr
905
+ line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
906
+ `stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
907
+ drain where the repeatable one belonged.
908
+
862
909
  ```python
863
910
  import log_foundry as lf
864
911
  from log_foundry.sinks.sqs import SQSSink
@@ -876,9 +923,11 @@ def handler(event, context):
876
923
  # would log nothing.
877
924
  drained = lf.flush()
878
925
  h = lf.health()
879
- if not drained or h.failed_batches or h.dropped or h.stopped_reason:
926
+ if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
880
927
  # `drained` covers this invocation's tail; the counters cover anything the worker
881
928
  # lost earlier — a batch its own interval trigger already gave up on, for instance.
929
+ # `h.retired` catches the mistake above: inside a handler it can only mean something
930
+ # called shutdown(), and from here on this container logs nothing.
882
931
  # Emitting this through your platform's own logger keeps it outside the pipeline
883
932
  # that just failed.
884
933
  print(f"log-foundry: undelivered logs ({drained=}, {h=})")
@@ -198,6 +198,34 @@ lf.configure(
198
198
  If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
199
199
  works with zero configuration.
200
200
 
201
+ **Passing `sink=` after logging has started swaps the live destination.** The background worker
202
+ captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
203
+ what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
204
+ what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
205
+ it, and points the worker at the new one:
206
+
207
+ ```python
208
+ lf.configure(sink=StdoutSink())
209
+ do_some_work() # these events go to stdout
210
+
211
+ lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
212
+ do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
213
+ ```
214
+
215
+ The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
216
+ you asked for that sink — but the previous sink is left **open** rather than closed, because the
217
+ drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
218
+ the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
219
+ do not hand it back to a later call.
220
+
221
+ The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
222
+ a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
223
+ an unreachable broker is the case to watch. Configure the sink before the first log where you can;
224
+ that path has no worker to retarget and nothing to close.
225
+
226
+ `configure()` is still a startup call. It is not thread-safe, and a span finishing on another
227
+ thread mid-swap may land on either sink.
228
+
201
229
  ### `@trace`
202
230
 
203
231
  Decorate any **synchronous** function. Usable bare or with arguments:
@@ -710,7 +738,11 @@ returns a snapshot of the worker's counters:
710
738
 
711
739
  ```python
712
740
  h = log_foundry.health()
713
- if h.dropped or h.failed_batches or h.stopped_reason or (h.sink and (h.sink.dropped or h.sink.failed)):
741
+ if (
742
+ h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
743
+ or (h.sink and (h.sink.dropped or h.sink.failed))
744
+ or (h.retired and h.submitted_after_shutdown)
745
+ ):
714
746
  ... # logs were silently lost — worth an alert
715
747
  ```
716
748
 
@@ -723,6 +755,8 @@ They tell you different things, and they want different responses:
723
755
  | `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
724
756
  | `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
725
757
  | `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
758
+ | `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
759
+ | `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
726
760
 
727
761
  `h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
728
762
  the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
@@ -744,8 +778,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
744
778
  thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
745
779
  pointing at the wrong fix.
746
780
 
781
+ `retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
782
+ logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
783
+ means every log line since the shutdown has gone nowhere. That state used to read as perfectly
784
+ healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
785
+
747
786
  Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
748
- fields over time — a fourth (`stopped_reason`) in `v0.7.0` and a fifth (`sink`) not yet in a tagged
787
+ fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
788
+ seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
749
789
  release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
750
790
  field keeps its position when a new one is appended, so attribute and index access stay stable.
751
791
 
@@ -823,6 +863,13 @@ getting them wrong is silent:
823
863
  the first invocation on a warm container would log and every later one would silently log
824
864
  nothing. That failure reads as "works locally, broken in production".
825
865
 
866
+ It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
867
+ `health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
868
+ pair, and only that pair, is this mistake. The first such submission also writes one stderr
869
+ line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
870
+ `stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
871
+ drain where the repeatable one belonged.
872
+
826
873
  ```python
827
874
  import log_foundry as lf
828
875
  from log_foundry.sinks.sqs import SQSSink
@@ -840,9 +887,11 @@ def handler(event, context):
840
887
  # would log nothing.
841
888
  drained = lf.flush()
842
889
  h = lf.health()
843
- if not drained or h.failed_batches or h.dropped or h.stopped_reason:
890
+ if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
844
891
  # `drained` covers this invocation's tail; the counters cover anything the worker
845
892
  # lost earlier — a batch its own interval trigger already gave up on, for instance.
893
+ # `h.retired` catches the mistake above: inside a handler it can only mean something
894
+ # called shutdown(), and from here on this container logs nothing.
846
895
  # Emitting this through your platform's own logger keeps it outside the pipeline
847
896
  # that just failed.
848
897
  print(f"log-foundry: undelivered logs ({drained=}, {h=})")
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev25"
23
+ version = "0.10.2.dev26"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -58,22 +58,35 @@ def health() -> Health:
58
58
  code non-blocking, and a non-zero ``failed_batches`` means a sink stayed broken through the
59
59
  whole retry budget — both are losses the library absorbs on purpose, and this is how you
60
60
  notice them. A non-``None`` ``stopped_reason`` is worse than either: the background thread
61
- died on that exception type, so nothing further will be delivered at all (SPEC-019)::
61
+ died on that exception type, so nothing further will be delivered at all (SPEC-019). A
62
+ ``retired`` worker still being handed events is the same total loss arrived at from the
63
+ other direction — someone called :func:`shutdown` in a process that logs again (SPEC-030)::
62
64
 
63
65
  h = log_foundry.health()
64
- if h.dropped or h.failed_batches or h.stopped_reason or (
66
+ if h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps or (
65
67
  h.sink and (h.sink.dropped or h.sink.failed)
66
- ):
68
+ ) or (h.retired and h.submitted_after_shutdown):
67
69
  ... # raise an alert; logs were silently lost
68
70
 
71
+ ``retired`` alone is not a fault — a process that shuts down and then stops logging is
72
+ doing the right thing, which is why it is paired with the count rather than alerted on.
73
+
69
74
  Args:
70
75
  None.
71
76
 
72
77
  Returns:
73
- The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason`` and
74
- ``sink``. The last is the configured sink's own
78
+ The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason``, ``sink``,
79
+ ``retired``, ``submitted_after_shutdown`` and ``incomplete_swaps``. ``retired`` says
80
+ :func:`shutdown` was called, and ``submitted_after_shutdown`` counts events accepted
81
+ afterwards, which are queued where nothing will drain them — non-zero together, that is
82
+ the ``shutdown()``-per-invocation mistake, and the remedy is :func:`flush`.
83
+ ``incomplete_swaps`` counts late ``configure(sink=...)`` calls whose drain of the
84
+ previous sink could not be confirmed, leaving that sink open. ``sink`` is the configured
85
+ sink's own
75
86
  :class:`~log_foundry.sinks.base.SinkLosses` — loss the sink absorbed rather than the
76
87
  worker (SPEC-026) — and is ``None`` when no worker exists or the sink reports nothing.
88
+ Note ``sink`` describes whichever sink is live now, so a swap takes the previous sink's
89
+ absorbed losses out of the snapshot with it.
77
90
  Its ``dropped`` is not the worker's: the worker's is backpressure at the queue, the
78
91
  sink's is an event that never reached the wire, and the stderr line names which. Its
79
92
  ``failed`` is an upper bound on loss rather than a count of it, since a sink that raises
@@ -98,6 +111,11 @@ def shutdown(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
98
111
  registered via ``atexit``, so call it explicitly only when you want to be certain the tail
99
112
  of the queue reached the sink before a fast process exit (SPEC-004 FR-005).
100
113
 
114
+ Logging afterwards is **accepted and undeliverable**: those events are queued and nothing
115
+ will drain them. It is not silent any more — :func:`health` then reports ``retired`` with a
116
+ non-zero ``submitted_after_shutdown``, and the first such submission writes one stderr line
117
+ (SPEC-030). That pair is the reading that catches the mistake above.
118
+
101
119
  Args:
102
120
  timeout: Seconds bounding the wait for the background thread (SPEC-027 FR-004). ``None``
103
121
  waits indefinitely, which is what this did unconditionally before and is still
@@ -76,11 +76,31 @@ def configure(
76
76
  the zero-dependency dev default (arch §8). Every ceiling is validated before anything is
77
77
  assigned, so a rejected call leaves the config exactly as it found it.
78
78
 
79
+ A ``sink=`` passed after logging has already started is the one argument that needs more
80
+ than an assignment, because the background worker captured its sink when it was built
81
+ (arch §7). It **swaps the live delivery target**: everything submitted so far is drained to
82
+ the previous sink, that sink is closed, and subsequent events go to the new one. The drains
83
+ are bounded, and a swap whose drain could not be confirmed leaves the previous sink open and
84
+ records ``health().incomplete_swaps`` (SPEC-030 FR-003) — so "repeated calls compose rather
85
+ than reset" holds for the sink too, at the cost of one bounded wait. Passing the sink that
86
+ is already live is a no-op: no drain, no close. The previous sink is closed and must not be
87
+ handed back to a later call.
88
+
89
+ The *closing* of the previous sink is not bounded, because ``Sink.close`` takes no timeout —
90
+ a destination that blocks in ``close()`` blocks this call. It is the same gap
91
+ ``architecture.md`` §13 already records for ``shutdown()``, and it has the same fix, which
92
+ is a change to the sink contract rather than to this function.
93
+
94
+ This is still a startup call. It is not thread-safe, and a span finishing on another thread
95
+ during a swap may land on either sink.
96
+
79
97
  Args:
80
98
  service: The service name stamped onto every event.
81
99
  version: The service version stamped onto every event.
82
100
  env: The deployment environment stamped onto every event.
83
- sink: The destination every event is delivered to.
101
+ sink: The destination every event is delivered to. Passed after the first log, it swaps
102
+ the live target as described above rather than only updating what ``get_config()``
103
+ reports.
84
104
  defaults: Fields merged into every event at the lowest precedence.
85
105
  max_value_bytes: Per-value ceiling, in UTF-8 bytes or rendered digits.
86
106
  max_stack_bytes: Ceiling for ``error.stack`` alone.
@@ -119,6 +139,37 @@ def configure(
119
139
 
120
140
  _ensure_sink()
121
141
 
142
+ if sink is not None:
143
+ _swap_live_sink(sink)
144
+
145
+
146
+ def _swap_live_sink(sink: Sink) -> None:
147
+ """Points an already-running worker at a newly configured sink (SPEC-030 FR-003).
148
+
149
+ The import is local for the reason ``_ensure_sink``'s is: ``decorator`` imports this module
150
+ at module scope, so reaching back the other way at import time would be a cycle. It also
151
+ keeps the dependency to the one call that needs it — ``configure()`` without a ``sink=``
152
+ never touches the worker at all.
153
+
154
+ The budget is passed explicitly rather than left to the parameter default, so that the
155
+ bound this call actually applies is resolved when it runs. A default argument is bound at
156
+ definition time, which would leave the end-to-end bound untestable without reaching past
157
+ the function under test.
158
+
159
+ Args:
160
+ sink: The sink just written to the config.
161
+
162
+ Returns:
163
+ None.
164
+
165
+ Raises:
166
+ None.
167
+ """
168
+ from log_foundry.decorator import _swap_sink
169
+ from log_foundry.worker import DEFAULT_SWAP_TIMEOUT
170
+
171
+ _swap_sink(sink, DEFAULT_SWAP_TIMEOUT)
172
+
122
173
 
123
174
  def get_config() -> Config:
124
175
  """Returns the current global config singleton.
@@ -20,11 +20,13 @@ from log_foundry.ids import (
20
20
  parse_traceparent,
21
21
  )
22
22
  from log_foundry.model import Span, backfill_baggage, end_event, start_event
23
- from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, Health, Worker
23
+ from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_SWAP_TIMEOUT, Health, Worker
24
24
 
25
25
  if TYPE_CHECKING:
26
26
  import contextvars
27
27
 
28
+ from log_foundry.sinks.base import Sink
29
+
28
30
  __all__ = ["continue_trace", "trace"]
29
31
 
30
32
  _worker: Worker | None = None
@@ -231,6 +233,37 @@ def _shutdown_worker(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
231
233
  _worker.shutdown(timeout)
232
234
 
233
235
 
236
+ def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> None:
237
+ """Retargets the process worker at a new sink, backing a late ``configure(sink=...)``.
238
+
239
+ Like :func:`_flush_worker` this deliberately does not call :func:`_get_worker`: a process
240
+ that has not logged has captured no sink, so there is nothing to swap and building a thread
241
+ to prove it would be pure cost — that is also the case where the old behaviour was already
242
+ correct (SPEC-030 FR-003).
243
+
244
+ Args:
245
+ new_sink: The sink already written to the config, to be made the live delivery target.
246
+ timeout: Seconds bounding the drain of the previous sink.
247
+
248
+ Returns:
249
+ None.
250
+
251
+ Raises:
252
+ None. This runs inside ``configure()``, which has never raised for anything but a
253
+ rejected ceiling, and a sink swap that fails must not become the reason an application
254
+ cannot start.
255
+ """
256
+ worker = _worker
257
+ if worker is None:
258
+ return
259
+ try:
260
+ worker.swap_sink(new_sink, timeout)
261
+ except Exception as exc:
262
+ _diag.absorbed(
263
+ "swapping the log sink", exc, "events may still be delivered to the previous sink"
264
+ )
265
+
266
+
234
267
  def _flush_worker(timeout: float | None = 5.0) -> bool:
235
268
  """Drains the process worker without retiring it, backing ``flush()`` (SPEC-013 FR-003).
236
269
 
@@ -26,6 +26,14 @@ and an unbounded join there is a hung process.
26
26
 
27
27
  _DROP_WARN_EVERY = 1000
28
28
 
29
+ DEFAULT_SWAP_TIMEOUT = 5.0
30
+ """Seconds a late ``configure(sink=...)`` will spend draining the previous sink (FR-003).
31
+
32
+ Shorter than the shutdown budget on purpose: this runs on the caller's thread inside a
33
+ configuration call, where a long stall is a startup that appears to hang, and what is at risk
34
+ is a sink swap rather than the tail of the whole process.
35
+ """
36
+
29
37
 
30
38
  def _bounded_seconds(timeout: float | None) -> str:
31
39
  """Renders a shutdown timeout for a diagnostic without trusting its ``__str__``.
@@ -52,9 +60,10 @@ def _bounded_seconds(timeout: float | None) -> str:
52
60
  class Health(NamedTuple):
53
61
  """A point-in-time snapshot of the worker's delivery counters (SPEC-017 FR-005).
54
62
 
55
- ``stopped_reason`` and ``sink`` are defaulted and appended in that order, so the zeroed
56
- snapshot in ``decorator._worker_health`` — and any third-party construction — keeps
57
- working, and attribute and index access to every earlier field stays as it was.
63
+ ``stopped_reason``, ``sink``, and SPEC-030's three are defaulted and appended in that
64
+ order, so the zeroed snapshot in ``decorator._worker_health`` — and any third-party
65
+ construction — keeps working, and attribute and index access to every earlier field stays
66
+ as it was.
58
67
 
59
68
  Attributes:
60
69
  queued: Submissions currently buffered. Approximate by nature: it is read without
@@ -72,6 +81,25 @@ class Health(NamedTuple):
72
81
  the sink reports nothing (SPEC-026 FR-003). Nested rather than folded into the
73
82
  integers above because they count different things: ``dropped`` here is backpressure
74
83
  at this queue, ``dropped`` on the sink is an event that never reached the wire.
84
+ retired: Whether :meth:`Worker.shutdown` has been called. It describes an action the
85
+ caller took, not a failure the library detected, which is why it is a boolean where
86
+ ``stopped_reason`` is a string — SPEC-019 rejected an ``alive`` flag because it would
87
+ read ``False`` for a process that never logged, and that objection does not apply to a
88
+ field which is simply ``False`` until someone calls ``shutdown()`` (SPEC-030 FR-001).
89
+ On its own it is not a fault: a process that shuts down and stops logging is correct.
90
+ submitted_after_shutdown: Submissions accepted after ``shutdown()`` and queued where
91
+ nothing will drain them. Non-zero alongside ``retired`` is the signature of the
92
+ serverless mistake — ``shutdown()`` called per invocation on a warm container, so the
93
+ first invocation logs and every later one silently does not. The count starts at the
94
+ moment ``shutdown()`` begins rather than when the drain thread ends, so a submission
95
+ racing the final drain is counted even if that drain carried it; erring toward
96
+ reporting is the right direction for a signal whose whole purpose is visibility.
97
+ incomplete_swaps: Late ``configure(sink=...)`` calls whose drain of the previous sink
98
+ did not complete (SPEC-030 FR-003). The swap still took effect, so the caller has the
99
+ sink it asked for, but two things could not be guaranteed: events submitted before the
100
+ call may have been carried to the new sink instead of the old one, and the old sink was
101
+ left **open** rather than closed, because the drain thread may still be inside its
102
+ ``emit`` — the reasoning SPEC-027 FR-004 applies to an expired ``shutdown()``.
75
103
  """
76
104
 
77
105
  queued: int
@@ -79,6 +107,9 @@ class Health(NamedTuple):
79
107
  failed_batches: int
80
108
  stopped_reason: str | None = None
81
109
  sink: SinkLosses | None = None
110
+ retired: bool = False
111
+ submitted_after_shutdown: int = 0
112
+ incomplete_swaps: int = 0
82
113
 
83
114
 
84
115
  class _FlushMarker:
@@ -170,6 +201,8 @@ class Worker:
170
201
  self.dropped = 0
171
202
  self.failed_batches = 0
172
203
  self.stopped_reason: str | None = None
204
+ self.submitted_after_shutdown = 0
205
+ self.incomplete_swaps = 0
173
206
  self._queue: queue.Queue[object] = queue.Queue(maxsize=max_queue)
174
207
  self._stop = threading.Event()
175
208
  self._shutdown_done = False
@@ -221,6 +254,15 @@ class Worker:
221
254
  blocking write would let a wedged console stall the drain path. Lines may therefore
222
255
  interleave out of order under concurrency, but the counts they carry are exact.
223
256
 
257
+ A submission arriving after :meth:`shutdown` is still accepted, and counted where it
258
+ can be seen (SPEC-030 FR-001): the worker is retired and nothing will drain the queue,
259
+ so this is total silent loss until something reports it. The check is a single unlocked
260
+ read of a flag that is only ever set, never cleared, which is what keeps the normal path
261
+ free — taking the lock here would put every submission behind the counter's contention
262
+ for a condition that is false in every correct program. SPEC-019's objection to a
263
+ liveness check in ``submit`` does not reach it: that was about probing the thread, this
264
+ is a boolean already in the object's dict.
265
+
224
266
  Args:
225
267
  events: The span's buffered events, submitted as one item.
226
268
 
@@ -230,6 +272,8 @@ class Worker:
230
272
  Raises:
231
273
  None.
232
274
  """
275
+ if self._shutdown_done:
276
+ self._count_undeliverable()
233
277
  try:
234
278
  self._queue.put_nowait(events)
235
279
  except queue.Full:
@@ -239,13 +283,44 @@ class Worker:
239
283
  if total == 1 or total % _DROP_WARN_EVERY == 0:
240
284
  _diag.lost("submission", total, "log queue full; count is cumulative")
241
285
 
286
+ def _count_undeliverable(self) -> None:
287
+ """Counts a post-shutdown submission and warns on the same throttle as overflow.
288
+
289
+ The two conditions warrant the same treatment for the same reason (FR-002): a caller
290
+ making this mistake makes it on every invocation, so a line per submission would be its
291
+ own outage, while total silence is how the mistake survives to production. The counter
292
+ moves before the line is attempted, per ``_diag``'s record-first rule, and the write
293
+ happens outside the lock so a wedged console cannot stall the drain path.
294
+
295
+ Args:
296
+ None.
297
+
298
+ Returns:
299
+ None.
300
+
301
+ Raises:
302
+ None.
303
+ """
304
+ with self._lock:
305
+ self.submitted_after_shutdown += 1
306
+ total = self.submitted_after_shutdown
307
+ if total == 1 or total % _DROP_WARN_EVERY == 0:
308
+ _diag.lost(
309
+ "submission",
310
+ total,
311
+ "logged after shutdown(), which is terminal; nothing will drain these. Use "
312
+ "flush() in a process that logs again. Count is cumulative",
313
+ )
314
+
242
315
  def health(self) -> Health:
243
316
  """Snapshots the delivery counters (SPEC-017 FR-005, SPEC-019 FR-003).
244
317
 
245
318
  This stays valid after :meth:`shutdown`: the counters are plain integers that outlive
246
319
  the thread, and the final drain consumes the queue, so ``queued`` reads 0 rather than a
247
320
  stale marker. The same applies to ``stopped_reason``, since a caller finding a dead
248
- worker will usually call ``shutdown()`` next.
321
+ worker will usually call ``shutdown()`` next. Reading it after a shutdown is in fact
322
+ the point of ``retired`` and ``submitted_after_shutdown`` (SPEC-030 FR-001), which
323
+ report a state only a retired worker can be in.
249
324
 
250
325
  Args:
251
326
  None.
@@ -259,12 +334,18 @@ class Worker:
259
334
  with self._lock:
260
335
  dropped, failed_batches = self.dropped, self.failed_batches
261
336
  stopped_reason = self.stopped_reason
337
+ retired = self._shutdown_done
338
+ submitted_after_shutdown = self.submitted_after_shutdown
339
+ incomplete_swaps = self.incomplete_swaps
262
340
  return Health(
263
341
  queued=self._queue.qsize(),
264
342
  dropped=dropped,
265
343
  failed_batches=failed_batches,
266
344
  stopped_reason=stopped_reason,
267
345
  sink=self._sink_losses(),
346
+ retired=retired,
347
+ submitted_after_shutdown=submitted_after_shutdown,
348
+ incomplete_swaps=incomplete_swaps,
268
349
  )
269
350
 
270
351
  def _sink_losses(self) -> SinkLosses | None:
@@ -332,6 +413,137 @@ class Worker:
332
413
  return False
333
414
  return marker.delivered
334
415
 
416
+ def swap_sink(self, new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> None:
417
+ """Retargets delivery at a new sink, draining and closing the previous one (FR-003).
418
+
419
+ This is what makes a late ``configure(sink=...)`` mean what it says. The sink was
420
+ captured once when the worker was built, so before SPEC-030 a later call updated the
421
+ config — which ``get_config().sink`` then reported — while every event continued to the
422
+ old sink: the config and the behaviour disagreed, and nothing said so.
423
+
424
+ The attribute is reassigned rather than the worker rebuilt, which keeps the queue, the
425
+ thread, the counters and the ``atexit`` registration intact; rebuilding would drop
426
+ whatever was queued and register a second drain. The order is the contract: drain first
427
+ so everything submitted before the call reaches the sink it was submitted for, swap,
428
+ then drain again before closing. That second drain is a fence rather than a delivery —
429
+ it proves the drain thread is not still inside the old sink's ``emit``, which is the one
430
+ way ``close()`` could be called under a writer.
431
+
432
+ On a drain that cannot be confirmed the swap still stands, because the caller asked for
433
+ the new sink and silently keeping the old one is the defect this method exists to fix.
434
+ What changes is that the old sink is left **open** and ``incomplete_swaps`` records it:
435
+ the drain thread may still be using it, and SPEC-027 FR-004 already settled that a
436
+ leaked resource beats a close raced against a write.
437
+
438
+ Both guards are re-taken after the first drain, which blocks and therefore cannot be
439
+ trusted to return into the state it left. Retirement is the one that bites: ``shutdown``
440
+ closes whatever ``self.sink`` was at that moment and latches its once-only flag, so a
441
+ swap reassigning afterwards would install a sink nothing will ever close, and then
442
+ report that the *old* one was left open when it had in fact just been closed.
443
+
444
+ ``configure()`` remains a startup call and this does not make it thread-safe. A span
445
+ finishing on another thread during the swap may land on either sink; what is guaranteed
446
+ is that everything submitted before the call was drained to the old one.
447
+
448
+ Args:
449
+ new_sink: The sink every subsequent batch is emitted to.
450
+ timeout: Seconds bounding both drains — one shared deadline, so a hung sink cannot
451
+ make ``configure()`` wait for twice the budget. It does **not** bound the closing of
452
+ the old sink, which has no timeout of its own; see :meth:`_close_swapped_out`.
453
+ ``None`` waits indefinitely.
454
+
455
+ Returns:
456
+ None. The outcome is reported through ``health().incomplete_swaps`` and one stderr
457
+ line rather than a return value, because the caller is ``configure()``, which has
458
+ never had one and whose callers do not check.
459
+
460
+ Raises:
461
+ None on a sink fault. A close that fails is announced, as everywhere else
462
+ (SPEC-025 FR-004).
463
+ """
464
+ with self._lock:
465
+ if self._shutdown_done or self.sink is new_sink:
466
+ return
467
+ deadline = None if timeout is None else time.monotonic() + timeout
468
+ drained = self.flush(timeout)
469
+ with self._lock:
470
+ if self._shutdown_done:
471
+ return
472
+ old = self.sink
473
+ if old is new_sink:
474
+ return
475
+ self.sink = new_sink
476
+ self._offer_stop_signal()
477
+ remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
478
+ if not (drained and self.flush(remaining)):
479
+ self._record_incomplete_swap(timeout)
480
+ return
481
+ self._close_swapped_out(old)
482
+
483
+ def _record_incomplete_swap(self, timeout: float | None) -> None:
484
+ """Counts a swap whose drain could not be confirmed, then announces it (FR-003).
485
+
486
+ Two things are reported at once because they have one cause: queued items may have been
487
+ carried to the new sink rather than the old one, and the old sink was left open. The
488
+ count is queued *items* — one per submitted span plus any marker — which makes it a
489
+ floor on the events involved, the useful direction for a reader deciding whether to care.
490
+
491
+ Args:
492
+ timeout: The budget the drain was given, rendered for the line.
493
+
494
+ Returns:
495
+ None.
496
+
497
+ Raises:
498
+ None.
499
+ """
500
+ with self._lock:
501
+ self.incomplete_swaps += 1
502
+ _diag.lost(
503
+ "item",
504
+ self._queued_or_unknown(),
505
+ f"the previous sink could not be confirmed drained within "
506
+ f"{_bounded_seconds(timeout)} of a configure(sink=...); it is left open, and "
507
+ f"queued items may reach the new sink instead",
508
+ )
509
+
510
+ def _close_swapped_out(self, sink: Sink) -> None:
511
+ """Closes a sink the worker no longer delivers to, absorbing a failure.
512
+
513
+ This is reached only once both drains have been confirmed, so the *drain thread* is
514
+ provably out of this sink's ``emit``. An orphan-path emitter on an application thread
515
+ is not covered: it resolves the sink through ``_ensure_sink`` before emitting, so one
516
+ that read the old sink before ``configure()`` reassigned it can still be inside its
517
+ ``emit`` — which is why ``sinks/base.py`` requires ``close()`` to tolerate exactly that
518
+ (SPEC-028 FR-001), and why the sinks holding transport state take their lock in both.
519
+
520
+ The close is **not** bounded by the swap's timeout: ``Sink.close`` has no timeout of its
521
+ own, so bounding it needs an interruptible close — a change to the sink contract, not to
522
+ this method. Of :meth:`_close_if_owed`'s two reasons for rejecting a threaded close, only
523
+ the second reaches this site: the daemon killed mid-``commit()`` is an interpreter-exit
524
+ hazard and a swap runs in a live process, but an expired join still cannot tell a
525
+ slow-but-successful close from a stuck one, and would report a loss for a swap that
526
+ completed. A destination whose ``close()`` blocks therefore blocks ``configure()``,
527
+ recorded in ``architecture.md`` §13 rather than papered over.
528
+
529
+ It is deliberately not :meth:`_close_sink`, which answers a different question — that
530
+ one closes the sink the worker still holds, exactly once, and only after the thread has
531
+ ended.
532
+
533
+ Args:
534
+ sink: The sink that was swapped out.
535
+
536
+ Returns:
537
+ None.
538
+
539
+ Raises:
540
+ None.
541
+ """
542
+ try:
543
+ sink.close()
544
+ except Exception as exc:
545
+ _diag.absorbed("closing a swapped-out sink", exc, "it may still hold its resources")
546
+
335
547
  def shutdown(self, timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
336
548
  """Stops the thread, drains and emits everything queued, then closes the sink.
337
549
 
@@ -348,6 +560,12 @@ class Worker:
348
560
  ``close()`` twice on a sink that may have partially released its resources; what
349
561
  SPEC-025 FR-004 changed is that the failure is announced rather than swallowed.
350
562
 
563
+ The worker does not come back, and :meth:`submit` keeps accepting afterwards — so a
564
+ caller that logs again queues events nothing will drain. That is reported rather than
565
+ prevented, through ``retired`` and ``submitted_after_shutdown`` (SPEC-030 FR-001) and
566
+ one stderr line; refusing the submission or restarting the thread were both rejected,
567
+ the second because a thread that resurrects itself fights a process trying to exit.
568
+
351
569
  Args:
352
570
  timeout: Seconds the join may take. ``None`` waits indefinitely, which is what this
353
571
  did unconditionally before and is still available on request.