log-foundry 0.10.2.dev28__tar.gz → 0.10.2.dev29__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/PKG-INFO +31 -7
  2. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/README.md +30 -6
  3. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/pyproject.toml +1 -1
  4. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/__init__.py +11 -3
  5. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/config.py +5 -4
  6. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/decorator.py +2 -1
  7. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/worker.py +142 -15
  8. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/LICENSE +0 -0
  9. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/_diag.py +0 -0
  10. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/api.py +0 -0
  11. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/console.py +0 -0
  12. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/context.py +0 -0
  13. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/ids.py +0 -0
  14. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/model.py +0 -0
  15. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/py.typed +0 -0
  16. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sanitize.py +0 -0
  17. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/__init__.py +0 -0
  18. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/_batch.py +0 -0
  19. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/_chunk.py +0 -0
  20. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/_retry.py +0 -0
  21. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/_socket.py +0 -0
  22. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/_time.py +0 -0
  23. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/base.py +0 -0
  24. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/callback.py +0 -0
  25. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/clickhouse.py +0 -0
  26. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/datadog.py +0 -0
  27. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/elasticsearch.py +0 -0
  28. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/eventhubs.py +0 -0
  29. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/file.py +0 -0
  30. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/filtering.py +0 -0
  31. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/firehose.py +0 -0
  32. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/honeycomb.py +0 -0
  33. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/http.py +0 -0
  34. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/kafka.py +0 -0
  35. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/kinesis.py +0 -0
  36. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/logging_sink.py +0 -0
  37. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/logstash.py +0 -0
  38. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/loki.py +0 -0
  39. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/mongodb.py +0 -0
  40. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/multi.py +0 -0
  41. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/nats.py +0 -0
  42. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/newrelic.py +0 -0
  43. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/postgres.py +0 -0
  44. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/pubsub.py +0 -0
  45. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/rabbitmq.py +0 -0
  46. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/redis.py +0 -0
  47. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/sentry.py +0 -0
  48. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/sns.py +0 -0
  49. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/splunk.py +0 -0
  50. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/sqlite.py +0 -0
  51. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/sqs.py +0 -0
  52. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/stdout.py +0 -0
  53. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/syslog.py +0 -0
  54. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/transform.py +0 -0
  55. {log_foundry-0.10.2.dev28 → log_foundry-0.10.2.dev29}/src/log_foundry/sinks/util.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev28
3
+ Version: 0.10.2.dev29
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -254,10 +254,30 @@ drain thread may still be inside its `emit`, and `health().incomplete_swaps` rec
254
254
  the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
255
255
  do not hand it back to a later call.
256
256
 
257
- The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
258
- a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
259
- an unreachable broker is the case to watch. Configure the sink before the first log where you can;
260
- that path has no worker to retarget and nothing to close.
257
+ The 5 s covers the **whole** call — both drains and the previous sink's `close()`. `Sink.close()`
258
+ takes no timeout of its own (`KafkaSink.close()` flushes its producer, so an unreachable broker
259
+ blocks it), so that close runs on its own daemon thread and is joined for whatever is left of the
260
+ budget. A hung `close()` therefore costs you the budget, once, and the close carries on in the
261
+ background afterwards.
262
+
263
+ Nothing is *reported* when that join expires — a slow close is not a failed swap, and a counter
264
+ that could not tell the two apart would be worse than none. What you get instead is a live
265
+ reading: `health().closing_sinks` is how many swapped-out sinks are inside `close()` at the moment
266
+ you ask. Non-zero once is a swap in progress; non-zero every time you look is a destination that
267
+ is not coming back, still holding its resources.
268
+
269
+ **What happens to that background close when the process exits.** `shutdown()` (which `atexit`
270
+ runs for you) drains and closes the live sink first, then gives any still-running swapped-out close
271
+ a short grace — 2 s, per `shutdown()` call, carved from that call's own timeout — to finish. A
272
+ close that was merely slow completes. One that is genuinely stuck is abandoned there, and **its
273
+ own buffered data is lost**: for a sink whose `close()` *is* its delivery, like
274
+ `KafkaSink.close()` flushing the producer, that is everything it had not yet sent.
275
+ `health().closing_sinks` is the only warning you get, which is why it is worth watching.
276
+
277
+ The closer runs as a daemon thread deliberately. A non-daemon one is worse: CPython joins
278
+ non-daemon threads **before** running `atexit`, so a single stuck `close()` would stop the exit
279
+ drain from ever running — the live sink never drained, and your own `atexit` handlers never run
280
+ either. The grace is what recovers the slow-close case that the daemon flag alone would lose.
261
281
 
262
282
  `configure()` is still a startup call. It is not thread-safe, and a span finishing on another
263
283
  thread mid-swap may land on either sink.
@@ -801,6 +821,9 @@ if (
801
821
  ... # logs were silently lost — worth an alert
802
822
  ```
803
823
 
824
+ `closing_sinks` is deliberately not a term here: it is briefly non-zero during a perfectly healthy
825
+ sink swap, so a single reading is not a fault. Watch it over time instead — see the table below.
826
+
804
827
  They tell you different things, and they want different responses:
805
828
 
806
829
  | Field | Means | What to do |
@@ -812,6 +835,7 @@ They tell you different things, and they want different responses:
812
835
  | `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
813
836
  | `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
814
837
  | `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
838
+ | `closing_sinks` | Swapped-out sinks inside `close()` **right now** — a live gauge, not a counter, and the only field that falls as well as rises. Non-zero on a single read is normal during a swap. | Nothing, unless it stays non-zero. That means a destination is stuck in `close()` and will not release its resources. |
815
839
 
816
840
  `h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
817
841
  the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
@@ -839,8 +863,8 @@ means every log line since the shutdown has gone nowhere. That state used to rea
839
863
  healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
840
864
 
841
865
  Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
842
- fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
843
- seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
866
+ fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus four more
867
+ (`retired`, `submitted_after_shutdown`, `incomplete_swaps`, `closing_sinks`) not yet in a tagged
844
868
  release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
845
869
  field keeps its position when a new one is appended, so attribute and index access stay stable.
846
870
 
@@ -218,10 +218,30 @@ drain thread may still be inside its `emit`, and `health().incomplete_swaps` rec
218
218
  the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
219
219
  do not hand it back to a later call.
220
220
 
221
- The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
222
- a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
223
- an unreachable broker is the case to watch. Configure the sink before the first log where you can;
224
- that path has no worker to retarget and nothing to close.
221
+ The 5 s covers the **whole** call — both drains and the previous sink's `close()`. `Sink.close()`
222
+ takes no timeout of its own (`KafkaSink.close()` flushes its producer, so an unreachable broker
223
+ blocks it), so that close runs on its own daemon thread and is joined for whatever is left of the
224
+ budget. A hung `close()` therefore costs you the budget, once, and the close carries on in the
225
+ background afterwards.
226
+
227
+ Nothing is *reported* when that join expires — a slow close is not a failed swap, and a counter
228
+ that could not tell the two apart would be worse than none. What you get instead is a live
229
+ reading: `health().closing_sinks` is how many swapped-out sinks are inside `close()` at the moment
230
+ you ask. Non-zero once is a swap in progress; non-zero every time you look is a destination that
231
+ is not coming back, still holding its resources.
232
+
233
+ **What happens to that background close when the process exits.** `shutdown()` (which `atexit`
234
+ runs for you) drains and closes the live sink first, then gives any still-running swapped-out close
235
+ a short grace — 2 s, per `shutdown()` call, carved from that call's own timeout — to finish. A
236
+ close that was merely slow completes. One that is genuinely stuck is abandoned there, and **its
237
+ own buffered data is lost**: for a sink whose `close()` *is* its delivery, like
238
+ `KafkaSink.close()` flushing the producer, that is everything it had not yet sent.
239
+ `health().closing_sinks` is the only warning you get, which is why it is worth watching.
240
+
241
+ The closer runs as a daemon thread deliberately. A non-daemon one is worse: CPython joins
242
+ non-daemon threads **before** running `atexit`, so a single stuck `close()` would stop the exit
243
+ drain from ever running — the live sink never drained, and your own `atexit` handlers never run
244
+ either. The grace is what recovers the slow-close case that the daemon flag alone would lose.
225
245
 
226
246
  `configure()` is still a startup call. It is not thread-safe, and a span finishing on another
227
247
  thread mid-swap may land on either sink.
@@ -765,6 +785,9 @@ if (
765
785
  ... # logs were silently lost — worth an alert
766
786
  ```
767
787
 
788
+ `closing_sinks` is deliberately not a term here: it is briefly non-zero during a perfectly healthy
789
+ sink swap, so a single reading is not a fault. Watch it over time instead — see the table below.
790
+
768
791
  They tell you different things, and they want different responses:
769
792
 
770
793
  | Field | Means | What to do |
@@ -776,6 +799,7 @@ They tell you different things, and they want different responses:
776
799
  | `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
777
800
  | `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
778
801
  | `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
802
+ | `closing_sinks` | Swapped-out sinks inside `close()` **right now** — a live gauge, not a counter, and the only field that falls as well as rises. Non-zero on a single read is normal during a swap. | Nothing, unless it stays non-zero. That means a destination is stuck in `close()` and will not release its resources. |
779
803
 
780
804
  `h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
781
805
  the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
@@ -803,8 +827,8 @@ means every log line since the shutdown has gone nowhere. That state used to rea
803
827
  healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
804
828
 
805
829
  Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
806
- fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
807
- seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
830
+ fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus four more
831
+ (`retired`, `submitted_after_shutdown`, `incomplete_swaps`, `closing_sinks`) not yet in a tagged
808
832
  release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
809
833
  field keeps its position when a new one is appended, so attribute and index access stay stable.
810
834
 
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev28"
23
+ version = "0.10.2.dev29"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -70,6 +70,9 @@ def health() -> Health:
70
70
 
71
71
  ``retired`` alone is not a fault — a process that shuts down and then stops logging is
72
72
  doing the right thing, which is why it is paired with the count rather than alerted on.
73
+ ``closing_sinks`` is deliberately absent for the same kind of reason: it is briefly non-zero
74
+ during a perfectly healthy sink swap, so alerting on a single reading would fire on correct
75
+ use. It is a gauge to watch over time, not a term in this test.
73
76
 
74
77
  Args:
75
78
  None.
@@ -81,8 +84,11 @@ def health() -> Health:
81
84
  afterwards, which are queued where nothing will drain them — non-zero together, that is
82
85
  the ``shutdown()``-per-invocation mistake, and the remedy is :func:`flush`.
83
86
  ``incomplete_swaps`` counts late ``configure(sink=...)`` calls whose drain of the
84
- previous sink could not be confirmed, leaving that sink open. ``sink`` is the configured
85
- sink's own
87
+ previous sink could not be confirmed, leaving that sink open. ``closing_sinks`` is the
88
+ odd one out — a live gauge rather than a counter, reporting how many swapped-out sinks
89
+ are inside ``close()`` right now, so a destination stuck there is visible at all; it
90
+ falls back to zero on its own, and only a *persistently* non-zero reading is a fault.
91
+ ``sink`` is the configured sink's own
86
92
  :class:`~log_foundry.sinks.base.SinkLosses` — loss the sink absorbed rather than the
87
93
  worker (SPEC-026) — and is ``None`` when no worker exists or the sink reports nothing.
88
94
  Note ``sink`` describes whichever sink is live now, so a swap takes the previous sink's
@@ -117,7 +123,9 @@ def shutdown(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
117
123
  (SPEC-030). That pair is the reading that catches the mistake above.
118
124
 
119
125
  Args:
120
- timeout: Seconds bounding the wait for the background thread (SPEC-027 FR-004). ``None``
126
+ timeout: Seconds bounding the wait for the background thread and, carved from the same
127
+ budget, a short grace for any sink still closing after a late ``configure(sink=...)``
128
+ (SPEC-027 FR-004, SPEC-030 FR-003). ``None``
121
129
  waits indefinitely, which is what this did unconditionally before and is still
122
130
  available on request, but is unsafe anywhere with an execution deadline — ``atexit`` is
123
131
  one such place, where a sink blocked in a network call would hold the process open. An
@@ -86,10 +86,11 @@ def configure(
86
86
  is already live is a no-op: no drain, no close. The previous sink is closed and must not be
87
87
  handed back to a later call.
88
88
 
89
- The *closing* of the previous sink is not bounded, because ``Sink.close`` takes no timeout
90
- a destination that blocks in ``close()`` blocks this call. It is the same gap
91
- ``architecture.md`` §13 already records for ``shutdown()``, and it has the same fix, which
92
- is a change to the sink contract rather than to this function.
89
+ The bound covers the whole call, the close included. ``Sink.close`` takes no timeout, so that
90
+ close runs on its own daemon thread and is joined for what is left of the budget: a
91
+ destination that hangs in ``close()`` delays this call by the budget and no more, then carries
92
+ on in the background. An expired join reports nothing, since a slow close and a stuck one
93
+ cannot be told apart at that moment; ``health().closing_sinks`` reports the live fact instead.
93
94
 
94
95
  This is still a startup call. It is not thread-safe, and a span finishing on another thread
95
96
  during a swap may land on either sink.
@@ -243,7 +243,8 @@ def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) ->
243
243
 
244
244
  Args:
245
245
  new_sink: The sink already written to the config, to be made the live delivery target.
246
- timeout: Seconds bounding the drain of the previous sink.
246
+ timeout: Seconds bounding the whole swap — both drains and the close of the previous
247
+ sink share it as one deadline.
247
248
 
248
249
  Returns:
249
250
  None.
@@ -26,6 +26,15 @@ and an unbounded join there is a hung process.
26
26
 
27
27
  _DROP_WARN_EVERY = 1000
28
28
 
29
+ DEFAULT_CLOSER_GRACE = 2.0
30
+ """Seconds :meth:`Worker.shutdown` gives an outstanding swapped-out close to finish.
31
+
32
+ Deliberately much smaller than the shutdown budget it is carved from. This is a last chance for
33
+ a close that is *nearly* done, not a second full attempt: it already had the swap's whole budget
34
+ (``DEFAULT_SWAP_TIMEOUT``) before ``shutdown`` was ever called, so one still running here is far
35
+ more likely stuck than slow, and every second spent on it is a second the process does not exit.
36
+ """
37
+
29
38
  DEFAULT_SWAP_TIMEOUT = 5.0
30
39
  """Seconds a late ``configure(sink=...)`` will spend draining the previous sink (FR-003).
31
40
 
@@ -100,6 +109,14 @@ class Health(NamedTuple):
100
109
  call may have been carried to the new sink instead of the old one, and the old sink was
101
110
  left **open** rather than closed, because the drain thread may still be inside its
102
111
  ``emit`` — the reasoning SPEC-027 FR-004 applies to an expired ``shutdown()``.
112
+ closing_sinks: Swapped-out sinks whose ``close()`` is running *at this instant* — a live
113
+ gauge, not a counter, and the only field here that can fall as well as rise. A close is
114
+ bounded only in how long ``configure()`` waits for it, so this is how a destination
115
+ stuck in ``close()`` becomes visible at all. Reading it non-zero once means a swap just
116
+ happened; reading it non-zero repeatedly means a close is not coming back, and that sink
117
+ still holds its resources. It is deliberately a live read rather than a count of expired
118
+ joins: a slow close and a stuck one are indistinguishable at the moment a join expires,
119
+ and SPEC-028 reverted a design that guessed.
103
120
  """
104
121
 
105
122
  queued: int
@@ -110,6 +127,7 @@ class Health(NamedTuple):
110
127
  retired: bool = False
111
128
  submitted_after_shutdown: int = 0
112
129
  incomplete_swaps: int = 0
130
+ closing_sinks: int = 0
113
131
 
114
132
 
115
133
  class _FlushMarker:
@@ -203,6 +221,7 @@ class Worker:
203
221
  self.stopped_reason: str | None = None
204
222
  self.submitted_after_shutdown = 0
205
223
  self.incomplete_swaps = 0
224
+ self._closers: list[threading.Thread] = []
206
225
  self._queue: queue.Queue[object] = queue.Queue(maxsize=max_queue)
207
226
  self._stop = threading.Event()
208
227
  self._shutdown_done = False
@@ -337,6 +356,8 @@ class Worker:
337
356
  retired = self._shutdown_done
338
357
  submitted_after_shutdown = self.submitted_after_shutdown
339
358
  incomplete_swaps = self.incomplete_swaps
359
+ self._closers = [closer for closer in self._closers if closer.is_alive()]
360
+ closing_sinks = len(self._closers)
340
361
  return Health(
341
362
  queued=self._queue.qsize(),
342
363
  dropped=dropped,
@@ -346,6 +367,7 @@ class Worker:
346
367
  retired=retired,
347
368
  submitted_after_shutdown=submitted_after_shutdown,
348
369
  incomplete_swaps=incomplete_swaps,
370
+ closing_sinks=closing_sinks,
349
371
  )
350
372
 
351
373
  def _sink_losses(self) -> SinkLosses | None:
@@ -447,10 +469,9 @@ class Worker:
447
469
 
448
470
  Args:
449
471
  new_sink: The sink every subsequent batch is emitted to.
450
- timeout: Seconds bounding both drains — one shared deadline, so a hung sink cannot
451
- make ``configure()`` wait for twice the budget. It does **not** bound the closing of
452
- the old sink, which has no timeout of its own; see :meth:`_close_swapped_out`.
453
- ``None`` waits indefinitely.
472
+ timeout: Seconds bounding the **whole** call — one deadline shared by both drains and
473
+ the close of the old sink, so a destination that hangs in any of the three cannot
474
+ hold ``configure()`` for a multiple of the budget. ``None`` waits indefinitely.
454
475
 
455
476
  Returns:
456
477
  None. The outcome is reported through ``health().incomplete_swaps`` and one stderr
@@ -478,7 +499,8 @@ class Worker:
478
499
  if not (drained and self.flush(remaining)):
479
500
  self._record_incomplete_swap(timeout)
480
501
  return
481
- self._close_swapped_out(old)
502
+ left = None if deadline is None else max(0.0, deadline - time.monotonic())
503
+ self._close_swapped_out(old, left)
482
504
 
483
505
  def _record_incomplete_swap(self, timeout: float | None) -> None:
484
506
  """Counts a swap whose drain could not be confirmed, then announces it (FR-003).
@@ -507,8 +529,8 @@ class Worker:
507
529
  f"queued items may reach the new sink instead",
508
530
  )
509
531
 
510
- def _close_swapped_out(self, sink: Sink) -> None:
511
- """Closes a sink the worker no longer delivers to, absorbing a failure.
532
+ def _close_swapped_out(self, sink: Sink, timeout: float | None) -> None:
533
+ """Closes a sink the worker no longer delivers to, waiting only for the budget.
512
534
 
513
535
  This is reached only once both drains have been confirmed, so the *drain thread* is
514
536
  provably out of this sink's ``emit``. An orphan-path emitter on an application thread
@@ -517,14 +539,24 @@ class Worker:
517
539
  ``emit`` — which is why ``sinks/base.py`` requires ``close()`` to tolerate exactly that
518
540
  (SPEC-028 FR-001), and why the sinks holding transport state take their lock in both.
519
541
 
520
- The close is **not** bounded by the swap's timeout: ``Sink.close`` has no timeout of its
521
- own, so bounding it needs an interruptible close a change to the sink contract, not to
522
- this method. Of :meth:`_close_if_owed`'s two reasons for rejecting a threaded close, only
523
- the second reaches this site: the daemon killed mid-``commit()`` is an interpreter-exit
524
- hazard and a swap runs in a live process, but an expired join still cannot tell a
525
- slow-but-successful close from a stuck one, and would report a loss for a swap that
526
- completed. A destination whose ``close()`` blocks therefore blocks ``configure()``,
527
- recorded in ``architecture.md`` §13 rather than papered over.
542
+ ``Sink.close`` takes no timeout, so the close is run on its own thread and joined for
543
+ what is left of the swap's budget. **An expired join decides only who waits** it moves
544
+ no counter and writes no line, which is what dissolves SPEC-028's objection that an
545
+ expired join cannot tell a slow-but-successful close from a stuck one and so reports a
546
+ loss for closes that completed. What *is* observable is a live fact rather than an
547
+ inference: ``health().closing_sinks`` counts the closes running at the moment it is read.
548
+
549
+ The thread is a **daemon**, and it is :meth:`_join_closers` that makes that safe rather
550
+ than merely available. A non-daemon thread was tried and is worse on its own: CPython
551
+ joins non-daemon threads *before* running ``atexit``, so one hung close stops the exit
552
+ drain from ever running and loses everything buffered in the **live** sink, along with
553
+ the application's own exit handlers. A daemon alone is also worse on its own, in the
554
+ opposite case: a close that is slow but *succeeding* is killed at exit, losing whatever
555
+ it was flushing. ``shutdown`` therefore drains and closes the live sink first, then
556
+ joins any outstanding closer for what is left of its budget — so a slow close finishes,
557
+ a hung one costs only the grace, and neither can reach the live sink. What SPEC-028
558
+ refused to abandon was the sink the worker was *still delivering to*; this one has been
559
+ fenced out of the delivery path by two confirmed drains.
528
560
 
529
561
  It is deliberately not :meth:`_close_sink`, which answers a different question — that
530
562
  one closes the sink the worker still holds, exactly once, and only after the thread has
@@ -532,6 +564,49 @@ class Worker:
532
564
 
533
565
  Args:
534
566
  sink: The sink that was swapped out.
567
+ timeout: Seconds to wait for the close before returning and letting it finish on its
568
+ own. ``None`` waits indefinitely.
569
+
570
+ Returns:
571
+ None.
572
+
573
+ Raises:
574
+ None. ``Thread.start`` raises when the platform will not give the process another
575
+ thread, and a swap that cannot spawn one must leave the sink open and say so rather
576
+ than fall back to an inline close — the fallback would reintroduce the unbounded
577
+ wait this method exists to remove, in the one situation where the process is
578
+ already under resource pressure.
579
+ """
580
+ closer = threading.Thread(
581
+ target=self._close_detached,
582
+ args=(sink,),
583
+ name="log-foundry-sink-close",
584
+ daemon=True,
585
+ )
586
+ try:
587
+ closer.start()
588
+ except Exception as exc:
589
+ _diag.absorbed(
590
+ "starting the thread that closes a swapped-out sink",
591
+ exc,
592
+ "it is left open and may still hold its resources",
593
+ )
594
+ return
595
+ with self._lock:
596
+ self._closers = [old for old in self._closers if old.is_alive()]
597
+ self._closers.append(closer)
598
+ closer.join(timeout)
599
+
600
+ def _close_detached(self, sink: Sink) -> None:
601
+ """Closes a swapped-out sink on its own thread, absorbing a failure.
602
+
603
+ The guard is what makes the thread safe to leave unattended: an exception escaping here
604
+ would reach CPython's thread bootstrap, which prints a full traceback carrying the
605
+ exception's message — the user data arch §6 keeps out of anything the library says about
606
+ itself, and the reason :meth:`shutdown` guards its own close.
607
+
608
+ Args:
609
+ sink: The sink to close.
535
610
 
536
611
  Returns:
537
612
  None.
@@ -578,11 +653,13 @@ class Worker:
578
653
  a full traceback carrying the exception's message, which arch §6 keeps out of
579
654
  anything the library says about itself.
580
655
  """
656
+ deadline = None if timeout is None else time.monotonic() + timeout
581
657
  with self._lock:
582
658
  first = not self._shutdown_done
583
659
  self._shutdown_done = True
584
660
  if not first:
585
661
  self._close_if_owed()
662
+ self._join_closers(None if deadline is None else max(0.0, deadline - time.monotonic()))
586
663
  return
587
664
  self._stop.set()
588
665
  try:
@@ -603,6 +680,56 @@ class Worker:
603
680
  )
604
681
  return
605
682
  self._close_if_owed()
683
+ self._join_closers(None if deadline is None else max(0.0, deadline - time.monotonic()))
684
+
685
+ def _join_closers(self, timeout: float | None) -> None:
686
+ """Gives a swapped-out sink's close its last chance before the process exits.
687
+
688
+ This is what makes the daemon closer of :meth:`_close_swapped_out` safe rather than
689
+ merely available. A daemon is killed wherever it has reached at interpreter exit, so
690
+ without this a close that was slow but *succeeding* — a ``KafkaSink`` flushing its
691
+ producer, where the close is the delivery — would lose its buffer, which a non-daemon
692
+ thread would not have. Measured both ways: daemon alone lost those events, non-daemon
693
+ alone lost everything in the *live* sink instead, and this join is what takes neither
694
+ loss.
695
+
696
+ **The cap is the mechanism.** The wait is the smaller of ``DEFAULT_CLOSER_GRACE`` and
697
+ what remains of ``shutdown``'s own budget: capped so a stuck close cannot hold a process
698
+ at exit for the whole shutdown budget, and carved from that budget so it cannot extend it
699
+ either. Running after :meth:`_close_if_owed` rather than before it is defence in depth
700
+ rather than the guarantee — measured, the two orders deliver the live sink identically,
701
+ because the cap returns control long before anything is at risk. It is still the right
702
+ order, and pinned by a test: it is what holds if an external deadline kills the process
703
+ *during* the grace, where the live sink would otherwise be the one left unclosed.
704
+
705
+ It runs on the idempotent path too. A first ``shutdown`` that expired on a wedged drain
706
+ thread returns before ever reaching here, and the ``atexit`` call that follows would
707
+ otherwise return instantly — denying the grace to a swapped-out close that is healthy and
708
+ moments from finishing, which is exactly the loss the grace exists to prevent. The closer
709
+ is independent of the worker thread, so a wedged worker is no reason to abandon it. The
710
+ expired path itself still skips it, and that costs nothing: the thread join consumed the
711
+ budget, so the remainder is zero.
712
+
713
+ Args:
714
+ timeout: Seconds remaining in ``shutdown``'s budget, further capped by
715
+ ``DEFAULT_CLOSER_GRACE`` and shared across every outstanding close. ``None`` takes
716
+ the cap rather than waiting indefinitely — an unbounded ``shutdown`` is a caller's
717
+ choice about draining events, not a licence for a stuck close to hold the exit.
718
+
719
+ Returns:
720
+ None.
721
+
722
+ Raises:
723
+ None. A join on a thread that has already finished is a no-op, and one that has not
724
+ is abandoned at the deadline — which is the daemon's contract, not a failure.
725
+ """
726
+ with self._lock:
727
+ closers = [closer for closer in self._closers if closer.is_alive()]
728
+ self._closers = closers
729
+ grace = DEFAULT_CLOSER_GRACE if timeout is None else min(timeout, DEFAULT_CLOSER_GRACE)
730
+ deadline = time.monotonic() + grace
731
+ for closer in closers:
732
+ closer.join(max(0.0, deadline - time.monotonic()))
606
733
 
607
734
  def _close_if_owed(self) -> None:
608
735
  """Closes the sink exactly once, and only once the drain thread has ended.