log-foundry 0.10.2.dev25__tar.gz → 0.10.2.dev26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/PKG-INFO +53 -4
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/README.md +52 -3
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/__init__.py +23 -5
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/config.py +52 -1
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/decorator.py +34 -1
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/worker.py +222 -4
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/LICENSE +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_retry.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_socket.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/base.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/callback.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/clickhouse.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/datadog.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/elasticsearch.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/eventhubs.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/file.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/filtering.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/firehose.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/honeycomb.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/http.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/kafka.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/kinesis.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/logging_sink.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/logstash.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/loki.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/mongodb.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/multi.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/nats.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/postgres.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/pubsub.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/rabbitmq.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/redis.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sentry.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sns.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/splunk.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sqlite.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/sqs.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/stdout.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/syslog.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/transform.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/util.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: log-foundry
|
|
3
|
-
Version: 0.10.2.
|
|
3
|
+
Version: 0.10.2.dev26
|
|
4
4
|
Summary: Generate logs for your console and JSON events for downstream consumption.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -234,6 +234,34 @@ lf.configure(
|
|
|
234
234
|
If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
|
|
235
235
|
works with zero configuration.
|
|
236
236
|
|
|
237
|
+
**Passing `sink=` after logging has started swaps the live destination.** The background worker
|
|
238
|
+
captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
|
|
239
|
+
what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
|
|
240
|
+
what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
|
|
241
|
+
it, and points the worker at the new one:
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
lf.configure(sink=StdoutSink())
|
|
245
|
+
do_some_work() # these events go to stdout
|
|
246
|
+
|
|
247
|
+
lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
|
|
248
|
+
do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
|
|
252
|
+
you asked for that sink — but the previous sink is left **open** rather than closed, because the
|
|
253
|
+
drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
|
|
254
|
+
the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
|
|
255
|
+
do not hand it back to a later call.
|
|
256
|
+
|
|
257
|
+
The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
|
|
258
|
+
a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
|
|
259
|
+
an unreachable broker is the case to watch. Configure the sink before the first log where you can;
|
|
260
|
+
that path has no worker to retarget and nothing to close.
|
|
261
|
+
|
|
262
|
+
`configure()` is still a startup call. It is not thread-safe, and a span finishing on another
|
|
263
|
+
thread mid-swap may land on either sink.
|
|
264
|
+
|
|
237
265
|
### `@trace`
|
|
238
266
|
|
|
239
267
|
Decorate any **synchronous** function. Usable bare or with arguments:
|
|
@@ -746,7 +774,11 @@ returns a snapshot of the worker's counters:
|
|
|
746
774
|
|
|
747
775
|
```python
|
|
748
776
|
h = log_foundry.health()
|
|
749
|
-
if
|
|
777
|
+
if (
|
|
778
|
+
h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
|
|
779
|
+
or (h.sink and (h.sink.dropped or h.sink.failed))
|
|
780
|
+
or (h.retired and h.submitted_after_shutdown)
|
|
781
|
+
):
|
|
750
782
|
... # logs were silently lost — worth an alert
|
|
751
783
|
```
|
|
752
784
|
|
|
@@ -759,6 +791,8 @@ They tell you different things, and they want different responses:
|
|
|
759
791
|
| `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
|
|
760
792
|
| `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
|
|
761
793
|
| `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
|
|
794
|
+
| `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
|
|
795
|
+
| `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
|
|
762
796
|
|
|
763
797
|
`h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
|
|
764
798
|
the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
|
|
@@ -780,8 +814,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
|
|
|
780
814
|
thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
|
|
781
815
|
pointing at the wrong fix.
|
|
782
816
|
|
|
817
|
+
`retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
|
|
818
|
+
logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
|
|
819
|
+
means every log line since the shutdown has gone nowhere. That state used to read as perfectly
|
|
820
|
+
healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
|
|
821
|
+
|
|
783
822
|
Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
|
|
784
|
-
fields over time — a fourth (`stopped_reason`) in `v0.7.0
|
|
823
|
+
fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
|
|
824
|
+
seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
|
|
785
825
|
release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
|
|
786
826
|
field keeps its position when a new one is appended, so attribute and index access stay stable.
|
|
787
827
|
|
|
@@ -859,6 +899,13 @@ getting them wrong is silent:
|
|
|
859
899
|
the first invocation on a warm container would log and every later one would silently log
|
|
860
900
|
nothing. That failure reads as "works locally, broken in production".
|
|
861
901
|
|
|
902
|
+
It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
|
|
903
|
+
`health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
|
|
904
|
+
pair, and only that pair, is this mistake. The first such submission also writes one stderr
|
|
905
|
+
line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
|
|
906
|
+
`stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
|
|
907
|
+
drain where the repeatable one belonged.
|
|
908
|
+
|
|
862
909
|
```python
|
|
863
910
|
import log_foundry as lf
|
|
864
911
|
from log_foundry.sinks.sqs import SQSSink
|
|
@@ -876,9 +923,11 @@ def handler(event, context):
|
|
|
876
923
|
# would log nothing.
|
|
877
924
|
drained = lf.flush()
|
|
878
925
|
h = lf.health()
|
|
879
|
-
if not drained or h.failed_batches or h.dropped or h.stopped_reason:
|
|
926
|
+
if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
|
|
880
927
|
# `drained` covers this invocation's tail; the counters cover anything the worker
|
|
881
928
|
# lost earlier — a batch its own interval trigger already gave up on, for instance.
|
|
929
|
+
# `h.retired` catches the mistake above: inside a handler it can only mean something
|
|
930
|
+
# called shutdown(), and from here on this container logs nothing.
|
|
882
931
|
# Emitting this through your platform's own logger keeps it outside the pipeline
|
|
883
932
|
# that just failed.
|
|
884
933
|
print(f"log-foundry: undelivered logs ({drained=}, {h=})")
|
|
@@ -198,6 +198,34 @@ lf.configure(
|
|
|
198
198
|
If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
|
|
199
199
|
works with zero configuration.
|
|
200
200
|
|
|
201
|
+
**Passing `sink=` after logging has started swaps the live destination.** The background worker
|
|
202
|
+
captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
|
|
203
|
+
what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
|
|
204
|
+
what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
|
|
205
|
+
it, and points the worker at the new one:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
lf.configure(sink=StdoutSink())
|
|
209
|
+
do_some_work() # these events go to stdout
|
|
210
|
+
|
|
211
|
+
lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
|
|
212
|
+
do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
|
|
216
|
+
you asked for that sink — but the previous sink is left **open** rather than closed, because the
|
|
217
|
+
drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
|
|
218
|
+
the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
|
|
219
|
+
do not hand it back to a later call.
|
|
220
|
+
|
|
221
|
+
The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
|
|
222
|
+
a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
|
|
223
|
+
an unreachable broker is the case to watch. Configure the sink before the first log where you can;
|
|
224
|
+
that path has no worker to retarget and nothing to close.
|
|
225
|
+
|
|
226
|
+
`configure()` is still a startup call. It is not thread-safe, and a span finishing on another
|
|
227
|
+
thread mid-swap may land on either sink.
|
|
228
|
+
|
|
201
229
|
### `@trace`
|
|
202
230
|
|
|
203
231
|
Decorate any **synchronous** function. Usable bare or with arguments:
|
|
@@ -710,7 +738,11 @@ returns a snapshot of the worker's counters:
|
|
|
710
738
|
|
|
711
739
|
```python
|
|
712
740
|
h = log_foundry.health()
|
|
713
|
-
if
|
|
741
|
+
if (
|
|
742
|
+
h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
|
|
743
|
+
or (h.sink and (h.sink.dropped or h.sink.failed))
|
|
744
|
+
or (h.retired and h.submitted_after_shutdown)
|
|
745
|
+
):
|
|
714
746
|
... # logs were silently lost — worth an alert
|
|
715
747
|
```
|
|
716
748
|
|
|
@@ -723,6 +755,8 @@ They tell you different things, and they want different responses:
|
|
|
723
755
|
| `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
|
|
724
756
|
| `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
|
|
725
757
|
| `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
|
|
758
|
+
| `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
|
|
759
|
+
| `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
|
|
726
760
|
|
|
727
761
|
`h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
|
|
728
762
|
the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
|
|
@@ -744,8 +778,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
|
|
|
744
778
|
thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
|
|
745
779
|
pointing at the wrong fix.
|
|
746
780
|
|
|
781
|
+
`retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
|
|
782
|
+
logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
|
|
783
|
+
means every log line since the shutdown has gone nowhere. That state used to read as perfectly
|
|
784
|
+
healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
|
|
785
|
+
|
|
747
786
|
Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
|
|
748
|
-
fields over time — a fourth (`stopped_reason`) in `v0.7.0
|
|
787
|
+
fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
|
|
788
|
+
seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
|
|
749
789
|
release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
|
|
750
790
|
field keeps its position when a new one is appended, so attribute and index access stay stable.
|
|
751
791
|
|
|
@@ -823,6 +863,13 @@ getting them wrong is silent:
|
|
|
823
863
|
the first invocation on a warm container would log and every later one would silently log
|
|
824
864
|
nothing. That failure reads as "works locally, broken in production".
|
|
825
865
|
|
|
866
|
+
It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
|
|
867
|
+
`health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
|
|
868
|
+
pair, and only that pair, is this mistake. The first such submission also writes one stderr
|
|
869
|
+
line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
|
|
870
|
+
`stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
|
|
871
|
+
drain where the repeatable one belonged.
|
|
872
|
+
|
|
826
873
|
```python
|
|
827
874
|
import log_foundry as lf
|
|
828
875
|
from log_foundry.sinks.sqs import SQSSink
|
|
@@ -840,9 +887,11 @@ def handler(event, context):
|
|
|
840
887
|
# would log nothing.
|
|
841
888
|
drained = lf.flush()
|
|
842
889
|
h = lf.health()
|
|
843
|
-
if not drained or h.failed_batches or h.dropped or h.stopped_reason:
|
|
890
|
+
if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
|
|
844
891
|
# `drained` covers this invocation's tail; the counters cover anything the worker
|
|
845
892
|
# lost earlier — a batch its own interval trigger already gave up on, for instance.
|
|
893
|
+
# `h.retired` catches the mistake above: inside a handler it can only mean something
|
|
894
|
+
# called shutdown(), and from here on this container logs nothing.
|
|
846
895
|
# Emitting this through your platform's own logger keeps it outside the pipeline
|
|
847
896
|
# that just failed.
|
|
848
897
|
print(f"log-foundry: undelivered logs ({drained=}, {h=})")
|
|
@@ -58,22 +58,35 @@ def health() -> Health:
|
|
|
58
58
|
code non-blocking, and a non-zero ``failed_batches`` means a sink stayed broken through the
|
|
59
59
|
whole retry budget — both are losses the library absorbs on purpose, and this is how you
|
|
60
60
|
notice them. A non-``None`` ``stopped_reason`` is worse than either: the background thread
|
|
61
|
-
died on that exception type, so nothing further will be delivered at all (SPEC-019)
|
|
61
|
+
died on that exception type, so nothing further will be delivered at all (SPEC-019). A
|
|
62
|
+
``retired`` worker still being handed events is the same total loss arrived at from the
|
|
63
|
+
other direction — someone called :func:`shutdown` in a process that logs again (SPEC-030)::
|
|
62
64
|
|
|
63
65
|
h = log_foundry.health()
|
|
64
|
-
if h.dropped or h.failed_batches or h.stopped_reason or (
|
|
66
|
+
if h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps or (
|
|
65
67
|
h.sink and (h.sink.dropped or h.sink.failed)
|
|
66
|
-
):
|
|
68
|
+
) or (h.retired and h.submitted_after_shutdown):
|
|
67
69
|
... # raise an alert; logs were silently lost
|
|
68
70
|
|
|
71
|
+
``retired`` alone is not a fault — a process that shuts down and then stops logging is
|
|
72
|
+
doing the right thing, which is why it is paired with the count rather than alerted on.
|
|
73
|
+
|
|
69
74
|
Args:
|
|
70
75
|
None.
|
|
71
76
|
|
|
72
77
|
Returns:
|
|
73
|
-
The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason``
|
|
74
|
-
``
|
|
78
|
+
The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason``, ``sink``,
|
|
79
|
+
``retired``, ``submitted_after_shutdown`` and ``incomplete_swaps``. ``retired`` says
|
|
80
|
+
:func:`shutdown` was called, and ``submitted_after_shutdown`` counts events accepted
|
|
81
|
+
afterwards, which are queued where nothing will drain them — non-zero together, that is
|
|
82
|
+
the ``shutdown()``-per-invocation mistake, and the remedy is :func:`flush`.
|
|
83
|
+
``incomplete_swaps`` counts late ``configure(sink=...)`` calls whose drain of the
|
|
84
|
+
previous sink could not be confirmed, leaving that sink open. ``sink`` is the configured
|
|
85
|
+
sink's own
|
|
75
86
|
:class:`~log_foundry.sinks.base.SinkLosses` — loss the sink absorbed rather than the
|
|
76
87
|
worker (SPEC-026) — and is ``None`` when no worker exists or the sink reports nothing.
|
|
88
|
+
Note ``sink`` describes whichever sink is live now, so a swap takes the previous sink's
|
|
89
|
+
absorbed losses out of the snapshot with it.
|
|
77
90
|
Its ``dropped`` is not the worker's: the worker's is backpressure at the queue, the
|
|
78
91
|
sink's is an event that never reached the wire, and the stderr line names which. Its
|
|
79
92
|
``failed`` is an upper bound on loss rather than a count of it, since a sink that raises
|
|
@@ -98,6 +111,11 @@ def shutdown(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
|
|
|
98
111
|
registered via ``atexit``, so call it explicitly only when you want to be certain the tail
|
|
99
112
|
of the queue reached the sink before a fast process exit (SPEC-004 FR-005).
|
|
100
113
|
|
|
114
|
+
Logging afterwards is **accepted and undeliverable**: those events are queued and nothing
|
|
115
|
+
will drain them. It is not silent any more — :func:`health` then reports ``retired`` with a
|
|
116
|
+
non-zero ``submitted_after_shutdown``, and the first such submission writes one stderr line
|
|
117
|
+
(SPEC-030). That pair is the reading that catches the mistake above.
|
|
118
|
+
|
|
101
119
|
Args:
|
|
102
120
|
timeout: Seconds bounding the wait for the background thread (SPEC-027 FR-004). ``None``
|
|
103
121
|
waits indefinitely, which is what this did unconditionally before and is still
|
|
@@ -76,11 +76,31 @@ def configure(
|
|
|
76
76
|
the zero-dependency dev default (arch §8). Every ceiling is validated before anything is
|
|
77
77
|
assigned, so a rejected call leaves the config exactly as it found it.
|
|
78
78
|
|
|
79
|
+
A ``sink=`` passed after logging has already started is the one argument that needs more
|
|
80
|
+
than an assignment, because the background worker captured its sink when it was built
|
|
81
|
+
(arch §7). It **swaps the live delivery target**: everything submitted so far is drained to
|
|
82
|
+
the previous sink, that sink is closed, and subsequent events go to the new one. The drains
|
|
83
|
+
are bounded, and a swap whose drain could not be confirmed leaves the previous sink open and
|
|
84
|
+
records ``health().incomplete_swaps`` (SPEC-030 FR-003) — so "repeated calls compose rather
|
|
85
|
+
than reset" holds for the sink too, at the cost of one bounded wait. Passing the sink that
|
|
86
|
+
is already live is a no-op: no drain, no close. The previous sink is closed and must not be
|
|
87
|
+
handed back to a later call.
|
|
88
|
+
|
|
89
|
+
The *closing* of the previous sink is not bounded, because ``Sink.close`` takes no timeout —
|
|
90
|
+
a destination that blocks in ``close()`` blocks this call. It is the same gap
|
|
91
|
+
``architecture.md`` §13 already records for ``shutdown()``, and it has the same fix, which
|
|
92
|
+
is a change to the sink contract rather than to this function.
|
|
93
|
+
|
|
94
|
+
This is still a startup call. It is not thread-safe, and a span finishing on another thread
|
|
95
|
+
during a swap may land on either sink.
|
|
96
|
+
|
|
79
97
|
Args:
|
|
80
98
|
service: The service name stamped onto every event.
|
|
81
99
|
version: The service version stamped onto every event.
|
|
82
100
|
env: The deployment environment stamped onto every event.
|
|
83
|
-
sink: The destination every event is delivered to.
|
|
101
|
+
sink: The destination every event is delivered to. Passed after the first log, it swaps
|
|
102
|
+
the live target as described above rather than only updating what ``get_config()``
|
|
103
|
+
reports.
|
|
84
104
|
defaults: Fields merged into every event at the lowest precedence.
|
|
85
105
|
max_value_bytes: Per-value ceiling, in UTF-8 bytes or rendered digits.
|
|
86
106
|
max_stack_bytes: Ceiling for ``error.stack`` alone.
|
|
@@ -119,6 +139,37 @@ def configure(
|
|
|
119
139
|
|
|
120
140
|
_ensure_sink()
|
|
121
141
|
|
|
142
|
+
if sink is not None:
|
|
143
|
+
_swap_live_sink(sink)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _swap_live_sink(sink: Sink) -> None:
|
|
147
|
+
"""Points an already-running worker at a newly configured sink (SPEC-030 FR-003).
|
|
148
|
+
|
|
149
|
+
The import is local for the reason ``_ensure_sink``'s is: ``decorator`` imports this module
|
|
150
|
+
at module scope, so reaching back the other way at import time would be a cycle. It also
|
|
151
|
+
keeps the dependency to the one call that needs it — ``configure()`` without a ``sink=``
|
|
152
|
+
never touches the worker at all.
|
|
153
|
+
|
|
154
|
+
The budget is passed explicitly rather than left to the parameter default, so that the
|
|
155
|
+
bound this call actually applies is resolved when it runs. A default argument is bound at
|
|
156
|
+
definition time, which would leave the end-to-end bound untestable without reaching past
|
|
157
|
+
the function under test.
|
|
158
|
+
|
|
159
|
+
Args:
|
|
160
|
+
sink: The sink just written to the config.
|
|
161
|
+
|
|
162
|
+
Returns:
|
|
163
|
+
None.
|
|
164
|
+
|
|
165
|
+
Raises:
|
|
166
|
+
None.
|
|
167
|
+
"""
|
|
168
|
+
from log_foundry.decorator import _swap_sink
|
|
169
|
+
from log_foundry.worker import DEFAULT_SWAP_TIMEOUT
|
|
170
|
+
|
|
171
|
+
_swap_sink(sink, DEFAULT_SWAP_TIMEOUT)
|
|
172
|
+
|
|
122
173
|
|
|
123
174
|
def get_config() -> Config:
|
|
124
175
|
"""Returns the current global config singleton.
|
|
@@ -20,11 +20,13 @@ from log_foundry.ids import (
|
|
|
20
20
|
parse_traceparent,
|
|
21
21
|
)
|
|
22
22
|
from log_foundry.model import Span, backfill_baggage, end_event, start_event
|
|
23
|
-
from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, Health, Worker
|
|
23
|
+
from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_SWAP_TIMEOUT, Health, Worker
|
|
24
24
|
|
|
25
25
|
if TYPE_CHECKING:
|
|
26
26
|
import contextvars
|
|
27
27
|
|
|
28
|
+
from log_foundry.sinks.base import Sink
|
|
29
|
+
|
|
28
30
|
__all__ = ["continue_trace", "trace"]
|
|
29
31
|
|
|
30
32
|
_worker: Worker | None = None
|
|
@@ -231,6 +233,37 @@ def _shutdown_worker(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
|
|
|
231
233
|
_worker.shutdown(timeout)
|
|
232
234
|
|
|
233
235
|
|
|
236
|
+
def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> None:
|
|
237
|
+
"""Retargets the process worker at a new sink, backing a late ``configure(sink=...)``.
|
|
238
|
+
|
|
239
|
+
Like :func:`_flush_worker` this deliberately does not call :func:`_get_worker`: a process
|
|
240
|
+
that has not logged has captured no sink, so there is nothing to swap and building a thread
|
|
241
|
+
to prove it would be pure cost — that is also the case where the old behaviour was already
|
|
242
|
+
correct (SPEC-030 FR-003).
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
new_sink: The sink already written to the config, to be made the live delivery target.
|
|
246
|
+
timeout: Seconds bounding the drain of the previous sink.
|
|
247
|
+
|
|
248
|
+
Returns:
|
|
249
|
+
None.
|
|
250
|
+
|
|
251
|
+
Raises:
|
|
252
|
+
None. This runs inside ``configure()``, which has never raised for anything but a
|
|
253
|
+
rejected ceiling, and a sink swap that fails must not become the reason an application
|
|
254
|
+
cannot start.
|
|
255
|
+
"""
|
|
256
|
+
worker = _worker
|
|
257
|
+
if worker is None:
|
|
258
|
+
return
|
|
259
|
+
try:
|
|
260
|
+
worker.swap_sink(new_sink, timeout)
|
|
261
|
+
except Exception as exc:
|
|
262
|
+
_diag.absorbed(
|
|
263
|
+
"swapping the log sink", exc, "events may still be delivered to the previous sink"
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
234
267
|
def _flush_worker(timeout: float | None = 5.0) -> bool:
|
|
235
268
|
"""Drains the process worker without retiring it, backing ``flush()`` (SPEC-013 FR-003).
|
|
236
269
|
|
|
@@ -26,6 +26,14 @@ and an unbounded join there is a hung process.
|
|
|
26
26
|
|
|
27
27
|
_DROP_WARN_EVERY = 1000
|
|
28
28
|
|
|
29
|
+
DEFAULT_SWAP_TIMEOUT = 5.0
|
|
30
|
+
"""Seconds a late ``configure(sink=...)`` will spend draining the previous sink (FR-003).
|
|
31
|
+
|
|
32
|
+
Shorter than the shutdown budget on purpose: this runs on the caller's thread inside a
|
|
33
|
+
configuration call, where a long stall is a startup that appears to hang, and what is at risk
|
|
34
|
+
is a sink swap rather than the tail of the whole process.
|
|
35
|
+
"""
|
|
36
|
+
|
|
29
37
|
|
|
30
38
|
def _bounded_seconds(timeout: float | None) -> str:
|
|
31
39
|
"""Renders a shutdown timeout for a diagnostic without trusting its ``__str__``.
|
|
@@ -52,9 +60,10 @@ def _bounded_seconds(timeout: float | None) -> str:
|
|
|
52
60
|
class Health(NamedTuple):
|
|
53
61
|
"""A point-in-time snapshot of the worker's delivery counters (SPEC-017 FR-005).
|
|
54
62
|
|
|
55
|
-
``stopped_reason`` and
|
|
56
|
-
snapshot in ``decorator._worker_health`` — and any third-party
|
|
57
|
-
working, and attribute and index access to every earlier field stays
|
|
63
|
+
``stopped_reason``, ``sink``, and SPEC-030's three are defaulted and appended in that
|
|
64
|
+
order, so the zeroed snapshot in ``decorator._worker_health`` — and any third-party
|
|
65
|
+
construction — keeps working, and attribute and index access to every earlier field stays
|
|
66
|
+
as it was.
|
|
58
67
|
|
|
59
68
|
Attributes:
|
|
60
69
|
queued: Submissions currently buffered. Approximate by nature: it is read without
|
|
@@ -72,6 +81,25 @@ class Health(NamedTuple):
|
|
|
72
81
|
the sink reports nothing (SPEC-026 FR-003). Nested rather than folded into the
|
|
73
82
|
integers above because they count different things: ``dropped`` here is backpressure
|
|
74
83
|
at this queue, ``dropped`` on the sink is an event that never reached the wire.
|
|
84
|
+
retired: Whether :meth:`Worker.shutdown` has been called. It describes an action the
|
|
85
|
+
caller took, not a failure the library detected, which is why it is a boolean where
|
|
86
|
+
``stopped_reason`` is a string — SPEC-019 rejected an ``alive`` flag because it would
|
|
87
|
+
read ``False`` for a process that never logged, and that objection does not apply to a
|
|
88
|
+
field which is simply ``False`` until someone calls ``shutdown()`` (SPEC-030 FR-001).
|
|
89
|
+
On its own it is not a fault: a process that shuts down and stops logging is correct.
|
|
90
|
+
submitted_after_shutdown: Submissions accepted after ``shutdown()`` and queued where
|
|
91
|
+
nothing will drain them. Non-zero alongside ``retired`` is the signature of the
|
|
92
|
+
serverless mistake — ``shutdown()`` called per invocation on a warm container, so the
|
|
93
|
+
first invocation logs and every later one silently does not. The count starts at the
|
|
94
|
+
moment ``shutdown()`` begins rather than when the drain thread ends, so a submission
|
|
95
|
+
racing the final drain is counted even if that drain carried it; erring toward
|
|
96
|
+
reporting is the right direction for a signal whose whole purpose is visibility.
|
|
97
|
+
incomplete_swaps: Late ``configure(sink=...)`` calls whose drain of the previous sink
|
|
98
|
+
did not complete (SPEC-030 FR-003). The swap still took effect, so the caller has the
|
|
99
|
+
sink it asked for, but two things could not be guaranteed: events submitted before the
|
|
100
|
+
call may have been carried to the new sink instead of the old one, and the old sink was
|
|
101
|
+
left **open** rather than closed, because the drain thread may still be inside its
|
|
102
|
+
``emit`` — the reasoning SPEC-027 FR-004 applies to an expired ``shutdown()``.
|
|
75
103
|
"""
|
|
76
104
|
|
|
77
105
|
queued: int
|
|
@@ -79,6 +107,9 @@ class Health(NamedTuple):
|
|
|
79
107
|
failed_batches: int
|
|
80
108
|
stopped_reason: str | None = None
|
|
81
109
|
sink: SinkLosses | None = None
|
|
110
|
+
retired: bool = False
|
|
111
|
+
submitted_after_shutdown: int = 0
|
|
112
|
+
incomplete_swaps: int = 0
|
|
82
113
|
|
|
83
114
|
|
|
84
115
|
class _FlushMarker:
|
|
@@ -170,6 +201,8 @@ class Worker:
|
|
|
170
201
|
self.dropped = 0
|
|
171
202
|
self.failed_batches = 0
|
|
172
203
|
self.stopped_reason: str | None = None
|
|
204
|
+
self.submitted_after_shutdown = 0
|
|
205
|
+
self.incomplete_swaps = 0
|
|
173
206
|
self._queue: queue.Queue[object] = queue.Queue(maxsize=max_queue)
|
|
174
207
|
self._stop = threading.Event()
|
|
175
208
|
self._shutdown_done = False
|
|
@@ -221,6 +254,15 @@ class Worker:
|
|
|
221
254
|
blocking write would let a wedged console stall the drain path. Lines may therefore
|
|
222
255
|
interleave out of order under concurrency, but the counts they carry are exact.
|
|
223
256
|
|
|
257
|
+
A submission arriving after :meth:`shutdown` is still accepted, and counted where it
|
|
258
|
+
can be seen (SPEC-030 FR-001): the worker is retired and nothing will drain the queue,
|
|
259
|
+
so this is total silent loss until something reports it. The check is a single unlocked
|
|
260
|
+
read of a flag that is only ever set, never cleared, which is what keeps the normal path
|
|
261
|
+
free — taking the lock here would put every submission behind the counter's contention
|
|
262
|
+
for a condition that is false in every correct program. SPEC-019's objection to a
|
|
263
|
+
liveness check in ``submit`` does not reach it: that was about probing the thread, this
|
|
264
|
+
is a boolean already in the object's dict.
|
|
265
|
+
|
|
224
266
|
Args:
|
|
225
267
|
events: The span's buffered events, submitted as one item.
|
|
226
268
|
|
|
@@ -230,6 +272,8 @@ class Worker:
|
|
|
230
272
|
Raises:
|
|
231
273
|
None.
|
|
232
274
|
"""
|
|
275
|
+
if self._shutdown_done:
|
|
276
|
+
self._count_undeliverable()
|
|
233
277
|
try:
|
|
234
278
|
self._queue.put_nowait(events)
|
|
235
279
|
except queue.Full:
|
|
@@ -239,13 +283,44 @@ class Worker:
|
|
|
239
283
|
if total == 1 or total % _DROP_WARN_EVERY == 0:
|
|
240
284
|
_diag.lost("submission", total, "log queue full; count is cumulative")
|
|
241
285
|
|
|
286
|
+
def _count_undeliverable(self) -> None:
|
|
287
|
+
"""Counts a post-shutdown submission and warns on the same throttle as overflow.
|
|
288
|
+
|
|
289
|
+
The two conditions warrant the same treatment for the same reason (FR-002): a caller
|
|
290
|
+
making this mistake makes it on every invocation, so a line per submission would be its
|
|
291
|
+
own outage, while total silence is how the mistake survives to production. The counter
|
|
292
|
+
moves before the line is attempted, per ``_diag``'s record-first rule, and the write
|
|
293
|
+
happens outside the lock so a wedged console cannot stall the drain path.
|
|
294
|
+
|
|
295
|
+
Args:
|
|
296
|
+
None.
|
|
297
|
+
|
|
298
|
+
Returns:
|
|
299
|
+
None.
|
|
300
|
+
|
|
301
|
+
Raises:
|
|
302
|
+
None.
|
|
303
|
+
"""
|
|
304
|
+
with self._lock:
|
|
305
|
+
self.submitted_after_shutdown += 1
|
|
306
|
+
total = self.submitted_after_shutdown
|
|
307
|
+
if total == 1 or total % _DROP_WARN_EVERY == 0:
|
|
308
|
+
_diag.lost(
|
|
309
|
+
"submission",
|
|
310
|
+
total,
|
|
311
|
+
"logged after shutdown(), which is terminal; nothing will drain these. Use "
|
|
312
|
+
"flush() in a process that logs again. Count is cumulative",
|
|
313
|
+
)
|
|
314
|
+
|
|
242
315
|
def health(self) -> Health:
|
|
243
316
|
"""Snapshots the delivery counters (SPEC-017 FR-005, SPEC-019 FR-003).
|
|
244
317
|
|
|
245
318
|
This stays valid after :meth:`shutdown`: the counters are plain integers that outlive
|
|
246
319
|
the thread, and the final drain consumes the queue, so ``queued`` reads 0 rather than a
|
|
247
320
|
stale marker. The same applies to ``stopped_reason``, since a caller finding a dead
|
|
248
|
-
worker will usually call ``shutdown()`` next.
|
|
321
|
+
worker will usually call ``shutdown()`` next. Reading it after a shutdown is in fact
|
|
322
|
+
the point of ``retired`` and ``submitted_after_shutdown`` (SPEC-030 FR-001), which
|
|
323
|
+
report a state only a retired worker can be in.
|
|
249
324
|
|
|
250
325
|
Args:
|
|
251
326
|
None.
|
|
@@ -259,12 +334,18 @@ class Worker:
|
|
|
259
334
|
with self._lock:
|
|
260
335
|
dropped, failed_batches = self.dropped, self.failed_batches
|
|
261
336
|
stopped_reason = self.stopped_reason
|
|
337
|
+
retired = self._shutdown_done
|
|
338
|
+
submitted_after_shutdown = self.submitted_after_shutdown
|
|
339
|
+
incomplete_swaps = self.incomplete_swaps
|
|
262
340
|
return Health(
|
|
263
341
|
queued=self._queue.qsize(),
|
|
264
342
|
dropped=dropped,
|
|
265
343
|
failed_batches=failed_batches,
|
|
266
344
|
stopped_reason=stopped_reason,
|
|
267
345
|
sink=self._sink_losses(),
|
|
346
|
+
retired=retired,
|
|
347
|
+
submitted_after_shutdown=submitted_after_shutdown,
|
|
348
|
+
incomplete_swaps=incomplete_swaps,
|
|
268
349
|
)
|
|
269
350
|
|
|
270
351
|
def _sink_losses(self) -> SinkLosses | None:
|
|
@@ -332,6 +413,137 @@ class Worker:
|
|
|
332
413
|
return False
|
|
333
414
|
return marker.delivered
|
|
334
415
|
|
|
416
|
+
def swap_sink(self, new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> None:
|
|
417
|
+
"""Retargets delivery at a new sink, draining and closing the previous one (FR-003).
|
|
418
|
+
|
|
419
|
+
This is what makes a late ``configure(sink=...)`` mean what it says. The sink was
|
|
420
|
+
captured once when the worker was built, so before SPEC-030 a later call updated the
|
|
421
|
+
config — which ``get_config().sink`` then reported — while every event continued to the
|
|
422
|
+
old sink: the config and the behaviour disagreed, and nothing said so.
|
|
423
|
+
|
|
424
|
+
The attribute is reassigned rather than the worker rebuilt, which keeps the queue, the
|
|
425
|
+
thread, the counters and the ``atexit`` registration intact; rebuilding would drop
|
|
426
|
+
whatever was queued and register a second drain. The order is the contract: drain first
|
|
427
|
+
so everything submitted before the call reaches the sink it was submitted for, swap,
|
|
428
|
+
then drain again before closing. That second drain is a fence rather than a delivery —
|
|
429
|
+
it proves the drain thread is not still inside the old sink's ``emit``, which is the one
|
|
430
|
+
way ``close()`` could be called under a writer.
|
|
431
|
+
|
|
432
|
+
On a drain that cannot be confirmed the swap still stands, because the caller asked for
|
|
433
|
+
the new sink and silently keeping the old one is the defect this method exists to fix.
|
|
434
|
+
What changes is that the old sink is left **open** and ``incomplete_swaps`` records it:
|
|
435
|
+
the drain thread may still be using it, and SPEC-027 FR-004 already settled that a
|
|
436
|
+
leaked resource beats a close raced against a write.
|
|
437
|
+
|
|
438
|
+
Both guards are re-taken after the first drain, which blocks and therefore cannot be
|
|
439
|
+
trusted to return into the state it left. Retirement is the one that bites: ``shutdown``
|
|
440
|
+
closes whatever ``self.sink`` was at that moment and latches its once-only flag, so a
|
|
441
|
+
swap reassigning afterwards would install a sink nothing will ever close, and then
|
|
442
|
+
report that the *old* one was left open when it had in fact just been closed.
|
|
443
|
+
|
|
444
|
+
``configure()`` remains a startup call and this does not make it thread-safe. A span
|
|
445
|
+
finishing on another thread during the swap may land on either sink; what is guaranteed
|
|
446
|
+
is that everything submitted before the call was drained to the old one.
|
|
447
|
+
|
|
448
|
+
Args:
|
|
449
|
+
new_sink: The sink every subsequent batch is emitted to.
|
|
450
|
+
timeout: Seconds bounding both drains — one shared deadline, so a hung sink cannot
|
|
451
|
+
make ``configure()`` wait for twice the budget. It does **not** bound the closing of
|
|
452
|
+
the old sink, which has no timeout of its own; see :meth:`_close_swapped_out`.
|
|
453
|
+
``None`` waits indefinitely.
|
|
454
|
+
|
|
455
|
+
Returns:
|
|
456
|
+
None. The outcome is reported through ``health().incomplete_swaps`` and one stderr
|
|
457
|
+
line rather than a return value, because the caller is ``configure()``, which has
|
|
458
|
+
never had one and whose callers do not check.
|
|
459
|
+
|
|
460
|
+
Raises:
|
|
461
|
+
None on a sink fault. A close that fails is announced, as everywhere else
|
|
462
|
+
(SPEC-025 FR-004).
|
|
463
|
+
"""
|
|
464
|
+
with self._lock:
|
|
465
|
+
if self._shutdown_done or self.sink is new_sink:
|
|
466
|
+
return
|
|
467
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
468
|
+
drained = self.flush(timeout)
|
|
469
|
+
with self._lock:
|
|
470
|
+
if self._shutdown_done:
|
|
471
|
+
return
|
|
472
|
+
old = self.sink
|
|
473
|
+
if old is new_sink:
|
|
474
|
+
return
|
|
475
|
+
self.sink = new_sink
|
|
476
|
+
self._offer_stop_signal()
|
|
477
|
+
remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
|
|
478
|
+
if not (drained and self.flush(remaining)):
|
|
479
|
+
self._record_incomplete_swap(timeout)
|
|
480
|
+
return
|
|
481
|
+
self._close_swapped_out(old)
|
|
482
|
+
|
|
483
|
+
def _record_incomplete_swap(self, timeout: float | None) -> None:
|
|
484
|
+
"""Counts a swap whose drain could not be confirmed, then announces it (FR-003).
|
|
485
|
+
|
|
486
|
+
Two things are reported at once because they have one cause: queued items may have been
|
|
487
|
+
carried to the new sink rather than the old one, and the old sink was left open. The
|
|
488
|
+
count is queued *items* — one per submitted span plus any marker — which makes it a
|
|
489
|
+
floor on the events involved, the useful direction for a reader deciding whether to care.
|
|
490
|
+
|
|
491
|
+
Args:
|
|
492
|
+
timeout: The budget the drain was given, rendered for the line.
|
|
493
|
+
|
|
494
|
+
Returns:
|
|
495
|
+
None.
|
|
496
|
+
|
|
497
|
+
Raises:
|
|
498
|
+
None.
|
|
499
|
+
"""
|
|
500
|
+
with self._lock:
|
|
501
|
+
self.incomplete_swaps += 1
|
|
502
|
+
_diag.lost(
|
|
503
|
+
"item",
|
|
504
|
+
self._queued_or_unknown(),
|
|
505
|
+
f"the previous sink could not be confirmed drained within "
|
|
506
|
+
f"{_bounded_seconds(timeout)} of a configure(sink=...); it is left open, and "
|
|
507
|
+
f"queued items may reach the new sink instead",
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
def _close_swapped_out(self, sink: Sink) -> None:
|
|
511
|
+
"""Closes a sink the worker no longer delivers to, absorbing a failure.
|
|
512
|
+
|
|
513
|
+
This is reached only once both drains have been confirmed, so the *drain thread* is
|
|
514
|
+
provably out of this sink's ``emit``. An orphan-path emitter on an application thread
|
|
515
|
+
is not covered: it resolves the sink through ``_ensure_sink`` before emitting, so one
|
|
516
|
+
that read the old sink before ``configure()`` reassigned it can still be inside its
|
|
517
|
+
``emit`` — which is why ``sinks/base.py`` requires ``close()`` to tolerate exactly that
|
|
518
|
+
(SPEC-028 FR-001), and why the sinks holding transport state take their lock in both.
|
|
519
|
+
|
|
520
|
+
The close is **not** bounded by the swap's timeout: ``Sink.close`` has no timeout of its
|
|
521
|
+
own, so bounding it needs an interruptible close — a change to the sink contract, not to
|
|
522
|
+
this method. Of :meth:`_close_if_owed`'s two reasons for rejecting a threaded close, only
|
|
523
|
+
the second reaches this site: the daemon killed mid-``commit()`` is an interpreter-exit
|
|
524
|
+
hazard and a swap runs in a live process, but an expired join still cannot tell a
|
|
525
|
+
slow-but-successful close from a stuck one, and would report a loss for a swap that
|
|
526
|
+
completed. A destination whose ``close()`` blocks therefore blocks ``configure()``,
|
|
527
|
+
recorded in ``architecture.md`` §13 rather than papered over.
|
|
528
|
+
|
|
529
|
+
It is deliberately not :meth:`_close_sink`, which answers a different question — that
|
|
530
|
+
one closes the sink the worker still holds, exactly once, and only after the thread has
|
|
531
|
+
ended.
|
|
532
|
+
|
|
533
|
+
Args:
|
|
534
|
+
sink: The sink that was swapped out.
|
|
535
|
+
|
|
536
|
+
Returns:
|
|
537
|
+
None.
|
|
538
|
+
|
|
539
|
+
Raises:
|
|
540
|
+
None.
|
|
541
|
+
"""
|
|
542
|
+
try:
|
|
543
|
+
sink.close()
|
|
544
|
+
except Exception as exc:
|
|
545
|
+
_diag.absorbed("closing a swapped-out sink", exc, "it may still hold its resources")
|
|
546
|
+
|
|
335
547
|
def shutdown(self, timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
|
|
336
548
|
"""Stops the thread, drains and emits everything queued, then closes the sink.
|
|
337
549
|
|
|
@@ -348,6 +560,12 @@ class Worker:
|
|
|
348
560
|
``close()`` twice on a sink that may have partially released its resources; what
|
|
349
561
|
SPEC-025 FR-004 changed is that the failure is announced rather than swallowed.
|
|
350
562
|
|
|
563
|
+
The worker does not come back, and :meth:`submit` keeps accepting afterwards — so a
|
|
564
|
+
caller that logs again queues events nothing will drain. That is reported rather than
|
|
565
|
+
prevented, through ``retired`` and ``submitted_after_shutdown`` (SPEC-030 FR-001) and
|
|
566
|
+
one stderr line; refusing the submission or restarting the thread were both rejected,
|
|
567
|
+
the second because a thread that resurrects itself fights a process trying to exit.
|
|
568
|
+
|
|
351
569
|
Args:
|
|
352
570
|
timeout: Seconds the join may take. ``None`` waits indefinitely, which is what this
|
|
353
571
|
did unconditionally before and is still available on request.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev26}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|