log-foundry 0.10.2.dev62__tar.gz → 0.10.2.dev64__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/PKG-INFO +24 -13
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/README.md +23 -12
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/_fork.py +60 -14
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/_lifecycle.py +39 -4
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/decorator.py +43 -1
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/base.py +24 -6
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/file.py +10 -5
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/worker.py +18 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/LICENSE +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/__init__.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/config.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/results.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/_retry.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/_socket.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/callback.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/clickhouse.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/datadog.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/elasticsearch.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/eventhubs.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/filtering.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/firehose.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/honeycomb.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/http.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/kafka.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/kinesis.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/logging_sink.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/logstash.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/loki.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/memory.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/mongodb.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/multi.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/nats.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/null.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/postgres.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/pubsub.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/rabbitmq.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/redis.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/sentry.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/sns.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/splunk.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/sqlite.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/sqs.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/stdout.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/syslog.py +0 -0
- {log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/transform.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: log-foundry
|
|
3
|
-
Version: 0.10.2.
|
|
3
|
+
Version: 0.10.2.dev64
|
|
4
4
|
Summary: Generate logs for your console and JSON events for downstream consumption.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -527,18 +527,26 @@ A few conventions hold across every sink below:
|
|
|
527
527
|
open for you to manage.
|
|
528
528
|
- **Forking.** A forked child repairs the library automatically — it rebuilds the worker so it keeps
|
|
529
529
|
delivering, re-initialises every lock (without which the child's *first* log call can deadlock),
|
|
530
|
-
and
|
|
531
|
-
do is give the child a sink of its own: the child inherits the same object, so
|
|
532
|
-
SQLite handle or one file is now written by two processes
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
530
|
+
and re-opens any buffered stream it inherited so the parent's pending bytes are not written twice.
|
|
531
|
+
What it does **not** do is give the child a sink of its own: the child inherits the same object, so
|
|
532
|
+
one socket, one SQLite handle or one file is now written by two processes. It will generally not
|
|
533
|
+
be *closed* by both — the library records which process it was handed each sink in, and a child
|
|
534
|
+
refuses to close one it inherited — but a shared connection is still a shared connection, and
|
|
535
|
+
there are two exceptions below.
|
|
536
|
+
**Under gunicorn, uWSGI or Celery, build a connection-holding sink in the worker process:**
|
|
537
|
+
`configure()` from gunicorn's `post_fork` hook rather than under preload, and don't log from the
|
|
538
|
+
master. Reconfiguring in the child is harmless **for a sink the library was handed in this
|
|
539
|
+
process** — but if the master *built* a connection sink and never called `configure()` with it,
|
|
540
|
+
a child that then does is the first process to hand it over, so it owns it and closes it at exit.
|
|
541
|
+
That is the one case the record cannot decide, and it is the case this advice avoids. A master
|
|
542
|
+
that must log should use a
|
|
543
|
+
sink whose `close()` costs nothing to share, such as `StdoutSink` or `FileSink`. A sink you wrote
|
|
544
|
+
yourself is repaired only if it subclasses `Sink` or a shipped sink; one that satisfies the
|
|
545
|
+
protocol structurally is outside the repair, along with any third-party client's own locks and
|
|
546
|
+
buffers. The second exception: if you subclass a shipped sink **and add a transport of your
|
|
547
|
+
own**, override `reacquire_after_fork()` — inheriting it claims the whole object on the strength
|
|
548
|
+
of re-opening only the part the parent class knows about, after which the child *will* close
|
|
549
|
+
your connection.
|
|
542
550
|
- **Never crashes the app.** A broken destination degrades logging and nothing more. A sink that
|
|
543
551
|
delivered *part* of a batch counts what it lost (`.failed`, `.dropped_oversized`,
|
|
544
552
|
`.dropped_unadjudicated`, …) and returns, since retrying would re-deliver what already landed.
|
|
@@ -951,6 +959,8 @@ if (
|
|
|
951
959
|
|
|
952
960
|
`closing_sinks` is deliberately not a term here: it is briefly non-zero during a perfectly healthy
|
|
953
961
|
sink swap, so a single reading is not a fault. Watch it over time instead — see the table below.
|
|
962
|
+
`inherited_sink` is not a term either, for a different reason: it reports a *state* the process is
|
|
963
|
+
in rather than a loss it took, and in a prefork deployment it is `True` on every worker by design.
|
|
954
964
|
|
|
955
965
|
They tell you different things, and they want different responses:
|
|
956
966
|
|
|
@@ -963,6 +973,7 @@ They tell you different things, and they want different responses:
|
|
|
963
973
|
| `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
|
|
964
974
|
| `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
|
|
965
975
|
| `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
|
|
976
|
+
| `inherited_sink` | This process is delivering to a sink it **inherited across a `fork`** and may not release, so it will not be closed here. Not a loss and not an alert term. | Nothing, usually. It explains a handle still open after `shutdown()`, and tells you a deployment shares one sink across a fork at all. `True` for a shared `StdoutSink` too, whose `close()` only flushes — so a `True` is not by itself evidence that anything is held. If you want the child to own its transport, build the sink in the worker process (see Forking). |
|
|
966
977
|
| `closing_sinks` | Swapped-out sinks inside `close()` **right now** — a live gauge, not a counter, and the only field that falls as well as rises. Non-zero on a single read is normal during a swap. | Nothing, unless it stays non-zero. That means a destination is stuck in `close()` and will not release its resources. |
|
|
967
978
|
|
|
968
979
|
`h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
|
|
@@ -491,18 +491,26 @@ A few conventions hold across every sink below:
|
|
|
491
491
|
open for you to manage.
|
|
492
492
|
- **Forking.** A forked child repairs the library automatically — it rebuilds the worker so it keeps
|
|
493
493
|
delivering, re-initialises every lock (without which the child's *first* log call can deadlock),
|
|
494
|
-
and
|
|
495
|
-
do is give the child a sink of its own: the child inherits the same object, so
|
|
496
|
-
SQLite handle or one file is now written by two processes
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
494
|
+
and re-opens any buffered stream it inherited so the parent's pending bytes are not written twice.
|
|
495
|
+
What it does **not** do is give the child a sink of its own: the child inherits the same object, so
|
|
496
|
+
one socket, one SQLite handle or one file is now written by two processes. It will generally not
|
|
497
|
+
be *closed* by both — the library records which process it was handed each sink in, and a child
|
|
498
|
+
refuses to close one it inherited — but a shared connection is still a shared connection, and
|
|
499
|
+
there are two exceptions below.
|
|
500
|
+
**Under gunicorn, uWSGI or Celery, build a connection-holding sink in the worker process:**
|
|
501
|
+
`configure()` from gunicorn's `post_fork` hook rather than under preload, and don't log from the
|
|
502
|
+
master. Reconfiguring in the child is harmless **for a sink the library was handed in this
|
|
503
|
+
process** — but if the master *built* a connection sink and never called `configure()` with it,
|
|
504
|
+
a child that then does is the first process to hand it over, so it owns it and closes it at exit.
|
|
505
|
+
That is the one case the record cannot decide, and it is the case this advice avoids. A master
|
|
506
|
+
that must log should use a
|
|
507
|
+
sink whose `close()` costs nothing to share, such as `StdoutSink` or `FileSink`. A sink you wrote
|
|
508
|
+
yourself is repaired only if it subclasses `Sink` or a shipped sink; one that satisfies the
|
|
509
|
+
protocol structurally is outside the repair, along with any third-party client's own locks and
|
|
510
|
+
buffers. The second exception: if you subclass a shipped sink **and add a transport of your
|
|
511
|
+
own**, override `reacquire_after_fork()` — inheriting it claims the whole object on the strength
|
|
512
|
+
of re-opening only the part the parent class knows about, after which the child *will* close
|
|
513
|
+
your connection.
|
|
506
514
|
- **Never crashes the app.** A broken destination degrades logging and nothing more. A sink that
|
|
507
515
|
delivered *part* of a batch counts what it lost (`.failed`, `.dropped_oversized`,
|
|
508
516
|
`.dropped_unadjudicated`, …) and returns, since retrying would re-deliver what already landed.
|
|
@@ -915,6 +923,8 @@ if (
|
|
|
915
923
|
|
|
916
924
|
`closing_sinks` is deliberately not a term here: it is briefly non-zero during a perfectly healthy
|
|
917
925
|
sink swap, so a single reading is not a fault. Watch it over time instead — see the table below.
|
|
926
|
+
`inherited_sink` is not a term either, for a different reason: it reports a *state* the process is
|
|
927
|
+
in rather than a loss it took, and in a prefork deployment it is `True` on every worker by design.
|
|
918
928
|
|
|
919
929
|
They tell you different things, and they want different responses:
|
|
920
930
|
|
|
@@ -927,6 +937,7 @@ They tell you different things, and they want different responses:
|
|
|
927
937
|
| `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
|
|
928
938
|
| `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
|
|
929
939
|
| `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
|
|
940
|
+
| `inherited_sink` | This process is delivering to a sink it **inherited across a `fork`** and may not release, so it will not be closed here. Not a loss and not an alert term. | Nothing, usually. It explains a handle still open after `shutdown()`, and tells you a deployment shares one sink across a fork at all. `True` for a shared `StdoutSink` too, whose `close()` only flushes — so a `True` is not by itself evidence that anything is held. If you want the child to own its transport, build the sink in the worker process (see Forking). |
|
|
930
941
|
| `closing_sinks` | Swapped-out sinks inside `close()` **right now** — a live gauge, not a counter, and the only field that falls as well as rises. Non-zero on a single read is normal during a swap. | Nothing, unless it stays non-zero. That means a destination is stuck in `close()` and will not release its resources. |
|
|
931
942
|
|
|
932
943
|
`h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
|
|
@@ -37,6 +37,36 @@ and all three would have to import this module for it to reach them; instead the
|
|
|
37
37
|
it, so this module imports nothing that imports it and there is no cycle to break later.
|
|
38
38
|
"""
|
|
39
39
|
|
|
40
|
+
_FORK_SKIP = ("reacquired_in_child",)
|
|
41
|
+
"""This module's own opt-out from the walk below (:data:`_SKIP_ATTRIBUTE`).
|
|
42
|
+
|
|
43
|
+
:data:`reacquired_in_child` holds **sinks**, so without this the next fork's walk enters the
|
|
44
|
+
previous fork's roster and re-collects every sink in it — including ones ``configure()``
|
|
45
|
+
superseded in between. Measured: a child's walk collected a sink two ``configure()`` calls out
|
|
46
|
+
of date, its hook was called, and the reclaim that follows then stamped it for the grandchild's
|
|
47
|
+
pid, so a sink no descendant ever acquired read ``releasable``. A ratchet, not a one-off: once a
|
|
48
|
+
hook-implementing sink enters a roster, every descendant's walk finds it again.
|
|
49
|
+
|
|
50
|
+
Exactly the hazard ``_lifecycle._owned`` already declares an opt-out for, reached by adding a
|
|
51
|
+
second global that holds sinks. The rebind below is the belt to this brace.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
reacquired_in_child: list[Any] = []
|
|
55
|
+
"""Sinks whose re-acquisition hook returned normally in this child (SPEC-042 FR-005).
|
|
56
|
+
|
|
57
|
+
Published rather than acted on, because what it means is an *ownership* claim and ownership is
|
|
58
|
+
``_lifecycle``'s: a sink that returns from the hook holds a transport of its own, so a child may
|
|
59
|
+
release it.
|
|
60
|
+
|
|
61
|
+
Rebound — never mutated in place — **before the repair walk runs**, so a second fork cannot
|
|
62
|
+
inherit the first one's roster even if the opt-out above were removed. The first version of this
|
|
63
|
+
claimed the rebind happened "at the start of every child repair" while doing it in the middle,
|
|
64
|
+
after the walk; both halves were false and nothing tested either.
|
|
65
|
+
|
|
66
|
+
This module still imports nothing but ``_diag`` (SPEC-039 FR-006); the reader registers a
|
|
67
|
+
handler and reads this, which is the same inversion :data:`_child_handlers` exists for.
|
|
68
|
+
"""
|
|
69
|
+
|
|
40
70
|
_installed = False
|
|
41
71
|
"""Whether :func:`install` has already registered the child handler in this process.
|
|
42
72
|
|
|
@@ -330,8 +360,12 @@ def _fresh_primitive(value: Any, memo: dict[int, Any], keepalive: list[Any]) ->
|
|
|
330
360
|
return fresh
|
|
331
361
|
|
|
332
362
|
|
|
333
|
-
|
|
334
|
-
"""The optional sink member a forked child asks to
|
|
363
|
+
_REACQUIRE_HOOK = "reacquire_after_fork"
|
|
364
|
+
"""The optional sink member a forked child asks to take its transport over.
|
|
365
|
+
|
|
366
|
+
Named for the larger half of what it does (SPEC-042 FR-005): stranding the parent's pending
|
|
367
|
+
bytes is one consequence, and re-acquiring the transport is the step — a sink that returns from
|
|
368
|
+
it has claimed the transport as this process's own, which is what makes releasing it safe.
|
|
335
369
|
|
|
336
370
|
Probed by name, as ``losses()`` and ``log_foundry_stop_signal`` are, so no existing sink stops
|
|
337
371
|
satisfying ``Sink`` by not having it. ``sinks/base.py`` states the contract; this is the only
|
|
@@ -339,7 +373,7 @@ place the name is read.
|
|
|
339
373
|
"""
|
|
340
374
|
|
|
341
375
|
|
|
342
|
-
def
|
|
376
|
+
def _offers_reacquire(holder: object) -> bool:
|
|
343
377
|
"""Whether this object carries FR-004's buffer hook, without letting the question raise.
|
|
344
378
|
|
|
345
379
|
Every other read the walk makes is guarded individually, and this one has to be too: an
|
|
@@ -361,7 +395,7 @@ def _offers_discard(holder: object) -> bool:
|
|
|
361
395
|
None.
|
|
362
396
|
"""
|
|
363
397
|
try:
|
|
364
|
-
return callable(getattr(holder,
|
|
398
|
+
return callable(getattr(holder, _REACQUIRE_HOOK, None))
|
|
365
399
|
except Exception:
|
|
366
400
|
return False
|
|
367
401
|
|
|
@@ -428,7 +462,7 @@ def _reinit_primitives() -> list[Any]:
|
|
|
428
462
|
None.
|
|
429
463
|
|
|
430
464
|
Returns:
|
|
431
|
-
The owned instances carrying :data:`
|
|
465
|
+
The owned instances carrying :data:`_REACQUIRE_HOOK`, in the order the walk reached them.
|
|
432
466
|
|
|
433
467
|
Raises:
|
|
434
468
|
None.
|
|
@@ -452,7 +486,7 @@ def _reinit_primitives() -> list[Any]:
|
|
|
452
486
|
if not isinstance(holder, types.ModuleType | type):
|
|
453
487
|
if not _is_owned(holder):
|
|
454
488
|
continue
|
|
455
|
-
if
|
|
489
|
+
if _offers_reacquire(holder):
|
|
456
490
|
buffered.append(holder)
|
|
457
491
|
skip = _skipped_names(holder)
|
|
458
492
|
for name, value in _namespace_items(holder):
|
|
@@ -466,19 +500,25 @@ def _reinit_primitives() -> list[Any]:
|
|
|
466
500
|
return buffered
|
|
467
501
|
|
|
468
502
|
|
|
469
|
-
def
|
|
470
|
-
"""Asks each sink that owns a buffered stream to
|
|
503
|
+
def _reacquire_transports(holders: list[Any]) -> list[Any]:
|
|
504
|
+
"""Asks each sink that owns a buffered stream to re-acquire it in this child (FR-004).
|
|
471
505
|
|
|
472
506
|
A fork landing inside ``emit`` — after the write loop, before the flush — leaves the child
|
|
473
507
|
holding the parent's unflushed bytes, which both processes then write: measured, the event
|
|
474
508
|
at the fork point appeared on disk twice. Without a ``before`` handler there is nowhere to
|
|
475
|
-
empty the buffer from (FR-001), so the child
|
|
509
|
+
empty the buffer from (FR-001), so the child re-opens instead, which strands the copy.
|
|
510
|
+
|
|
511
|
+
**Which hooks returned normally is published rather than acted on** (SPEC-042 FR-005). A
|
|
512
|
+
sink that returns from the hook has claimed the transport as this process's own, which is an
|
|
513
|
+
ownership fact — but ownership lives in ``_lifecycle``, and this module imports nothing but
|
|
514
|
+
``_diag``. So the roster is handed back and the handler ``_lifecycle`` registers re-stamps
|
|
515
|
+
it.
|
|
476
516
|
|
|
477
517
|
Args:
|
|
478
518
|
holders: What :func:`_reinit_primitives` collected on its walk.
|
|
479
519
|
|
|
480
520
|
Returns:
|
|
481
|
-
|
|
521
|
+
The holders whose hook returned normally, in the order they were called.
|
|
482
522
|
|
|
483
523
|
Raises:
|
|
484
524
|
None. One sink's failure is absorbed separately from the rest, so a hook that cannot
|
|
@@ -486,15 +526,19 @@ def _discard_buffers(holders: list[Any]) -> None:
|
|
|
486
526
|
sink its repair — and the consequence is named, because an absorbed failure is
|
|
487
527
|
invisible apart from that line.
|
|
488
528
|
"""
|
|
529
|
+
reacquired: list[Any] = []
|
|
489
530
|
for holder in holders:
|
|
490
531
|
try:
|
|
491
|
-
getattr(holder,
|
|
532
|
+
getattr(holder, _REACQUIRE_HOOK)()
|
|
533
|
+
reacquired.append(holder)
|
|
492
534
|
except Exception as exc:
|
|
493
535
|
_diag.absorbed(
|
|
494
|
-
"
|
|
536
|
+
"re-acquiring a transport after a fork",
|
|
495
537
|
exc,
|
|
496
|
-
f"{type(holder).__name__} may write the parent's pending bytes again"
|
|
538
|
+
f"{type(holder).__name__} may write the parent's pending bytes again, and this "
|
|
539
|
+
f"child will not release it",
|
|
497
540
|
)
|
|
541
|
+
return reacquired
|
|
498
542
|
|
|
499
543
|
|
|
500
544
|
def register_child_handler(fn: Callable[[], None]) -> None:
|
|
@@ -555,12 +599,14 @@ def _reinit_after_fork() -> None:
|
|
|
555
599
|
registers another simply causes it to run — and :func:`register_child_handler` is the
|
|
556
600
|
only writer, appending only.
|
|
557
601
|
"""
|
|
602
|
+
global reacquired_in_child
|
|
603
|
+
reacquired_in_child = []
|
|
558
604
|
buffered: list[Any] = []
|
|
559
605
|
try:
|
|
560
606
|
buffered = _reinit_primitives()
|
|
561
607
|
except Exception as exc:
|
|
562
608
|
_diag.absorbed("repairing the library after a fork", exc, "this child may block or lose")
|
|
563
|
-
|
|
609
|
+
reacquired_in_child = _reacquire_transports(buffered)
|
|
564
610
|
for handler in _child_handlers:
|
|
565
611
|
try:
|
|
566
612
|
handler()
|
|
@@ -347,7 +347,12 @@ def _mark_inherited() -> None:
|
|
|
347
347
|
destructive close. **That covers less than it appears to**: every read the walk makes is
|
|
348
348
|
already absorbed one level down in ``_fork``, which returns empty and announces, so the
|
|
349
349
|
common failure is a *partial* walk that raises nothing and leaves the flag clear. Sinks
|
|
350
|
-
it did not reach are then unrecorded rather than marked. The
|
|
350
|
+
it did not reach are then unrecorded rather than marked. **The reclaim below runs
|
|
351
|
+
either way**: a sink that returned from its hook provably holds its own transport, and a
|
|
352
|
+
walk that failed is no reason to refuse it forever — which is the outcome
|
|
353
|
+
:func:`reclaim`'s own docstring says a ``setdefault`` would wrongly cause. The ceiling
|
|
354
|
+
path already reached it; the exception path did not, and the two differed silently.
|
|
355
|
+
The outer guard is for a fault
|
|
351
356
|
in this function's own frame — resolving the roots, or an object whose ``__class__``
|
|
352
357
|
property raises, which makes ``isinstance`` raise here. Not a hostile *metaclass*: a
|
|
353
358
|
value's ``__instancecheck__`` is never consulted, since ``Sink``'s own ``_ProtocolMeta``
|
|
@@ -384,6 +389,9 @@ def _mark_inherited() -> None:
|
|
|
384
389
|
for _name, value in _fork._namespace_items(holder):
|
|
385
390
|
if _fork._is_container(value) or _may_be_a_sink(value):
|
|
386
391
|
stack.append(value)
|
|
392
|
+
with _owned_lock:
|
|
393
|
+
for inherited in found:
|
|
394
|
+
_owned.setdefault(id(inherited), (_FOREIGN, inherited))
|
|
387
395
|
except Exception as exc:
|
|
388
396
|
_marking_failed = True
|
|
389
397
|
_diag.absorbed(
|
|
@@ -391,10 +399,37 @@ def _mark_inherited() -> None:
|
|
|
391
399
|
exc,
|
|
392
400
|
"this child will refuse to close any sink it has no record of",
|
|
393
401
|
)
|
|
394
|
-
|
|
402
|
+
for reacquired in _fork.reacquired_in_child:
|
|
403
|
+
reclaim(reacquired)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def reclaim(sink: object) -> None:
|
|
407
|
+
"""Records that a sink re-acquired its transport in this process (SPEC-042 FR-005).
|
|
408
|
+
|
|
409
|
+
The one write that **overrides** an existing record, and it has to be: :func:`_mark_inherited`
|
|
410
|
+
has already stamped everything inherited ``_FOREIGN`` by the time the hook roster is read, so
|
|
411
|
+
a ``setdefault`` here would leave a sink that provably holds its own descriptor refused
|
|
412
|
+
forever.
|
|
413
|
+
|
|
414
|
+
**It re-stamps the sink that re-acquired, and nothing above it** (FR-005 AC-8). A child
|
|
415
|
+
inheriting ``MultiSink(FileSink, FileSink)`` re-stamps the two children — only they implement
|
|
416
|
+
the hook — while the wrapper keeps the parent's mark and stays refused, which leaves the
|
|
417
|
+
re-acquired children reachable only through a wrapper nothing will release. That is a leak
|
|
418
|
+
and loses nothing, since ``FileSink.emit`` flushes at the end of every batch, but it is
|
|
419
|
+
stated rather than discovered.
|
|
420
|
+
|
|
421
|
+
Args:
|
|
422
|
+
sink: The sink whose hook returned normally.
|
|
423
|
+
|
|
424
|
+
Returns:
|
|
425
|
+
None.
|
|
426
|
+
|
|
427
|
+
Raises:
|
|
428
|
+
None.
|
|
429
|
+
"""
|
|
430
|
+
pid = os.getpid()
|
|
395
431
|
with _owned_lock:
|
|
396
|
-
|
|
397
|
-
_owned.setdefault(id(inherited), (_FOREIGN, inherited))
|
|
432
|
+
_owned[id(sink)] = (pid, sink)
|
|
398
433
|
|
|
399
434
|
|
|
400
435
|
def stamp(sink: object) -> None:
|
|
@@ -12,7 +12,7 @@ from time import monotonic
|
|
|
12
12
|
from typing import TYPE_CHECKING, Any, TypeVar, cast, overload
|
|
13
13
|
|
|
14
14
|
from log_foundry import _diag, _fork, _lifecycle, context
|
|
15
|
-
from log_foundry.config import _ensure_sink
|
|
15
|
+
from log_foundry.config import _ensure_sink, _live_config
|
|
16
16
|
from log_foundry.ids import (
|
|
17
17
|
is_valid_span_id,
|
|
18
18
|
is_valid_trace_id,
|
|
@@ -728,6 +728,47 @@ def _flush_worker(timeout: float | None = 5.0) -> FlushResult:
|
|
|
728
728
|
return FlushResult(ok=False, reason="thread-died")
|
|
729
729
|
|
|
730
730
|
|
|
731
|
+
def _delivering_to_an_inherited_sink() -> bool:
|
|
732
|
+
"""Whether the sink this process last installed for delivery is one it may not release.
|
|
733
|
+
|
|
734
|
+
Answerable with **no worker**, which is what makes it truthful in a process that only ever
|
|
735
|
+
logs outside a span — the same refusal :func:`_worker_health` already makes for ``retired``,
|
|
736
|
+
and for the same reason: standing up a thread to answer ``health()`` is forbidden.
|
|
737
|
+
|
|
738
|
+
The three candidates are asked in delivery order: the worker's sink if a worker exists,
|
|
739
|
+
else the sink an orphan emit reached, else the configured one. SPEC-033's measured
|
|
740
|
+
disagreement is worker-versus-config, which the **first** term already covers — and the
|
|
741
|
+
second cannot currently produce a distinct answer, because with no worker ``_orphan_sink``
|
|
742
|
+
is either ``None`` or the same object the config holds (``_swap_sink`` re-points both, and
|
|
743
|
+
where it returns early the record is ``None``). It is kept rather than removed, and recorded
|
|
744
|
+
rather than tested: every other orphan-path read in this module treats ``_orphan_sink`` as
|
|
745
|
+
the authority on which sink an emit reached, and dropping it here would make one site read
|
|
746
|
+
the config while its neighbours read the record — resting on a property of ``_swap_sink``'s
|
|
747
|
+
current body that nothing states as an invariant. A test would have to arrange an
|
|
748
|
+
unreachable state by hand, which is the fixture-built vacuity this repo keeps finding.
|
|
749
|
+
|
|
750
|
+
With no sink resolved at all there is nothing installed and nothing inherited, so the answer
|
|
751
|
+
is ``False`` rather than a guess.
|
|
752
|
+
|
|
753
|
+
Args:
|
|
754
|
+
None.
|
|
755
|
+
|
|
756
|
+
Returns:
|
|
757
|
+
Whether that one sink carries another process's ownership record.
|
|
758
|
+
|
|
759
|
+
Raises:
|
|
760
|
+
None. ``health()`` is a diagnostic and must not be the reason a caller fails; an
|
|
761
|
+
unanswerable question reports ``False``, the same direction as a process that never
|
|
762
|
+
forked.
|
|
763
|
+
"""
|
|
764
|
+
try:
|
|
765
|
+
worker = _worker
|
|
766
|
+
sink = worker.sink if worker is not None else (_orphan_sink or _live_config().sink)
|
|
767
|
+
return sink is not None and not _lifecycle.releasable(sink)
|
|
768
|
+
except Exception:
|
|
769
|
+
return False
|
|
770
|
+
|
|
771
|
+
|
|
731
772
|
def _worker_health() -> Health:
|
|
732
773
|
"""Snapshots the process worker's counters, or zeros if none was ever created.
|
|
733
774
|
|
|
@@ -772,6 +813,7 @@ def _worker_health() -> Health:
|
|
|
772
813
|
failed_batches=0,
|
|
773
814
|
retired=_orphan_retired,
|
|
774
815
|
closing_sinks=_lifecycle.closing_count(),
|
|
816
|
+
inherited_sink=_delivering_to_an_inherited_sink(),
|
|
775
817
|
)
|
|
776
818
|
health = worker.health()
|
|
777
819
|
if _orphan_retired and not health.retired:
|
|
@@ -90,13 +90,31 @@ class Sink(Protocol):
|
|
|
90
90
|
accident, and the cost is one verbose attribute on the sinks that want interruptibility.
|
|
91
91
|
|
|
92
92
|
A fifth is optional in the same way and exists for one event only:
|
|
93
|
-
``
|
|
93
|
+
``reacquire_after_fork() -> None``. A sink that owns a **buffered** stream — one it
|
|
94
94
|
opened itself, rather than the process's ``sys.stdout`` — inherits the parent's unflushed
|
|
95
|
-
bytes in a forked child, and both processes then write them (SPEC-039 FR-004).
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
95
|
+
bytes in a forked child, and both processes then write them (SPEC-039 FR-004). It is
|
|
96
|
+
*called* rather than assigned, which is why it carries no ``log_foundry_`` prefix: the
|
|
97
|
+
collision the fourth member's name guards against is an attribute the library writes onto an
|
|
98
|
+
object it does not own.
|
|
99
|
+
|
|
100
|
+
**The contract has two halves and the name says the larger one** (SPEC-042 FR-005). Strand
|
|
101
|
+
what you inherited — the parent's pending bytes are not yours to write — *and*, by returning
|
|
102
|
+
normally, **claim the transport as this process's own**. ``FileSink`` satisfies both in one
|
|
103
|
+
step, because reopening the path is what strands the buffer: measured, parent and child then
|
|
104
|
+
hold different descriptors. That second half is what the library acts on, so a sink that
|
|
105
|
+
merely dropped a buffer without re-acquiring would satisfy a name describing only the
|
|
106
|
+
discard — which this member carried until SPEC-042 — while making a destructive close look
|
|
107
|
+
safe. A sink that does **not** implement this keeps whatever it inherited, so a child will
|
|
108
|
+
not release it; one that raises is treated the same way, since a failed re-acquisition is
|
|
109
|
+
not a claim.
|
|
110
|
+
|
|
111
|
+
**Inheriting the hook claims the whole object, so a subclass that adds a transport must
|
|
112
|
+
override it.** ``class MySink(FileSink)`` that also holds a socket inherits a hook which
|
|
113
|
+
re-acquires only the *file*, returns normally, and thereby tells the library the child owns
|
|
114
|
+
everything — after which the child's ``close()`` runs against the parent's connection.
|
|
115
|
+
Measured. Returning normally is the claim; the library cannot check what was actually
|
|
116
|
+
re-acquired, which is why the obligation sits here. A subclass that cannot honour it should
|
|
117
|
+
define the member to raise, which is refused and therefore safe.
|
|
100
118
|
|
|
101
119
|
**Which sinks are actually asked is narrower than "whichever define it", and the boundary is
|
|
102
120
|
the same one every fork repair has.** The child's repair walks this package's own objects —
|
|
@@ -124,8 +124,8 @@ class FileSink:
|
|
|
124
124
|
self._stream.write(json.dumps(event) + "\n")
|
|
125
125
|
self._stream.flush()
|
|
126
126
|
|
|
127
|
-
def
|
|
128
|
-
"""
|
|
127
|
+
def reacquire_after_fork(self) -> None:
|
|
128
|
+
"""Re-opens the file so this child holds its own descriptor (SPEC-039 FR-004, SPEC-042).
|
|
129
129
|
|
|
130
130
|
``emit`` writes a whole batch into a **buffered** stream and flushes once at the end, so
|
|
131
131
|
a fork landing inside it leaves both processes holding the same pending bytes and both
|
|
@@ -138,6 +138,11 @@ class FileSink:
|
|
|
138
138
|
taking it could only wait on a holder that cannot exist. A hook that blocks here blocks a
|
|
139
139
|
child that has not yet returned from ``fork``, where no watchdog can reach it.
|
|
140
140
|
|
|
141
|
+
A **closed** sink returns without re-acquiring anything, which is a trivially true claim
|
|
142
|
+
rather than an empty one: there is no transport left to hold, so nothing a later close
|
|
143
|
+
could destroy. Stated because ``sinks/base.py`` says returning normally *is* the claim,
|
|
144
|
+
and this is the one shipped sink that can return having done nothing.
|
|
145
|
+
|
|
141
146
|
Args:
|
|
142
147
|
None.
|
|
143
148
|
|
|
@@ -290,10 +295,10 @@ class RotatingFileSink:
|
|
|
290
295
|
self._size += data
|
|
291
296
|
self._stream.flush()
|
|
292
297
|
|
|
293
|
-
def
|
|
294
|
-
"""
|
|
298
|
+
def reacquire_after_fork(self) -> None:
|
|
299
|
+
"""Re-opens the file so this child holds its own descriptor (SPEC-039 FR-004, SPEC-042).
|
|
295
300
|
|
|
296
|
-
Identical to :meth:`FileSink.
|
|
301
|
+
Identical to :meth:`FileSink.reacquire_after_fork` and measured on this class
|
|
297
302
|
too, because the window is the same one: a whole batch written into a buffered stream
|
|
298
303
|
and flushed once at the end.
|
|
299
304
|
|
|
@@ -128,6 +128,22 @@ class Health:
|
|
|
128
128
|
still holds its resources. It is deliberately a live read rather than a count of expired
|
|
129
129
|
joins: a slow close and a stuck one are indistinguishable at the moment a join expires,
|
|
130
130
|
and SPEC-028 reverted a design that guessed.
|
|
131
|
+
inherited_sink: Whether the sink this process **last installed for delivery** is one it
|
|
132
|
+
may not release — one it inherited across a ``fork`` (SPEC-042 FR-004). "Last
|
|
133
|
+
installed", not "would deliver to now": after ``shutdown()`` the process delivers
|
|
134
|
+
nowhere, and reporting ``True`` there is the point rather than a wrinkle, since an
|
|
135
|
+
inherited sink left open at exit is exactly what this explains. **The referent is one
|
|
136
|
+
object, named
|
|
137
|
+
here because SPEC-033 measured three candidates disagreeing**: the worker's sink if a
|
|
138
|
+
worker exists, else the sink the orphan path recorded, else the configured one. It
|
|
139
|
+
describes that object and *not* the graph beneath it, so a child that wraps an
|
|
140
|
+
inherited sink in a ``MultiSink`` of its own reads ``False`` here while the wrapper's
|
|
141
|
+
child is still refused — stated because the opposite reading is the natural one.
|
|
142
|
+
It is a **state, not a fault**, and deliberately not a term in the documented alert
|
|
143
|
+
idiom, which is the call ``closing_sinks`` got. It explains a handle still open after
|
|
144
|
+
``shutdown()``, and it is the signal that a deployment shares a sink across a fork at
|
|
145
|
+
all. ``True`` for a shared ``StdoutSink`` too, whose ``close()`` only flushes — so a
|
|
146
|
+
``True`` is not by itself evidence that anything is held.
|
|
131
147
|
"""
|
|
132
148
|
|
|
133
149
|
queued: int
|
|
@@ -139,6 +155,7 @@ class Health:
|
|
|
139
155
|
submitted_after_shutdown: int = 0
|
|
140
156
|
incomplete_swaps: int = 0
|
|
141
157
|
closing_sinks: int = 0
|
|
158
|
+
inherited_sink: bool = False
|
|
142
159
|
|
|
143
160
|
|
|
144
161
|
class _FlushMarker:
|
|
@@ -543,6 +560,7 @@ class Worker:
|
|
|
543
560
|
submitted_after_shutdown=submitted_after_shutdown,
|
|
544
561
|
incomplete_swaps=incomplete_swaps,
|
|
545
562
|
closing_sinks=_lifecycle.closing_count(),
|
|
563
|
+
inherited_sink=not _lifecycle.releasable(self.sink),
|
|
546
564
|
)
|
|
547
565
|
|
|
548
566
|
def _sink_losses(self) -> SinkLosses | None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{log_foundry-0.10.2.dev62 → log_foundry-0.10.2.dev64}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|