log-foundry 0.10.2.dev69__tar.gz → 0.10.2.dev71__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/PKG-INFO +10 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/README.md +9 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/__init__.py +2 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/decorator.py +59 -8
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/results.py +7 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/_socket.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/base.py +48 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/callback.py +5 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/clickhouse.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/datadog.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/elasticsearch.py +8 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/eventhubs.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/file.py +8 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/filtering.py +27 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/firehose.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/honeycomb.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/http.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/kafka.py +35 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/kinesis.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/logging_sink.py +51 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/logstash.py +5 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/loki.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/memory.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/mongodb.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/multi.py +46 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/nats.py +47 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/newrelic.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/null.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/postgres.py +5 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/pubsub.py +73 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/rabbitmq.py +6 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/redis.py +14 -2
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/sentry.py +39 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/sns.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/splunk.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/sqlite.py +5 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/sqs.py +4 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/stdout.py +8 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/syslog.py +5 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/transform.py +27 -1
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/LICENSE +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/_fork.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/_lifecycle.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/config.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/_retry.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: log-foundry
|
|
3
|
-
Version: 0.10.2.
|
|
3
|
+
Version: 0.10.2.dev71
|
|
4
4
|
Summary: Generate logs for your console and JSON events for downstream consumption.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -233,6 +233,15 @@ is FIFO, everything submitted before the call is necessarily ahead of that marke
|
|
|
233
233
|
exactly why the guarantee is "events submitted before this call", and why concurrent
|
|
234
234
|
submissions from other threads may or may not be included.
|
|
235
235
|
|
|
236
|
+
**`flush()` also empties the sink's own client buffer**, when the sink has one. A sink that
|
|
237
|
+
buffers in a driver rather than writing through — `KafkaSink` hands to librdkafka, `PubSubSink`
|
|
238
|
+
appends an unresolved future, `SentrySink` hands to the SDK's background transport — used to be
|
|
239
|
+
unreachable this way, so `flush()` could return truthy with events sitting in a client. If that
|
|
240
|
+
client cannot be emptied the result is falsy with `reason="sink-flush"`, which is distinct from
|
|
241
|
+
`"abandoned"`: the events are past this library and inside a driver. A **custom sink** that
|
|
242
|
+
buffers should implement `flush()` — it is optional, probed by name, and a sink without one is
|
|
243
|
+
unaffected.
|
|
244
|
+
|
|
236
245
|
**`flush()` also sweeps the spans that are still open**, so an in-span event does not have to wait
|
|
237
246
|
for its span to close to be delivered. The span stays open and usable afterwards: its events go
|
|
238
247
|
now and its `span.end` arrives later, in its own batch. Two consequences worth knowing. Boundary
|
|
@@ -197,6 +197,15 @@ is FIFO, everything submitted before the call is necessarily ahead of that marke
|
|
|
197
197
|
exactly why the guarantee is "events submitted before this call", and why concurrent
|
|
198
198
|
submissions from other threads may or may not be included.
|
|
199
199
|
|
|
200
|
+
**`flush()` also empties the sink's own client buffer**, when the sink has one. A sink that
|
|
201
|
+
buffers in a driver rather than writing through — `KafkaSink` hands to librdkafka, `PubSubSink`
|
|
202
|
+
appends an unresolved future, `SentrySink` hands to the SDK's background transport — used to be
|
|
203
|
+
unreachable this way, so `flush()` could return truthy with events sitting in a client. If that
|
|
204
|
+
client cannot be emptied the result is falsy with `reason="sink-flush"`, which is distinct from
|
|
205
|
+
`"abandoned"`: the events are past this library and inside a driver. A **custom sink** that
|
|
206
|
+
buffers should implement `flush()` — it is optional, probed by name, and a sink without one is
|
|
207
|
+
unaffected.
|
|
208
|
+
|
|
200
209
|
**`flush()` also sweeps the spans that are still open**, so an in-span event does not have to wait
|
|
201
210
|
for its span to close to be delivered. The span stays open and usable afterwards: its events go
|
|
202
211
|
now and its `span.end` arrives later, in its own batch. Two consequences worth knowing. Boundary
|
|
@@ -44,7 +44,8 @@ def flush(timeout: float | None = 5.0) -> FlushResult:
|
|
|
44
44
|
A :class:`FlushResult`. Truthy when the events submitted before this call reached the
|
|
45
45
|
sink — so a truthy result means they were delivered, not merely that a drain took place.
|
|
46
46
|
Falsy carries a ``reason`` naming which outcome occurred: ``"timed-out"``, ``"retired"``,
|
|
47
|
-
``"thread-died"``, ``"queue-full"`` or ``"
|
|
47
|
+
``"thread-died"``, ``"queue-full"``, ``"abandoned"`` or ``"sink-flush"`` — the last meaning
|
|
48
|
+
the queue drained but the sink could not empty its own client buffer (SPEC-036 FR-002). It is falsy rather than ``False``:
|
|
48
49
|
``if flush():`` is unchanged, but ``flush() is True`` can no longer hold, which is why the
|
|
49
50
|
type had to change before ``1.0.0`` rather than after (SPEC-034 FR-007). Events submitted
|
|
50
51
|
concurrently by another thread may or may not be included, since the caller cannot have
|
|
@@ -22,6 +22,7 @@ from log_foundry.ids import (
|
|
|
22
22
|
)
|
|
23
23
|
from log_foundry.model import Span, backfill_baggage, end_event, start_event
|
|
24
24
|
from log_foundry.results import ContinueResult, FlushResult
|
|
25
|
+
from log_foundry.sinks.base import flush_sink
|
|
25
26
|
from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_SWAP_TIMEOUT, Health, Worker
|
|
26
27
|
|
|
27
28
|
if TYPE_CHECKING:
|
|
@@ -836,6 +837,44 @@ def _sweep_open_spans() -> None:
|
|
|
836
837
|
worker.submit(buffered)
|
|
837
838
|
|
|
838
839
|
|
|
840
|
+
def _flush_live_sink() -> bool:
|
|
841
|
+
"""Drains whatever the delivering sink holds in its own client (SPEC-036 FR-002).
|
|
842
|
+
|
|
843
|
+
Called **after** the queue drain, because the queue's events have to reach the client buffer
|
|
844
|
+
before it is emptied. A sink with no ``flush`` of its own is unaffected, which is what keeps
|
|
845
|
+
every pre-SPEC-036 sink satisfying the protocol.
|
|
846
|
+
|
|
847
|
+
Which sink is asked follows the ownership rule the rest of this module uses (SPEC-033): a
|
|
848
|
+
live worker's sink if there is one, otherwise the sink an orphan emit actually **reached**.
|
|
849
|
+
Not "a sink has been resolved" — ``configure()`` runs ``_ensure_sink()`` unconditionally, so a
|
|
850
|
+
bare ``configure(service=...)`` has already built a ``StdoutSink`` that nothing was ever
|
|
851
|
+
written to, and materialising a flush against it is the cost SPEC-031 FR-006 declined for the
|
|
852
|
+
close path for the same reason. So a ``flush()`` in a process that has never logged touches
|
|
853
|
+
no sink, which is what FR-001 AC-6 needs to stay true.
|
|
854
|
+
|
|
855
|
+
Args:
|
|
856
|
+
None.
|
|
857
|
+
|
|
858
|
+
Returns:
|
|
859
|
+
Whether the sink's own flush succeeded. ``True`` also when there was no sink to ask, or
|
|
860
|
+
when it holds nothing of its own.
|
|
861
|
+
|
|
862
|
+
Raises:
|
|
863
|
+
None. A failure is reported as a ``FlushResult`` reason by the caller, never raised: a
|
|
864
|
+
flush is the call most likely to be made in a ``finally``.
|
|
865
|
+
"""
|
|
866
|
+
worker = _worker
|
|
867
|
+
sink = worker.sink if worker is not None and not worker.retired else _orphan_sink
|
|
868
|
+
if sink is None:
|
|
869
|
+
return True
|
|
870
|
+
try:
|
|
871
|
+
flush_sink(sink)
|
|
872
|
+
except Exception as exc:
|
|
873
|
+
_diag.absorbed("flushing the sink's own buffer", exc, "its client still holds events")
|
|
874
|
+
return False
|
|
875
|
+
return True
|
|
876
|
+
|
|
877
|
+
|
|
839
878
|
def _flush_worker(timeout: float | None = 5.0) -> FlushResult:
|
|
840
879
|
"""Drains the process worker without retiring it, backing ``flush()`` (SPEC-013 FR-003).
|
|
841
880
|
|
|
@@ -859,6 +898,12 @@ def _flush_worker(timeout: float | None = 5.0) -> FlushResult:
|
|
|
859
898
|
exact shape the spec was written to remove. The drain still runs, so whatever was
|
|
860
899
|
submitted before the failure is not held back by it.
|
|
861
900
|
|
|
901
|
+
The sink's own buffer is drained **whichever way the earlier steps went**, and the failure
|
|
902
|
+
reasons are decided afterwards. A draft returned early on a failed sweep or a dead drain
|
|
903
|
+
thread, which skipped it — and by then ``worker.flush`` had already pushed the queue *into*
|
|
904
|
+
that buffer, so the events most worth saving before a freeze were the ones left there. The
|
|
905
|
+
reason reported is the most upstream failure, because that is the one to fix.
|
|
906
|
+
|
|
862
907
|
Raises:
|
|
863
908
|
None. A flush is the call most likely to be made in a ``finally``, so the library must
|
|
864
909
|
never be the reason a caller's function fails; a failure is reported by the return
|
|
@@ -871,15 +916,21 @@ def _flush_worker(timeout: float | None = 5.0) -> FlushResult:
|
|
|
871
916
|
_diag.absorbed("sweeping open spans for a flush", exc, "buffered events were not swept")
|
|
872
917
|
swept = False
|
|
873
918
|
worker = _worker
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
919
|
+
drained: FlushResult = FlushResult(ok=True)
|
|
920
|
+
thread_died = False
|
|
921
|
+
if worker is not None:
|
|
922
|
+
try:
|
|
923
|
+
drained = worker.flush(timeout)
|
|
924
|
+
except Exception:
|
|
925
|
+
thread_died = True
|
|
926
|
+
sink_drained = _flush_live_sink()
|
|
927
|
+
if not swept:
|
|
928
|
+
return FlushResult(ok=False, reason="abandoned")
|
|
929
|
+
if thread_died:
|
|
881
930
|
return FlushResult(ok=False, reason="thread-died")
|
|
882
|
-
|
|
931
|
+
if not sink_drained:
|
|
932
|
+
return FlushResult(ok=False, reason="sink-flush")
|
|
933
|
+
return drained
|
|
883
934
|
|
|
884
935
|
|
|
885
936
|
def _note_orphan_loss() -> None:
|
|
@@ -50,7 +50,13 @@ class FlushResult(_Result):
|
|
|
50
50
|
"""What :func:`log_foundry.flush` returns.
|
|
51
51
|
|
|
52
52
|
``reason`` is ``None`` on success. The tokens it can carry today are ``"timed-out"``,
|
|
53
|
-
``"retired"``, ``"thread-died"``, ``"queue-full"`` and ``"
|
|
53
|
+
``"retired"``, ``"thread-died"``, ``"queue-full"``, ``"abandoned"`` and ``"sink-flush"``.
|
|
54
|
+
|
|
55
|
+
``"sink-flush"`` is the one SPEC-036 added (FR-002 AC-8) and it is worth distinguishing: the
|
|
56
|
+
queue drained cleanly and the **sink's own client buffer** did not, so the events are past
|
|
57
|
+
this library and inside a driver. ``"abandoned"`` is the neighbouring case where this call
|
|
58
|
+
could not hand them over at all. New tokens may appear in any release, which is what this
|
|
59
|
+
type exists for — branch on ``bool()``.
|
|
54
60
|
"""
|
|
55
61
|
|
|
56
62
|
|
|
@@ -118,6 +118,10 @@ class SocketTransport:
|
|
|
118
118
|
failed: Messages abandoned past the reconnect-retry bound.
|
|
119
119
|
dropped_oversized: UDP datagrams discarded before any send for exceeding
|
|
120
120
|
``max_datagram_bytes``.
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``send_all`` puts the bytes on the
|
|
124
|
+
socket before it returns, and no client object outlives it holding data.
|
|
121
125
|
"""
|
|
122
126
|
|
|
123
127
|
def __init__(
|
|
@@ -6,7 +6,7 @@ from abc import abstractmethod
|
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from typing import Protocol, runtime_checkable
|
|
8
8
|
|
|
9
|
-
__all__ = ["Sink", "SinkDeliveryError", "SinkLosses", "read_losses"]
|
|
9
|
+
__all__ = ["Sink", "SinkDeliveryError", "SinkLosses", "flush_sink", "read_losses"]
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class SinkDeliveryError(Exception):
|
|
@@ -142,6 +142,24 @@ class Sink(Protocol):
|
|
|
142
142
|
the call an operator makes when a destination is already hanging, and sharing one lock
|
|
143
143
|
would make that poll wait for an in-flight emit and its retry backoff. Where a sink holds
|
|
144
144
|
both, the order is always transport then counter, never the reverse.
|
|
145
|
+
A sixth is optional in the same way: ``flush() -> None``, which drains whatever the sink is
|
|
146
|
+
holding in its **client** without closing it (SPEC-036 FR-002). A sink that buffers in a
|
|
147
|
+
driver rather than writing through — ``KafkaSink`` hands to librdkafka, ``GooglePubSubSink``
|
|
148
|
+
appends an unresolved future — is unreachable through ``log_foundry.flush()`` without it:
|
|
149
|
+
measured against a stand-in with that shape, ``flush() -> True``, on the wire 0, in the
|
|
150
|
+
client buffer 3, ``health()`` all zeros. It is called **after** the queue drain, so the
|
|
151
|
+
queue's events have reached the client buffer before it is emptied.
|
|
152
|
+
|
|
153
|
+
It is **not a close**, and the difference is the whole point: the sink keeps its transport and
|
|
154
|
+
goes on accepting events afterwards. Like :meth:`emit` it must tolerate being called
|
|
155
|
+
concurrently with an emit (SPEC-028), and like :meth:`emit` it must **raise** when it could
|
|
156
|
+
not deliver what it was holding — that is the only channel by which ``log_foundry.flush()``
|
|
157
|
+
can report ``reason="sink-flush"`` instead of success. :func:`flush_sink` is the probe, and it
|
|
158
|
+
deliberately does **not** behave like :func:`read_losses`: that one swallows a raising
|
|
159
|
+
accessor because a broken reporter must not take ``health()`` down, while this one propagates,
|
|
160
|
+
because a swallowed flush failure is exactly the "sink the worker believes" this file exists
|
|
161
|
+
to prevent.
|
|
162
|
+
|
|
145
163
|
"""
|
|
146
164
|
|
|
147
165
|
@abstractmethod
|
|
@@ -260,3 +278,32 @@ def read_losses(sink: object) -> SinkLosses | None:
|
|
|
260
278
|
except Exception:
|
|
261
279
|
return None
|
|
262
280
|
return losses if isinstance(losses, SinkLosses) else None
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def flush_sink(sink: object) -> bool:
|
|
284
|
+
"""Calls a sink's optional ``flush()``, letting any failure propagate (SPEC-036 FR-002).
|
|
285
|
+
|
|
286
|
+
The sibling of :func:`read_losses`, written here for the same reason — the probe and its
|
|
287
|
+
guarantees belong in one place — and with the **opposite** failure rule, which is the part
|
|
288
|
+
worth reading twice. ``read_losses`` swallows a raising accessor because a broken reporter
|
|
289
|
+
must not take ``health()`` down with it. This one must not swallow anything: a sink's flush
|
|
290
|
+
failure reaches the caller only through ``log_foundry.flush()``'s result, so absorbing it here
|
|
291
|
+
would produce the exact "sink the worker believes" this module exists to prevent — a
|
|
292
|
+
``flush()`` reporting success over a client buffer that never went out.
|
|
293
|
+
|
|
294
|
+
Args:
|
|
295
|
+
sink: The sink to probe, of any type.
|
|
296
|
+
|
|
297
|
+
Returns:
|
|
298
|
+
Whether the sink had a ``flush`` to call. ``False`` means it holds nothing of its own, and
|
|
299
|
+
the queue drain was the whole of the flush.
|
|
300
|
+
|
|
301
|
+
Raises:
|
|
302
|
+
Exception: Whatever the sink's ``flush`` raises, deliberately unguarded. The caller turns
|
|
303
|
+
it into a ``FlushResult`` reason; see ``decorator._flush_live_sink``.
|
|
304
|
+
"""
|
|
305
|
+
accessor = getattr(sink, "flush", None)
|
|
306
|
+
if not callable(accessor):
|
|
307
|
+
return False
|
|
308
|
+
accessor()
|
|
309
|
+
return True
|
|
@@ -22,6 +22,11 @@ class CallbackSink:
|
|
|
22
22
|
(SPEC-032 FR-003). Both decisions belong to the callable: this class holds nothing, and what
|
|
23
23
|
a hook releases is not knowable from here — a callable needing either guarantee must provide
|
|
24
24
|
it, exactly as a hand-written ``Sink`` implementation would.
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
It keeps **no** client buffer (SPEC-036 FR-002): it hands each event to a *function*, which
|
|
28
|
+
has returned by the time ``emit`` does. Unlike the three wrapper sinks it wraps no sink, so
|
|
29
|
+
there is nothing to forward a flush to.
|
|
25
30
|
"""
|
|
26
31
|
|
|
27
32
|
def __init__(
|
|
@@ -47,6 +47,10 @@ class ClickHouseSink:
|
|
|
47
47
|
default auto-generated session, so it is squarely in that case and the lock is required
|
|
48
48
|
rather than merely prudent. One client per thread would be the alternative, and that is the
|
|
49
49
|
connection-pool design FR-002 puts out of scope.
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
53
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
50
54
|
"""
|
|
51
55
|
|
|
52
56
|
def __init__(
|
|
@@ -28,6 +28,10 @@ class DatadogSink(HTTPSink):
|
|
|
28
28
|
sink in the family whose per-event limit is stricter than its request limit, so without
|
|
29
29
|
it a 2 MB event passes the 5 MB request budget and is rejected by a limit the budget
|
|
30
30
|
cannot see. All three are the vendor's own figures, from the Logs API's send-logs limits.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
34
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
31
35
|
"""
|
|
32
36
|
|
|
33
37
|
MAX_BATCH_COUNT = 1000
|
{log_foundry-0.10.2.dev69 → log_foundry-0.10.2.dev71}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
@@ -38,6 +38,10 @@ class ElasticsearchSink(HTTPSink):
|
|
|
38
38
|
bulk guidance is to find a working size by experiment rather than to send the largest
|
|
39
39
|
request the server will accept, and a 100 MB bulk is a poor default for a log shipper.
|
|
40
40
|
Raise it with ``max_batch_bytes=``.
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
44
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
41
45
|
"""
|
|
42
46
|
|
|
43
47
|
MAX_BATCH_COUNT = 1000
|
|
@@ -201,4 +205,8 @@ class OpenSearchSink(ElasticsearchSink):
|
|
|
201
205
|
"""OpenSearch reuses the Elasticsearch ``_bulk`` protocol verbatim (FR-003).
|
|
202
206
|
|
|
203
207
|
Endpoint and auth differ only by configuration, so this is a straight reuse.
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
211
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
204
212
|
"""
|
|
@@ -23,6 +23,10 @@ class AzureEventHubsSink:
|
|
|
23
23
|
1 MB per-batch limit, which the SDK signals by raising ``ValueError`` from ``add``. The
|
|
24
24
|
worst-case delay (SPEC-027 FR-005) is ``max_retries`` interruptible waits per batch, 0.7 s at
|
|
25
25
|
the defaults.
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
29
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
26
30
|
"""
|
|
27
31
|
|
|
28
32
|
def __init__(
|
|
@@ -71,6 +71,10 @@ class FileSink:
|
|
|
71
71
|
concurrently (SPEC-028 FR-002) — this module claimed a single worker thread until that spec
|
|
72
72
|
measured the orphan path emitting on application threads at the same time. Cross-*process*
|
|
73
73
|
coordination remains out of scope: two processes appending to one path are on their own.
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``emit`` flushes the stream before it
|
|
77
|
+
returns, so nothing of this sink's is left pending between calls.
|
|
74
78
|
"""
|
|
75
79
|
|
|
76
80
|
def __init__(self, path: str, *, encoding: str = "utf-8") -> None:
|
|
@@ -211,6 +215,10 @@ class RotatingFileSink:
|
|
|
211
215
|
damage: a second thread mid-``emit`` could write to the handle rotation had just closed, or
|
|
212
216
|
to the pre-rotation file it had already renamed away. Both are serialized on a lock
|
|
213
217
|
(SPEC-028 FR-002).
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``emit`` flushes the stream before it
|
|
221
|
+
returns, so nothing of this sink's is left pending between calls.
|
|
214
222
|
"""
|
|
215
223
|
|
|
216
224
|
def __init__(
|
|
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
from typing import TYPE_CHECKING
|
|
6
6
|
|
|
7
7
|
from log_foundry import _diag, _lifecycle
|
|
8
|
-
from log_foundry.sinks.base import read_losses
|
|
8
|
+
from log_foundry.sinks.base import flush_sink, read_losses
|
|
9
9
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
11
11
|
import threading
|
|
@@ -28,6 +28,13 @@ class FilteringSink:
|
|
|
28
28
|
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
29
29
|
(SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
|
|
30
30
|
belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
It holds **no** client buffer of its own, but it **forwards** ``flush()`` to what it wraps
|
|
34
|
+
(SPEC-036 FR-002). Holding nothing is not the same as having nothing to do: a wrapper that
|
|
35
|
+
did not forward would leave a buffering child unreachable through ``log_foundry.flush()``
|
|
36
|
+
while looking fine — the SPEC-027 lesson about ``log_foundry_stop_signal``, that a signal
|
|
37
|
+
stopped at a wrapper reaches nothing and moves the defect rather than fixing it.
|
|
31
38
|
"""
|
|
32
39
|
|
|
33
40
|
def __init__(
|
|
@@ -147,6 +154,25 @@ class FilteringSink:
|
|
|
147
154
|
f"{type(self._inner).__name__} stays uninterruptible",
|
|
148
155
|
)
|
|
149
156
|
|
|
157
|
+
def flush(self) -> None:
|
|
158
|
+
"""Forwards the flush to the wrapped sink (SPEC-036 FR-002).
|
|
159
|
+
|
|
160
|
+
This wrapper holds nothing itself, but a wrapper that did not forward would leave a
|
|
161
|
+
buffering inner sink unreachable through ``log_foundry.flush()`` while looking fine —
|
|
162
|
+
the SPEC-027 lesson that a signal stopped at a wrapper reaches nothing.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
None.
|
|
166
|
+
|
|
167
|
+
Returns:
|
|
168
|
+
None.
|
|
169
|
+
|
|
170
|
+
Raises:
|
|
171
|
+
Exception: Whatever the wrapped sink raises, so the failure reaches the caller as a
|
|
172
|
+
``FlushResult`` reason rather than being swallowed here.
|
|
173
|
+
"""
|
|
174
|
+
flush_sink(self._inner)
|
|
175
|
+
|
|
150
176
|
def losses(self) -> SinkLosses | None:
|
|
151
177
|
"""Reports the inner sink's losses (SPEC-026 FR-002).
|
|
152
178
|
|
|
@@ -40,6 +40,10 @@ class FirehoseSink:
|
|
|
40
40
|
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
41
41
|
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
42
42
|
afterwards still reaches the delivery stream.
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
46
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
43
47
|
"""
|
|
44
48
|
|
|
45
49
|
MAX_RECORDS = 500
|
|
@@ -24,6 +24,10 @@ class HoneycombSink(HTTPSink):
|
|
|
24
24
|
maximum event count for the batch endpoint.
|
|
25
25
|
MAX_BATCH_BYTES: 1,000,000 — Honeycomb's documented 1 MB of uncompressed JSON for the
|
|
26
26
|
Create Events endpoint.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
30
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
27
31
|
"""
|
|
28
32
|
|
|
29
33
|
MAX_BATCH_COUNT = 1000
|
|
@@ -201,6 +201,10 @@ class HTTPSink:
|
|
|
201
201
|
And it **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` releases nothing —
|
|
202
202
|
a batch emitted afterwards still reaches the endpoint, and refusing it would be loss the
|
|
203
203
|
library invented rather than loss it reported.
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
207
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
204
208
|
"""
|
|
205
209
|
|
|
206
210
|
MAX_BATCH_COUNT = DEFAULT_MAX_BATCH_COUNT
|
|
@@ -228,6 +228,41 @@ class KafkaSink:
|
|
|
228
228
|
self._closed = True
|
|
229
229
|
self._flush_bounded()
|
|
230
230
|
|
|
231
|
+
def flush(self) -> None:
|
|
232
|
+
"""Drains the producer's local buffer without closing the sink (SPEC-036 FR-002).
|
|
233
|
+
|
|
234
|
+
``emit`` hands to librdkafka and returns; only ``flush()`` drains it. Before this hook
|
|
235
|
+
existed the buffer was unreachable through ``log_foundry.flush()`` — the whole point of
|
|
236
|
+
that call in a process about to be frozen — and went out only at ``close()``.
|
|
237
|
+
|
|
238
|
+
It **raises** when the producer still holds messages after ``flush_timeout``, which is what
|
|
239
|
+
makes ``log_foundry.flush()`` report ``reason="sink-flush"`` rather than success. Nothing
|
|
240
|
+
is counted as lost: unlike the close path those messages are still queued and the next
|
|
241
|
+
flush or the close may yet deliver them, so booking them against ``failed`` would report a
|
|
242
|
+
loss that has not happened. The remainder is named in the error instead.
|
|
243
|
+
|
|
244
|
+
Refuses after ``close()`` on SPEC-032's rule — the producer has been flushed and released.
|
|
245
|
+
|
|
246
|
+
Args:
|
|
247
|
+
None.
|
|
248
|
+
|
|
249
|
+
Returns:
|
|
250
|
+
None.
|
|
251
|
+
|
|
252
|
+
Raises:
|
|
253
|
+
SinkDeliveryError: The sink is closed, or the producer could not be drained inside
|
|
254
|
+
``flush_timeout``.
|
|
255
|
+
Exception: Whatever the producer raises.
|
|
256
|
+
"""
|
|
257
|
+
if self._closed:
|
|
258
|
+
raise SinkDeliveryError("KafkaSink cannot flush: the sink is closed")
|
|
259
|
+
remaining = self.producer.flush(self.flush_timeout)
|
|
260
|
+
if type(remaining) is int and remaining > 0:
|
|
261
|
+
raise SinkDeliveryError(
|
|
262
|
+
f"KafkaSink flushed within {self.flush_timeout}s with {remaining} "
|
|
263
|
+
"message(s) still queued"
|
|
264
|
+
)
|
|
265
|
+
|
|
231
266
|
def _flush_bounded(self) -> None:
|
|
232
267
|
"""Flushes the producer within a bound, counting whatever it could not deliver (FR-006).
|
|
233
268
|
|
|
@@ -40,6 +40,10 @@ class KinesisSink:
|
|
|
40
40
|
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
41
41
|
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
42
42
|
afterwards still reaches the stream.
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
46
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
43
47
|
"""
|
|
44
48
|
|
|
45
49
|
MAX_RECORDS = 500
|
|
@@ -4,6 +4,8 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
6
|
|
|
7
|
+
from log_foundry import _diag
|
|
8
|
+
|
|
7
9
|
__all__ = ["LoggingSink"]
|
|
8
10
|
|
|
9
11
|
_LEVELS = {
|
|
@@ -41,6 +43,14 @@ class LoggingSink:
|
|
|
41
43
|
and this sink holds nothing else. And it **adds no post-close guard** (SPEC-032 FR-003) —
|
|
42
44
|
``close()`` is a no-op by design, since tearing down handlers this sink did not configure is
|
|
43
45
|
not its to do, so a later batch still reaches the framework.
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
Its transport is the **handler chain**, and it forwards ``flush()`` onto it (SPEC-036
|
|
49
|
+
FR-002). A first pass filed this alongside ``MemorySink`` and ``NullSink`` as having nothing
|
|
50
|
+
underneath, which is wrong: ``logging.Handler.flush`` exists precisely because handlers
|
|
51
|
+
buffer, ``logging.handlers.MemoryHandler`` does nothing else, and ``QueueHandler`` and most
|
|
52
|
+
third-party handlers are the same shape. Measured against a ``MemoryHandler``: three events
|
|
53
|
+
emitted, nothing on the stream, and everything on it after one ``flush()``.
|
|
44
54
|
"""
|
|
45
55
|
|
|
46
56
|
def __init__(
|
|
@@ -79,6 +89,47 @@ class LoggingSink:
|
|
|
79
89
|
for event in batch:
|
|
80
90
|
self._logger.handle(self._to_record(event))
|
|
81
91
|
|
|
92
|
+
def flush(self) -> None:
|
|
93
|
+
"""Flushes the logger's handlers, and its ancestors' unless propagation is off.
|
|
94
|
+
|
|
95
|
+
The handler chain is this sink's transport, so this walks it the way ``logging`` itself
|
|
96
|
+
dispatches a record — the current logger's handlers, then each ancestor's, stopping where
|
|
97
|
+
``propagate`` is ``False``. Anything else would flush a handler the events never reached,
|
|
98
|
+
or miss the one they did.
|
|
99
|
+
|
|
100
|
+
Every handler is attempted before anything is raised, and then the first failure is —
|
|
101
|
+
``MultiSink.flush``'s rule, for its reason: one broken handler must not leave a healthy
|
|
102
|
+
one downstream of it unflushed. Failures are **not** absorbed, because a handler that
|
|
103
|
+
could not flush is a client buffer that did not go out, which ``log_foundry.flush()``
|
|
104
|
+
reports as ``reason="sink-flush"``.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
None.
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
None.
|
|
111
|
+
|
|
112
|
+
Raises:
|
|
113
|
+
Exception: The first handler's exception, when any handler could not be flushed.
|
|
114
|
+
"""
|
|
115
|
+
first_error: Exception | None = None
|
|
116
|
+
logger: logging.Logger | None = self._logger
|
|
117
|
+
while logger is not None:
|
|
118
|
+
for handler in logger.handlers:
|
|
119
|
+
try:
|
|
120
|
+
handler.flush()
|
|
121
|
+
except Exception as err:
|
|
122
|
+
if first_error is None:
|
|
123
|
+
first_error = err
|
|
124
|
+
_diag.absorbed(
|
|
125
|
+
"flushing a logging handler",
|
|
126
|
+
err,
|
|
127
|
+
f"{type(handler).__name__} still holds records",
|
|
128
|
+
)
|
|
129
|
+
logger = logger.parent if logger.propagate else None
|
|
130
|
+
if first_error is not None:
|
|
131
|
+
raise first_error
|
|
132
|
+
|
|
82
133
|
def close(self) -> None:
|
|
83
134
|
"""Does nothing, since the sink does not own the user's logging configuration (FR-005).
|
|
84
135
|
|
|
@@ -44,6 +44,11 @@ class LogstashSink:
|
|
|
44
44
|
``SinkDeliveryError`` and reopens nothing, while in HTTP mode ``close()`` released nothing
|
|
45
45
|
and the batch still ships. Both are the backend's answer, correctly, rather than one this
|
|
46
46
|
class invents on top.
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` hands its bytes to the
|
|
50
|
+
socket before it returns, and no client object outlives it holding data. What the *network*
|
|
51
|
+
then does with a datagram is not something a flush could hurry.
|
|
47
52
|
"""
|
|
48
53
|
|
|
49
54
|
def __init__(
|
|
@@ -38,6 +38,10 @@ class LokiSink(HTTPSink):
|
|
|
38
38
|
(``distributor.max_recv_msg_size``, 100 MB for the compressed body), but that is an
|
|
39
39
|
operator-tunable server setting and a hosted Loki is routinely configured well below it,
|
|
40
40
|
so the default here is the conservative one. Raise it with ``max_batch_bytes=``.
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
44
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
41
45
|
"""
|
|
42
46
|
|
|
43
47
|
MAX_BATCH_COUNT = 1000
|
|
@@ -16,6 +16,10 @@ class MemorySink:
|
|
|
16
16
|
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
17
17
|
(SPEC-032 FR-003): there is no transport, and ``close()`` releases nothing — a test that
|
|
18
18
|
closes the sink and then asserts on a later batch still sees it in ``.events``.
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
It keeps **no** client buffer (SPEC-036 FR-002): there is no transport under it for a flush
|
|
22
|
+
to push anything onto.
|
|
19
23
|
"""
|
|
20
24
|
|
|
21
25
|
def __init__(self, maxlen: int | None = None) -> None:
|
|
@@ -35,6 +35,10 @@ class MongoDBSink:
|
|
|
35
35
|
Attributes:
|
|
36
36
|
failed: Documents the server rejected, or a whole batch abandoned past the retry bound.
|
|
37
37
|
dropped_oversized: Documents dropped for exceeding MongoDB's 16 MB per-document limit.
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
41
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
38
42
|
"""
|
|
39
43
|
|
|
40
44
|
def __init__(
|
|
@@ -6,7 +6,7 @@ import threading
|
|
|
6
6
|
from typing import TYPE_CHECKING
|
|
7
7
|
|
|
8
8
|
from log_foundry import _diag, _lifecycle
|
|
9
|
-
from log_foundry.sinks.base import SinkLosses, read_losses
|
|
9
|
+
from log_foundry.sinks.base import SinkLosses, flush_sink, read_losses
|
|
10
10
|
|
|
11
11
|
if TYPE_CHECKING:
|
|
12
12
|
from log_foundry.sinks.base import Sink
|
|
@@ -36,6 +36,13 @@ class MultiSink:
|
|
|
36
36
|
``close()`` here only forwards, so a guard added at this level would refuse batches the
|
|
37
37
|
children would have taken, while a child that must refuse already does and is counted here
|
|
38
38
|
like any other failure.
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
It holds **no** client buffer of its own, but it **forwards** ``flush()`` to what it wraps
|
|
42
|
+
(SPEC-036 FR-002). Holding nothing is not the same as having nothing to do: a wrapper that
|
|
43
|
+
did not forward would leave a buffering child unreachable through ``log_foundry.flush()``
|
|
44
|
+
while looking fine — the SPEC-027 lesson about ``log_foundry_stop_signal``, that a signal
|
|
45
|
+
stopped at a wrapper reaches nothing and moves the defect rather than fixing it.
|
|
39
46
|
"""
|
|
40
47
|
|
|
41
48
|
def __init__(self, *sinks: Sink) -> None:
|
|
@@ -155,6 +162,44 @@ class MultiSink:
|
|
|
155
162
|
f"{type(sink).__name__} stays uninterruptible",
|
|
156
163
|
)
|
|
157
164
|
|
|
165
|
+
def flush(self) -> None:
|
|
166
|
+
"""Forwards the flush to every child, so a buffering one is actually reached.
|
|
167
|
+
|
|
168
|
+
SPEC-036 FR-002. Without this a ``MultiSink(StdoutSink(), KafkaSink(...))`` — the
|
|
169
|
+
composition the README itself shows — left every Kafka message in librdkafka's buffer
|
|
170
|
+
while ``log_foundry.flush()`` reported success: measured, 0 on the wire and 3 in the
|
|
171
|
+
client. That is the SPEC-027 shape exactly, where a stop signal set on a wrapper reached
|
|
172
|
+
nothing.
|
|
173
|
+
|
|
174
|
+
Every child is attempted before anything is raised, so one failing child cannot stop a
|
|
175
|
+
healthy sibling being drained — the isolation :meth:`emit` applies, for the same reason.
|
|
176
|
+
The raise rule is **different** from ``emit``'s, and deliberately: ``emit`` raises only on
|
|
177
|
+
total failure because the worker retries a raised batch and a partial retry duplicates.
|
|
178
|
+
Nothing retries a flush, and the caller asked whether everything is out, so **any** child
|
|
179
|
+
that could not be drained makes the answer no.
|
|
180
|
+
|
|
181
|
+
Args:
|
|
182
|
+
None.
|
|
183
|
+
|
|
184
|
+
Returns:
|
|
185
|
+
None.
|
|
186
|
+
|
|
187
|
+
Raises:
|
|
188
|
+
Exception: The first child's exception, when any child could not be flushed.
|
|
189
|
+
"""
|
|
190
|
+
first_error: Exception | None = None
|
|
191
|
+
for sink in self._sinks:
|
|
192
|
+
try:
|
|
193
|
+
flush_sink(sink)
|
|
194
|
+
except Exception as err:
|
|
195
|
+
if first_error is None:
|
|
196
|
+
first_error = err
|
|
197
|
+
_diag.absorbed(
|
|
198
|
+
"flushing a MultiSink child", err, f"{type(sink).__name__} still holds events"
|
|
199
|
+
)
|
|
200
|
+
if first_error is not None:
|
|
201
|
+
raise first_error
|
|
202
|
+
|
|
158
203
|
def losses(self) -> SinkLosses | None:
|
|
159
204
|
"""Sums the children's losses so a fan-out reports the whole tree (SPEC-026 FR-002).
|
|
160
205
|
|
|
@@ -152,6 +152,53 @@ class NATSSink:
|
|
|
152
152
|
if batch and not published:
|
|
153
153
|
raise SinkDeliveryError(f"NATSSink published none of {len(batch)} event(s)")
|
|
154
154
|
|
|
155
|
+
def flush(self) -> None:
|
|
156
|
+
"""Pushes the client's outbound buffer onto the wire without closing (SPEC-036 FR-002).
|
|
157
|
+
|
|
158
|
+
Core ``publish()`` writes into the client's own outbound buffer and returns; the network
|
|
159
|
+
write happens on the driver's flusher task. That is why :meth:`close` drains, and why
|
|
160
|
+
``log_foundry.flush()`` could not reach a published-but-unwritten event before this hook.
|
|
161
|
+
Under JetStream ``publish()`` awaits an ack, so there is nothing pending and this costs a
|
|
162
|
+
round trip at worst.
|
|
163
|
+
|
|
164
|
+
Takes the same lock :meth:`emit` does, for the reason recorded there: an ``asyncio`` loop
|
|
165
|
+
is single-entry, and a second thread calling ``run_until_complete`` on a running loop can
|
|
166
|
+
leave a thread never returning at all.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
None.
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
None.
|
|
173
|
+
|
|
174
|
+
Raises:
|
|
175
|
+
SinkDeliveryError: The sink is closed.
|
|
176
|
+
Exception: Whatever the driver raises while flushing.
|
|
177
|
+
"""
|
|
178
|
+
with self._lock:
|
|
179
|
+
if self._loop.is_closed():
|
|
180
|
+
raise SinkDeliveryError("NATSSink cannot flush: the sink is closed")
|
|
181
|
+
self._loop.run_until_complete(self._flush_client())
|
|
182
|
+
|
|
183
|
+
async def _flush_client(self) -> None:
|
|
184
|
+
"""Flushes the client if the driver offers one, mirroring :meth:`_drain`'s probe.
|
|
185
|
+
|
|
186
|
+
Probed by name for the same reason ``drain`` is: the sink is written against a driver it
|
|
187
|
+
does not own, and a client without the method has nothing buffered to push.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
None.
|
|
191
|
+
|
|
192
|
+
Returns:
|
|
193
|
+
None.
|
|
194
|
+
|
|
195
|
+
Raises:
|
|
196
|
+
Exception: Whatever the driver raises while flushing.
|
|
197
|
+
"""
|
|
198
|
+
flush = getattr(self._client, "flush", None)
|
|
199
|
+
if flush is not None:
|
|
200
|
+
await flush()
|
|
201
|
+
|
|
155
202
|
async def _drain(self) -> None:
|
|
156
203
|
"""Drains the client if the driver offers a drain.
|
|
157
204
|
|
|
@@ -20,6 +20,10 @@ class NewRelicSink(HTTPSink):
|
|
|
20
20
|
maximum entry count, only a payload size.
|
|
21
21
|
MAX_BATCH_BYTES: 1,000,000 — the Log API's documented "1MB (10^6 bytes) maximum per POST".
|
|
22
22
|
Measured uncompressed here, which is the conservative reading when ``gzip=True``.
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
26
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
23
27
|
"""
|
|
24
28
|
|
|
25
29
|
MAX_BATCH_COUNT = 1000
|
|
@@ -19,6 +19,10 @@ class NullSink:
|
|
|
19
19
|
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
20
20
|
(SPEC-032 FR-003): there is no transport and ``close()`` releases nothing, so discarding a
|
|
21
21
|
batch after close is the same operation as discarding one before it.
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
It keeps **no** client buffer (SPEC-036 FR-002): there is no transport under it for a flush
|
|
25
|
+
to push anything onto.
|
|
22
26
|
"""
|
|
23
27
|
|
|
24
28
|
def __init__(self) -> None:
|
|
@@ -32,6 +32,11 @@ class PostgresSink:
|
|
|
32
32
|
``commit`` publishes the other's half-written batch, and its ``rollback`` on a failure
|
|
33
33
|
discards rows the other had already inserted and is about to report as delivered. A lock
|
|
34
34
|
gives the sequence the exclusivity it was written for.
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``emit`` commits its own transaction, so
|
|
38
|
+
nothing is uncommitted once it returns. The commit in ``close`` is belt and braces, not a
|
|
39
|
+
buffer.
|
|
35
40
|
"""
|
|
36
41
|
|
|
37
42
|
def __init__(
|
|
@@ -403,6 +403,79 @@ class GooglePubSubSink:
|
|
|
403
403
|
_diag.lost("event", 1, f"GooglePubSubSink publish unconfirmed, {type(err).__name__}")
|
|
404
404
|
return True
|
|
405
405
|
|
|
406
|
+
def flush(self) -> None:
|
|
407
|
+
"""Resolves the outstanding publish futures without closing the sink (SPEC-036 FR-002).
|
|
408
|
+
|
|
409
|
+
``emit`` appends an unresolved future and returns; before this hook existed nothing but
|
|
410
|
+
``close()`` ever called ``result()`` on them, so ``log_foundry.flush()`` could not reach a
|
|
411
|
+
single one — the call whose whole purpose is delivery before a freeze.
|
|
412
|
+
|
|
413
|
+
**It is :meth:`_await_overflow` applied to the whole pending list, and the three rules it
|
|
414
|
+
obeys are that method's, each earned by a measured defect.** One ``deadline`` covers the
|
|
415
|
+
list rather than a timeout per future: at the shipped ``max_pending`` a per-future wait is
|
|
416
|
+
not a bound at all, and a stalled destination would hold ``log_foundry.flush()`` for
|
|
417
|
+
hours. ``_Unboundable`` is **caught**, because a future whose ``result()`` takes no
|
|
418
|
+
``timeout`` cannot be waited on within one — and letting it escape here would abandon the
|
|
419
|
+
entire list, which has already been swapped out and is referenced by nothing else. And
|
|
420
|
+
``_futures_lock`` is **not** held across a ``result()``, because ``emit`` takes it per
|
|
421
|
+
event and an application thread on the orphan path would block behind it.
|
|
422
|
+
|
|
423
|
+
A future that did not settle is put back rather than dropped: the sink stays open, so it
|
|
424
|
+
is unfinished, not unconfirmed, and the next flush or the close waits on it again. It
|
|
425
|
+
**raises** when any remained, which is what makes ``log_foundry.flush()`` report
|
|
426
|
+
``reason="sink-flush"``. A future that settled *failed* is already counted by
|
|
427
|
+
:meth:`_resolve` and reported through ``losses()``, per SPEC-026.
|
|
428
|
+
|
|
429
|
+
Args:
|
|
430
|
+
None.
|
|
431
|
+
|
|
432
|
+
Returns:
|
|
433
|
+
None.
|
|
434
|
+
|
|
435
|
+
Raises:
|
|
436
|
+
SinkDeliveryError: The sink is closed, or a publish was still in flight afterwards.
|
|
437
|
+
"""
|
|
438
|
+
if self._closed:
|
|
439
|
+
raise SinkDeliveryError("GooglePubSubSink cannot flush: the sink is closed")
|
|
440
|
+
with self._futures_lock:
|
|
441
|
+
pending, self._futures = self._futures, []
|
|
442
|
+
if not pending:
|
|
443
|
+
return
|
|
444
|
+
|
|
445
|
+
deadline = time.monotonic() + self.overflow_timeout
|
|
446
|
+
unresolved: list[Any] = []
|
|
447
|
+
for index, future in enumerate(pending):
|
|
448
|
+
settled = False
|
|
449
|
+
while not self._out_of_time(deadline):
|
|
450
|
+
began = time.monotonic()
|
|
451
|
+
slice_ = min(deadline - began, _POLL_INTERVAL)
|
|
452
|
+
try:
|
|
453
|
+
settled = self._resolve(future, slice_)
|
|
454
|
+
except _Unboundable:
|
|
455
|
+
break
|
|
456
|
+
if settled:
|
|
457
|
+
break
|
|
458
|
+
wait(slice_ - (time.monotonic() - began), self.log_foundry_stop_signal)
|
|
459
|
+
if settled:
|
|
460
|
+
continue
|
|
461
|
+
unresolved.append(future)
|
|
462
|
+
if self._out_of_time(deadline):
|
|
463
|
+
unresolved.extend(pending[index + 1 :])
|
|
464
|
+
break
|
|
465
|
+
|
|
466
|
+
if not unresolved:
|
|
467
|
+
return
|
|
468
|
+
with self._futures_lock:
|
|
469
|
+
closed = self._closed
|
|
470
|
+
if not closed:
|
|
471
|
+
self._futures[:0] = unresolved
|
|
472
|
+
if closed:
|
|
473
|
+
for future in unresolved:
|
|
474
|
+
self._resolve(future)
|
|
475
|
+
raise SinkDeliveryError(
|
|
476
|
+
f"GooglePubSubSink flushed with {len(unresolved)} publish(es) still in flight"
|
|
477
|
+
)
|
|
478
|
+
|
|
406
479
|
def close(self) -> None:
|
|
407
480
|
"""Resolves all pending publish futures, counting and logging errors (FR-008).
|
|
408
481
|
|
|
@@ -33,6 +33,12 @@ class RabbitMQSink:
|
|
|
33
33
|
event is published as a persistent message to the configured exchange and routing key, and a
|
|
34
34
|
dropped or closed connection is re-established within a bounded retry. The worst-case delay
|
|
35
35
|
(SPEC-027 FR-005) is ``max_retries`` interruptible waits per message, 0.7 s at the defaults.
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``basic_publish`` writes the frame
|
|
39
|
+
before it returns, so nothing is queued locally between emits. This sink does not enable
|
|
40
|
+
publisher confirms, so "written" is not "acknowledged" — but that is a delivery-guarantee
|
|
41
|
+
question, not a buffer a flush could empty.
|
|
36
42
|
"""
|
|
37
43
|
|
|
38
44
|
def __init__(
|
|
@@ -34,6 +34,10 @@ class _RedisSink:
|
|
|
34
34
|
command, so a batch emitted after ``shutdown()`` opened a connection nothing would ever
|
|
35
35
|
reap — the same leak SPEC-028's review found in ``RabbitMQSink``, whose ``_active_channel``
|
|
36
36
|
reopened whatever ``close()`` had released.
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
40
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
37
41
|
"""
|
|
38
42
|
|
|
39
43
|
def __init__(self, *, client: Any, url: str | None, max_retries: int) -> None:
|
|
@@ -172,7 +176,11 @@ class _RedisSink:
|
|
|
172
176
|
|
|
173
177
|
|
|
174
178
|
class RedisStreamsSink(_RedisSink):
|
|
175
|
-
"""Appends each event to a Redis stream via ``XADD``, pipelined per batch (FR-005).
|
|
179
|
+
"""Appends each event to a Redis stream via ``XADD``, pipelined per batch (FR-005).
|
|
180
|
+
|
|
181
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
182
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
183
|
+
"""
|
|
176
184
|
|
|
177
185
|
def __init__(
|
|
178
186
|
self,
|
|
@@ -235,7 +243,11 @@ class RedisStreamsSink(_RedisSink):
|
|
|
235
243
|
|
|
236
244
|
|
|
237
245
|
class RedisListSink(_RedisSink):
|
|
238
|
-
"""Pushes each event onto a Redis list via ``RPUSH``, pipelined per batch (FR-005).
|
|
246
|
+
"""Pushes each event onto a Redis list via ``RPUSH``, pipelined per batch (FR-005).
|
|
247
|
+
|
|
248
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
249
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
250
|
+
"""
|
|
239
251
|
|
|
240
252
|
def __init__(
|
|
241
253
|
self,
|
|
@@ -122,6 +122,45 @@ class SentrySink:
|
|
|
122
122
|
f"SentrySink delivered none of {attempted} qualifying event(s)"
|
|
123
123
|
)
|
|
124
124
|
|
|
125
|
+
def flush(self) -> None:
|
|
126
|
+
"""Pushes the Sentry SDK's own transport queue, which nothing else here ever does.
|
|
127
|
+
|
|
128
|
+
SPEC-036 FR-002, and a case SPEC-042's measured roster of five did not reach: that roster
|
|
129
|
+
was derived from what a *refused close* costs, and :meth:`close` releases nothing here, so
|
|
130
|
+
this sink never appeared in it. ``capture_event`` hands to the SDK's **background
|
|
131
|
+
transport** and returns, so without this hook an event accepted by Sentry's client was
|
|
132
|
+
unreachable through ``log_foundry.flush()`` and went out only when the SDK's own timer or
|
|
133
|
+
interpreter exit got to it.
|
|
134
|
+
|
|
135
|
+
Only the injected-or-imported SDK client has a queue. The ``urllib`` fallback posts an
|
|
136
|
+
envelope per event and holds nothing, so with no client this is correctly a no-op.
|
|
137
|
+
|
|
138
|
+
``Client.flush`` is probed by name, as every optional member the library calls on an object
|
|
139
|
+
it does not own is: a stand-in ``client=`` satisfying only ``capture_event`` stays valid,
|
|
140
|
+
which is what the injected-client tests use.
|
|
141
|
+
|
|
142
|
+
**It cannot report a failure, and that is the SDK's shape rather than a choice here.**
|
|
143
|
+
``sentry_sdk.Client.flush`` logs a warning and returns ``None`` when its own timeout
|
|
144
|
+
expires, so a queue the SDK just gave up on is indistinguishable from one it drained, and
|
|
145
|
+
``log_foundry.flush()`` reports success either way. Recorded rather than worked around: the
|
|
146
|
+
alternatives are reading a private attribute or timing the call, and both would invent a
|
|
147
|
+
verdict the SDK declines to give.
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
None.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
None.
|
|
154
|
+
|
|
155
|
+
Raises:
|
|
156
|
+
Exception: Whatever the SDK raises while flushing.
|
|
157
|
+
"""
|
|
158
|
+
if self.client is None:
|
|
159
|
+
return
|
|
160
|
+
flush = getattr(self.client, "flush", None)
|
|
161
|
+
if callable(flush):
|
|
162
|
+
flush()
|
|
163
|
+
|
|
125
164
|
def close(self) -> None:
|
|
126
165
|
"""Forwards to the HTTP fallback, whose own close releases nothing (FR-012).
|
|
127
166
|
|
|
@@ -33,6 +33,10 @@ class SNSSink:
|
|
|
33
33
|
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
34
34
|
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
35
35
|
afterwards still reaches the topic.
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
39
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
36
40
|
"""
|
|
37
41
|
|
|
38
42
|
MAX_BATCH = 10
|
|
@@ -28,6 +28,10 @@ class SplunkHECSink(HTTPSink):
|
|
|
28
28
|
it is ``max_content_length`` on the receiving instance, so there is no vendor figure to
|
|
29
29
|
cite and the default is chosen rather than documented. Raise it with
|
|
30
30
|
``max_batch_bytes=`` to match your deployment.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` is a request that has
|
|
34
|
+
completed by the time it returns, and no client object outlives it holding data.
|
|
31
35
|
"""
|
|
32
36
|
|
|
33
37
|
MAX_BATCH_COUNT = 1000
|
|
@@ -28,6 +28,11 @@ class SQLiteSink:
|
|
|
28
28
|
implicit transaction, so one thread's rollback discards rows the other had already inserted.
|
|
29
29
|
A lock restores the one-writer-at-a-time the guard used to enforce. Cross-*process* writers
|
|
30
30
|
to one database file remain out of scope.
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``emit`` commits its own transaction, so
|
|
34
|
+
nothing is uncommitted once it returns. The commit in ``close`` is belt and braces, not a
|
|
35
|
+
buffer.
|
|
31
36
|
"""
|
|
32
37
|
|
|
33
38
|
def __init__(
|
|
@@ -95,6 +95,10 @@ class SQSSink:
|
|
|
95
95
|
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
96
96
|
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
97
97
|
afterwards still reaches the queue.
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
It keeps **no** client buffer (SPEC-036 FR-002): the driver call returns only once the
|
|
101
|
+
destination has the batch, so nothing is queued locally between emits.
|
|
98
102
|
"""
|
|
99
103
|
|
|
100
104
|
MAX_BATCH = 10
|
|
@@ -26,6 +26,10 @@ class StdoutSink:
|
|
|
26
26
|
|
|
27
27
|
It **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` only flushes — the
|
|
28
28
|
stream belongs to the process, not to this sink, so a later batch still lands.
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``emit`` flushes the stream before it
|
|
32
|
+
returns, so nothing of this sink's is left pending between calls.
|
|
29
33
|
"""
|
|
30
34
|
|
|
31
35
|
def __init__(self, stream: TextIO | None = None) -> None:
|
|
@@ -84,6 +88,10 @@ class StderrSink(StdoutSink):
|
|
|
84
88
|
It writes each event as one ``json.dumps`` line and flushes, exactly like ``StdoutSink`` —
|
|
85
89
|
only the default stream differs, following the twelve-factor convention of logs on stderr
|
|
86
90
|
and app output on stdout.
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
It keeps **no** client buffer (SPEC-036 FR-002): ``emit`` flushes the stream before it
|
|
94
|
+
returns, so nothing of this sink's is left pending between calls.
|
|
87
95
|
"""
|
|
88
96
|
|
|
89
97
|
def __init__(self, stream: TextIO | None = None) -> None:
|
|
@@ -51,6 +51,11 @@ class SyslogSink:
|
|
|
51
51
|
post-close refusal comes from there too (SPEC-032 FR-004) — a batch emitted after
|
|
52
52
|
``close()`` reaches ``send_all`` and is refused with ``SinkDeliveryError`` without the
|
|
53
53
|
socket being reopened, so a guard here would only duplicate one that already holds.
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
It keeps **no** client buffer (SPEC-036 FR-002): each ``emit`` hands its bytes to the
|
|
57
|
+
socket before it returns, and no client object outlives it holding data. What the *network*
|
|
58
|
+
then does with a datagram is not something a flush could hurry.
|
|
54
59
|
"""
|
|
55
60
|
|
|
56
61
|
def __init__(
|
|
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
from typing import TYPE_CHECKING
|
|
6
6
|
|
|
7
7
|
from log_foundry import _diag, _lifecycle
|
|
8
|
-
from log_foundry.sinks.base import read_losses
|
|
8
|
+
from log_foundry.sinks.base import flush_sink, read_losses
|
|
9
9
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
11
11
|
import threading
|
|
@@ -27,6 +27,13 @@ class TransformSink:
|
|
|
27
27
|
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
28
28
|
(SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
|
|
29
29
|
belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
It holds **no** client buffer of its own, but it **forwards** ``flush()`` to what it wraps
|
|
33
|
+
(SPEC-036 FR-002). Holding nothing is not the same as having nothing to do: a wrapper that
|
|
34
|
+
did not forward would leave a buffering child unreachable through ``log_foundry.flush()``
|
|
35
|
+
while looking fine — the SPEC-027 lesson about ``log_foundry_stop_signal``, that a signal
|
|
36
|
+
stopped at a wrapper reaches nothing and moves the defect rather than fixing it.
|
|
30
37
|
"""
|
|
31
38
|
|
|
32
39
|
def __init__(
|
|
@@ -115,6 +122,25 @@ class TransformSink:
|
|
|
115
122
|
f"{type(self._inner).__name__} stays uninterruptible",
|
|
116
123
|
)
|
|
117
124
|
|
|
125
|
+
def flush(self) -> None:
|
|
126
|
+
"""Forwards the flush to the wrapped sink (SPEC-036 FR-002).
|
|
127
|
+
|
|
128
|
+
This wrapper holds nothing itself, but a wrapper that did not forward would leave a
|
|
129
|
+
buffering inner sink unreachable through ``log_foundry.flush()`` while looking fine —
|
|
130
|
+
the SPEC-027 lesson that a signal stopped at a wrapper reaches nothing.
|
|
131
|
+
|
|
132
|
+
Args:
|
|
133
|
+
None.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
None.
|
|
137
|
+
|
|
138
|
+
Raises:
|
|
139
|
+
Exception: Whatever the wrapped sink raises, so the failure reaches the caller as a
|
|
140
|
+
``FlushResult`` reason rather than being swallowed here.
|
|
141
|
+
"""
|
|
142
|
+
flush_sink(self._inner)
|
|
143
|
+
|
|
118
144
|
def losses(self) -> SinkLosses | None:
|
|
119
145
|
"""Reports the inner sink's losses (SPEC-026 FR-002).
|
|
120
146
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|