log-foundry 0.10.2.dev26__tar.gz → 0.10.2.dev28__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/PKG-INFO +21 -2
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/README.md +20 -1
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/base.py +23 -1
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/callback.py +5 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/datadog.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/elasticsearch.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/filtering.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/firehose.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/honeycomb.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/http.py +7 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/kafka.py +42 -3
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/kinesis.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/logging_sink.py +5 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/logstash.py +8 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/loki.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/multi.py +7 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/pubsub.py +53 -5
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/redis.py +35 -6
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sentry.py +8 -2
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sns.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/splunk.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sqs.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/stdout.py +12 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/syslog.py +6 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/transform.py +4 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/util.py +8 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/LICENSE +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/__init__.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/config.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/decorator.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_retry.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_socket.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/clickhouse.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/eventhubs.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/file.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/mongodb.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/nats.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/postgres.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/rabbitmq.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sqlite.py +0 -0
- {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: log-foundry
|
|
3
|
-
Version: 0.10.2.
|
|
3
|
+
Version: 0.10.2.dev28
|
|
4
4
|
Summary: Generate logs for your console and JSON events for downstream consumption.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -696,7 +696,7 @@ in `CallbackSink`.
|
|
|
696
696
|
|
|
697
697
|
#### Writing your own sink
|
|
698
698
|
|
|
699
|
-
`Sink` is two required methods, `emit(batch)` and `close()`, plus
|
|
699
|
+
`Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
|
|
700
700
|
fails and one about *when* it is called. They are not stylistic — the library's whole
|
|
701
701
|
loss-reporting apparatus is built on them:
|
|
702
702
|
|
|
@@ -716,6 +716,15 @@ loss-reporting apparatus is built on them:
|
|
|
716
716
|
- **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
|
|
717
717
|
partial success re-delivers the records that already arrived, and duplicates downstream are worse
|
|
718
718
|
than a counted loss.
|
|
719
|
+
- **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
|
|
720
|
+
released transport has delivered nothing, so it is the first rule again by another route — and
|
|
721
|
+
it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
|
|
722
|
+
wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
|
|
723
|
+
one appended a delivery future nothing would resolve, and one transparently *reconnected*,
|
|
724
|
+
leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
|
|
725
|
+
If your `close()` releases nothing — you open a fresh connection per request, or the client is
|
|
726
|
+
the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
|
|
727
|
+
invented. `emit([])` stays a no-op either way.
|
|
719
728
|
|
|
720
729
|
A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
|
|
721
730
|
never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
|
|
@@ -732,12 +741,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
|
732
741
|
class MySink:
|
|
733
742
|
def __init__(self) -> None:
|
|
734
743
|
self._dropped = self._failed = 0
|
|
744
|
+
self._closed = False
|
|
735
745
|
self._lock = threading.Lock() # transport state
|
|
736
746
|
self._counter_lock = threading.Lock() # counters only, never held across I/O
|
|
737
747
|
|
|
738
748
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
749
|
+
if not batch:
|
|
750
|
+
return
|
|
739
751
|
delivered = 0
|
|
740
752
|
with self._lock: # your connection, socket or stream
|
|
753
|
+
if self._closed: # refuse: nothing here can deliver it
|
|
754
|
+
raise SinkDeliveryError(
|
|
755
|
+
f"MySink delivered none of {len(batch)} event(s): the sink is closed"
|
|
756
|
+
)
|
|
741
757
|
for chunk in self._chunks(batch):
|
|
742
758
|
if self._send(chunk): # your own bounded retry
|
|
743
759
|
delivered += len(chunk)
|
|
@@ -753,6 +769,9 @@ class MySink:
|
|
|
753
769
|
|
|
754
770
|
def close(self) -> None:
|
|
755
771
|
with self._lock: # never release under an active writer
|
|
772
|
+
if self._closed: # idempotent: atexit races your own cleanup
|
|
773
|
+
return
|
|
774
|
+
self._closed = True # set the flag, *then* release
|
|
756
775
|
...
|
|
757
776
|
```
|
|
758
777
|
|
|
@@ -660,7 +660,7 @@ in `CallbackSink`.
|
|
|
660
660
|
|
|
661
661
|
#### Writing your own sink
|
|
662
662
|
|
|
663
|
-
`Sink` is two required methods, `emit(batch)` and `close()`, plus
|
|
663
|
+
`Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
|
|
664
664
|
fails and one about *when* it is called. They are not stylistic — the library's whole
|
|
665
665
|
loss-reporting apparatus is built on them:
|
|
666
666
|
|
|
@@ -680,6 +680,15 @@ loss-reporting apparatus is built on them:
|
|
|
680
680
|
- **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
|
|
681
681
|
partial success re-delivers the records that already arrived, and duplicates downstream are worse
|
|
682
682
|
than a counted loss.
|
|
683
|
+
- **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
|
|
684
|
+
released transport has delivered nothing, so it is the first rule again by another route — and
|
|
685
|
+
it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
|
|
686
|
+
wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
|
|
687
|
+
one appended a delivery future nothing would resolve, and one transparently *reconnected*,
|
|
688
|
+
leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
|
|
689
|
+
If your `close()` releases nothing — you open a fresh connection per request, or the client is
|
|
690
|
+
the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
|
|
691
|
+
invented. `emit([])` stays a no-op either way.
|
|
683
692
|
|
|
684
693
|
A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
|
|
685
694
|
never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
|
|
@@ -696,12 +705,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
|
696
705
|
class MySink:
|
|
697
706
|
def __init__(self) -> None:
|
|
698
707
|
self._dropped = self._failed = 0
|
|
708
|
+
self._closed = False
|
|
699
709
|
self._lock = threading.Lock() # transport state
|
|
700
710
|
self._counter_lock = threading.Lock() # counters only, never held across I/O
|
|
701
711
|
|
|
702
712
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
713
|
+
if not batch:
|
|
714
|
+
return
|
|
703
715
|
delivered = 0
|
|
704
716
|
with self._lock: # your connection, socket or stream
|
|
717
|
+
if self._closed: # refuse: nothing here can deliver it
|
|
718
|
+
raise SinkDeliveryError(
|
|
719
|
+
f"MySink delivered none of {len(batch)} event(s): the sink is closed"
|
|
720
|
+
)
|
|
705
721
|
for chunk in self._chunks(batch):
|
|
706
722
|
if self._send(chunk): # your own bounded retry
|
|
707
723
|
delivered += len(chunk)
|
|
@@ -717,6 +733,9 @@ class MySink:
|
|
|
717
733
|
|
|
718
734
|
def close(self) -> None:
|
|
719
735
|
with self._lock: # never release under an active writer
|
|
736
|
+
if self._closed: # idempotent: atexit races your own cleanup
|
|
737
|
+
return
|
|
738
|
+
self._closed = True # set the flag, *then* release
|
|
720
739
|
...
|
|
721
740
|
```
|
|
722
741
|
|
|
@@ -94,9 +94,25 @@ class Sink(Protocol):
|
|
|
94
94
|
re-delivered wholesale by the worker's retry, and duplicates downstream are worse than
|
|
95
95
|
the counted loss (SPEC-017 FR-004, SPEC-018). Report that through ``losses()`` instead.
|
|
96
96
|
|
|
97
|
+
**After ``close()`` has released or invalidated something, raise rather than absorb**
|
|
98
|
+
(SPEC-032 FR-002). This is the same obligation applied to the sink's own lifecycle
|
|
99
|
+
rather than the destination's: an absorbed batch is one the worker believes, so the
|
|
100
|
+
retry never engages, ``failed_batches`` stays at zero and ``flush()`` returns True while
|
|
101
|
+
the events are gone — reached not through a destination that is down but through a
|
|
102
|
+
transport the library itself has already let go. Three shipped sinks failed this rule
|
|
103
|
+
for four specs, in three different ways: a produce into a batch nothing would flush
|
|
104
|
+
again, a future nothing would resolve, and a client that quietly reconnected.
|
|
105
|
+
|
|
106
|
+
The converse is equally binding: a sink holding nothing to release **keeps accepting**.
|
|
107
|
+
``close()`` on a sink that opens a fresh connection per request, or whose client belongs
|
|
108
|
+
to the caller, has invalidated nothing, so a later batch still delivers and refusing it
|
|
109
|
+
would be loss the library invented rather than loss it reported. Which of the two
|
|
110
|
+
applies is a property of the sink, so each shipped sink records its answer in its class
|
|
111
|
+
docstring and a test holds it to it.
|
|
112
|
+
|
|
97
113
|
Args:
|
|
98
114
|
batch: The events to ship. ``emit([])`` is a no-op and never raises, since an empty
|
|
99
|
-
batch has not failed to deliver.
|
|
115
|
+
batch has not failed to deliver — closed or not.
|
|
100
116
|
|
|
101
117
|
Returns:
|
|
102
118
|
None.
|
|
@@ -116,6 +132,12 @@ class Sink(Protocol):
|
|
|
116
132
|
succeeding. Taking the same lock ``emit`` takes satisfies this; so does an idempotent
|
|
117
133
|
guard checked under that lock.
|
|
118
134
|
|
|
135
|
+
The flag this sets is the one a later :meth:`emit` reads, so the two are one decision
|
|
136
|
+
rather than two (SPEC-032 FR-002). Set it *before* releasing anything: an emit arriving
|
|
137
|
+
during a release must be refused rather than handed a half-released transport. Make the
|
|
138
|
+
release idempotent for the same reason a second ``close()`` is expected at all —
|
|
139
|
+
``atexit`` racing a caller's own cleanup is the documented case.
|
|
140
|
+
|
|
119
141
|
Args:
|
|
120
142
|
None.
|
|
121
143
|
|
|
@@ -17,6 +17,11 @@ class CallbackSink:
|
|
|
17
17
|
function, without writing a ``Sink`` implementation. Attributes are internal; the observable
|
|
18
18
|
contract is that ``emit`` hands the batch to the callable unchanged and ``close`` invokes
|
|
19
19
|
the close hook once when one was supplied.
|
|
20
|
+
|
|
21
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
22
|
+
(SPEC-032 FR-003). Both decisions belong to the callable: this class holds nothing, and what
|
|
23
|
+
a hook releases is not knowable from here — a callable needing either guarantee must provide
|
|
24
|
+
it, exactly as a hand-written ``Sink`` implementation would.
|
|
20
25
|
"""
|
|
21
26
|
|
|
22
27
|
def __init__(
|
|
@@ -16,6 +16,10 @@ class DatadogSink(HTTPSink):
|
|
|
16
16
|
|
|
17
17
|
The batch goes as a JSON array to the region-specific intake with a ``DD-API-KEY`` header,
|
|
18
18
|
each entry enriched with ``ddsource``, ``service`` and ``ddtags``.
|
|
19
|
+
|
|
20
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
21
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
22
|
+
is no transport held and ``close()`` releases nothing.
|
|
19
23
|
"""
|
|
20
24
|
|
|
21
25
|
def __init__(
|
{log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
@@ -25,6 +25,10 @@ class ElasticsearchSink(HTTPSink):
|
|
|
25
25
|
dropped_unadjudicated: Events whose outcome a ``_bulk`` response did not describe, because
|
|
26
26
|
its ``items`` array did not line up with the batch sent. Abandoned rather than retried,
|
|
27
27
|
for SPEC-018's reason: the request succeeded, so re-sending would duplicate what landed.
|
|
28
|
+
|
|
29
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
30
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
31
|
+
is no transport held and ``close()`` releases nothing.
|
|
28
32
|
"""
|
|
29
33
|
|
|
30
34
|
def __init__(self, url: str, *, index: str, auth: str | tuple[str, str] | None = None,
|
|
@@ -24,6 +24,10 @@ class FilteringSink:
|
|
|
24
24
|
This is a static, emit-time filter in front of an inner sink, not the reserved
|
|
25
25
|
tail-sampling ``should_send`` seam (arch §10) — that stays deferred and owns rate policy at
|
|
26
26
|
span-decision time, while this only reshapes an already-built batch on its way to a sink.
|
|
27
|
+
|
|
28
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
29
|
+
(SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
|
|
30
|
+
belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
|
|
27
31
|
"""
|
|
28
32
|
|
|
29
33
|
def __init__(
|
|
@@ -36,6 +36,10 @@ class FirehoseSink:
|
|
|
36
36
|
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
37
37
|
lock. ``boto3`` clients are documented thread-safe, this one is built once in
|
|
38
38
|
``__init__``, and the sink rebinds nothing after construction.
|
|
39
|
+
|
|
40
|
+
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
41
|
+
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
42
|
+
afterwards still reaches the delivery stream.
|
|
39
43
|
"""
|
|
40
44
|
|
|
41
45
|
MAX_RECORDS = 500
|
|
@@ -14,6 +14,10 @@ class HoneycombSink(HTTPSink):
|
|
|
14
14
|
|
|
15
15
|
The request goes to ``/1/batch/<dataset>`` with an ``X-Honeycomb-Team`` header, in
|
|
16
16
|
Honeycomb's ``[{"data": <event>}, ...]`` batch shape.
|
|
17
|
+
|
|
18
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
19
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
20
|
+
is no transport held and ``close()`` releases nothing.
|
|
17
21
|
"""
|
|
18
22
|
|
|
19
23
|
def __init__(
|
|
@@ -74,6 +74,13 @@ class HTTPSink:
|
|
|
74
74
|
failed: Requests abandoned past the retry bound.
|
|
75
75
|
dropped_oversized: Events dropped for exceeding a destination's hard size limit, used by
|
|
76
76
|
subclasses that enforce one; the generic core imposes no universal limit.
|
|
77
|
+
|
|
78
|
+
The two lifecycle decisions this class records, inherited by every platform subclass. It
|
|
79
|
+
takes **no** transport lock (SPEC-028 FR-002): there is no transport to guard, since
|
|
80
|
+
``urllib`` opens a fresh connection per request and nothing is rebound after construction.
|
|
81
|
+
And it **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` releases nothing —
|
|
82
|
+
a batch emitted afterwards still reaches the endpoint, and refusing it would be loss the
|
|
83
|
+
library invented rather than loss it reported.
|
|
77
84
|
"""
|
|
78
85
|
|
|
79
86
|
def __init__(
|
|
@@ -28,6 +28,12 @@ class KafkaSink:
|
|
|
28
28
|
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
29
29
|
lock. ``confluent-kafka`` documents its ``Producer`` as thread-safe, and this sink adds no state
|
|
30
30
|
of its own to guard — ``produce()`` is a local hand-off into the client's internal queue.
|
|
31
|
+
|
|
32
|
+
It refuses an emit after :meth:`close` (SPEC-032 FR-001), which is not the housekeeping it
|
|
33
|
+
looks like: ``flush()`` is the only thing that drains the producer's local batch and services
|
|
34
|
+
its delivery callbacks, so a message produced after it dies with the process — accepted,
|
|
35
|
+
uncounted, and never the subject of a callback. ``emit`` returning normally is what made that
|
|
36
|
+
silent, since the worker then never retried and ``flush()`` reported success.
|
|
31
37
|
"""
|
|
32
38
|
|
|
33
39
|
def __init__(
|
|
@@ -67,6 +73,8 @@ class KafkaSink:
|
|
|
67
73
|
self.failed = 0
|
|
68
74
|
self.rejected = 0
|
|
69
75
|
self._counter_lock = threading.Lock()
|
|
76
|
+
self._closed = False
|
|
77
|
+
self._close_lock = threading.Lock()
|
|
70
78
|
|
|
71
79
|
def losses(self) -> SinkLosses:
|
|
72
80
|
"""Reports refused and undelivered messages (SPEC-026 FR-002).
|
|
@@ -98,6 +106,15 @@ class KafkaSink:
|
|
|
98
106
|
observe (FR-002). A partial refusal is counted and left alone, since the messages that
|
|
99
107
|
were accepted are already on their way.
|
|
100
108
|
|
|
109
|
+
A closed sink refuses the batch before touching the producer (SPEC-032 FR-001). Refusing
|
|
110
|
+
is not a loss this sink absorbed, so it moves no counter here: it is a failure reported
|
|
111
|
+
to the worker, which records it in ``health().failed_batches``. The check is an unlocked
|
|
112
|
+
read of a write-once flag rather than a lock held across the produce loop, which would
|
|
113
|
+
serialize a client ``confluent-kafka`` documents as thread-safe. A close landing between
|
|
114
|
+
the check and the produce therefore still loses that message, exactly as ``MongoDBSink``
|
|
115
|
+
documents for its own check; what the flag ends is the far larger case of a sink closed
|
|
116
|
+
long before, which is every log written after ``shutdown()``.
|
|
117
|
+
|
|
101
118
|
Args:
|
|
102
119
|
batch: The events to produce.
|
|
103
120
|
|
|
@@ -105,9 +122,16 @@ class KafkaSink:
|
|
|
105
122
|
None.
|
|
106
123
|
|
|
107
124
|
Raises:
|
|
108
|
-
SinkDeliveryError: When every message was refused
|
|
109
|
-
the total failure the worker's retry exists for
|
|
125
|
+
SinkDeliveryError: When the sink is closed, or when every message was refused so the
|
|
126
|
+
batch reached nothing — the total failure the worker's retry exists for
|
|
127
|
+
(SPEC-026 FR-001).
|
|
110
128
|
"""
|
|
129
|
+
if not batch:
|
|
130
|
+
return
|
|
131
|
+
if self._closed:
|
|
132
|
+
raise SinkDeliveryError(
|
|
133
|
+
f"KafkaSink produced none of {len(batch)} message(s): the sink is closed"
|
|
134
|
+
)
|
|
111
135
|
accepted = 0
|
|
112
136
|
for event in batch:
|
|
113
137
|
body = json.dumps(event).encode("utf-8")
|
|
@@ -127,6 +151,17 @@ class KafkaSink:
|
|
|
127
151
|
def close(self) -> None:
|
|
128
152
|
"""Flushes the producer so buffered messages are delivered before exit (FR-002).
|
|
129
153
|
|
|
154
|
+
Idempotent, and the lock is held *across* the flush rather than only around the flag, as
|
|
155
|
+
every other guarded sink holds its own (SPEC-032 FR-001). Releasing it early would let a
|
|
156
|
+
second concurrent ``close()`` return while the first is still draining, so a caller would
|
|
157
|
+
believe the producer was flushed when it was not — and ``atexit`` racing user cleanup is
|
|
158
|
+
the documented case for a second call. The cost is that a second close waits on an
|
|
159
|
+
unreachable broker, which is the ``architecture.md`` §13 constraint that already applies
|
|
160
|
+
to every sink here.
|
|
161
|
+
|
|
162
|
+
The flag is set *before* the flush rather than after, so an emit arriving during a flush
|
|
163
|
+
is refused rather than producing into a batch the flush has already walked past.
|
|
164
|
+
|
|
130
165
|
Args:
|
|
131
166
|
None.
|
|
132
167
|
|
|
@@ -136,7 +171,11 @@ class KafkaSink:
|
|
|
136
171
|
Raises:
|
|
137
172
|
Exception: Whatever the producer raises on flush.
|
|
138
173
|
"""
|
|
139
|
-
self.
|
|
174
|
+
with self._close_lock:
|
|
175
|
+
if self._closed:
|
|
176
|
+
return
|
|
177
|
+
self._closed = True
|
|
178
|
+
self.producer.flush()
|
|
140
179
|
|
|
141
180
|
def _key(self, event: dict[str, object]) -> bytes | None:
|
|
142
181
|
"""Derives one message's partition key from the configured field.
|
|
@@ -36,6 +36,10 @@ class KinesisSink:
|
|
|
36
36
|
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
37
37
|
lock. ``boto3`` clients are documented thread-safe, this one is built once in
|
|
38
38
|
``__init__``, and the sink rebinds nothing after construction.
|
|
39
|
+
|
|
40
|
+
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
41
|
+
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
42
|
+
afterwards still reaches the stream.
|
|
39
43
|
"""
|
|
40
44
|
|
|
41
45
|
MAX_RECORDS = 500
|
|
@@ -36,6 +36,11 @@ class LoggingSink:
|
|
|
36
36
|
no new dependency. The sink only emits into that pipeline: it never configures loggers,
|
|
37
37
|
handlers or formatters, and never tears the framework down. The module is named
|
|
38
38
|
``logging_sink`` so it never shadows the stdlib module.
|
|
39
|
+
|
|
40
|
+
It takes **no** transport lock (SPEC-028 FR-002): ``logging`` serializes its own handlers,
|
|
41
|
+
and this sink holds nothing else. And it **adds no post-close guard** (SPEC-032 FR-003) —
|
|
42
|
+
``close()`` is a no-op by design, since tearing down handlers this sink did not configure is
|
|
43
|
+
not its to do, so a later batch still reaches the framework.
|
|
39
44
|
"""
|
|
40
45
|
|
|
41
46
|
def __init__(
|
|
@@ -23,6 +23,14 @@ class LogstashSink:
|
|
|
23
23
|
one newline-terminated line over a raw TCP or UDP socket, reusing
|
|
24
24
|
:class:`~log_foundry.sinks._socket.SocketTransport`. Either backend handles its own bounded
|
|
25
25
|
retry and raises on total failure of its own accord, so ``emit`` needs no rule of its own.
|
|
26
|
+
|
|
27
|
+
It takes **no** transport lock (SPEC-028 FR-002) of its own: whichever backend it built owns
|
|
28
|
+
that decision — ``SocketTransport`` locks its sends, ``HTTPSink`` holds no transport to
|
|
29
|
+
guard. The post-close rule follows the same split (SPEC-032 FR-004), and the two modes
|
|
30
|
+
genuinely differ: in socket mode a batch after ``close()`` is refused with
|
|
31
|
+
``SinkDeliveryError`` and reopens nothing, while in HTTP mode ``close()`` released nothing
|
|
32
|
+
and the batch still ships. Both are the backend's answer, correctly, rather than one this
|
|
33
|
+
class invents on top.
|
|
26
34
|
"""
|
|
27
35
|
|
|
28
36
|
def __init__(
|
|
@@ -18,6 +18,10 @@ class LokiSink(HTTPSink):
|
|
|
18
18
|
Events are grouped into streams by a configurable set of label keys, each stream carrying
|
|
19
19
|
values of ``[<nanosecond_timestamp_str>, <log_line>]``. Timestamps are derived by parsing the
|
|
20
20
|
event's ISO-8601 ``timestamp``, falling back to emit-time now.
|
|
21
|
+
|
|
22
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
23
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
24
|
+
is no transport held and ``close()`` releases nothing.
|
|
21
25
|
"""
|
|
22
26
|
|
|
23
27
|
def __init__(self, url: str, *, labels: tuple[str, ...] = ("service", "env", "level"),
|
|
@@ -29,6 +29,13 @@ class MultiSink:
|
|
|
29
29
|
batch, so one batch against an all-down fan-out of *n* children increments this by *n*
|
|
30
30
|
per attempt, with a stderr line each. That is the visible cost of the loss no longer
|
|
31
31
|
being silent.
|
|
32
|
+
|
|
33
|
+
It takes **no** transport lock (SPEC-028 FR-002) — it holds no transport, and a lock spanning
|
|
34
|
+
a child's ``emit`` would serialize every destination behind the slowest one. And it
|
|
35
|
+
**adds no post-close guard** (SPEC-032 FR-003), because the post-close rule is each child's:
|
|
36
|
+
``close()`` here only forwards, so a guard added at this level would refuse batches the
|
|
37
|
+
children would have taken, while a child that must refuse already does and is counted here
|
|
38
|
+
like any other failure.
|
|
32
39
|
"""
|
|
33
40
|
|
|
34
41
|
def __init__(self, *sinks: Sink) -> None:
|
|
@@ -27,6 +27,11 @@ class GooglePubSubSink:
|
|
|
27
27
|
appended after the loop passed its index was dropped unresolved: an unconfirmed publish
|
|
28
28
|
never counted in ``failed`` and never reported by :meth:`losses`, which is precisely the
|
|
29
29
|
silent loss SPEC-026 exists to end.
|
|
30
|
+
|
|
31
|
+
That same loss was reachable from *outside* ``close`` until SPEC-032 FR-001: nothing stopped
|
|
32
|
+
a later ``emit`` appending to the fresh list, and nothing would ever call ``result()`` on
|
|
33
|
+
it. The sink now refuses a batch once closed, and the append re-checks under the futures
|
|
34
|
+
lock, so a publish cannot land on a list the swap has already taken.
|
|
30
35
|
"""
|
|
31
36
|
|
|
32
37
|
def __init__(self, topic: str, *, client: Any = None) -> None:
|
|
@@ -53,6 +58,7 @@ class GooglePubSubSink:
|
|
|
53
58
|
self._counter_lock = threading.Lock()
|
|
54
59
|
self._futures_lock = threading.Lock()
|
|
55
60
|
self._futures: list[Any] = []
|
|
61
|
+
self._closed = False
|
|
56
62
|
|
|
57
63
|
def losses(self) -> SinkLosses:
|
|
58
64
|
"""Reports refused publishes and futures that resolved to an error (FR-002).
|
|
@@ -80,6 +86,26 @@ class GooglePubSubSink:
|
|
|
80
86
|
the retry would duplicate them. The stderr line names which counter moved, since
|
|
81
87
|
"refused" and "unconfirmed" mean different things.
|
|
82
88
|
|
|
89
|
+
A closed sink refuses the batch before touching the client (SPEC-032 FR-001), because
|
|
90
|
+
nothing will resolve a future appended after :meth:`close` swapped the list out.
|
|
91
|
+
Refusing moves no counter here: it is a failure reported to the worker, which records it
|
|
92
|
+
in ``health().failed_batches``, not loss this sink absorbed. The flag is read twice on
|
|
93
|
+
purpose — once for the batch, and again under ``_futures_lock`` for each append, which
|
|
94
|
+
is the lock ``close`` swaps under. Without the second read a close landing mid-loop
|
|
95
|
+
would leave exactly the orphaned future this guard exists to prevent, and the read costs
|
|
96
|
+
nothing because the append already takes that lock. Only the read happens under it: the
|
|
97
|
+
counter bump and the stderr line are taken *after* the lock is released, because
|
|
98
|
+
``close`` waits on the same lock and a blocked stderr would otherwise hold it — an I/O
|
|
99
|
+
call inside a transport lock is what SPEC-028's two-lock decision exists to avoid.
|
|
100
|
+
|
|
101
|
+
A close that lands mid-batch does **not** raise, even when it catches every event. Each
|
|
102
|
+
one had ``publish()`` called on it and may well land, so raising would have the worker
|
|
103
|
+
re-send them and duplicate whatever did — the SPEC-018 rule that only a *provable*
|
|
104
|
+
non-delivery may be retried. They are counted as unconfirmed instead, which is exactly
|
|
105
|
+
what they are. That is why the total-failure raise below tests the refusals rather than
|
|
106
|
+
the successes: "nothing was published" and "nothing was confirmed" are different claims,
|
|
107
|
+
and only the first is safe to retry.
|
|
108
|
+
|
|
83
109
|
Args:
|
|
84
110
|
batch: The events to publish.
|
|
85
111
|
|
|
@@ -87,21 +113,36 @@ class GooglePubSubSink:
|
|
|
87
113
|
None.
|
|
88
114
|
|
|
89
115
|
Raises:
|
|
90
|
-
SinkDeliveryError: When
|
|
116
|
+
SinkDeliveryError: When the sink was already closed on entry, or when every event was
|
|
117
|
+
refused (SPEC-026 FR-001).
|
|
91
118
|
"""
|
|
92
|
-
|
|
119
|
+
if not batch:
|
|
120
|
+
return
|
|
121
|
+
if self._closed:
|
|
122
|
+
raise SinkDeliveryError(
|
|
123
|
+
f"GooglePubSubSink published none of {len(batch)} event(s): the sink is closed"
|
|
124
|
+
)
|
|
125
|
+
refused = 0
|
|
93
126
|
for event in batch:
|
|
94
127
|
try:
|
|
95
128
|
future = self.client.publish(self.topic, data=json.dumps(event).encode("utf-8"))
|
|
96
129
|
except Exception as err:
|
|
130
|
+
refused += 1
|
|
97
131
|
with self._counter_lock:
|
|
98
132
|
self.rejected += 1
|
|
99
133
|
_diag.lost("event", 1, f"GooglePubSubSink refused the publish, {type(err).__name__}")
|
|
100
134
|
continue
|
|
101
135
|
with self._futures_lock:
|
|
102
|
-
self.
|
|
103
|
-
|
|
104
|
-
|
|
136
|
+
orphaned = self._closed
|
|
137
|
+
if not orphaned:
|
|
138
|
+
self._futures.append(future)
|
|
139
|
+
if orphaned:
|
|
140
|
+
with self._counter_lock:
|
|
141
|
+
self.failed += 1
|
|
142
|
+
_diag.lost(
|
|
143
|
+
"event", 1, "GooglePubSubSink publish unconfirmed, the sink closed mid-batch"
|
|
144
|
+
)
|
|
145
|
+
if refused == len(batch):
|
|
105
146
|
raise SinkDeliveryError(
|
|
106
147
|
f"GooglePubSubSink published none of {len(batch)} event(s)"
|
|
107
148
|
)
|
|
@@ -116,6 +157,12 @@ class GooglePubSubSink:
|
|
|
116
157
|
and never reported by ``losses()``. That is the silent loss SPEC-026 exists to end,
|
|
117
158
|
reached through the one piece of shared state this sink has.
|
|
118
159
|
|
|
160
|
+
The closed flag is set in the same critical section as the swap (SPEC-032 FR-001), which
|
|
161
|
+
is what makes :meth:`emit`'s second check exact: an append winning the lock before this
|
|
162
|
+
point is resolved by the loop below, and one arriving after sees the flag. The futures
|
|
163
|
+
are resolved outside the lock, so a slow ``result()`` never blocks an emit that is about
|
|
164
|
+
to be refused anyway.
|
|
165
|
+
|
|
119
166
|
Args:
|
|
120
167
|
None.
|
|
121
168
|
|
|
@@ -127,6 +174,7 @@ class GooglePubSubSink:
|
|
|
127
174
|
(FR-011).
|
|
128
175
|
"""
|
|
129
176
|
with self._futures_lock:
|
|
177
|
+
self._closed = True
|
|
130
178
|
pending, self._futures = self._futures, []
|
|
131
179
|
for future in pending:
|
|
132
180
|
try:
|
|
@@ -28,6 +28,12 @@ class _RedisSink:
|
|
|
28
28
|
execution never mutates the client. Its documented exceptions are ``PubSub`` and
|
|
29
29
|
``Pipeline`` objects, which must not be passed between threads — which is why the pipeline
|
|
30
30
|
here is built and executed inside a single ``emit`` call and never stored on the instance.
|
|
31
|
+
|
|
32
|
+
It refuses an emit after :meth:`close` (SPEC-032 FR-001). Left unguarded this sink did not
|
|
33
|
+
fail after close, it *succeeded*: ``redis-py``'s pool reconnects transparently on the next
|
|
34
|
+
command, so a batch emitted after ``shutdown()`` opened a connection nothing would ever
|
|
35
|
+
reap — the same leak SPEC-028's review found in ``RabbitMQSink``, whose ``_active_channel``
|
|
36
|
+
reopened whatever ``close()`` had released.
|
|
31
37
|
"""
|
|
32
38
|
|
|
33
39
|
def __init__(self, *, client: Any, url: str | None, max_retries: int) -> None:
|
|
@@ -56,6 +62,8 @@ class _RedisSink:
|
|
|
56
62
|
self.stop_signal: threading.Event | None = None
|
|
57
63
|
self.failed = 0
|
|
58
64
|
self._counter_lock = threading.Lock()
|
|
65
|
+
self._closed = False
|
|
66
|
+
self._close_lock = threading.Lock()
|
|
59
67
|
|
|
60
68
|
def losses(self) -> SinkLosses:
|
|
61
69
|
"""Reports events abandoned past the retry bound (SPEC-026 FR-002).
|
|
@@ -75,6 +83,13 @@ class _RedisSink:
|
|
|
75
83
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
76
84
|
"""Pipelines the whole batch into one round trip, retrying on error (FR-005).
|
|
77
85
|
|
|
86
|
+
A closed sink refuses the batch before asking for a pipeline (SPEC-032 FR-001), since
|
|
87
|
+
asking would silently reopen the connection ``close()`` released. The refusal does not
|
|
88
|
+
depend on whether the sink owned the client: a borrowed client surviving its sink is the
|
|
89
|
+
caller's business, and is not permission to keep writing through a sink that has been
|
|
90
|
+
released. Refusing moves no counter here — it is a failure reported to the worker, which
|
|
91
|
+
records it in ``health().failed_batches``, not loss this sink absorbed.
|
|
92
|
+
|
|
78
93
|
Args:
|
|
79
94
|
batch: The events to buffer. An empty batch is a no-op.
|
|
80
95
|
|
|
@@ -82,13 +97,18 @@ class _RedisSink:
|
|
|
82
97
|
None.
|
|
83
98
|
|
|
84
99
|
Raises:
|
|
85
|
-
SinkDeliveryError: When the
|
|
86
|
-
so such a failure delivered nothing: it is counted and
|
|
87
|
-
its retry and ``health()`` the loss (SPEC-026 FR-001).
|
|
88
|
-
protect, because the pipeline is all or nothing.
|
|
100
|
+
SinkDeliveryError: When the sink is closed. Also when the retry bound is spent: the
|
|
101
|
+
batch travels as one pipeline, so such a failure delivered nothing: it is counted and
|
|
102
|
+
then raised, giving the worker its retry and ``health()`` the loss (SPEC-026 FR-001).
|
|
103
|
+
There is no partial case to protect, because the pipeline is all or nothing.
|
|
89
104
|
"""
|
|
90
105
|
if not batch:
|
|
91
106
|
return
|
|
107
|
+
if self._closed:
|
|
108
|
+
raise SinkDeliveryError(
|
|
109
|
+
f"{type(self).__name__} delivered none of {len(batch)} event(s): "
|
|
110
|
+
"the sink is closed"
|
|
111
|
+
)
|
|
92
112
|
for attempt in range(self.max_retries + 1):
|
|
93
113
|
try:
|
|
94
114
|
pipe = self.client.pipeline()
|
|
@@ -114,6 +134,11 @@ class _RedisSink:
|
|
|
114
134
|
def close(self) -> None:
|
|
115
135
|
"""Closes the connection only if the sink owns it (FR-005).
|
|
116
136
|
|
|
137
|
+
Idempotent, with the flag set under a lock so two concurrent calls cannot both reach
|
|
138
|
+
``client.close()`` — ``atexit`` racing user code is the documented case. The flag is set
|
|
139
|
+
whether or not the client is owned, because it marks *this sink* as released rather than
|
|
140
|
+
the connection (SPEC-032 FR-001).
|
|
141
|
+
|
|
117
142
|
Args:
|
|
118
143
|
None.
|
|
119
144
|
|
|
@@ -123,8 +148,12 @@ class _RedisSink:
|
|
|
123
148
|
Raises:
|
|
124
149
|
Exception: Whatever the client raises on close.
|
|
125
150
|
"""
|
|
126
|
-
|
|
127
|
-
self.
|
|
151
|
+
with self._close_lock:
|
|
152
|
+
if self._closed:
|
|
153
|
+
return
|
|
154
|
+
self._closed = True
|
|
155
|
+
if self._owns_client:
|
|
156
|
+
self.client.close()
|
|
128
157
|
|
|
129
158
|
def _stage(self, pipe: Any, event: dict[str, object]) -> None:
|
|
130
159
|
"""Stages one event onto the pipeline, in whatever form the subclass writes.
|
|
@@ -38,6 +38,9 @@ class SentrySink:
|
|
|
38
38
|
lock. This sink owns no transport: it delegates to a ``sentry_sdk`` built for capture from any
|
|
39
39
|
thread, or to an ``HTTPSink`` that builds a fresh request per call and rebinds nothing.
|
|
40
40
|
Its counters take the counter lock like every other sink's.
|
|
41
|
+
|
|
42
|
+
It also **adds no post-close guard** (SPEC-032 FR-003), for the same reason: neither backend
|
|
43
|
+
holds anything ``close()`` releases, so a batch emitted afterwards still reaches Sentry.
|
|
41
44
|
"""
|
|
42
45
|
|
|
43
46
|
def __init__(
|
|
@@ -120,9 +123,12 @@ class SentrySink:
|
|
|
120
123
|
)
|
|
121
124
|
|
|
122
125
|
def close(self) -> None:
|
|
123
|
-
"""
|
|
126
|
+
"""Forwards to the HTTP fallback, whose own close releases nothing (FR-012).
|
|
124
127
|
|
|
125
|
-
Idempotent
|
|
128
|
+
Idempotent, and it releases nothing — which is why the class docstring's post-close claim
|
|
129
|
+
holds despite this method calling a ``close()``. ``HTTPSink.close`` is a documented no-op,
|
|
130
|
+
since ``urllib`` builds a fresh connection per request; the forward exists so a future
|
|
131
|
+
``HTTPSink`` that *did* hold a pool would be released here rather than leaked.
|
|
126
132
|
|
|
127
133
|
Args:
|
|
128
134
|
None.
|
|
@@ -29,6 +29,10 @@ class SNSSink:
|
|
|
29
29
|
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
30
30
|
lock. ``boto3`` clients are documented thread-safe, this one is built once in
|
|
31
31
|
``__init__``, and the sink rebinds nothing after construction.
|
|
32
|
+
|
|
33
|
+
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
34
|
+
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
35
|
+
afterwards still reaches the topic.
|
|
32
36
|
"""
|
|
33
37
|
|
|
34
38
|
MAX_BATCH = 10
|
|
@@ -17,6 +17,10 @@ class SplunkHECSink(HTTPSink):
|
|
|
17
17
|
concatenated-JSON-objects body with an ``Authorization: Splunk <token>`` header. The
|
|
18
18
|
envelope's epoch-seconds ``time`` is parsed from the event's ISO-8601 ``timestamp``, falling
|
|
19
19
|
back to emit-time now.
|
|
20
|
+
|
|
21
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
22
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
23
|
+
is no transport held and ``close()`` releases nothing.
|
|
20
24
|
"""
|
|
21
25
|
|
|
22
26
|
def __init__(
|
|
@@ -91,6 +91,10 @@ class SQSSink:
|
|
|
91
91
|
``emit`` — which is what keeps it clear of the documented caveat that calling
|
|
92
92
|
``boto3.client()`` concurrently can produce response-ordering faults. Nothing else here is
|
|
93
93
|
rebound after construction.
|
|
94
|
+
|
|
95
|
+
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
96
|
+
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
97
|
+
afterwards still reaches the queue.
|
|
94
98
|
"""
|
|
95
99
|
|
|
96
100
|
MAX_BATCH = 10
|
|
@@ -14,6 +14,18 @@ class StdoutSink:
|
|
|
14
14
|
|
|
15
15
|
This is the zero-dependency default sink, for local dev and container log scraping. Like
|
|
16
16
|
every sink it receives already-built event dicts and knows nothing about spans or context.
|
|
17
|
+
|
|
18
|
+
It takes **no** transport lock (SPEC-028 FR-002), and the honest reason is that locking it
|
|
19
|
+
was out of scope there rather than that it needs none: ``emit`` writes one line per event, so
|
|
20
|
+
two threads' *batches* do interleave, exactly as they did in ``FileSink`` before its lock.
|
|
21
|
+
Individual lines survive — ``TextIOWrapper.write`` holds its own lock — but line integrity is
|
|
22
|
+
not the guarantee at stake, as ``test_file_sink_keeps_each_batch_contiguous_under_concurrent
|
|
23
|
+
_emitters`` records. Interleaved batches are harmless in a line-oriented stream that a
|
|
24
|
+
scraper reads per line, which is what this sink is for; a caller needing contiguity should
|
|
25
|
+
use ``FileSink``.
|
|
26
|
+
|
|
27
|
+
It **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` only flushes — the
|
|
28
|
+
stream belongs to the process, not to this sink, so a later batch still lands.
|
|
17
29
|
"""
|
|
18
30
|
|
|
19
31
|
def __init__(self, stream: TextIO | None = None) -> None:
|
|
@@ -33,6 +33,12 @@ class SyslogSink:
|
|
|
33
33
|
Each event becomes an RFC 5424 message whose ``PRI`` is derived from a configurable facility
|
|
34
34
|
and a severity mapped from the event's level. UDP sends one datagram per event, TCP uses
|
|
35
35
|
octet-counted framing (RFC 6587), and the whole sink is dependency-free.
|
|
36
|
+
|
|
37
|
+
It takes **no** transport lock (SPEC-028 FR-002) of its own: the socket it holds is a
|
|
38
|
+
:class:`~log_foundry.sinks._socket.SocketTransport`, which locks its own sends. Its
|
|
39
|
+
post-close refusal comes from there too (SPEC-032 FR-004) — a batch emitted after
|
|
40
|
+
``close()`` reaches ``send_all`` and is refused with ``SinkDeliveryError`` without the
|
|
41
|
+
socket being reopened, so a guard here would only duplicate one that already holds.
|
|
36
42
|
"""
|
|
37
43
|
|
|
38
44
|
def __init__(
|
|
@@ -23,6 +23,10 @@ class TransformSink:
|
|
|
23
23
|
rename keys — and returning ``None`` drops that event. The caller's batch and event dicts
|
|
24
24
|
are never mutated in place: only the function's return values are forwarded, so a transform
|
|
25
25
|
must copy before mutating.
|
|
26
|
+
|
|
27
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
28
|
+
(SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
|
|
29
|
+
belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
|
|
26
30
|
"""
|
|
27
31
|
|
|
28
32
|
def __init__(
|
|
@@ -43,6 +43,10 @@ class NullSink:
|
|
|
43
43
|
what this sink is for, and reporting it as loss would make ``health()``'s alert idiom fire
|
|
44
44
|
on every batch for anyone who chose this sink to turn logging off. The counter stays
|
|
45
45
|
readable on the instance.
|
|
46
|
+
|
|
47
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
48
|
+
(SPEC-032 FR-003): there is no transport and ``close()`` releases nothing, so discarding a
|
|
49
|
+
batch after close is the same operation as discarding one before it.
|
|
46
50
|
"""
|
|
47
51
|
|
|
48
52
|
def __init__(self) -> None:
|
|
@@ -96,6 +100,10 @@ class MemorySink:
|
|
|
96
100
|
eyeballing in a notebook. With ``maxlen`` set it behaves as a bounded ring, keeping only the
|
|
97
101
|
most recent events; the list object identity is stable, so a held reference keeps seeing
|
|
98
102
|
updates.
|
|
103
|
+
|
|
104
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
105
|
+
(SPEC-032 FR-003): there is no transport, and ``close()`` releases nothing — a test that
|
|
106
|
+
closes the sink and then asserts on a later batch still sees it in ``.events``.
|
|
99
107
|
"""
|
|
100
108
|
|
|
101
109
|
def __init__(self, maxlen: int | None = None) -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|