log-foundry 0.10.2.dev25__tar.gz → 0.10.2.dev27__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/PKG-INFO +73 -5
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/README.md +72 -4
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/__init__.py +23 -5
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/config.py +52 -1
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/decorator.py +34 -1
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/base.py +23 -1
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/callback.py +5 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/datadog.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/elasticsearch.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/filtering.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/firehose.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/honeycomb.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/http.py +7 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/kafka.py +42 -3
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/kinesis.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/logging_sink.py +5 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/logstash.py +8 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/loki.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/multi.py +7 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/pubsub.py +53 -5
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/redis.py +35 -6
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sentry.py +8 -2
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sns.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/splunk.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sqs.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/stdout.py +12 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/syslog.py +6 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/transform.py +4 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/util.py +8 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/worker.py +222 -4
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/LICENSE +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_retry.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_socket.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/clickhouse.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/eventhubs.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/file.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/mongodb.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/nats.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/postgres.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/rabbitmq.py +0 -0
- {log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/sqlite.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: log-foundry
|
|
3
|
-
Version: 0.10.2.
|
|
3
|
+
Version: 0.10.2.dev27
|
|
4
4
|
Summary: Generate logs for your console and JSON events for downstream consumption.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -234,6 +234,34 @@ lf.configure(
|
|
|
234
234
|
If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
|
|
235
235
|
works with zero configuration.
|
|
236
236
|
|
|
237
|
+
**Passing `sink=` after logging has started swaps the live destination.** The background worker
|
|
238
|
+
captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
|
|
239
|
+
what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
|
|
240
|
+
what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
|
|
241
|
+
it, and points the worker at the new one:
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
lf.configure(sink=StdoutSink())
|
|
245
|
+
do_some_work() # these events go to stdout
|
|
246
|
+
|
|
247
|
+
lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
|
|
248
|
+
do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
|
|
252
|
+
you asked for that sink — but the previous sink is left **open** rather than closed, because the
|
|
253
|
+
drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
|
|
254
|
+
the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
|
|
255
|
+
do not hand it back to a later call.
|
|
256
|
+
|
|
257
|
+
The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
|
|
258
|
+
a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
|
|
259
|
+
an unreachable broker is the case to watch. Configure the sink before the first log where you can;
|
|
260
|
+
that path has no worker to retarget and nothing to close.
|
|
261
|
+
|
|
262
|
+
`configure()` is still a startup call. It is not thread-safe, and a span finishing on another
|
|
263
|
+
thread mid-swap may land on either sink.
|
|
264
|
+
|
|
237
265
|
### `@trace`
|
|
238
266
|
|
|
239
267
|
Decorate any **synchronous** function. Usable bare or with arguments:
|
|
@@ -668,7 +696,7 @@ in `CallbackSink`.
|
|
|
668
696
|
|
|
669
697
|
#### Writing your own sink
|
|
670
698
|
|
|
671
|
-
`Sink` is two required methods, `emit(batch)` and `close()`, plus
|
|
699
|
+
`Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
|
|
672
700
|
fails and one about *when* it is called. They are not stylistic — the library's whole
|
|
673
701
|
loss-reporting apparatus is built on them:
|
|
674
702
|
|
|
@@ -688,6 +716,15 @@ loss-reporting apparatus is built on them:
|
|
|
688
716
|
- **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
|
|
689
717
|
partial success re-delivers the records that already arrived, and duplicates downstream are worse
|
|
690
718
|
than a counted loss.
|
|
719
|
+
- **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
|
|
720
|
+
released transport has delivered nothing, so it is the first rule again by another route — and
|
|
721
|
+
it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
|
|
722
|
+
wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
|
|
723
|
+
one appended a delivery future nothing would resolve, and one transparently *reconnected*,
|
|
724
|
+
leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
|
|
725
|
+
If your `close()` releases nothing — you open a fresh connection per request, or the client is
|
|
726
|
+
the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
|
|
727
|
+
invented. `emit([])` stays a no-op either way.
|
|
691
728
|
|
|
692
729
|
A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
|
|
693
730
|
never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
|
|
@@ -704,12 +741,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
|
704
741
|
class MySink:
|
|
705
742
|
def __init__(self) -> None:
|
|
706
743
|
self._dropped = self._failed = 0
|
|
744
|
+
self._closed = False
|
|
707
745
|
self._lock = threading.Lock() # transport state
|
|
708
746
|
self._counter_lock = threading.Lock() # counters only, never held across I/O
|
|
709
747
|
|
|
710
748
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
749
|
+
if not batch:
|
|
750
|
+
return
|
|
711
751
|
delivered = 0
|
|
712
752
|
with self._lock: # your connection, socket or stream
|
|
753
|
+
if self._closed: # refuse: nothing here can deliver it
|
|
754
|
+
raise SinkDeliveryError(
|
|
755
|
+
f"MySink delivered none of {len(batch)} event(s): the sink is closed"
|
|
756
|
+
)
|
|
713
757
|
for chunk in self._chunks(batch):
|
|
714
758
|
if self._send(chunk): # your own bounded retry
|
|
715
759
|
delivered += len(chunk)
|
|
@@ -725,6 +769,9 @@ class MySink:
|
|
|
725
769
|
|
|
726
770
|
def close(self) -> None:
|
|
727
771
|
with self._lock: # never release under an active writer
|
|
772
|
+
if self._closed: # idempotent: atexit races your own cleanup
|
|
773
|
+
return
|
|
774
|
+
self._closed = True # set the flag, *then* release
|
|
728
775
|
...
|
|
729
776
|
```
|
|
730
777
|
|
|
@@ -746,7 +793,11 @@ returns a snapshot of the worker's counters:
|
|
|
746
793
|
|
|
747
794
|
```python
|
|
748
795
|
h = log_foundry.health()
|
|
749
|
-
if
|
|
796
|
+
if (
|
|
797
|
+
h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
|
|
798
|
+
or (h.sink and (h.sink.dropped or h.sink.failed))
|
|
799
|
+
or (h.retired and h.submitted_after_shutdown)
|
|
800
|
+
):
|
|
750
801
|
... # logs were silently lost — worth an alert
|
|
751
802
|
```
|
|
752
803
|
|
|
@@ -759,6 +810,8 @@ They tell you different things, and they want different responses:
|
|
|
759
810
|
| `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
|
|
760
811
|
| `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
|
|
761
812
|
| `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
|
|
813
|
+
| `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
|
|
814
|
+
| `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
|
|
762
815
|
|
|
763
816
|
`h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
|
|
764
817
|
the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
|
|
@@ -780,8 +833,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
|
|
|
780
833
|
thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
|
|
781
834
|
pointing at the wrong fix.
|
|
782
835
|
|
|
836
|
+
`retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
|
|
837
|
+
logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
|
|
838
|
+
means every log line since the shutdown has gone nowhere. That state used to read as perfectly
|
|
839
|
+
healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
|
|
840
|
+
|
|
783
841
|
Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
|
|
784
|
-
fields over time — a fourth (`stopped_reason`) in `v0.7.0
|
|
842
|
+
fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
|
|
843
|
+
seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
|
|
785
844
|
release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
|
|
786
845
|
field keeps its position when a new one is appended, so attribute and index access stay stable.
|
|
787
846
|
|
|
@@ -859,6 +918,13 @@ getting them wrong is silent:
|
|
|
859
918
|
the first invocation on a warm container would log and every later one would silently log
|
|
860
919
|
nothing. That failure reads as "works locally, broken in production".
|
|
861
920
|
|
|
921
|
+
It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
|
|
922
|
+
`health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
|
|
923
|
+
pair, and only that pair, is this mistake. The first such submission also writes one stderr
|
|
924
|
+
line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
|
|
925
|
+
`stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
|
|
926
|
+
drain where the repeatable one belonged.
|
|
927
|
+
|
|
862
928
|
```python
|
|
863
929
|
import log_foundry as lf
|
|
864
930
|
from log_foundry.sinks.sqs import SQSSink
|
|
@@ -876,9 +942,11 @@ def handler(event, context):
|
|
|
876
942
|
# would log nothing.
|
|
877
943
|
drained = lf.flush()
|
|
878
944
|
h = lf.health()
|
|
879
|
-
if not drained or h.failed_batches or h.dropped or h.stopped_reason:
|
|
945
|
+
if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
|
|
880
946
|
# `drained` covers this invocation's tail; the counters cover anything the worker
|
|
881
947
|
# lost earlier — a batch its own interval trigger already gave up on, for instance.
|
|
948
|
+
# `h.retired` catches the mistake above: inside a handler it can only mean something
|
|
949
|
+
# called shutdown(), and from here on this container logs nothing.
|
|
882
950
|
# Emitting this through your platform's own logger keeps it outside the pipeline
|
|
883
951
|
# that just failed.
|
|
884
952
|
print(f"log-foundry: undelivered logs ({drained=}, {h=})")
|
|
@@ -198,6 +198,34 @@ lf.configure(
|
|
|
198
198
|
If you never set a `sink`, the first decorated call falls back to `StdoutSink()`, so `@trace`
|
|
199
199
|
works with zero configuration.
|
|
200
200
|
|
|
201
|
+
**Passing `sink=` after logging has started swaps the live destination.** The background worker
|
|
202
|
+
captures its sink when it is built, so a later `configure(sink=...)` has to do more than update
|
|
203
|
+
what `get_config()` reports — otherwise the config and the behaviour disagree silently, which is
|
|
204
|
+
what it used to do. The swap drains everything submitted so far to the **previous** sink, closes
|
|
205
|
+
it, and points the worker at the new one:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
lf.configure(sink=StdoutSink())
|
|
209
|
+
do_some_work() # these events go to stdout
|
|
210
|
+
|
|
211
|
+
lf.configure(sink=SQSSink(queue_url=QUEUE_URL))
|
|
212
|
+
do_some_more_work() # these go to SQS; the earlier ones were drained to stdout
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The drain is bounded at 5 s. If it cannot be confirmed in that time the swap still takes effect —
|
|
216
|
+
you asked for that sink — but the previous sink is left **open** rather than closed, because the
|
|
217
|
+
drain thread may still be inside its `emit`, and `health().incomplete_swaps` records it. Passing
|
|
218
|
+
the sink that is already live is a no-op: no drain, no close. The previous sink **is** closed, so
|
|
219
|
+
do not hand it back to a later call.
|
|
220
|
+
|
|
221
|
+
The 5 s covers the drain, **not** the previous sink's `close()`, which has no timeout of its own —
|
|
222
|
+
a destination that blocks there blocks `configure()`. `KafkaSink.close()` flushes its producer, so
|
|
223
|
+
an unreachable broker is the case to watch. Configure the sink before the first log where you can;
|
|
224
|
+
that path has no worker to retarget and nothing to close.
|
|
225
|
+
|
|
226
|
+
`configure()` is still a startup call. It is not thread-safe, and a span finishing on another
|
|
227
|
+
thread mid-swap may land on either sink.
|
|
228
|
+
|
|
201
229
|
### `@trace`
|
|
202
230
|
|
|
203
231
|
Decorate any **synchronous** function. Usable bare or with arguments:
|
|
@@ -632,7 +660,7 @@ in `CallbackSink`.
|
|
|
632
660
|
|
|
633
661
|
#### Writing your own sink
|
|
634
662
|
|
|
635
|
-
`Sink` is two required methods, `emit(batch)` and `close()`, plus
|
|
663
|
+
`Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
|
|
636
664
|
fails and one about *when* it is called. They are not stylistic — the library's whole
|
|
637
665
|
loss-reporting apparatus is built on them:
|
|
638
666
|
|
|
@@ -652,6 +680,15 @@ loss-reporting apparatus is built on them:
|
|
|
652
680
|
- **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
|
|
653
681
|
partial success re-delivers the records that already arrived, and duplicates downstream are worse
|
|
654
682
|
than a counted loss.
|
|
683
|
+
- **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
|
|
684
|
+
released transport has delivered nothing, so it is the first rule again by another route — and
|
|
685
|
+
it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
|
|
686
|
+
wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
|
|
687
|
+
one appended a delivery future nothing would resolve, and one transparently *reconnected*,
|
|
688
|
+
leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
|
|
689
|
+
If your `close()` releases nothing — you open a fresh connection per request, or the client is
|
|
690
|
+
the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
|
|
691
|
+
invented. `emit([])` stays a no-op either way.
|
|
655
692
|
|
|
656
693
|
A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
|
|
657
694
|
never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
|
|
@@ -668,12 +705,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
|
668
705
|
class MySink:
|
|
669
706
|
def __init__(self) -> None:
|
|
670
707
|
self._dropped = self._failed = 0
|
|
708
|
+
self._closed = False
|
|
671
709
|
self._lock = threading.Lock() # transport state
|
|
672
710
|
self._counter_lock = threading.Lock() # counters only, never held across I/O
|
|
673
711
|
|
|
674
712
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
713
|
+
if not batch:
|
|
714
|
+
return
|
|
675
715
|
delivered = 0
|
|
676
716
|
with self._lock: # your connection, socket or stream
|
|
717
|
+
if self._closed: # refuse: nothing here can deliver it
|
|
718
|
+
raise SinkDeliveryError(
|
|
719
|
+
f"MySink delivered none of {len(batch)} event(s): the sink is closed"
|
|
720
|
+
)
|
|
677
721
|
for chunk in self._chunks(batch):
|
|
678
722
|
if self._send(chunk): # your own bounded retry
|
|
679
723
|
delivered += len(chunk)
|
|
@@ -689,6 +733,9 @@ class MySink:
|
|
|
689
733
|
|
|
690
734
|
def close(self) -> None:
|
|
691
735
|
with self._lock: # never release under an active writer
|
|
736
|
+
if self._closed: # idempotent: atexit races your own cleanup
|
|
737
|
+
return
|
|
738
|
+
self._closed = True # set the flag, *then* release
|
|
692
739
|
...
|
|
693
740
|
```
|
|
694
741
|
|
|
@@ -710,7 +757,11 @@ returns a snapshot of the worker's counters:
|
|
|
710
757
|
|
|
711
758
|
```python
|
|
712
759
|
h = log_foundry.health()
|
|
713
|
-
if
|
|
760
|
+
if (
|
|
761
|
+
h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps
|
|
762
|
+
or (h.sink and (h.sink.dropped or h.sink.failed))
|
|
763
|
+
or (h.retired and h.submitted_after_shutdown)
|
|
764
|
+
):
|
|
714
765
|
... # logs were silently lost — worth an alert
|
|
715
766
|
```
|
|
716
767
|
|
|
@@ -723,6 +774,8 @@ They tell you different things, and they want different responses:
|
|
|
723
774
|
| `stopped_reason` | The background thread **died** on that exception type. Nothing further will be delivered, ever. | Restart the process; investigate the named exception. |
|
|
724
775
|
| `sink.dropped` | The sink discarded events **before** attempting delivery — an oversized record, or one the client refused outright. | Read the stderr line: it names the cause. An oversized record means shrink what you log; a refused local produce/publish (Kafka, Pub/Sub) points at the client — a saturated buffer, a bad topic, a credential. |
|
|
725
776
|
| `sink.failed` | The sink attempted delivery and could not confirm it — abandoned requests, partially-failed batches, responses it could not adjudicate. | Fix the destination. |
|
|
777
|
+
| `retired` + `submitted_after_shutdown` | `shutdown()` was called and the process **kept logging**. Those events are queued where nothing will drain them — total loss, for as long as the process runs. | Use `flush()`, not `shutdown()`, in a process that logs again. This is the serverless mistake below. |
|
|
778
|
+
| `incomplete_swaps` | A late `configure(sink=...)` could not confirm the previous sink was drained. The swap took effect; that sink was left open and some queued events may have gone to the new one. | Investigate the previous sink — it was hung or failing. Configure the sink before the first log where you can. |
|
|
726
779
|
|
|
727
780
|
`h.sink` is a `SinkLosses(dropped, failed)` or `None` — `None` when no worker exists yet, or when
|
|
728
781
|
the configured sink reports nothing (`losses()` is optional). Note the two `dropped` fields count
|
|
@@ -744,8 +797,14 @@ logged, and after a clean `shutdown()`, so a plain truthiness check is safe. Wit
|
|
|
744
797
|
thread showed up only indirectly, as `dropped` climbing once the queue filled — the wrong signal,
|
|
745
798
|
pointing at the wrong fix.
|
|
746
799
|
|
|
800
|
+
`retired` is deliberately **not** alerted on by itself. A process that shuts down and then stops
|
|
801
|
+
logging is doing the right thing; it is the *pair* — retired, and still being handed events — that
|
|
802
|
+
means every log line since the shutdown has gone nowhere. That state used to read as perfectly
|
|
803
|
+
healthy: `stopped_reason` is `None` after a clean shutdown, and the queue simply grows.
|
|
804
|
+
|
|
747
805
|
Read a snapshot by attribute (`h.dropped`), as above. `Health` is a `NamedTuple` and has gained
|
|
748
|
-
fields over time — a fourth (`stopped_reason`) in `v0.7.0
|
|
806
|
+
fields over time — a fourth (`stopped_reason`) in `v0.7.0`, and a fifth (`sink`) plus a sixth,
|
|
807
|
+
seventh and eighth (`retired`, `submitted_after_shutdown`, `incomplete_swaps`) not yet in a tagged
|
|
749
808
|
release — so unpacking it whole (`queued, dropped, failed = health()`) raises `ValueError`. Every
|
|
750
809
|
field keeps its position when a new one is appended, so attribute and index access stay stable.
|
|
751
810
|
|
|
@@ -823,6 +882,13 @@ getting them wrong is silent:
|
|
|
823
882
|
the first invocation on a warm container would log and every later one would silently log
|
|
824
883
|
nothing. That failure reads as "works locally, broken in production".
|
|
825
884
|
|
|
885
|
+
It is no longer silent. Logging after `shutdown()` is accepted and undeliverable, and
|
|
886
|
+
`health()` says so: **`retired` is `True` and `submitted_after_shutdown` is non-zero** — that
|
|
887
|
+
pair, and only that pair, is this mistake. The first such submission also writes one stderr
|
|
888
|
+
line naming `flush()` as the remedy, throttled to the first and every thousandth after it.
|
|
889
|
+
`stopped_reason` stays `None` throughout, because nothing failed; someone used the terminal
|
|
890
|
+
drain where the repeatable one belonged.
|
|
891
|
+
|
|
826
892
|
```python
|
|
827
893
|
import log_foundry as lf
|
|
828
894
|
from log_foundry.sinks.sqs import SQSSink
|
|
@@ -840,9 +906,11 @@ def handler(event, context):
|
|
|
840
906
|
# would log nothing.
|
|
841
907
|
drained = lf.flush()
|
|
842
908
|
h = lf.health()
|
|
843
|
-
if not drained or h.failed_batches or h.dropped or h.stopped_reason:
|
|
909
|
+
if not drained or h.failed_batches or h.dropped or h.stopped_reason or h.retired:
|
|
844
910
|
# `drained` covers this invocation's tail; the counters cover anything the worker
|
|
845
911
|
# lost earlier — a batch its own interval trigger already gave up on, for instance.
|
|
912
|
+
# `h.retired` catches the mistake above: inside a handler it can only mean something
|
|
913
|
+
# called shutdown(), and from here on this container logs nothing.
|
|
846
914
|
# Emitting this through your platform's own logger keeps it outside the pipeline
|
|
847
915
|
# that just failed.
|
|
848
916
|
print(f"log-foundry: undelivered logs ({drained=}, {h=})")
|
|
@@ -58,22 +58,35 @@ def health() -> Health:
|
|
|
58
58
|
code non-blocking, and a non-zero ``failed_batches`` means a sink stayed broken through the
|
|
59
59
|
whole retry budget — both are losses the library absorbs on purpose, and this is how you
|
|
60
60
|
notice them. A non-``None`` ``stopped_reason`` is worse than either: the background thread
|
|
61
|
-
died on that exception type, so nothing further will be delivered at all (SPEC-019)
|
|
61
|
+
died on that exception type, so nothing further will be delivered at all (SPEC-019). A
|
|
62
|
+
``retired`` worker still being handed events is the same total loss arrived at from the
|
|
63
|
+
other direction — someone called :func:`shutdown` in a process that logs again (SPEC-030)::
|
|
62
64
|
|
|
63
65
|
h = log_foundry.health()
|
|
64
|
-
if h.dropped or h.failed_batches or h.stopped_reason or (
|
|
66
|
+
if h.dropped or h.failed_batches or h.stopped_reason or h.incomplete_swaps or (
|
|
65
67
|
h.sink and (h.sink.dropped or h.sink.failed)
|
|
66
|
-
):
|
|
68
|
+
) or (h.retired and h.submitted_after_shutdown):
|
|
67
69
|
... # raise an alert; logs were silently lost
|
|
68
70
|
|
|
71
|
+
``retired`` alone is not a fault — a process that shuts down and then stops logging is
|
|
72
|
+
doing the right thing, which is why it is paired with the count rather than alerted on.
|
|
73
|
+
|
|
69
74
|
Args:
|
|
70
75
|
None.
|
|
71
76
|
|
|
72
77
|
Returns:
|
|
73
|
-
The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason``
|
|
74
|
-
``
|
|
78
|
+
The snapshot: ``queued``, ``dropped``, ``failed_batches``, ``stopped_reason``, ``sink``,
|
|
79
|
+
``retired``, ``submitted_after_shutdown`` and ``incomplete_swaps``. ``retired`` says
|
|
80
|
+
:func:`shutdown` was called, and ``submitted_after_shutdown`` counts events accepted
|
|
81
|
+
afterwards, which are queued where nothing will drain them — non-zero together, that is
|
|
82
|
+
the ``shutdown()``-per-invocation mistake, and the remedy is :func:`flush`.
|
|
83
|
+
``incomplete_swaps`` counts late ``configure(sink=...)`` calls whose drain of the
|
|
84
|
+
previous sink could not be confirmed, leaving that sink open. ``sink`` is the configured
|
|
85
|
+
sink's own
|
|
75
86
|
:class:`~log_foundry.sinks.base.SinkLosses` — loss the sink absorbed rather than the
|
|
76
87
|
worker (SPEC-026) — and is ``None`` when no worker exists or the sink reports nothing.
|
|
88
|
+
Note ``sink`` describes whichever sink is live now, so a swap takes the previous sink's
|
|
89
|
+
absorbed losses out of the snapshot with it.
|
|
77
90
|
Its ``dropped`` is not the worker's: the worker's is backpressure at the queue, the
|
|
78
91
|
sink's is an event that never reached the wire, and the stderr line names which. Its
|
|
79
92
|
``failed`` is an upper bound on loss rather than a count of it, since a sink that raises
|
|
@@ -98,6 +111,11 @@ def shutdown(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
|
|
|
98
111
|
registered via ``atexit``, so call it explicitly only when you want to be certain the tail
|
|
99
112
|
of the queue reached the sink before a fast process exit (SPEC-004 FR-005).
|
|
100
113
|
|
|
114
|
+
Logging afterwards is **accepted and undeliverable**: those events are queued and nothing
|
|
115
|
+
will drain them. It is not silent any more — :func:`health` then reports ``retired`` with a
|
|
116
|
+
non-zero ``submitted_after_shutdown``, and the first such submission writes one stderr line
|
|
117
|
+
(SPEC-030). That pair is the reading that catches the mistake above.
|
|
118
|
+
|
|
101
119
|
Args:
|
|
102
120
|
timeout: Seconds bounding the wait for the background thread (SPEC-027 FR-004). ``None``
|
|
103
121
|
waits indefinitely, which is what this did unconditionally before and is still
|
|
@@ -76,11 +76,31 @@ def configure(
|
|
|
76
76
|
the zero-dependency dev default (arch §8). Every ceiling is validated before anything is
|
|
77
77
|
assigned, so a rejected call leaves the config exactly as it found it.
|
|
78
78
|
|
|
79
|
+
A ``sink=`` passed after logging has already started is the one argument that needs more
|
|
80
|
+
than an assignment, because the background worker captured its sink when it was built
|
|
81
|
+
(arch §7). It **swaps the live delivery target**: everything submitted so far is drained to
|
|
82
|
+
the previous sink, that sink is closed, and subsequent events go to the new one. The drains
|
|
83
|
+
are bounded, and a swap whose drain could not be confirmed leaves the previous sink open and
|
|
84
|
+
records ``health().incomplete_swaps`` (SPEC-030 FR-003) — so "repeated calls compose rather
|
|
85
|
+
than reset" holds for the sink too, at the cost of one bounded wait. Passing the sink that
|
|
86
|
+
is already live is a no-op: no drain, no close. The previous sink is closed and must not be
|
|
87
|
+
handed back to a later call.
|
|
88
|
+
|
|
89
|
+
The *closing* of the previous sink is not bounded, because ``Sink.close`` takes no timeout —
|
|
90
|
+
a destination that blocks in ``close()`` blocks this call. It is the same gap
|
|
91
|
+
``architecture.md`` §13 already records for ``shutdown()``, and it has the same fix, which
|
|
92
|
+
is a change to the sink contract rather than to this function.
|
|
93
|
+
|
|
94
|
+
This is still a startup call. It is not thread-safe, and a span finishing on another thread
|
|
95
|
+
during a swap may land on either sink.
|
|
96
|
+
|
|
79
97
|
Args:
|
|
80
98
|
service: The service name stamped onto every event.
|
|
81
99
|
version: The service version stamped onto every event.
|
|
82
100
|
env: The deployment environment stamped onto every event.
|
|
83
|
-
sink: The destination every event is delivered to.
|
|
101
|
+
sink: The destination every event is delivered to. Passed after the first log, it swaps
|
|
102
|
+
the live target as described above rather than only updating what ``get_config()``
|
|
103
|
+
reports.
|
|
84
104
|
defaults: Fields merged into every event at the lowest precedence.
|
|
85
105
|
max_value_bytes: Per-value ceiling, in UTF-8 bytes or rendered digits.
|
|
86
106
|
max_stack_bytes: Ceiling for ``error.stack`` alone.
|
|
@@ -119,6 +139,37 @@ def configure(
|
|
|
119
139
|
|
|
120
140
|
_ensure_sink()
|
|
121
141
|
|
|
142
|
+
if sink is not None:
|
|
143
|
+
_swap_live_sink(sink)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _swap_live_sink(sink: Sink) -> None:
|
|
147
|
+
"""Points an already-running worker at a newly configured sink (SPEC-030 FR-003).
|
|
148
|
+
|
|
149
|
+
The import is local for the reason ``_ensure_sink``'s is: ``decorator`` imports this module
|
|
150
|
+
at module scope, so reaching back the other way at import time would be a cycle. It also
|
|
151
|
+
keeps the dependency to the one call that needs it — ``configure()`` without a ``sink=``
|
|
152
|
+
never touches the worker at all.
|
|
153
|
+
|
|
154
|
+
The budget is passed explicitly rather than left to the parameter default, so that the
|
|
155
|
+
bound this call actually applies is resolved when it runs. A default argument is bound at
|
|
156
|
+
definition time, which would leave the end-to-end bound untestable without reaching past
|
|
157
|
+
the function under test.
|
|
158
|
+
|
|
159
|
+
Args:
|
|
160
|
+
sink: The sink just written to the config.
|
|
161
|
+
|
|
162
|
+
Returns:
|
|
163
|
+
None.
|
|
164
|
+
|
|
165
|
+
Raises:
|
|
166
|
+
None.
|
|
167
|
+
"""
|
|
168
|
+
from log_foundry.decorator import _swap_sink
|
|
169
|
+
from log_foundry.worker import DEFAULT_SWAP_TIMEOUT
|
|
170
|
+
|
|
171
|
+
_swap_sink(sink, DEFAULT_SWAP_TIMEOUT)
|
|
172
|
+
|
|
122
173
|
|
|
123
174
|
def get_config() -> Config:
|
|
124
175
|
"""Returns the current global config singleton.
|
|
@@ -20,11 +20,13 @@ from log_foundry.ids import (
|
|
|
20
20
|
parse_traceparent,
|
|
21
21
|
)
|
|
22
22
|
from log_foundry.model import Span, backfill_baggage, end_event, start_event
|
|
23
|
-
from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, Health, Worker
|
|
23
|
+
from log_foundry.worker import DEFAULT_SHUTDOWN_TIMEOUT, DEFAULT_SWAP_TIMEOUT, Health, Worker
|
|
24
24
|
|
|
25
25
|
if TYPE_CHECKING:
|
|
26
26
|
import contextvars
|
|
27
27
|
|
|
28
|
+
from log_foundry.sinks.base import Sink
|
|
29
|
+
|
|
28
30
|
__all__ = ["continue_trace", "trace"]
|
|
29
31
|
|
|
30
32
|
_worker: Worker | None = None
|
|
@@ -231,6 +233,37 @@ def _shutdown_worker(timeout: float | None = DEFAULT_SHUTDOWN_TIMEOUT) -> None:
|
|
|
231
233
|
_worker.shutdown(timeout)
|
|
232
234
|
|
|
233
235
|
|
|
236
|
+
def _swap_sink(new_sink: Sink, timeout: float | None = DEFAULT_SWAP_TIMEOUT) -> None:
|
|
237
|
+
"""Retargets the process worker at a new sink, backing a late ``configure(sink=...)``.
|
|
238
|
+
|
|
239
|
+
Like :func:`_flush_worker` this deliberately does not call :func:`_get_worker`: a process
|
|
240
|
+
that has not logged has captured no sink, so there is nothing to swap and building a thread
|
|
241
|
+
to prove it would be pure cost — that is also the case where the old behaviour was already
|
|
242
|
+
correct (SPEC-030 FR-003).
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
new_sink: The sink already written to the config, to be made the live delivery target.
|
|
246
|
+
timeout: Seconds bounding the drain of the previous sink.
|
|
247
|
+
|
|
248
|
+
Returns:
|
|
249
|
+
None.
|
|
250
|
+
|
|
251
|
+
Raises:
|
|
252
|
+
None. This runs inside ``configure()``, which has never raised for anything but a
|
|
253
|
+
rejected ceiling, and a sink swap that fails must not become the reason an application
|
|
254
|
+
cannot start.
|
|
255
|
+
"""
|
|
256
|
+
worker = _worker
|
|
257
|
+
if worker is None:
|
|
258
|
+
return
|
|
259
|
+
try:
|
|
260
|
+
worker.swap_sink(new_sink, timeout)
|
|
261
|
+
except Exception as exc:
|
|
262
|
+
_diag.absorbed(
|
|
263
|
+
"swapping the log sink", exc, "events may still be delivered to the previous sink"
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
|
|
234
267
|
def _flush_worker(timeout: float | None = 5.0) -> bool:
|
|
235
268
|
"""Drains the process worker without retiring it, backing ``flush()`` (SPEC-013 FR-003).
|
|
236
269
|
|
|
@@ -94,9 +94,25 @@ class Sink(Protocol):
|
|
|
94
94
|
re-delivered wholesale by the worker's retry, and duplicates downstream are worse than
|
|
95
95
|
the counted loss (SPEC-017 FR-004, SPEC-018). Report that through ``losses()`` instead.
|
|
96
96
|
|
|
97
|
+
**After ``close()`` has released or invalidated something, raise rather than absorb**
|
|
98
|
+
(SPEC-032 FR-002). This is the same obligation applied to the sink's own lifecycle
|
|
99
|
+
rather than the destination's: an absorbed batch is one the worker believes, so the
|
|
100
|
+
retry never engages, ``failed_batches`` stays at zero and ``flush()`` returns True while
|
|
101
|
+
the events are gone — reached not through a destination that is down but through a
|
|
102
|
+
transport the library itself has already let go. Three shipped sinks failed this rule
|
|
103
|
+
for four specs, in three different ways: a produce into a batch nothing would flush
|
|
104
|
+
again, a future nothing would resolve, and a client that quietly reconnected.
|
|
105
|
+
|
|
106
|
+
The converse is equally binding: a sink holding nothing to release **keeps accepting**.
|
|
107
|
+
``close()`` on a sink that opens a fresh connection per request, or whose client belongs
|
|
108
|
+
to the caller, has invalidated nothing, so a later batch still delivers and refusing it
|
|
109
|
+
would be loss the library invented rather than loss it reported. Which of the two
|
|
110
|
+
applies is a property of the sink, so each shipped sink records its answer in its class
|
|
111
|
+
docstring and a test holds it to it.
|
|
112
|
+
|
|
97
113
|
Args:
|
|
98
114
|
batch: The events to ship. ``emit([])`` is a no-op and never raises, since an empty
|
|
99
|
-
batch has not failed to deliver.
|
|
115
|
+
batch has not failed to deliver — closed or not.
|
|
100
116
|
|
|
101
117
|
Returns:
|
|
102
118
|
None.
|
|
@@ -116,6 +132,12 @@ class Sink(Protocol):
|
|
|
116
132
|
succeeding. Taking the same lock ``emit`` takes satisfies this; so does an idempotent
|
|
117
133
|
guard checked under that lock.
|
|
118
134
|
|
|
135
|
+
The flag this sets is the one a later :meth:`emit` reads, so the two are one decision
|
|
136
|
+
rather than two (SPEC-032 FR-002). Set it *before* releasing anything: an emit arriving
|
|
137
|
+
during a release must be refused rather than handed a half-released transport. Make the
|
|
138
|
+
release idempotent for the same reason a second ``close()`` is expected at all —
|
|
139
|
+
``atexit`` racing a caller's own cleanup is the documented case.
|
|
140
|
+
|
|
119
141
|
Args:
|
|
120
142
|
None.
|
|
121
143
|
|
|
@@ -17,6 +17,11 @@ class CallbackSink:
|
|
|
17
17
|
function, without writing a ``Sink`` implementation. Attributes are internal; the observable
|
|
18
18
|
contract is that ``emit`` hands the batch to the callable unchanged and ``close`` invokes
|
|
19
19
|
the close hook once when one was supplied.
|
|
20
|
+
|
|
21
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
22
|
+
(SPEC-032 FR-003). Both decisions belong to the callable: this class holds nothing, and what
|
|
23
|
+
a hook releases is not knowable from here — a callable needing either guarantee must provide
|
|
24
|
+
it, exactly as a hand-written ``Sink`` implementation would.
|
|
20
25
|
"""
|
|
21
26
|
|
|
22
27
|
def __init__(
|
|
@@ -16,6 +16,10 @@ class DatadogSink(HTTPSink):
|
|
|
16
16
|
|
|
17
17
|
The batch goes as a JSON array to the region-specific intake with a ``DD-API-KEY`` header,
|
|
18
18
|
each entry enriched with ``ddsource``, ``service`` and ``ddtags``.
|
|
19
|
+
|
|
20
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
21
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
22
|
+
is no transport held and ``close()`` releases nothing.
|
|
19
23
|
"""
|
|
20
24
|
|
|
21
25
|
def __init__(
|
{log_foundry-0.10.2.dev25 → log_foundry-0.10.2.dev27}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
@@ -25,6 +25,10 @@ class ElasticsearchSink(HTTPSink):
|
|
|
25
25
|
dropped_unadjudicated: Events whose outcome a ``_bulk`` response did not describe, because
|
|
26
26
|
its ``items`` array did not line up with the batch sent. Abandoned rather than retried,
|
|
27
27
|
for SPEC-018's reason: the request succeeded, so re-sending would duplicate what landed.
|
|
28
|
+
|
|
29
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
30
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
31
|
+
is no transport held and ``close()`` releases nothing.
|
|
28
32
|
"""
|
|
29
33
|
|
|
30
34
|
def __init__(self, url: str, *, index: str, auth: str | tuple[str, str] | None = None,
|
|
@@ -24,6 +24,10 @@ class FilteringSink:
|
|
|
24
24
|
This is a static, emit-time filter in front of an inner sink, not the reserved
|
|
25
25
|
tail-sampling ``should_send`` seam (arch §10) — that stays deferred and owns rate policy at
|
|
26
26
|
span-decision time, while this only reshapes an already-built batch on its way to a sink.
|
|
27
|
+
|
|
28
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
29
|
+
(SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
|
|
30
|
+
belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
|
|
27
31
|
"""
|
|
28
32
|
|
|
29
33
|
def __init__(
|
|
@@ -36,6 +36,10 @@ class FirehoseSink:
|
|
|
36
36
|
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
37
37
|
lock. ``boto3`` clients are documented thread-safe, this one is built once in
|
|
38
38
|
``__init__``, and the sink rebinds nothing after construction.
|
|
39
|
+
|
|
40
|
+
It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
|
|
41
|
+
because the client is the caller's to release or the SDK's to reap, so a batch emitted
|
|
42
|
+
afterwards still reaches the delivery stream.
|
|
39
43
|
"""
|
|
40
44
|
|
|
41
45
|
MAX_RECORDS = 500
|
|
@@ -14,6 +14,10 @@ class HoneycombSink(HTTPSink):
|
|
|
14
14
|
|
|
15
15
|
The request goes to ``/1/batch/<dataset>`` with an ``X-Honeycomb-Team`` header, in
|
|
16
16
|
Honeycomb's ``[{"data": <event>}, ...]`` batch shape.
|
|
17
|
+
|
|
18
|
+
It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
|
|
19
|
+
(SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
|
|
20
|
+
is no transport held and ``close()`` releases nothing.
|
|
17
21
|
"""
|
|
18
22
|
|
|
19
23
|
def __init__(
|
|
@@ -74,6 +74,13 @@ class HTTPSink:
|
|
|
74
74
|
failed: Requests abandoned past the retry bound.
|
|
75
75
|
dropped_oversized: Events dropped for exceeding a destination's hard size limit, used by
|
|
76
76
|
subclasses that enforce one; the generic core imposes no universal limit.
|
|
77
|
+
|
|
78
|
+
The two lifecycle decisions this class records, inherited by every platform subclass. It
|
|
79
|
+
takes **no** transport lock (SPEC-028 FR-002): there is no transport to guard, since
|
|
80
|
+
``urllib`` opens a fresh connection per request and nothing is rebound after construction.
|
|
81
|
+
And it **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` releases nothing —
|
|
82
|
+
a batch emitted afterwards still reaches the endpoint, and refusing it would be loss the
|
|
83
|
+
library invented rather than loss it reported.
|
|
77
84
|
"""
|
|
78
85
|
|
|
79
86
|
def __init__(
|