log-foundry 0.10.2.dev50__tar.gz → 0.10.2.dev52__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/PKG-INFO +12 -12
  2. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/README.md +11 -11
  3. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/pyproject.toml +1 -1
  4. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/_socket.py +86 -5
  5. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/file.py +22 -5
  6. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/firehose.py +14 -2
  7. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/kafka.py +96 -2
  8. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/kinesis.py +24 -1
  9. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/logstash.py +11 -2
  10. log_foundry-0.10.2.dev52/src/log_foundry/sinks/memory.py +63 -0
  11. log_foundry-0.10.2.dev52/src/log_foundry/sinks/null.py +65 -0
  12. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/redis.py +48 -3
  13. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/stdout.py +30 -1
  14. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/syslog.py +11 -2
  15. log_foundry-0.10.2.dev50/src/log_foundry/sinks/util.py +0 -156
  16. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/LICENSE +0 -0
  17. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/__init__.py +0 -0
  18. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/_diag.py +0 -0
  19. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/_lifecycle.py +0 -0
  20. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/api.py +0 -0
  21. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/config.py +0 -0
  22. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/console.py +0 -0
  23. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/context.py +0 -0
  24. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/decorator.py +0 -0
  25. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/ids.py +0 -0
  26. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/model.py +0 -0
  27. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/py.typed +0 -0
  28. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/results.py +0 -0
  29. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sanitize.py +0 -0
  30. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/__init__.py +0 -0
  31. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/_batch.py +0 -0
  32. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/_chunk.py +0 -0
  33. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/_retry.py +0 -0
  34. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/_time.py +0 -0
  35. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/base.py +0 -0
  36. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/callback.py +0 -0
  37. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/clickhouse.py +0 -0
  38. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/datadog.py +0 -0
  39. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/elasticsearch.py +0 -0
  40. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/eventhubs.py +0 -0
  41. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/filtering.py +0 -0
  42. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/honeycomb.py +0 -0
  43. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/http.py +0 -0
  44. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/logging_sink.py +0 -0
  45. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/loki.py +0 -0
  46. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/mongodb.py +0 -0
  47. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/multi.py +0 -0
  48. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/nats.py +0 -0
  49. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/newrelic.py +0 -0
  50. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/postgres.py +0 -0
  51. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/pubsub.py +0 -0
  52. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/rabbitmq.py +0 -0
  53. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/sentry.py +0 -0
  54. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/sns.py +0 -0
  55. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/splunk.py +0 -0
  56. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/sqlite.py +0 -0
  57. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/sqs.py +0 -0
  58. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/sinks/transform.py +0 -0
  59. {log_foundry-0.10.2.dev50 → log_foundry-0.10.2.dev52}/src/log_foundry/worker.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev50
3
+ Version: 0.10.2.dev52
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -274,9 +274,9 @@ the sink that is already live is a no-op: no drain, no close. The previous sink
274
274
  do not hand it back to a later call.
275
275
 
276
276
  The 5 s covers the **whole** call — both drains and the previous sink's `close()`. `Sink.close()`
277
- takes no timeout of its own (`KafkaSink.close()` flushes its producer, so an unreachable broker
278
- blocks it), so that close runs on its own daemon thread and is joined for whatever is left of the
279
- budget. A hung `close()` therefore costs you the budget, once, and the close carries on in the
277
+ takes no timeout of its own — though `KafkaSink` bounds its own flush at `flush_timeout=10.0`
278
+ and counts whatever is still queued — so that close runs on its own daemon thread and is joined
279
+ for whatever is left of the budget. A hung `close()` therefore costs you the budget, once, and the close carries on in the
280
280
  background afterwards.
281
281
 
282
282
  Nothing is *reported* when that join expires — a slow close is not a failed swap, and a counter
@@ -541,9 +541,9 @@ A few conventions hold across every sink below:
541
541
  | Sink | Import from | Configure |
542
542
  |---|---|---|
543
543
  | `StdoutSink` | `log_foundry.sinks.stdout` | `StdoutSink(stream=sys.stdout)` — one JSON line per event; the zero-config default |
544
- | `StderrSink` | `log_foundry.sinks.util` | `StderrSink(stream=sys.stderr)` — same, on stderr (twelve-factor) |
545
- | `NullSink` | `log_foundry.sinks.util` | `NullSink()` — discard everything; `.dropped` counts events |
546
- | `MemorySink` | `log_foundry.sinks.util` | `MemorySink(maxlen=None)` — collect into `.events` (a bounded ring when `maxlen` is set) |
544
+ | `StderrSink` | `log_foundry.sinks.stdout` | `StderrSink(stream=sys.stderr)` — same, on stderr (twelve-factor) |
545
+ | `NullSink` | `log_foundry.sinks.null` | `NullSink()` — discard everything; `.dropped` counts events |
546
+ | `MemorySink` | `log_foundry.sinks.memory` | `MemorySink(maxlen=None)` — collect into `.events` (a bounded ring when `maxlen` is set) |
547
547
 
548
548
  ```python
549
549
  from log_foundry.sinks.stdout import StdoutSink
@@ -592,7 +592,7 @@ attached to each record; the sink never configures or tears down logging itself.
592
592
  | Sink | Import from | Configure |
593
593
  |---|---|---|
594
594
  | `FileSink` | `log_foundry.sinks.file` | `FileSink(path, *, encoding="utf-8")` — append NDJSON to one file |
595
- | `RotatingFileSink` | `log_foundry.sinks.file` | `RotatingFileSink(path, *, max_bytes=0, backup_count=0, when=None, interval=1)` — rotate by size and/or time, keeping `backup_count` numbered backups |
595
+ | `RotatingFileSink` | `log_foundry.sinks.file` | `RotatingFileSink(path, *, max_bytes=0, backup_count=1, when=None, interval=1)` — rotate by size and/or time, keeping `backup_count` numbered backups. **`backup_count=0` truncates**: every event since the last rotation is destroyed. The default keeps one generation, costing 2 × `max_bytes` on disk under a size trigger, or one full rollover period under a time-only one — and `max_bytes` defaults to `0`, which bounds nothing |
596
596
  | `SQLiteSink` | `log_foundry.sinks.sqlite` | `SQLiteSink(database, *, table="log_events", create_table=True)` — batch-insert into an embedded SQLite DB |
597
597
 
598
598
  `RotatingFileSink`'s time trigger uses a `when` unit code — `"S"`/`"M"`/`"H"`/`"D"` — times `interval`
@@ -647,7 +647,7 @@ completed, not that every chunk of it landed.
647
647
  | `OpenSearchSink` | `log_foundry.sinks.elasticsearch` | same signature as `ElasticsearchSink` (identical bulk protocol) |
648
648
  | `LokiSink` | `log_foundry.sinks.loki` | `LokiSink(url, *, labels=("service", "env", "level"), **http_kwargs)` — Grafana Loki push API |
649
649
  | `LogstashSink` | `log_foundry.sinks.logstash` | `LogstashSink(url=…, **http_kwargs)` for HTTP, **or** `LogstashSink(host=…, port=…, transport="tcp")` for a raw TCP/UDP socket |
650
- | `SyslogSink` | `log_foundry.sinks.syslog` | `SyslogSink(host, port=514, *, transport="udp", facility="user", app_name="log-foundry")` — RFC 5424 over UDP/TCP |
650
+ | `SyslogSink` | `log_foundry.sinks.syslog` | `SyslogSink(host, port=514, *, transport="udp", facility="user", app_name="log-foundry", max_datagram_bytes=65507)` — RFC 5424 over UDP/TCP. A UDP frame over the limit is dropped and counted rather than sent, retried and abandoned; TCP is a stream and is unaffected |
651
651
 
652
652
  ```python
653
653
  from log_foundry.sinks.elasticsearch import ElasticsearchSink
@@ -753,9 +753,9 @@ Each needs its own extra (lazy-imported). All publish + retry within a bound and
753
753
 
754
754
  | Sink | Import from | Extra | Configure |
755
755
  |---|---|---|---|
756
- | `KafkaSink` | `log_foundry.sinks.kafka` | `kafka` | `KafkaSink(topic, *, bootstrap_servers="…", key_field="trace_id")` |
757
- | `RedisStreamsSink` | `log_foundry.sinks.redis` | `redis` | `RedisStreamsSink(stream, *, url=None)` — `XADD` |
758
- | `RedisListSink` | `log_foundry.sinks.redis` | `redis` | `RedisListSink(key, *, url=None)` — `RPUSH` |
756
+ | `KafkaSink` | `log_foundry.sinks.kafka` | `kafka` | `KafkaSink(topic, *, flush_timeout=10.0, bootstrap_servers="…", key_field="trace_id")` |
757
+ | `RedisStreamsSink` | `log_foundry.sinks.redis` | `redis` | `RedisStreamsSink(stream, *, url=None, maxlen=None)` — `XADD`. `maxlen` caps the stream (`approximate=True`); trimming happens **at Redis**, after delivery, so it is invisible to `health()` — which is why the default is unbounded |
758
+ | `RedisListSink` | `log_foundry.sinks.redis` | `redis` | `RedisListSink(key, *, url=None, maxlen=None)` — `RPUSH` + `LTRIM` to the newest `maxlen`; same destination-side trimming caveat |
759
759
  | `RabbitMQSink` | `log_foundry.sinks.rabbitmq` | `amqp` | `RabbitMQSink(*, exchange, routing_key, url=None)` — persistent messages |
760
760
  | `NATSSink` | `log_foundry.sinks.nats` | `nats` | `NATSSink(subject, *, jetstream=False, servers=None)` |
761
761
  | `GooglePubSubSink` | `log_foundry.sinks.pubsub` | `gcp-pubsub` | `GooglePubSubSink(topic)` |
@@ -238,9 +238,9 @@ the sink that is already live is a no-op: no drain, no close. The previous sink
238
238
  do not hand it back to a later call.
239
239
 
240
240
  The 5 s covers the **whole** call — both drains and the previous sink's `close()`. `Sink.close()`
241
- takes no timeout of its own (`KafkaSink.close()` flushes its producer, so an unreachable broker
242
- blocks it), so that close runs on its own daemon thread and is joined for whatever is left of the
243
- budget. A hung `close()` therefore costs you the budget, once, and the close carries on in the
241
+ takes no timeout of its own — though `KafkaSink` bounds its own flush at `flush_timeout=10.0`
242
+ and counts whatever is still queued — so that close runs on its own daemon thread and is joined
243
+ for whatever is left of the budget. A hung `close()` therefore costs you the budget, once, and the close carries on in the
244
244
  background afterwards.
245
245
 
246
246
  Nothing is *reported* when that join expires — a slow close is not a failed swap, and a counter
@@ -505,9 +505,9 @@ A few conventions hold across every sink below:
505
505
  | Sink | Import from | Configure |
506
506
  |---|---|---|
507
507
  | `StdoutSink` | `log_foundry.sinks.stdout` | `StdoutSink(stream=sys.stdout)` — one JSON line per event; the zero-config default |
508
- | `StderrSink` | `log_foundry.sinks.util` | `StderrSink(stream=sys.stderr)` — same, on stderr (twelve-factor) |
509
- | `NullSink` | `log_foundry.sinks.util` | `NullSink()` — discard everything; `.dropped` counts events |
510
- | `MemorySink` | `log_foundry.sinks.util` | `MemorySink(maxlen=None)` — collect into `.events` (a bounded ring when `maxlen` is set) |
508
+ | `StderrSink` | `log_foundry.sinks.stdout` | `StderrSink(stream=sys.stderr)` — same, on stderr (twelve-factor) |
509
+ | `NullSink` | `log_foundry.sinks.null` | `NullSink()` — discard everything; `.dropped` counts events |
510
+ | `MemorySink` | `log_foundry.sinks.memory` | `MemorySink(maxlen=None)` — collect into `.events` (a bounded ring when `maxlen` is set) |
511
511
 
512
512
  ```python
513
513
  from log_foundry.sinks.stdout import StdoutSink
@@ -556,7 +556,7 @@ attached to each record; the sink never configures or tears down logging itself.
556
556
  | Sink | Import from | Configure |
557
557
  |---|---|---|
558
558
  | `FileSink` | `log_foundry.sinks.file` | `FileSink(path, *, encoding="utf-8")` — append NDJSON to one file |
559
- | `RotatingFileSink` | `log_foundry.sinks.file` | `RotatingFileSink(path, *, max_bytes=0, backup_count=0, when=None, interval=1)` — rotate by size and/or time, keeping `backup_count` numbered backups |
559
+ | `RotatingFileSink` | `log_foundry.sinks.file` | `RotatingFileSink(path, *, max_bytes=0, backup_count=1, when=None, interval=1)` — rotate by size and/or time, keeping `backup_count` numbered backups. **`backup_count=0` truncates**: every event since the last rotation is destroyed. The default keeps one generation, costing 2 × `max_bytes` on disk under a size trigger, or one full rollover period under a time-only one — and `max_bytes` defaults to `0`, which bounds nothing |
560
560
  | `SQLiteSink` | `log_foundry.sinks.sqlite` | `SQLiteSink(database, *, table="log_events", create_table=True)` — batch-insert into an embedded SQLite DB |
561
561
 
562
562
  `RotatingFileSink`'s time trigger uses a `when` unit code — `"S"`/`"M"`/`"H"`/`"D"` — times `interval`
@@ -611,7 +611,7 @@ completed, not that every chunk of it landed.
611
611
  | `OpenSearchSink` | `log_foundry.sinks.elasticsearch` | same signature as `ElasticsearchSink` (identical bulk protocol) |
612
612
  | `LokiSink` | `log_foundry.sinks.loki` | `LokiSink(url, *, labels=("service", "env", "level"), **http_kwargs)` — Grafana Loki push API |
613
613
  | `LogstashSink` | `log_foundry.sinks.logstash` | `LogstashSink(url=…, **http_kwargs)` for HTTP, **or** `LogstashSink(host=…, port=…, transport="tcp")` for a raw TCP/UDP socket |
614
- | `SyslogSink` | `log_foundry.sinks.syslog` | `SyslogSink(host, port=514, *, transport="udp", facility="user", app_name="log-foundry")` — RFC 5424 over UDP/TCP |
614
+ | `SyslogSink` | `log_foundry.sinks.syslog` | `SyslogSink(host, port=514, *, transport="udp", facility="user", app_name="log-foundry", max_datagram_bytes=65507)` — RFC 5424 over UDP/TCP. A UDP frame over the limit is dropped and counted rather than sent, retried and abandoned; TCP is a stream and is unaffected |
615
615
 
616
616
  ```python
617
617
  from log_foundry.sinks.elasticsearch import ElasticsearchSink
@@ -717,9 +717,9 @@ Each needs its own extra (lazy-imported). All publish + retry within a bound and
717
717
 
718
718
  | Sink | Import from | Extra | Configure |
719
719
  |---|---|---|---|
720
- | `KafkaSink` | `log_foundry.sinks.kafka` | `kafka` | `KafkaSink(topic, *, bootstrap_servers="…", key_field="trace_id")` |
721
- | `RedisStreamsSink` | `log_foundry.sinks.redis` | `redis` | `RedisStreamsSink(stream, *, url=None)` — `XADD` |
722
- | `RedisListSink` | `log_foundry.sinks.redis` | `redis` | `RedisListSink(key, *, url=None)` — `RPUSH` |
720
+ | `KafkaSink` | `log_foundry.sinks.kafka` | `kafka` | `KafkaSink(topic, *, flush_timeout=10.0, bootstrap_servers="…", key_field="trace_id")` |
721
+ | `RedisStreamsSink` | `log_foundry.sinks.redis` | `redis` | `RedisStreamsSink(stream, *, url=None, maxlen=None)` — `XADD`. `maxlen` caps the stream (`approximate=True`); trimming happens **at Redis**, after delivery, so it is invisible to `health()` — which is why the default is unbounded |
722
+ | `RedisListSink` | `log_foundry.sinks.redis` | `redis` | `RedisListSink(key, *, url=None, maxlen=None)` — `RPUSH` + `LTRIM` to the newest `maxlen`; same destination-side trimming caveat |
723
723
  | `RabbitMQSink` | `log_foundry.sinks.rabbitmq` | `amqp` | `RabbitMQSink(*, exchange, routing_key, url=None)` — persistent messages |
724
724
  | `NATSSink` | `log_foundry.sinks.nats` | `nats` | `NATSSink(subject, *, jetstream=False, servers=None)` |
725
725
  | `GooglePubSubSink` | `log_foundry.sinks.pubsub` | `gcp-pubsub` | `GooglePubSubSink(topic)` |
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev50"
23
+ version = "0.10.2.dev52"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import errno
5
6
  import socket
6
7
  import threading
7
8
 
@@ -13,6 +14,26 @@ __all__ = ["SocketTransport"]
13
14
 
14
15
  _BACKOFF_BASE = 0.1
15
16
 
17
+ DEFAULT_MAX_DATAGRAM_BYTES = 65507
18
+ """Bytes one UDP datagram may carry, the IPv4 payload maximum (SPEC-038 FR-007).
19
+
20
+ 65,535 less the 20-byte IP header and the 8-byte UDP header. That is the **IPv4** figure, and
21
+ it is used for IPv6 too: IPv6 does not count its own header in the payload length, so its true
22
+ maximum is 65,527 — a 20-byte discrepancy no real path can reach, since no MTU carries either.
23
+ It is the ceiling the *protocol* imposes; a path's real limit is often far lower, which is why
24
+ it is configurable. TCP is a stream and has no such limit, so this bounds nothing there.
25
+ """
26
+
27
+ _PERMANENT_ERRNOS = frozenset({errno.EMSGSIZE})
28
+ """Socket errnos that describe the message rather than the destination (FR-007 AC-3).
29
+
30
+ ``EMSGSIZE`` alone. Every other errno this transport can raise — ``ECONNREFUSED``,
31
+ ``EHOSTUNREACH``, ``ENETDOWN``, ``EPIPE``, ``ETIMEDOUT`` — describes a destination that may come
32
+ back, and treating any of those as permanent would turn a transient outage into silent loss. A
33
+ set stated here rather than a condition inferred at the call site, so adding to it is a decision
34
+ someone makes deliberately.
35
+ """
36
+
16
37
 
17
38
  def _make_tcp(host: str, port: int, timeout: float) -> socket.socket:
18
39
  """Opens a connected TCP socket.
@@ -95,6 +116,8 @@ class SocketTransport:
95
116
 
96
117
  Attributes:
97
118
  failed: Messages abandoned past the reconnect-retry bound.
119
+ dropped_oversized: UDP datagrams discarded before any send for exceeding
120
+ ``max_datagram_bytes``.
98
121
  """
99
122
 
100
123
  def __init__(
@@ -105,6 +128,7 @@ class SocketTransport:
105
128
  transport: str = "tcp",
106
129
  timeout: float = 5.0,
107
130
  max_retries: int = 3,
131
+ max_datagram_bytes: int = DEFAULT_MAX_DATAGRAM_BYTES,
108
132
  ) -> None:
109
133
  """Configures the destination and retry bound without opening a socket yet.
110
134
 
@@ -116,6 +140,8 @@ class SocketTransport:
116
140
  max_retries: Reconnect retries per message, floored at zero for the reason
117
141
  ``Worker._emit`` floors its own (SPEC-021) — a negative value made no attempt at all
118
142
  and abandoned the message without moving ``failed``.
143
+ max_datagram_bytes: The largest UDP datagram to attempt, defaulting to
144
+ :data:`DEFAULT_MAX_DATAGRAM_BYTES`. Ignored for TCP, which is a stream.
119
145
 
120
146
  Returns:
121
147
  None.
@@ -130,8 +156,10 @@ class SocketTransport:
130
156
  self._transport = transport
131
157
  self._timeout = timeout
132
158
  self._max_retries = max(max_retries, 0)
159
+ self._max_datagram_bytes = max_datagram_bytes
133
160
  self._sock: socket.socket | None = None
134
161
  self.failed = 0
162
+ self.dropped_oversized = 0
135
163
  self._counter_lock = threading.Lock()
136
164
  self.log_foundry_stop_signal: threading.Event | None = None
137
165
  self._lock = threading.Lock()
@@ -170,15 +198,62 @@ class SocketTransport:
170
198
  f"SocketTransport delivered none of {len(messages)} message(s): "
171
199
  f"the transport is closed"
172
200
  )
201
+ sendable = self._sendable(messages)
173
202
  delivered = 0
174
- for message in messages:
203
+ for message in sendable:
175
204
  if self._send_one(message):
176
205
  delivered += 1
177
- if delivered == 0:
206
+ if sendable and delivered == 0:
178
207
  raise SinkDeliveryError(
179
- f"SocketTransport delivered none of {len(messages)} message(s)"
208
+ f"SocketTransport delivered none of {len(sendable)} message(s)"
180
209
  )
181
210
 
211
+ def _sendable(self, messages: list[bytes]) -> list[bytes]:
212
+ """Discards UDP datagrams too large to send, before any attempt (FR-007 AC-1).
213
+
214
+ A 70 KB event produced ``OSError errno=40`` (EMSGSIZE), which the retry loop treated as
215
+ transient: four sends with backoff, counted as ``failed``, then a raise that sent the
216
+ worker round for three more rounds — roughly sixteen futile sends and seconds of backoff
217
+ on the single drain thread, never converging. The size is knowable *before* sending, and
218
+ every other size-limited sink here drops and counts first, so this one does too.
219
+
220
+ Dropping is right rather than raising: the datagram is permanently unsendable on this
221
+ path, so there is nothing for a retry to fix, and reporting it as a delivery failure
222
+ would have the worker re-send the rest of the batch alongside it.
223
+
224
+ **``dropped_oversized`` inflates in one case, and it is recorded rather than hidden.**
225
+ If a batch holds an oversized frame *and* its sendable remainder then fails totally, the
226
+ emit raises, the worker retries the whole batch, and this filter re-frames and re-drops
227
+ the same event once per attempt — up to four times for one unsendable frame. Everywhere
228
+ else ``dropped`` is an exact count, so the exception matters. Nothing here can fix it:
229
+ the worker owns the retry and hands back the original events, so the sink cannot know it
230
+ has seen them before. ``FirehoseSink._records`` and ``KinesisSink._records`` have the
231
+ same shape and predate this, which is why the fix belongs one level up if it is ever
232
+ taken.
233
+
234
+ Args:
235
+ messages: The framed messages, in order.
236
+
237
+ Returns:
238
+ Those within the datagram limit. All of them under TCP, which is a stream.
239
+
240
+ Raises:
241
+ None.
242
+ """
243
+ if self._transport != "udp":
244
+ return messages
245
+ sendable = [m for m in messages if len(m) <= self._max_datagram_bytes]
246
+ dropped = len(messages) - len(sendable)
247
+ if dropped:
248
+ with self._counter_lock:
249
+ self.dropped_oversized += dropped
250
+ _diag.lost(
251
+ "message",
252
+ dropped,
253
+ f"SocketTransport, over the {self._max_datagram_bytes}-byte datagram limit",
254
+ )
255
+ return sendable
256
+
182
257
  def losses(self) -> SinkLosses:
183
258
  """Reports messages abandoned past the reconnect-retry bound (SPEC-026 FR-002).
184
259
 
@@ -192,7 +267,7 @@ class SocketTransport:
192
267
  None.
193
268
  """
194
269
  with self._counter_lock:
195
- return SinkLosses(dropped=0, failed=self.failed)
270
+ return SinkLosses(dropped=self.dropped_oversized, failed=self.failed)
196
271
 
197
272
  def close(self) -> None:
198
273
  """Closes the held socket, if any (FR-005, FR-012).
@@ -224,6 +299,12 @@ class SocketTransport:
224
299
  refused" from "host unknown", and the code is an integer from the OS rather than caller
225
300
  data.
226
301
 
302
+ An errno in ``_PERMANENT_ERRNOS`` skips the remaining attempts (FR-007 AC-3): a re-send
303
+ of identical bytes to the same destination can only earn the same answer, so retrying
304
+ buys nothing and costs the drain thread its backoff. :meth:`_sendable` normally catches
305
+ the oversized case first; this is the backstop for a path MTU smaller than the datagram
306
+ limit, which no local check can know.
307
+
227
308
  Args:
228
309
  message: The exact bytes to put on the wire.
229
310
 
@@ -242,7 +323,7 @@ class SocketTransport:
242
323
  return True
243
324
  except OSError as err:
244
325
  self._reset()
245
- if attempt < self._max_retries:
326
+ if attempt < self._max_retries and err.errno not in _PERMANENT_ERRNOS:
246
327
  wait(_BACKOFF_BASE * (2**attempt), self.log_foundry_stop_signal)
247
328
  continue
248
329
  with self._counter_lock:
@@ -120,9 +120,18 @@ class RotatingFileSink:
120
120
  Rotation renames the active file through numbered backups, prunes any beyond the backup
121
121
  count, and opens a fresh active file — a backup count of zero keeps none, simply replacing
122
122
  the active file. Backups are numbered rather than timestamped, so no filename derives from
123
- a clock at all and the monotonic deadline has no naming consequence. No event is lost across
124
- a rotation, because the rotate happens before the pending event is written and the event
125
- lands in the fresh file.
123
+ a clock at all and the monotonic deadline has no naming consequence. **The pending event is
124
+ not lost across a rotation**, because the rotate happens before it is written and it lands in
125
+ the fresh file (SPEC-038 FR-012 AC-5). That is the whole of the claim: it says nothing about
126
+ events *already written*, which retention governs — at ``backup_count=0`` every one of them
127
+ is destroyed at each rollover, which is why that is no longer the default.
128
+
129
+ No counter is added for what retention discards, at any ``backup_count``. This sink is a
130
+ bounded ring buffer, retention *is* the configuration, and discarding the oldest generation
131
+ is that configuration working — the precedent being ``MemorySink(maxlen)``, which behaves as
132
+ a bounded ring, counts nothing and implements no ``losses()``. Neither ``dropped`` (defined
133
+ as discarded *before* attempting delivery) nor ``failed`` fits an event that was written and
134
+ flushed to disk.
126
135
 
127
136
  A rotation rebinds the active stream, so it is the sink where concurrent writers did real
128
137
  damage: a second thread mid-``emit`` could write to the handle rotation had just closed, or
@@ -135,7 +144,7 @@ class RotatingFileSink:
135
144
  path: str,
136
145
  *,
137
146
  max_bytes: int = 0,
138
- backup_count: int = 0,
147
+ backup_count: int = 1,
139
148
  when: str | None = None,
140
149
  interval: int = 1,
141
150
  ) -> None:
@@ -148,7 +157,15 @@ class RotatingFileSink:
148
157
  Args:
149
158
  path: The active file to append to.
150
159
  max_bytes: The size trigger, or 0 to disable it.
151
- backup_count: How many numbered backups to retain.
160
+ backup_count: How many numbered backups to retain. **The default is 1** (SPEC-038
161
+ FR-012): at ``0`` a rotation calls ``os.remove`` on the active file, so
162
+ ``RotatingFileSink("app.log", max_bytes=10_000_000)`` silently threw away 10 MB at
163
+ every rollover. ``0`` still truncates, unchanged, for a caller who wants that.
164
+
165
+ The cost of the new default is disk: **2 x max_bytes under a size trigger, or one
166
+ full rollover period under a time-only one** — and ``max_bytes`` defaults to ``0``,
167
+ which bounds nothing, so a time-triggered sink's ceiling is whatever one period
168
+ writes.
152
169
  when: The time-trigger unit code, matched case-insensitively, or ``None`` to disable
153
170
  it. The vocabulary mirrors a subset of the stdlib ``TimedRotatingFileHandler``'s.
154
171
  interval: How many units make up one rollover period.
@@ -44,7 +44,12 @@ class FirehoseSink:
44
44
 
45
45
  MAX_RECORDS = 500
46
46
  MAX_REQUEST_BYTES = 4 * 1024 * 1024
47
- MAX_RECORD_BYTES = 1024 * 1024
47
+ MAX_RECORD_BYTES = 1_024_000
48
+ """Firehose's documented per-record ceiling: 1,000 KiB, which is **not** ``1024 * 1024``.
49
+
50
+ The 24,576-byte gap between the two is a band of records this sink used to pass through and
51
+ the service then rejected (SPEC-038 FR-009).
52
+ """
48
53
 
49
54
  def __init__(self, delivery_stream: str, *, client: Any = None, max_retries: int = 3) -> None:
50
55
  """Binds the sink to a delivery stream.
@@ -155,6 +160,13 @@ class FirehoseSink:
155
160
  def _records(self, batch: list[dict[str, object]]) -> list[dict[str, Any]]:
156
161
  """Builds the request entries, dropping any record too large to ever fit (FR-011).
157
162
 
163
+ Each record ends with a newline (SPEC-038 FR-005). Firehose concatenates record payloads
164
+ verbatim into the delivery buffer and the *producer* must supply the separator, so
165
+ without it the S3 objects read ``{"a":1}{"b":2}`` — unparseable by Athena, Glue and
166
+ OpenSearch ingest, and inconsistent with the NDJSON every other sink here emits. The
167
+ newline is part of ``Data``, so it is charged to both the per-record limit checked below
168
+ and the per-request budget :meth:`emit` chunks by.
169
+
158
170
  Args:
159
171
  batch: The events to convert.
160
172
 
@@ -166,7 +178,7 @@ class FirehoseSink:
166
178
  """
167
179
  records: list[dict[str, Any]] = []
168
180
  for event in batch:
169
- data = json.dumps(event).encode("utf-8")
181
+ data = json.dumps(event).encode("utf-8") + b"\n"
170
182
  if len(data) > self.MAX_RECORD_BYTES:
171
183
  with self._counter_lock:
172
184
  self.dropped_oversized += 1
@@ -12,6 +12,39 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
12
12
  __all__ = ["KafkaSink"]
13
13
 
14
14
 
15
+ def _usable_timeout(value: float) -> float:
16
+ """Returns a flush timeout that is actually a bound, or the default.
17
+
18
+ ``0`` is the value that switched this sink's exit delivery off, and ``inf`` is the unbounded
19
+ wait FR-006 exists to remove, so neither is accepted from a caller. The test is
20
+ ``not (0 < value < inf)`` rather than a pair of comparisons, because ``NaN`` compares
21
+ ``False`` to everything and would otherwise slip through (SPEC-027 FR-001's reasoning for
22
+ ``Retry-After``).
23
+
24
+ Args:
25
+ value: The caller's timeout.
26
+
27
+ Returns:
28
+ The value if it bounds anything, else :data:`DEFAULT_FLUSH_TIMEOUT`.
29
+
30
+ Raises:
31
+ None.
32
+ """
33
+ return value if 0 < value < float("inf") else DEFAULT_FLUSH_TIMEOUT
34
+
35
+ DEFAULT_FLUSH_TIMEOUT = 10.0
36
+ """Seconds :meth:`KafkaSink.close` waits for the producer to drain (SPEC-038 FR-006).
37
+
38
+ Ten rather than ``message.timeout.ms``'s five minutes.
39
+
40
+ It is spent **beside** ``shutdown()``'s budget, not from it: ``Worker.shutdown`` joins the drain
41
+ thread against its deadline and *then* closes the sink inline, with no remaining-budget argument
42
+ (arch §13, since ``Sink.close`` takes no timeout). Measured, ``shutdown(timeout=2.0)`` against a
43
+ dead broker takes 10.01 s. Ten seconds is chosen to keep that overrun small rather than to fit
44
+ inside a budget it cannot see — five minutes is what it replaces.
45
+ """
46
+
47
+
15
48
  class KafkaSink:
16
49
  """A :class:`~log_foundry.sinks.base.Sink` that produces events to a Kafka topic.
17
50
 
@@ -20,8 +53,17 @@ class KafkaSink:
20
53
  returns without blocking, while the delivery result arrives asynchronously on a callback
21
54
  serviced by ``poll()`` and ``flush()``.
22
55
 
56
+ The worst case (SPEC-027 FR-005) is not a retry loop — this sink has none — but its
57
+ ``close()``: one ``flush_timeout`` wait, 10 s at the default, spent **beside**
58
+ ``shutdown()``'s budget rather than from it. ``Worker.shutdown`` joins the drain thread
59
+ against its deadline and *then* closes the sink inline with no remaining-budget argument
60
+ (arch §13), so the two add up: measured, ``shutdown(timeout=2.0)`` against a dead broker
61
+ takes 10.01 s.
62
+
23
63
  Attributes:
24
- failed: Messages whose delivery callback reported an error.
64
+ flush_timeout: Seconds :meth:`close` waits for the producer to drain.
65
+ failed: Messages whose delivery callback reported an error, plus any still queued when
66
+ that close flush expired — the count ``flush()`` returns.
25
67
  rejected: Messages ``produce()`` itself refused — a full local queue, a serialization fault
26
68
  — which never reached the producer's batch at all.
27
69
 
@@ -43,6 +85,7 @@ class KafkaSink:
43
85
  producer: Any = None,
44
86
  bootstrap_servers: str | None = None,
45
87
  key_field: str = "trace_id",
88
+ flush_timeout: float = DEFAULT_FLUSH_TIMEOUT,
46
89
  ) -> None:
47
90
  """Binds the sink to a topic and a producer.
48
91
 
@@ -51,6 +94,13 @@ class KafkaSink:
51
94
  producer: A ``confluent-kafka``-shaped producer, or ``None`` to build one.
52
95
  bootstrap_servers: The broker list, required when no producer is injected.
53
96
  key_field: The event key used as the message key, or empty for no key.
97
+ flush_timeout: Seconds :meth:`close` waits for the producer to drain before counting
98
+ what is left as lost (SPEC-038 FR-006). Unbounded, an unreachable broker held
99
+ process exit for ``message.timeout.ms`` — five minutes by default — despite
100
+ ``shutdown(timeout=30)``. Non-positive or non-finite values fall back to the
101
+ default, as ``Worker._emit`` floors its own retries (SPEC-021): ``0`` reinstates
102
+ exactly the ``flush(0)`` that switched this sink's exit delivery off, and ``inf``
103
+ reinstates the unbounded wait.
54
104
 
55
105
  Returns:
56
106
  None.
@@ -70,6 +120,7 @@ class KafkaSink:
70
120
  self.topic = topic
71
121
  self.producer = producer
72
122
  self.key_field = key_field
123
+ self.flush_timeout = _usable_timeout(flush_timeout)
73
124
  self.failed = 0
74
125
  self.rejected = 0
75
126
  self._counter_lock = threading.Lock()
@@ -175,7 +226,50 @@ class KafkaSink:
175
226
  if self._closed:
176
227
  return
177
228
  self._closed = True
178
- self.producer.flush()
229
+ self._flush_bounded()
230
+
231
+ def _flush_bounded(self) -> None:
232
+ """Flushes the producer within a bound, counting whatever it could not deliver (FR-006).
233
+
234
+ ``Producer.flush()`` with no timeout waits for ``message.timeout.ms`` — five minutes by
235
+ default — and ``Worker.shutdown`` closes the live sink **inline and unbounded** (arch
236
+ §13), so an unreachable broker held process exit for five minutes despite
237
+ ``shutdown(timeout=30)``. The wait is now capped by ``flush_timeout``.
238
+
239
+ **The stop signal is deliberately not consulted here.** A revision cut the timeout to
240
+ zero while a shutdown was in progress, reasoning that the events were lost either way.
241
+ They were not: ``produce()`` is a local hand-off and ``flush()`` is the only thing that
242
+ drains the producer's batch, so with the stop event set — which it always is by the time
243
+ ``close()`` runs, since ``Worker.shutdown`` sets it *before* the join — Kafka's exit
244
+ delivery was switched off entirely. Measured through a real ``shutdown()``: eleven buffered
245
+ messages — nine ``info()`` calls plus the span's start and end — ``flush(0)``, **zero
246
+ delivered**, all eleven booked as ``failed``. That is worse
247
+ than the hang this bound exists to fix, and it is the same mistake FR-001 AC-4a records
248
+ for ``HTTPSink`` — skipping *work* during the exit drain, rather than skipping a *wait*.
249
+
250
+ ``flush()`` *returns* the number of messages still queued, which is exactly the count
251
+ lost at exit — the old call discarded it, so the loss was silent. It is counted into
252
+ ``failed`` and announced once, with a count rather than one line per message.
253
+
254
+ Args:
255
+ None.
256
+
257
+ Returns:
258
+ None.
259
+
260
+ Raises:
261
+ Exception: Whatever the producer raises, which ``close``'s caller already handles.
262
+ """
263
+ remaining = self.producer.flush(self.flush_timeout)
264
+ if type(remaining) is not int or remaining <= 0:
265
+ return
266
+ with self._counter_lock:
267
+ self.failed += remaining
268
+ _diag.lost(
269
+ "message",
270
+ remaining,
271
+ f"KafkaSink, still queued when the {self.flush_timeout}s close flush expired",
272
+ )
179
273
 
180
274
  def _key(self, event: dict[str, object]) -> bytes | None:
181
275
  """Derives one message's partition key from the configured field.
@@ -116,7 +116,7 @@ class KinesisSink:
116
116
  records,
117
117
  max_count=self.MAX_RECORDS,
118
118
  max_bytes=self.MAX_REQUEST_BYTES,
119
- size_of=lambda record: len(record["Data"]),
119
+ size_of=_record_size,
120
120
  ):
121
121
  chunks += 1
122
122
  outcome = self._send(chunk)
@@ -242,3 +242,26 @@ class KinesisSink:
242
242
  )
243
243
  return sent - len(records)
244
244
  return 0
245
+
246
+
247
+ def _record_size(record: dict[str, Any]) -> int:
248
+ """Measures one request entry, partition key included (SPEC-038 FR-009).
249
+
250
+ ``PutRecords`` charges the partition key against the 5 MiB request limit, and a key may be up
251
+ to 256 bytes, so a 500-record request could understate itself by ~128 KB — enough to have the
252
+ service reject a chunk this sink believed was inside the budget. ``SQSSink`` charges its FIFO
253
+ ids for the same reason and records the same rationale.
254
+
255
+ This applies to Kinesis alone: a Firehose record is ``{"Data": data}`` and that API has no
256
+ partition key, so there is nothing there to charge.
257
+
258
+ Args:
259
+ record: One ``PutRecords`` entry.
260
+
261
+ Returns:
262
+ The bytes it contributes to the request.
263
+
264
+ Raises:
265
+ None.
266
+ """
267
+ return len(record["Data"]) + len(record["PartitionKey"])
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
8
8
  if TYPE_CHECKING:
9
9
  import threading
10
10
 
11
- from log_foundry.sinks._socket import SocketTransport
11
+ from log_foundry.sinks._socket import DEFAULT_MAX_DATAGRAM_BYTES, SocketTransport
12
12
  from log_foundry.sinks.base import SinkLosses
13
13
  from log_foundry.sinks.http import HTTPSink
14
14
 
@@ -54,6 +54,7 @@ class LogstashSink:
54
54
  transport: str = "tcp",
55
55
  timeout: float = 5.0,
56
56
  max_retries: int = 3,
57
+ max_datagram_bytes: int = DEFAULT_MAX_DATAGRAM_BYTES,
57
58
  **http_kwargs: object,
58
59
  ) -> None:
59
60
  """Selects and builds exactly one backend.
@@ -65,6 +66,9 @@ class LogstashSink:
65
66
  transport: ``"tcp"`` or ``"udp"``, in socket mode.
66
67
  timeout: Seconds allowed per request or connection.
67
68
  max_retries: Retries the chosen backend makes.
69
+ max_datagram_bytes: In UDP socket mode, the largest datagram to attempt; a frame over
70
+ it is dropped and counted rather than sent, retried and abandoned (SPEC-038 FR-007).
71
+ Ignored in HTTP mode and over TCP, which is a stream.
68
72
  **http_kwargs: Forwarded to :class:`~log_foundry.sinks.http.HTTPSink` in HTTP mode.
69
73
 
70
74
  Returns:
@@ -82,7 +86,12 @@ class LogstashSink:
82
86
  elif host is not None and port is not None:
83
87
  self._http = None
84
88
  self._socket = SocketTransport(
85
- host, port, transport=transport, timeout=timeout, max_retries=max_retries
89
+ host,
90
+ port,
91
+ transport=transport,
92
+ timeout=timeout,
93
+ max_retries=max_retries,
94
+ max_datagram_bytes=max_datagram_bytes,
86
95
  )
87
96
  else:
88
97
  raise ValueError("LogstashSink requires either url= (HTTP) or host= + port= (socket)")
@@ -0,0 +1,63 @@
1
+ """MemorySink — collect events in process, for tests and inspection (arch §8, SPEC-008)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __all__ = ["MemorySink"]
6
+
7
+
8
+ class MemorySink:
9
+ """A :class:`~log_foundry.sinks.base.Sink` that collects events into a list (FR-006).
10
+
11
+ ``.events`` is a plain list exposing every event in arrival order, for asserting in tests or
12
+ eyeballing in a notebook. With ``maxlen`` set it behaves as a bounded ring, keeping only the
13
+ most recent events; the list object identity is stable, so a held reference keeps seeing
14
+ updates.
15
+
16
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
17
+ (SPEC-032 FR-003): there is no transport, and ``close()`` releases nothing — a test that
18
+ closes the sink and then asserts on a later batch still sees it in ``.events``.
19
+ """
20
+
21
+ def __init__(self, maxlen: int | None = None) -> None:
22
+ """Starts an empty collection, optionally bounded.
23
+
24
+ Args:
25
+ maxlen: The most events to keep, or ``None`` to keep all of them.
26
+
27
+ Returns:
28
+ None.
29
+
30
+ Raises:
31
+ None.
32
+ """
33
+ self.events: list[dict[str, object]] = []
34
+ self._maxlen = maxlen
35
+
36
+ def emit(self, batch: list[dict[str, object]]) -> None:
37
+ """Appends the batch in order, trimming to the most recent events if bounded (FR-006).
38
+
39
+ Args:
40
+ batch: The events to collect.
41
+
42
+ Returns:
43
+ None.
44
+
45
+ Raises:
46
+ None.
47
+ """
48
+ self.events.extend(batch)
49
+ if self._maxlen is not None and len(self.events) > self._maxlen:
50
+ del self.events[: len(self.events) - self._maxlen]
51
+
52
+ def close(self) -> None:
53
+ """Does nothing, since collected events remain readable after close (FR-006).
54
+
55
+ Args:
56
+ None.
57
+
58
+ Returns:
59
+ None.
60
+
61
+ Raises:
62
+ None.
63
+ """
@@ -0,0 +1,65 @@
1
+ """NullSink — discard every event, counting what it discarded (arch §8, SPEC-008)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import threading
6
+
7
+ __all__ = ["NullSink"]
8
+
9
+
10
+ class NullSink:
11
+ """A :class:`~log_foundry.sinks.base.Sink` that discards every event (FR-005).
12
+
13
+ This is useful to disable output without unwiring the pipeline, or to benchmark everything
14
+ up to the sink. It deliberately exposes no ``losses()`` (SPEC-026 FR-002): discarding is
15
+ what this sink is for, and reporting it as loss would make ``health()``'s alert idiom fire
16
+ on every batch for anyone who chose this sink to turn logging off. The counter stays
17
+ readable on the instance.
18
+
19
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
20
+ (SPEC-032 FR-003): there is no transport and ``close()`` releases nothing, so discarding a
21
+ batch after close is the same operation as discarding one before it.
22
+ """
23
+
24
+ def __init__(self) -> None:
25
+ """Starts the discarded-event counter at zero.
26
+
27
+ Args:
28
+ None.
29
+
30
+ Returns:
31
+ None.
32
+
33
+ Raises:
34
+ None.
35
+ """
36
+ self.dropped = 0
37
+ self._counter_lock = threading.Lock()
38
+
39
+ def emit(self, batch: list[dict[str, object]]) -> None:
40
+ """Discards the batch, counting the events dropped (FR-005).
41
+
42
+ Args:
43
+ batch: The events to discard.
44
+
45
+ Returns:
46
+ None.
47
+
48
+ Raises:
49
+ None.
50
+ """
51
+ with self._counter_lock:
52
+ self.dropped += len(batch)
53
+
54
+ def close(self) -> None:
55
+ """Does nothing, since nothing is held (FR-005).
56
+
57
+ Args:
58
+ None.
59
+
60
+ Returns:
61
+ None.
62
+
63
+ Raises:
64
+ None.
65
+ """
@@ -175,7 +175,13 @@ class RedisStreamsSink(_RedisSink):
175
175
  """Appends each event to a Redis stream via ``XADD``, pipelined per batch (FR-005)."""
176
176
 
177
177
  def __init__(
178
- self, stream: str, *, client: Any = None, url: str | None = None, max_retries: int = 3
178
+ self,
179
+ stream: str,
180
+ *,
181
+ client: Any = None,
182
+ url: str | None = None,
183
+ max_retries: int = 3,
184
+ maxlen: int | None = None,
179
185
  ) -> None:
180
186
  """Binds the sink to a stream.
181
187
 
@@ -184,6 +190,18 @@ class RedisStreamsSink(_RedisSink):
184
190
  client: A ``redis-py``-shaped client to borrow, or ``None`` to open one.
185
191
  url: The connection URL used when opening a client.
186
192
  max_retries: Retries per batch.
193
+ maxlen: A ceiling on the destination's length, or ``None`` for no ceiling.
194
+
195
+ The default is ``None`` — today's unbounded behaviour — because silently discarding
196
+ a user's buffered logs is not a default this library may choose (SPEC-038 FR-008).
197
+ Used as arch §9.1 recommends, as the durable buffer in front of a consumer, a stalled
198
+ consumer otherwise OOMs the Redis instance with no ceiling the operator can set from
199
+ here.
200
+
201
+ **Trimming discards at the *destination*, outside anything :meth:`losses` can see.**
202
+ Redis drops the oldest entries itself, after this sink has already reported the write
203
+ as delivered, so those events are invisible to ``health()`` — which is the trade a
204
+ bounded buffer makes, and is why it is opt-in.
187
205
 
188
206
  Returns:
189
207
  None.
@@ -192,6 +210,7 @@ class RedisStreamsSink(_RedisSink):
192
210
  ImportError: If the ``redis`` extra is not installed.
193
211
  """
194
212
  self._stream = stream
213
+ self.maxlen = maxlen
195
214
  super().__init__(client=client, url=url, max_retries=max_retries)
196
215
 
197
216
  def _stage(self, pipe: Any, event: dict[str, object]) -> None:
@@ -207,14 +226,25 @@ class RedisStreamsSink(_RedisSink):
207
226
  Raises:
208
227
  Exception: Whatever the client raises.
209
228
  """
210
- pipe.xadd(self._stream, {"event": json.dumps(event)})
229
+ if self.maxlen is None:
230
+ pipe.xadd(self._stream, {"event": json.dumps(event)})
231
+ else:
232
+ pipe.xadd(
233
+ self._stream, {"event": json.dumps(event)}, maxlen=self.maxlen, approximate=True
234
+ )
211
235
 
212
236
 
213
237
  class RedisListSink(_RedisSink):
214
238
  """Pushes each event onto a Redis list via ``RPUSH``, pipelined per batch (FR-005)."""
215
239
 
216
240
  def __init__(
217
- self, key: str, *, client: Any = None, url: str | None = None, max_retries: int = 3
241
+ self,
242
+ key: str,
243
+ *,
244
+ client: Any = None,
245
+ url: str | None = None,
246
+ max_retries: int = 3,
247
+ maxlen: int | None = None,
218
248
  ) -> None:
219
249
  """Binds the sink to a list.
220
250
 
@@ -223,6 +253,18 @@ class RedisListSink(_RedisSink):
223
253
  client: A ``redis-py``-shaped client to borrow, or ``None`` to open one.
224
254
  url: The connection URL used when opening a client.
225
255
  max_retries: Retries per batch.
256
+ maxlen: A ceiling on the destination's length, or ``None`` for no ceiling.
257
+
258
+ The default is ``None`` — today's unbounded behaviour — because silently discarding
259
+ a user's buffered logs is not a default this library may choose (SPEC-038 FR-008).
260
+ Used as arch §9.1 recommends, as the durable buffer in front of a consumer, a stalled
261
+ consumer otherwise OOMs the Redis instance with no ceiling the operator can set from
262
+ here.
263
+
264
+ **Trimming discards at the *destination*, outside anything :meth:`losses` can see.**
265
+ Redis drops the oldest entries itself, after this sink has already reported the write
266
+ as delivered, so those events are invisible to ``health()`` — which is the trade a
267
+ bounded buffer makes, and is why it is opt-in.
226
268
 
227
269
  Returns:
228
270
  None.
@@ -231,6 +273,7 @@ class RedisListSink(_RedisSink):
231
273
  ImportError: If the ``redis`` extra is not installed.
232
274
  """
233
275
  self._key = key
276
+ self.maxlen = maxlen
234
277
  super().__init__(client=client, url=url, max_retries=max_retries)
235
278
 
236
279
  def _stage(self, pipe: Any, event: dict[str, object]) -> None:
@@ -247,3 +290,5 @@ class RedisListSink(_RedisSink):
247
290
  Exception: Whatever the client raises.
248
291
  """
249
292
  pipe.rpush(self._key, json.dumps(event))
293
+ if self.maxlen is not None:
294
+ pipe.ltrim(self._key, -self.maxlen, -1)
@@ -6,7 +6,7 @@ import json
6
6
  import sys
7
7
  from typing import TextIO
8
8
 
9
- __all__ = ["StdoutSink"]
9
+ __all__ = ["StderrSink", "StdoutSink"]
10
10
 
11
11
 
12
12
  class StdoutSink:
@@ -76,3 +76,32 @@ class StdoutSink:
76
76
  Exception: Whatever the stream raises on flush.
77
77
  """
78
78
  self._stream.flush()
79
+
80
+
81
+ class StderrSink(StdoutSink):
82
+ """The :class:`~log_foundry.sinks.stdout.StdoutSink` shape, defaulting to stderr (FR-004).
83
+
84
+ It writes each event as one ``json.dumps`` line and flushes, exactly like ``StdoutSink`` —
85
+ only the default stream differs, following the twelve-factor convention of logs on stderr
86
+ and app output on stdout.
87
+ """
88
+
89
+ def __init__(self, stream: TextIO | None = None) -> None:
90
+ """Binds the sink to an output stream, once, at construction.
91
+
92
+ The binding is resolved here and not re-read per write, so a later
93
+ ``contextlib.redirect_stderr`` is not honoured — the same property
94
+ :class:`~log_foundry.sinks.stdout.StdoutSink` documents (SPEC-031 FR-003), restated
95
+ because this override means none of that docstring is inherited.
96
+
97
+ Args:
98
+ stream: The stream to write to, defaulting to ``sys.stderr`` as resolved now. An
99
+ explicit one, such as a ``StringIO``, can be injected for capture.
100
+
101
+ Returns:
102
+ None.
103
+
104
+ Raises:
105
+ None.
106
+ """
107
+ super().__init__(stream if stream is not None else sys.stderr)
@@ -7,7 +7,7 @@ import os
7
7
  import socket
8
8
  from typing import TYPE_CHECKING
9
9
 
10
- from log_foundry.sinks._socket import SocketTransport
10
+ from log_foundry.sinks._socket import DEFAULT_MAX_DATAGRAM_BYTES, SocketTransport
11
11
 
12
12
  if TYPE_CHECKING:
13
13
  import threading
@@ -63,6 +63,7 @@ class SyslogSink:
63
63
  app_name: str = "log-foundry",
64
64
  timeout: float = 5.0,
65
65
  max_retries: int = 3,
66
+ max_datagram_bytes: int = DEFAULT_MAX_DATAGRAM_BYTES,
66
67
  ) -> None:
67
68
  """Configures the destination, framing and message identity.
68
69
 
@@ -74,6 +75,9 @@ class SyslogSink:
74
75
  app_name: The ``APP-NAME`` field of every message.
75
76
  timeout: Seconds allowed for a TCP connection.
76
77
  max_retries: Reconnect retries per message.
78
+ max_datagram_bytes: The largest UDP datagram to attempt; a frame over it is dropped
79
+ and counted rather than sent, retried and abandoned (SPEC-038 FR-007). Ignored for
80
+ TCP, which is a stream.
77
81
 
78
82
  Returns:
79
83
  None.
@@ -89,7 +93,12 @@ class SyslogSink:
89
93
  self._hostname = socket.gethostname() or "-"
90
94
  self._procid = str(os.getpid())
91
95
  self._socket = SocketTransport(
92
- host, port, transport=transport, timeout=timeout, max_retries=max_retries
96
+ host,
97
+ port,
98
+ transport=transport,
99
+ timeout=timeout,
100
+ max_retries=max_retries,
101
+ max_datagram_bytes=max_datagram_bytes,
93
102
  )
94
103
  self._stop_signal: threading.Event | None = None
95
104
 
@@ -1,156 +0,0 @@
1
- """Small utility sinks: StderrSink, NullSink, MemorySink (arch §8, SPEC-008)."""
2
-
3
- from __future__ import annotations
4
-
5
- import sys
6
- import threading
7
- from typing import TextIO
8
-
9
- from log_foundry.sinks.stdout import StdoutSink
10
-
11
- __all__ = ["MemorySink", "NullSink", "StderrSink"]
12
-
13
-
14
- class StderrSink(StdoutSink):
15
- """The :class:`~log_foundry.sinks.stdout.StdoutSink` shape, defaulting to stderr (FR-004).
16
-
17
- It writes each event as one ``json.dumps`` line and flushes, exactly like ``StdoutSink`` —
18
- only the default stream differs, following the twelve-factor convention of logs on stderr
19
- and app output on stdout.
20
- """
21
-
22
- def __init__(self, stream: TextIO | None = None) -> None:
23
- """Binds the sink to an output stream, once, at construction.
24
-
25
- The binding is resolved here and not re-read per write, so a later
26
- ``contextlib.redirect_stderr`` is not honoured — the same property
27
- :class:`~log_foundry.sinks.stdout.StdoutSink` documents (SPEC-031 FR-003), restated
28
- because this override means none of that docstring is inherited.
29
-
30
- Args:
31
- stream: The stream to write to, defaulting to ``sys.stderr`` as resolved now. An
32
- explicit one, such as a ``StringIO``, can be injected for capture.
33
-
34
- Returns:
35
- None.
36
-
37
- Raises:
38
- None.
39
- """
40
- super().__init__(stream if stream is not None else sys.stderr)
41
-
42
-
43
- class NullSink:
44
- """A :class:`~log_foundry.sinks.base.Sink` that discards every event (FR-005).
45
-
46
- This is useful to disable output without unwiring the pipeline, or to benchmark everything
47
- up to the sink. It deliberately exposes no ``losses()`` (SPEC-026 FR-002): discarding is
48
- what this sink is for, and reporting it as loss would make ``health()``'s alert idiom fire
49
- on every batch for anyone who chose this sink to turn logging off. The counter stays
50
- readable on the instance.
51
-
52
- It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
53
- (SPEC-032 FR-003): there is no transport and ``close()`` releases nothing, so discarding a
54
- batch after close is the same operation as discarding one before it.
55
- """
56
-
57
- def __init__(self) -> None:
58
- """Starts the discarded-event counter at zero.
59
-
60
- Args:
61
- None.
62
-
63
- Returns:
64
- None.
65
-
66
- Raises:
67
- None.
68
- """
69
- self.dropped = 0
70
- self._counter_lock = threading.Lock()
71
-
72
- def emit(self, batch: list[dict[str, object]]) -> None:
73
- """Discards the batch, counting the events dropped (FR-005).
74
-
75
- Args:
76
- batch: The events to discard.
77
-
78
- Returns:
79
- None.
80
-
81
- Raises:
82
- None.
83
- """
84
- with self._counter_lock:
85
- self.dropped += len(batch)
86
-
87
- def close(self) -> None:
88
- """Does nothing, since nothing is held (FR-005).
89
-
90
- Args:
91
- None.
92
-
93
- Returns:
94
- None.
95
-
96
- Raises:
97
- None.
98
- """
99
-
100
-
101
- class MemorySink:
102
- """A :class:`~log_foundry.sinks.base.Sink` that collects events into a list (FR-006).
103
-
104
- ``.events`` is a plain list exposing every event in arrival order, for asserting in tests or
105
- eyeballing in a notebook. With ``maxlen`` set it behaves as a bounded ring, keeping only the
106
- most recent events; the list object identity is stable, so a held reference keeps seeing
107
- updates.
108
-
109
- It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
110
- (SPEC-032 FR-003): there is no transport, and ``close()`` releases nothing — a test that
111
- closes the sink and then asserts on a later batch still sees it in ``.events``.
112
- """
113
-
114
- def __init__(self, maxlen: int | None = None) -> None:
115
- """Starts an empty collection, optionally bounded.
116
-
117
- Args:
118
- maxlen: The most events to keep, or ``None`` to keep all of them.
119
-
120
- Returns:
121
- None.
122
-
123
- Raises:
124
- None.
125
- """
126
- self.events: list[dict[str, object]] = []
127
- self._maxlen = maxlen
128
-
129
- def emit(self, batch: list[dict[str, object]]) -> None:
130
- """Appends the batch in order, trimming to the most recent events if bounded (FR-006).
131
-
132
- Args:
133
- batch: The events to collect.
134
-
135
- Returns:
136
- None.
137
-
138
- Raises:
139
- None.
140
- """
141
- self.events.extend(batch)
142
- if self._maxlen is not None and len(self.events) > self._maxlen:
143
- del self.events[: len(self.events) - self._maxlen]
144
-
145
- def close(self) -> None:
146
- """Does nothing, since collected events remain readable after close (FR-006).
147
-
148
- Args:
149
- None.
150
-
151
- Returns:
152
- None.
153
-
154
- Raises:
155
- None.
156
- """