log-foundry 0.10.2.dev26__tar.gz → 0.10.2.dev28__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/PKG-INFO +21 -2
  2. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/README.md +20 -1
  3. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/pyproject.toml +1 -1
  4. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/base.py +23 -1
  5. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/callback.py +5 -0
  6. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/datadog.py +4 -0
  7. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/elasticsearch.py +4 -0
  8. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/filtering.py +4 -0
  9. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/firehose.py +4 -0
  10. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/honeycomb.py +4 -0
  11. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/http.py +7 -0
  12. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/kafka.py +42 -3
  13. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/kinesis.py +4 -0
  14. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/logging_sink.py +5 -0
  15. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/logstash.py +8 -0
  16. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/loki.py +4 -0
  17. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/multi.py +7 -0
  18. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/pubsub.py +53 -5
  19. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/redis.py +35 -6
  20. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sentry.py +8 -2
  21. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sns.py +4 -0
  22. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/splunk.py +4 -0
  23. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sqs.py +4 -0
  24. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/stdout.py +12 -0
  25. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/syslog.py +6 -0
  26. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/transform.py +4 -0
  27. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/util.py +8 -0
  28. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/LICENSE +0 -0
  29. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/__init__.py +0 -0
  30. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/_diag.py +0 -0
  31. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/api.py +0 -0
  32. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/config.py +0 -0
  33. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/console.py +0 -0
  34. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/context.py +0 -0
  35. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/decorator.py +0 -0
  36. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/ids.py +0 -0
  37. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/model.py +0 -0
  38. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/py.typed +0 -0
  39. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sanitize.py +0 -0
  40. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/__init__.py +0 -0
  41. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_batch.py +0 -0
  42. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_chunk.py +0 -0
  43. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_retry.py +0 -0
  44. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_socket.py +0 -0
  45. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/_time.py +0 -0
  46. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/clickhouse.py +0 -0
  47. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/eventhubs.py +0 -0
  48. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/file.py +0 -0
  49. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/mongodb.py +0 -0
  50. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/nats.py +0 -0
  51. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/newrelic.py +0 -0
  52. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/postgres.py +0 -0
  53. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/rabbitmq.py +0 -0
  54. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/sinks/sqlite.py +0 -0
  55. {log_foundry-0.10.2.dev26 → log_foundry-0.10.2.dev28}/src/log_foundry/worker.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev26
3
+ Version: 0.10.2.dev28
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -696,7 +696,7 @@ in `CallbackSink`.
696
696
 
697
697
  #### Writing your own sink
698
698
 
699
- `Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
699
+ `Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
700
700
  fails and one about *when* it is called. They are not stylistic — the library's whole
701
701
  loss-reporting apparatus is built on them:
702
702
 
@@ -716,6 +716,15 @@ loss-reporting apparatus is built on them:
716
716
  - **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
717
717
  partial success re-delivers the records that already arrived, and duplicates downstream are worse
718
718
  than a counted loss.
719
+ - **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
720
+ released transport has delivered nothing, so it is the first rule again by another route — and
721
+ it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
722
+ wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
723
+ one appended a delivery future nothing would resolve, and one transparently *reconnected*,
724
+ leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
725
+ If your `close()` releases nothing — you open a fresh connection per request, or the client is
726
+ the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
727
+ invented. `emit([])` stays a no-op either way.
719
728
 
720
729
  A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
721
730
  never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
@@ -732,12 +741,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
732
741
  class MySink:
733
742
  def __init__(self) -> None:
734
743
  self._dropped = self._failed = 0
744
+ self._closed = False
735
745
  self._lock = threading.Lock() # transport state
736
746
  self._counter_lock = threading.Lock() # counters only, never held across I/O
737
747
 
738
748
  def emit(self, batch: list[dict[str, object]]) -> None:
749
+ if not batch:
750
+ return
739
751
  delivered = 0
740
752
  with self._lock: # your connection, socket or stream
753
+ if self._closed: # refuse: nothing here can deliver it
754
+ raise SinkDeliveryError(
755
+ f"MySink delivered none of {len(batch)} event(s): the sink is closed"
756
+ )
741
757
  for chunk in self._chunks(batch):
742
758
  if self._send(chunk): # your own bounded retry
743
759
  delivered += len(chunk)
@@ -753,6 +769,9 @@ class MySink:
753
769
 
754
770
  def close(self) -> None:
755
771
  with self._lock: # never release under an active writer
772
+ if self._closed: # idempotent: atexit races your own cleanup
773
+ return
774
+ self._closed = True # set the flag, *then* release
756
775
  ...
757
776
  ```
758
777
 
@@ -660,7 +660,7 @@ in `CallbackSink`.
660
660
 
661
661
  #### Writing your own sink
662
662
 
663
- `Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
663
+ `Sink` is two required methods, `emit(batch)` and `close()`, plus three rules about *how* `emit`
664
664
  fails and one about *when* it is called. They are not stylistic — the library's whole
665
665
  loss-reporting apparatus is built on them:
666
666
 
@@ -680,6 +680,15 @@ loss-reporting apparatus is built on them:
680
680
  - **Do not raise when you delivered some of it.** The worker retries whole batches, so raising on a
681
681
  partial success re-delivers the records that already arrived, and duplicates downstream are worse
682
682
  than a counted loss.
683
+ - **Raise after your own `close()`, if `close()` released anything.** A batch handed to a
684
+ released transport has delivered nothing, so it is the first rule again by another route — and
685
+ it is easy to miss, because the sink looks like it worked. Three of the shipped sinks got this
686
+ wrong for four releases: one accepted a produce into a client buffer nothing would flush again,
687
+ one appended a delivery future nothing would resolve, and one transparently *reconnected*,
688
+ leaking a connection nothing would reap. Set the flag before you release, and read it in `emit`.
689
+ If your `close()` releases nothing — you open a fresh connection per request, or the client is
690
+ the caller's — then **keep accepting**: refusing a batch you would have delivered is loss you
691
+ invented. `emit([])` stays a no-op either way.
683
692
 
684
693
  A sink that absorbs a total failure and returns normally is a sink the worker believes: the retry
685
694
  never engages, `failed_batches` stays at zero, and `flush()` returns `True` while every event is
@@ -696,12 +705,19 @@ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
696
705
  class MySink:
697
706
  def __init__(self) -> None:
698
707
  self._dropped = self._failed = 0
708
+ self._closed = False
699
709
  self._lock = threading.Lock() # transport state
700
710
  self._counter_lock = threading.Lock() # counters only, never held across I/O
701
711
 
702
712
  def emit(self, batch: list[dict[str, object]]) -> None:
713
+ if not batch:
714
+ return
703
715
  delivered = 0
704
716
  with self._lock: # your connection, socket or stream
717
+ if self._closed: # refuse: nothing here can deliver it
718
+ raise SinkDeliveryError(
719
+ f"MySink delivered none of {len(batch)} event(s): the sink is closed"
720
+ )
705
721
  for chunk in self._chunks(batch):
706
722
  if self._send(chunk): # your own bounded retry
707
723
  delivered += len(chunk)
@@ -717,6 +733,9 @@ class MySink:
717
733
 
718
734
  def close(self) -> None:
719
735
  with self._lock: # never release under an active writer
736
+ if self._closed: # idempotent: atexit races your own cleanup
737
+ return
738
+ self._closed = True # set the flag, *then* release
720
739
  ...
721
740
  ```
722
741
 
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev26"
23
+ version = "0.10.2.dev28"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -94,9 +94,25 @@ class Sink(Protocol):
94
94
  re-delivered wholesale by the worker's retry, and duplicates downstream are worse than
95
95
  the counted loss (SPEC-017 FR-004, SPEC-018). Report that through ``losses()`` instead.
96
96
 
97
+ **After ``close()`` has released or invalidated something, raise rather than absorb**
98
+ (SPEC-032 FR-002). This is the same obligation applied to the sink's own lifecycle
99
+ rather than the destination's: an absorbed batch is one the worker believes, so the
100
+ retry never engages, ``failed_batches`` stays at zero and ``flush()`` returns True while
101
+ the events are gone — reached not through a destination that is down but through a
102
+ transport the library itself has already let go. Three shipped sinks failed this rule
103
+ for four specs, in three different ways: a produce into a batch nothing would flush
104
+ again, a future nothing would resolve, and a client that quietly reconnected.
105
+
106
+ The converse is equally binding: a sink holding nothing to release **keeps accepting**.
107
+ ``close()`` on a sink that opens a fresh connection per request, or whose client belongs
108
+ to the caller, has invalidated nothing, so a later batch still delivers and refusing it
109
+ would be loss the library invented rather than loss it reported. Which of the two
110
+ applies is a property of the sink, so each shipped sink records its answer in its class
111
+ docstring and a test holds it to it.
112
+
97
113
  Args:
98
114
  batch: The events to ship. ``emit([])`` is a no-op and never raises, since an empty
99
- batch has not failed to deliver.
115
+ batch has not failed to deliver — closed or not.
100
116
 
101
117
  Returns:
102
118
  None.
@@ -116,6 +132,12 @@ class Sink(Protocol):
116
132
  succeeding. Taking the same lock ``emit`` takes satisfies this; so does an idempotent
117
133
  guard checked under that lock.
118
134
 
135
+ The flag this sets is the one a later :meth:`emit` reads, so the two are one decision
136
+ rather than two (SPEC-032 FR-002). Set it *before* releasing anything: an emit arriving
137
+ during a release must be refused rather than handed a half-released transport. Make the
138
+ release idempotent for the same reason a second ``close()`` is expected at all —
139
+ ``atexit`` racing a caller's own cleanup is the documented case.
140
+
119
141
  Args:
120
142
  None.
121
143
 
@@ -17,6 +17,11 @@ class CallbackSink:
17
17
  function, without writing a ``Sink`` implementation. Attributes are internal; the observable
18
18
  contract is that ``emit`` hands the batch to the callable unchanged and ``close`` invokes
19
19
  the close hook once when one was supplied.
20
+
21
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
22
+ (SPEC-032 FR-003). Both decisions belong to the callable: this class holds nothing, and what
23
+ a hook releases is not knowable from here — a callable needing either guarantee must provide
24
+ it, exactly as a hand-written ``Sink`` implementation would.
20
25
  """
21
26
 
22
27
  def __init__(
@@ -16,6 +16,10 @@ class DatadogSink(HTTPSink):
16
16
 
17
17
  The batch goes as a JSON array to the region-specific intake with a ``DD-API-KEY`` header,
18
18
  each entry enriched with ``ddsource``, ``service`` and ``ddtags``.
19
+
20
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
21
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
22
+ is no transport held and ``close()`` releases nothing.
19
23
  """
20
24
 
21
25
  def __init__(
@@ -25,6 +25,10 @@ class ElasticsearchSink(HTTPSink):
25
25
  dropped_unadjudicated: Events whose outcome a ``_bulk`` response did not describe, because
26
26
  its ``items`` array did not line up with the batch sent. Abandoned rather than retried,
27
27
  for SPEC-018's reason: the request succeeded, so re-sending would duplicate what landed.
28
+
29
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
30
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
31
+ is no transport held and ``close()`` releases nothing.
28
32
  """
29
33
 
30
34
  def __init__(self, url: str, *, index: str, auth: str | tuple[str, str] | None = None,
@@ -24,6 +24,10 @@ class FilteringSink:
24
24
  This is a static, emit-time filter in front of an inner sink, not the reserved
25
25
  tail-sampling ``should_send`` seam (arch §10) — that stays deferred and owns rate policy at
26
26
  span-decision time, while this only reshapes an already-built batch on its way to a sink.
27
+
28
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
29
+ (SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
30
+ belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
27
31
  """
28
32
 
29
33
  def __init__(
@@ -36,6 +36,10 @@ class FirehoseSink:
36
36
  The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
37
37
  lock. ``boto3`` clients are documented thread-safe, this one is built once in
38
38
  ``__init__``, and the sink rebinds nothing after construction.
39
+
40
+ It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
41
+ because the client is the caller's to release or the SDK's to reap, so a batch emitted
42
+ afterwards still reaches the delivery stream.
39
43
  """
40
44
 
41
45
  MAX_RECORDS = 500
@@ -14,6 +14,10 @@ class HoneycombSink(HTTPSink):
14
14
 
15
15
  The request goes to ``/1/batch/<dataset>`` with an ``X-Honeycomb-Team`` header, in
16
16
  Honeycomb's ``[{"data": <event>}, ...]`` batch shape.
17
+
18
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
19
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
20
+ is no transport held and ``close()`` releases nothing.
17
21
  """
18
22
 
19
23
  def __init__(
@@ -74,6 +74,13 @@ class HTTPSink:
74
74
  failed: Requests abandoned past the retry bound.
75
75
  dropped_oversized: Events dropped for exceeding a destination's hard size limit, used by
76
76
  subclasses that enforce one; the generic core imposes no universal limit.
77
+
78
+ The two lifecycle decisions this class records, inherited by every platform subclass. It
79
+ takes **no** transport lock (SPEC-028 FR-002): there is no transport to guard, since
80
+ ``urllib`` opens a fresh connection per request and nothing is rebound after construction.
81
+ And it **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` releases nothing —
82
+ a batch emitted afterwards still reaches the endpoint, and refusing it would be loss the
83
+ library invented rather than loss it reported.
77
84
  """
78
85
 
79
86
  def __init__(
@@ -28,6 +28,12 @@ class KafkaSink:
28
28
  The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
29
29
  lock. ``confluent-kafka`` documents its ``Producer`` as thread-safe, and this sink adds no state
30
30
  of its own to guard — ``produce()`` is a local hand-off into the client's internal queue.
31
+
32
+ It refuses an emit after :meth:`close` (SPEC-032 FR-001), which is not the housekeeping it
33
+ looks like: ``flush()`` is the only thing that drains the producer's local batch and services
34
+ its delivery callbacks, so a message produced after it dies with the process — accepted,
35
+ uncounted, and never the subject of a callback. ``emit`` returning normally is what made that
36
+ silent, since the worker then never retried and ``flush()`` reported success.
31
37
  """
32
38
 
33
39
  def __init__(
@@ -67,6 +73,8 @@ class KafkaSink:
67
73
  self.failed = 0
68
74
  self.rejected = 0
69
75
  self._counter_lock = threading.Lock()
76
+ self._closed = False
77
+ self._close_lock = threading.Lock()
70
78
 
71
79
  def losses(self) -> SinkLosses:
72
80
  """Reports refused and undelivered messages (SPEC-026 FR-002).
@@ -98,6 +106,15 @@ class KafkaSink:
98
106
  observe (FR-002). A partial refusal is counted and left alone, since the messages that
99
107
  were accepted are already on their way.
100
108
 
109
+ A closed sink refuses the batch before touching the producer (SPEC-032 FR-001). Refusing
110
+ is not a loss this sink absorbed, so it moves no counter here: it is a failure reported
111
+ to the worker, which records it in ``health().failed_batches``. The check is an unlocked
112
+ read of a write-once flag rather than a lock held across the produce loop, which would
113
+ serialize a client ``confluent-kafka`` documents as thread-safe. A close landing between
114
+ the check and the produce therefore still loses that message, exactly as ``MongoDBSink``
115
+ documents for its own check; what the flag ends is the far larger case of a sink closed
116
+ long before, which is every log written after ``shutdown()``.
117
+
101
118
  Args:
102
119
  batch: The events to produce.
103
120
 
@@ -105,9 +122,16 @@ class KafkaSink:
105
122
  None.
106
123
 
107
124
  Raises:
108
- SinkDeliveryError: When every message was refused, so the batch reached nothing —
109
- the total failure the worker's retry exists for (SPEC-026 FR-001).
125
+ SinkDeliveryError: When the sink is closed, or when every message was refused so the
126
+ batch reached nothing — the total failure the worker's retry exists for
127
+ (SPEC-026 FR-001).
110
128
  """
129
+ if not batch:
130
+ return
131
+ if self._closed:
132
+ raise SinkDeliveryError(
133
+ f"KafkaSink produced none of {len(batch)} message(s): the sink is closed"
134
+ )
111
135
  accepted = 0
112
136
  for event in batch:
113
137
  body = json.dumps(event).encode("utf-8")
@@ -127,6 +151,17 @@ class KafkaSink:
127
151
  def close(self) -> None:
128
152
  """Flushes the producer so buffered messages are delivered before exit (FR-002).
129
153
 
154
+ Idempotent, and the lock is held *across* the flush rather than only around the flag, as
155
+ every other guarded sink holds its own (SPEC-032 FR-001). Releasing it early would let a
156
+ second concurrent ``close()`` return while the first is still draining, so a caller would
157
+ believe the producer was flushed when it was not — and ``atexit`` racing user cleanup is
158
+ the documented case for a second call. The cost is that a second close waits on an
159
+ unreachable broker, which is the ``architecture.md`` §13 constraint that already applies
160
+ to every sink here.
161
+
162
+ The flag is set *before* the flush rather than after, so an emit arriving during a flush
163
+ is refused rather than producing into a batch the flush has already walked past.
164
+
130
165
  Args:
131
166
  None.
132
167
 
@@ -136,7 +171,11 @@ class KafkaSink:
136
171
  Raises:
137
172
  Exception: Whatever the producer raises on flush.
138
173
  """
139
- self.producer.flush()
174
+ with self._close_lock:
175
+ if self._closed:
176
+ return
177
+ self._closed = True
178
+ self.producer.flush()
140
179
 
141
180
  def _key(self, event: dict[str, object]) -> bytes | None:
142
181
  """Derives one message's partition key from the configured field.
@@ -36,6 +36,10 @@ class KinesisSink:
36
36
  The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
37
37
  lock. ``boto3`` clients are documented thread-safe, this one is built once in
38
38
  ``__init__``, and the sink rebinds nothing after construction.
39
+
40
+ It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
41
+ because the client is the caller's to release or the SDK's to reap, so a batch emitted
42
+ afterwards still reaches the stream.
39
43
  """
40
44
 
41
45
  MAX_RECORDS = 500
@@ -36,6 +36,11 @@ class LoggingSink:
36
36
  no new dependency. The sink only emits into that pipeline: it never configures loggers,
37
37
  handlers or formatters, and never tears the framework down. The module is named
38
38
  ``logging_sink`` so it never shadows the stdlib module.
39
+
40
+ It takes **no** transport lock (SPEC-028 FR-002): ``logging`` serializes its own handlers,
41
+ and this sink holds nothing else. And it **adds no post-close guard** (SPEC-032 FR-003) —
42
+ ``close()`` is a no-op by design, since tearing down handlers this sink did not configure is
43
+ not its to do, so a later batch still reaches the framework.
39
44
  """
40
45
 
41
46
  def __init__(
@@ -23,6 +23,14 @@ class LogstashSink:
23
23
  one newline-terminated line over a raw TCP or UDP socket, reusing
24
24
  :class:`~log_foundry.sinks._socket.SocketTransport`. Either backend handles its own bounded
25
25
  retry and raises on total failure of its own accord, so ``emit`` needs no rule of its own.
26
+
27
+ It takes **no** transport lock (SPEC-028 FR-002) of its own: whichever backend it built owns
28
+ that decision — ``SocketTransport`` locks its sends, ``HTTPSink`` holds no transport to
29
+ guard. The post-close rule follows the same split (SPEC-032 FR-004), and the two modes
30
+ genuinely differ: in socket mode a batch after ``close()`` is refused with
31
+ ``SinkDeliveryError`` and reopens nothing, while in HTTP mode ``close()`` released nothing
32
+ and the batch still ships. Both are the backend's answer, correctly, rather than one this
33
+ class invents on top.
26
34
  """
27
35
 
28
36
  def __init__(
@@ -18,6 +18,10 @@ class LokiSink(HTTPSink):
18
18
  Events are grouped into streams by a configurable set of label keys, each stream carrying
19
19
  values of ``[<nanosecond_timestamp_str>, <log_line>]``. Timestamps are derived by parsing the
20
20
  event's ISO-8601 ``timestamp``, falling back to emit-time now.
21
+
22
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
23
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
24
+ is no transport held and ``close()`` releases nothing.
21
25
  """
22
26
 
23
27
  def __init__(self, url: str, *, labels: tuple[str, ...] = ("service", "env", "level"),
@@ -29,6 +29,13 @@ class MultiSink:
29
29
  batch, so one batch against an all-down fan-out of *n* children increments this by *n*
30
30
  per attempt, with a stderr line each. That is the visible cost of the loss no longer
31
31
  being silent.
32
+
33
+ It takes **no** transport lock (SPEC-028 FR-002) — it holds no transport, and a lock spanning
34
+ a child's ``emit`` would serialize every destination behind the slowest one. And it
35
+ **adds no post-close guard** (SPEC-032 FR-003), because the post-close rule is each child's:
36
+ ``close()`` here only forwards, so a guard added at this level would refuse batches the
37
+ children would have taken, while a child that must refuse already does and is counted here
38
+ like any other failure.
32
39
  """
33
40
 
34
41
  def __init__(self, *sinks: Sink) -> None:
@@ -27,6 +27,11 @@ class GooglePubSubSink:
27
27
  appended after the loop passed its index was dropped unresolved: an unconfirmed publish
28
28
  never counted in ``failed`` and never reported by :meth:`losses`, which is precisely the
29
29
  silent loss SPEC-026 exists to end.
30
+
31
+ That same loss was reachable from *outside* ``close`` until SPEC-032 FR-001: nothing stopped
32
+ a later ``emit`` appending to the fresh list, and nothing would ever call ``result()`` on
33
+ it. The sink now refuses a batch once closed, and the append re-checks under the futures
34
+ lock, so a publish cannot land on a list the swap has already taken.
30
35
  """
31
36
 
32
37
  def __init__(self, topic: str, *, client: Any = None) -> None:
@@ -53,6 +58,7 @@ class GooglePubSubSink:
53
58
  self._counter_lock = threading.Lock()
54
59
  self._futures_lock = threading.Lock()
55
60
  self._futures: list[Any] = []
61
+ self._closed = False
56
62
 
57
63
  def losses(self) -> SinkLosses:
58
64
  """Reports refused publishes and futures that resolved to an error (FR-002).
@@ -80,6 +86,26 @@ class GooglePubSubSink:
80
86
  the retry would duplicate them. The stderr line names which counter moved, since
81
87
  "refused" and "unconfirmed" mean different things.
82
88
 
89
+ A closed sink refuses the batch before touching the client (SPEC-032 FR-001), because
90
+ nothing will resolve a future appended after :meth:`close` swapped the list out.
91
+ Refusing moves no counter here: it is a failure reported to the worker, which records it
92
+ in ``health().failed_batches``, not loss this sink absorbed. The flag is read twice on
93
+ purpose — once for the batch, and again under ``_futures_lock`` for each append, which
94
+ is the lock ``close`` swaps under. Without the second read a close landing mid-loop
95
+ would leave exactly the orphaned future this guard exists to prevent, and the read costs
96
+ nothing because the append already takes that lock. Only the read happens under it: the
97
+ counter bump and the stderr line are taken *after* the lock is released, because
98
+ ``close`` waits on the same lock and a blocked stderr would otherwise hold it — an I/O
99
+ call inside a transport lock is what SPEC-028's two-lock decision exists to avoid.
100
+
101
+ A close that lands mid-batch does **not** raise, even when it catches every event. Each
102
+ one had ``publish()`` called on it and may well land, so raising would have the worker
103
+ re-send them and duplicate whatever did — the SPEC-018 rule that only a *provable*
104
+ non-delivery may be retried. They are counted as unconfirmed instead, which is exactly
105
+ what they are. That is why the total-failure raise below tests the refusals rather than
106
+ the successes: "nothing was published" and "nothing was confirmed" are different claims,
107
+ and only the first is safe to retry.
108
+
83
109
  Args:
84
110
  batch: The events to publish.
85
111
 
@@ -87,21 +113,36 @@ class GooglePubSubSink:
87
113
  None.
88
114
 
89
115
  Raises:
90
- SinkDeliveryError: When every event was refused (SPEC-026 FR-001).
116
+ SinkDeliveryError: When the sink was already closed on entry, or when every event was
117
+ refused (SPEC-026 FR-001).
91
118
  """
92
- published = 0
119
+ if not batch:
120
+ return
121
+ if self._closed:
122
+ raise SinkDeliveryError(
123
+ f"GooglePubSubSink published none of {len(batch)} event(s): the sink is closed"
124
+ )
125
+ refused = 0
93
126
  for event in batch:
94
127
  try:
95
128
  future = self.client.publish(self.topic, data=json.dumps(event).encode("utf-8"))
96
129
  except Exception as err:
130
+ refused += 1
97
131
  with self._counter_lock:
98
132
  self.rejected += 1
99
133
  _diag.lost("event", 1, f"GooglePubSubSink refused the publish, {type(err).__name__}")
100
134
  continue
101
135
  with self._futures_lock:
102
- self._futures.append(future)
103
- published += 1
104
- if batch and not published:
136
+ orphaned = self._closed
137
+ if not orphaned:
138
+ self._futures.append(future)
139
+ if orphaned:
140
+ with self._counter_lock:
141
+ self.failed += 1
142
+ _diag.lost(
143
+ "event", 1, "GooglePubSubSink publish unconfirmed, the sink closed mid-batch"
144
+ )
145
+ if refused == len(batch):
105
146
  raise SinkDeliveryError(
106
147
  f"GooglePubSubSink published none of {len(batch)} event(s)"
107
148
  )
@@ -116,6 +157,12 @@ class GooglePubSubSink:
116
157
  and never reported by ``losses()``. That is the silent loss SPEC-026 exists to end,
117
158
  reached through the one piece of shared state this sink has.
118
159
 
160
+ The closed flag is set in the same critical section as the swap (SPEC-032 FR-001), which
161
+ is what makes :meth:`emit`'s second check exact: an append winning the lock before this
162
+ point is resolved by the loop below, and one arriving after sees the flag. The futures
163
+ are resolved outside the lock, so a slow ``result()`` never blocks an emit that is about
164
+ to be refused anyway.
165
+
119
166
  Args:
120
167
  None.
121
168
 
@@ -127,6 +174,7 @@ class GooglePubSubSink:
127
174
  (FR-011).
128
175
  """
129
176
  with self._futures_lock:
177
+ self._closed = True
130
178
  pending, self._futures = self._futures, []
131
179
  for future in pending:
132
180
  try:
@@ -28,6 +28,12 @@ class _RedisSink:
28
28
  execution never mutates the client. Its documented exceptions are ``PubSub`` and
29
29
  ``Pipeline`` objects, which must not be passed between threads — which is why the pipeline
30
30
  here is built and executed inside a single ``emit`` call and never stored on the instance.
31
+
32
+ It refuses an emit after :meth:`close` (SPEC-032 FR-001). Left unguarded this sink did not
33
+ fail after close, it *succeeded*: ``redis-py``'s pool reconnects transparently on the next
34
+ command, so a batch emitted after ``shutdown()`` opened a connection nothing would ever
35
+ reap — the same leak SPEC-028's review found in ``RabbitMQSink``, whose ``_active_channel``
36
+ reopened whatever ``close()`` had released.
31
37
  """
32
38
 
33
39
  def __init__(self, *, client: Any, url: str | None, max_retries: int) -> None:
@@ -56,6 +62,8 @@ class _RedisSink:
56
62
  self.stop_signal: threading.Event | None = None
57
63
  self.failed = 0
58
64
  self._counter_lock = threading.Lock()
65
+ self._closed = False
66
+ self._close_lock = threading.Lock()
59
67
 
60
68
  def losses(self) -> SinkLosses:
61
69
  """Reports events abandoned past the retry bound (SPEC-026 FR-002).
@@ -75,6 +83,13 @@ class _RedisSink:
75
83
  def emit(self, batch: list[dict[str, object]]) -> None:
76
84
  """Pipelines the whole batch into one round trip, retrying on error (FR-005).
77
85
 
86
+ A closed sink refuses the batch before asking for a pipeline (SPEC-032 FR-001), since
87
+ asking would silently reopen the connection ``close()`` released. The refusal does not
88
+ depend on whether the sink owned the client: a borrowed client surviving its sink is the
89
+ caller's business, and is not permission to keep writing through a sink that has been
90
+ released. Refusing moves no counter here — it is a failure reported to the worker, which
91
+ records it in ``health().failed_batches``, not loss this sink absorbed.
92
+
78
93
  Args:
79
94
  batch: The events to buffer. An empty batch is a no-op.
80
95
 
@@ -82,13 +97,18 @@ class _RedisSink:
82
97
  None.
83
98
 
84
99
  Raises:
85
- SinkDeliveryError: When the retry bound is spent. The batch travels as one pipeline,
86
- so such a failure delivered nothing: it is counted and then raised, giving the worker
87
- its retry and ``health()`` the loss (SPEC-026 FR-001). There is no partial case to
88
- protect, because the pipeline is all or nothing.
100
+ SinkDeliveryError: When the sink is closed. Also when the retry bound is spent: the
101
+ batch travels as one pipeline, so such a failure delivered nothing: it is counted and
102
+ then raised, giving the worker its retry and ``health()`` the loss (SPEC-026 FR-001).
103
+ There is no partial case to protect, because the pipeline is all or nothing.
89
104
  """
90
105
  if not batch:
91
106
  return
107
+ if self._closed:
108
+ raise SinkDeliveryError(
109
+ f"{type(self).__name__} delivered none of {len(batch)} event(s): "
110
+ "the sink is closed"
111
+ )
92
112
  for attempt in range(self.max_retries + 1):
93
113
  try:
94
114
  pipe = self.client.pipeline()
@@ -114,6 +134,11 @@ class _RedisSink:
114
134
  def close(self) -> None:
115
135
  """Closes the connection only if the sink owns it (FR-005).
116
136
 
137
+ Idempotent, with the flag set under a lock so two concurrent calls cannot both reach
138
+ ``client.close()`` — ``atexit`` racing user code is the documented case. The flag is set
139
+ whether or not the client is owned, because it marks *this sink* as released rather than
140
+ the connection (SPEC-032 FR-001).
141
+
117
142
  Args:
118
143
  None.
119
144
 
@@ -123,8 +148,12 @@ class _RedisSink:
123
148
  Raises:
124
149
  Exception: Whatever the client raises on close.
125
150
  """
126
- if self._owns_client:
127
- self.client.close()
151
+ with self._close_lock:
152
+ if self._closed:
153
+ return
154
+ self._closed = True
155
+ if self._owns_client:
156
+ self.client.close()
128
157
 
129
158
  def _stage(self, pipe: Any, event: dict[str, object]) -> None:
130
159
  """Stages one event onto the pipeline, in whatever form the subclass writes.
@@ -38,6 +38,9 @@ class SentrySink:
38
38
  lock. This sink owns no transport: it delegates to a ``sentry_sdk`` built for capture from any
39
39
  thread, or to an ``HTTPSink`` that builds a fresh request per call and rebinds nothing.
40
40
  Its counters take the counter lock like every other sink's.
41
+
42
+ It also **adds no post-close guard** (SPEC-032 FR-003), for the same reason: neither backend
43
+ holds anything ``close()`` releases, so a batch emitted afterwards still reaches Sentry.
41
44
  """
42
45
 
43
46
  def __init__(
@@ -120,9 +123,12 @@ class SentrySink:
120
123
  )
121
124
 
122
125
  def close(self) -> None:
123
- """Releases the HTTP fallback resource, if any (FR-012).
126
+ """Forwards to the HTTP fallback, whose own close releases nothing (FR-012).
124
127
 
125
- Idempotent.
128
+ Idempotent, and it releases nothing — which is why the class docstring's post-close claim
129
+ holds despite this method calling a ``close()``. ``HTTPSink.close`` is a documented no-op,
130
+ since ``urllib`` builds a fresh connection per request; the forward exists so a future
131
+ ``HTTPSink`` that *did* hold a pool would be released here rather than leaked.
126
132
 
127
133
  Args:
128
134
  None.
@@ -29,6 +29,10 @@ class SNSSink:
29
29
  The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
30
30
  lock. ``boto3`` clients are documented thread-safe, this one is built once in
31
31
  ``__init__``, and the sink rebinds nothing after construction.
32
+
33
+ It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
34
+ because the client is the caller's to release or the SDK's to reap, so a batch emitted
35
+ afterwards still reaches the topic.
32
36
  """
33
37
 
34
38
  MAX_BATCH = 10
@@ -17,6 +17,10 @@ class SplunkHECSink(HTTPSink):
17
17
  concatenated-JSON-objects body with an ``Authorization: Splunk <token>`` header. The
18
18
  envelope's epoch-seconds ``time`` is parsed from the event's ISO-8601 ``timestamp``, falling
19
19
  back to emit-time now.
20
+
21
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
22
+ (SPEC-032 FR-003), for the reasons :class:`~log_foundry.sinks.http.HTTPSink` records: there
23
+ is no transport held and ``close()`` releases nothing.
20
24
  """
21
25
 
22
26
  def __init__(
@@ -91,6 +91,10 @@ class SQSSink:
91
91
  ``emit`` — which is what keeps it clear of the documented caveat that calling
92
92
  ``boto3.client()`` concurrently can produce response-ordering faults. Nothing else here is
93
93
  rebound after construction.
94
+
95
+ It also **adds no post-close guard** (SPEC-032 FR-003): ``close()`` is a documented no-op,
96
+ because the client is the caller's to release or the SDK's to reap, so a batch emitted
97
+ afterwards still reaches the queue.
94
98
  """
95
99
 
96
100
  MAX_BATCH = 10
@@ -14,6 +14,18 @@ class StdoutSink:
14
14
 
15
15
  This is the zero-dependency default sink, for local dev and container log scraping. Like
16
16
  every sink it receives already-built event dicts and knows nothing about spans or context.
17
+
18
+ It takes **no** transport lock (SPEC-028 FR-002), and the honest reason is that locking it
19
+ was out of scope there rather than that it needs none: ``emit`` writes one line per event, so
20
+ two threads' *batches* do interleave, exactly as they did in ``FileSink`` before its lock.
21
+ Individual lines survive — ``TextIOWrapper.write`` holds its own lock — but line integrity is
22
+ not the guarantee at stake, as ``test_file_sink_keeps_each_batch_contiguous_under_concurrent
23
+ _emitters`` records. Interleaved batches are harmless in a line-oriented stream that a
24
+ scraper reads per line, which is what this sink is for; a caller needing contiguity should
25
+ use ``FileSink``.
26
+
27
+ It **adds no post-close guard** (SPEC-032 FR-003), because ``close()`` only flushes — the
28
+ stream belongs to the process, not to this sink, so a later batch still lands.
17
29
  """
18
30
 
19
31
  def __init__(self, stream: TextIO | None = None) -> None:
@@ -33,6 +33,12 @@ class SyslogSink:
33
33
  Each event becomes an RFC 5424 message whose ``PRI`` is derived from a configurable facility
34
34
  and a severity mapped from the event's level. UDP sends one datagram per event, TCP uses
35
35
  octet-counted framing (RFC 6587), and the whole sink is dependency-free.
36
+
37
+ It takes **no** transport lock (SPEC-028 FR-002) of its own: the socket it holds is a
38
+ :class:`~log_foundry.sinks._socket.SocketTransport`, which locks its own sends. Its
39
+ post-close refusal comes from there too (SPEC-032 FR-004) — a batch emitted after
40
+ ``close()`` reaches ``send_all`` and is refused with ``SinkDeliveryError`` without the
41
+ socket being reopened, so a guard here would only duplicate one that already holds.
36
42
  """
37
43
 
38
44
  def __init__(
@@ -23,6 +23,10 @@ class TransformSink:
23
23
  rename keys — and returning ``None`` drops that event. The caller's batch and event dicts
24
24
  are never mutated in place: only the function's return values are forwarded, so a transform
25
25
  must copy before mutating.
26
+
27
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
28
+ (SPEC-032 FR-003): it holds no transport and its ``close()`` only forwards, so both decisions
29
+ belong to the inner sink. A guard here would refuse batches the inner sink would have taken.
26
30
  """
27
31
 
28
32
  def __init__(
@@ -43,6 +43,10 @@ class NullSink:
43
43
  what this sink is for, and reporting it as loss would make ``health()``'s alert idiom fire
44
44
  on every batch for anyone who chose this sink to turn logging off. The counter stays
45
45
  readable on the instance.
46
+
47
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
48
+ (SPEC-032 FR-003): there is no transport and ``close()`` releases nothing, so discarding a
49
+ batch after close is the same operation as discarding one before it.
46
50
  """
47
51
 
48
52
  def __init__(self) -> None:
@@ -96,6 +100,10 @@ class MemorySink:
96
100
  eyeballing in a notebook. With ``maxlen`` set it behaves as a bounded ring, keeping only the
97
101
  most recent events; the list object identity is stable, so a held reference keeps seeing
98
102
  updates.
103
+
104
+ It takes **no** transport lock (SPEC-028 FR-002) and **adds no post-close guard**
105
+ (SPEC-032 FR-003): there is no transport, and ``close()`` releases nothing — a test that
106
+ closes the sink and then asserts on a later batch still sees it in ``.events``.
99
107
  """
100
108
 
101
109
  def __init__(self, maxlen: int | None = None) -> None: