log-foundry 0.10.2.dev24__tar.gz → 0.10.2.dev25__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/PKG-INFO +1 -1
  2. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/pyproject.toml +1 -1
  3. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_socket.py +16 -2
  4. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/clickhouse.py +10 -5
  5. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/eventhubs.py +36 -1
  6. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/file.py +16 -2
  7. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/firehose.py +4 -0
  8. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/kafka.py +4 -0
  9. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/kinesis.py +4 -0
  10. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/mongodb.py +28 -7
  11. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/nats.py +21 -7
  12. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/postgres.py +5 -1
  13. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/pubsub.py +21 -4
  14. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/rabbitmq.py +33 -10
  15. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/redis.py +7 -0
  16. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sentry.py +5 -0
  17. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sns.py +4 -0
  18. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sqlite.py +37 -4
  19. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sqs.py +7 -0
  20. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/worker.py +19 -4
  21. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/LICENSE +0 -0
  22. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/README.md +0 -0
  23. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/__init__.py +0 -0
  24. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/_diag.py +0 -0
  25. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/api.py +0 -0
  26. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/config.py +0 -0
  27. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/console.py +0 -0
  28. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/context.py +0 -0
  29. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/decorator.py +0 -0
  30. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/ids.py +0 -0
  31. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/model.py +0 -0
  32. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/py.typed +0 -0
  33. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sanitize.py +0 -0
  34. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/__init__.py +0 -0
  35. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_batch.py +0 -0
  36. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_chunk.py +0 -0
  37. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_retry.py +0 -0
  38. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_time.py +0 -0
  39. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/base.py +0 -0
  40. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/callback.py +0 -0
  41. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/datadog.py +0 -0
  42. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/elasticsearch.py +0 -0
  43. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/filtering.py +0 -0
  44. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/honeycomb.py +0 -0
  45. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/http.py +0 -0
  46. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/logging_sink.py +0 -0
  47. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/logstash.py +0 -0
  48. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/loki.py +0 -0
  49. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/multi.py +0 -0
  50. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/newrelic.py +0 -0
  51. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/splunk.py +0 -0
  52. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/stdout.py +0 -0
  53. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/syslog.py +0 -0
  54. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/transform.py +0 -0
  55. {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/util.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev24
3
+ Version: 0.10.2.dev25
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev24"
23
+ version = "0.10.2.dev25"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -111,6 +111,7 @@ class SocketTransport:
111
111
  self._counter_lock = threading.Lock()
112
112
  self.stop_signal: threading.Event | None = None
113
113
  self._lock = threading.Lock()
114
+ self._closed = False
114
115
 
115
116
  def send_all(self, messages: list[bytes]) -> None:
116
117
  """Sends each pre-framed message, reconnecting on error (FR-005, FR-006).
@@ -119,6 +120,11 @@ class SocketTransport:
119
120
  reconnect rebinds, so releasing between messages would let another thread send on a
120
121
  socket this one is about to reset (SPEC-028 FR-002).
121
122
 
123
+ A closed transport refuses rather than reconnecting. ``_socket`` opens a connection
124
+ whenever it holds none and ``close`` only drops the one it has, so without this a single
125
+ ``log_foundry.info()`` after ``shutdown()`` would open a TCP connection nothing will ever
126
+ reap — measured, and the same leak ``RabbitMQSink`` had (SPEC-028).
127
+
122
128
  Args:
123
129
  messages: The exact bytes to put on the wire, one call per message.
124
130
 
@@ -130,14 +136,21 @@ class SocketTransport:
130
136
  this transport propagate a dead destination to the worker instead of reporting
131
137
  success (SPEC-026 FR-001). A partial send does not raise, because the worker's retry
132
138
  would re-send the messages that already landed, and an empty call is a no-op rather
133
- than a total failure.
139
+ than a total failure. Also when the transport is already closed.
134
140
  """
141
+ if not messages:
142
+ return
135
143
  with self._lock:
144
+ if self._closed:
145
+ raise SinkDeliveryError(
146
+ f"SocketTransport delivered none of {len(messages)} message(s): "
147
+ f"the transport is closed"
148
+ )
136
149
  delivered = 0
137
150
  for message in messages:
138
151
  if self._send_one(message):
139
152
  delivered += 1
140
- if messages and delivered == 0:
153
+ if delivered == 0:
141
154
  raise SinkDeliveryError(
142
155
  f"SocketTransport delivered none of {len(messages)} message(s)"
143
156
  )
@@ -175,6 +188,7 @@ class SocketTransport:
175
188
  None.
176
189
  """
177
190
  with self._lock:
191
+ self._closed = True
178
192
  self._reset()
179
193
 
180
194
  def _send_one(self, message: bytes) -> bool:
@@ -41,10 +41,11 @@ class ClickHouseSink:
41
41
  sink is write-only, and the worst-case delay (SPEC-027 FR-005) is ``max_retries``
42
42
  interruptible waits per chunk, 0.7 s at the defaults.
43
43
 
44
- The driver requirement satisfied (SPEC-028 FR-002): a ``clickhouse-connect`` client holds
45
- per-session state across an insert and the project does not publish it as safe to share
46
- between threads, so this sink serializes its use rather than assuming otherwise. A lock is
47
- the conservative reading one client per thread would be the alternative, and that is the
44
+ The driver requirement satisfied (SPEC-028 FR-002): ``clickhouse-connect`` states that a
45
+ client is **not** thread-safe when using session ids, and that concurrent queries in one
46
+ session raise ``ProgrammingError``. This sink calls ``get_client(dsn=...)``, which takes the
47
+ default auto-generated session, so it is squarely in that case and the lock is required
48
+ rather than merely prudent. One client per thread would be the alternative, and that is the
48
49
  connection-pool design FR-002 puts out of scope.
49
50
  """
50
51
 
@@ -133,6 +134,10 @@ class ClickHouseSink:
133
134
  return
134
135
  chunks = inserted = 0
135
136
  with self._lock:
137
+ if self._closed:
138
+ raise SinkDeliveryError(
139
+ f"ClickHouseSink inserted none of {len(batch)} event(s): the sink is closed"
140
+ )
136
141
  for chunk in chunk_list(batch, self._chunk_size):
137
142
  chunks += 1
138
143
  inserted += self._insert([self._row(event) for event in chunk])
@@ -157,9 +162,9 @@ class ClickHouseSink:
157
162
  with self._lock:
158
163
  if self._closed:
159
164
  return
165
+ self._closed = True
160
166
  if self._owns_client:
161
167
  self.client.close()
162
- self._closed = True
163
168
 
164
169
  def _row(self, event: dict[str, object]) -> list[object]:
165
170
  """Builds one row: the extracted columns, then the whole event as JSON.
@@ -68,6 +68,8 @@ class AzureEventHubsSink:
68
68
  self.failed = 0
69
69
  self.dropped_oversized = 0
70
70
  self._counter_lock = threading.Lock()
71
+ self._lock = threading.Lock()
72
+ self._closed = False
71
73
 
72
74
  def losses(self) -> SinkLosses:
73
75
  """Reports oversized drops and events in a batch abandoned past the bound (FR-002).
@@ -106,6 +108,32 @@ class AzureEventHubsSink:
106
108
  """
107
109
  if not batch:
108
110
  return
111
+ with self._lock:
112
+ if self._closed:
113
+ raise SinkDeliveryError(
114
+ f"AzureEventHubsSink sent none of {len(batch)} event(s): the sink is closed"
115
+ )
116
+ self._send_batch(batch)
117
+
118
+ def _send_batch(self, batch: list[dict[str, object]]) -> None:
119
+ """Packs and sends the batch, with the emit lock already held.
120
+
121
+ Split out so the lock's extent is one line in :meth:`emit`. The driver requirement
122
+ satisfied (SPEC-028 FR-002): Microsoft affirmatively states that the
123
+ ``EventHubProducerClient`` is not thread-safe and recommends guarding it with a
124
+ ``threading.Lock``; the ``EventDataBatch`` this builds up across the loop is single-owner
125
+ state besides — two threads packing into batches from one producer interleave
126
+ ``create_batch``/``add``/``send`` on it.
127
+
128
+ Args:
129
+ batch: The events to send, known non-empty.
130
+
131
+ Returns:
132
+ None.
133
+
134
+ Raises:
135
+ SinkDeliveryError: When every attempted send failed.
136
+ """
109
137
  event_data_cls = _event_data_cls()
110
138
  current = self.producer.create_batch()
111
139
  attempted = delivered = 0
@@ -132,6 +160,9 @@ class AzureEventHubsSink:
132
160
  def close(self) -> None:
133
161
  """Closes the producer (FR-009).
134
162
 
163
+ Idempotent, and takes the emit lock so the producer is never closed out from under an
164
+ in-flight send (SPEC-028 FR-002).
165
+
135
166
  Args:
136
167
  None.
137
168
 
@@ -141,7 +172,11 @@ class AzureEventHubsSink:
141
172
  Raises:
142
173
  Exception: Whatever the producer raises on close.
143
174
  """
144
- self.producer.close()
175
+ with self._lock:
176
+ if self._closed:
177
+ return
178
+ self._closed = True
179
+ self.producer.close()
145
180
 
146
181
  def _send(self, event_batch: Any) -> int:
147
182
  """Sends one ``EventDataBatch``, retrying failures (FR-009, FR-011).
@@ -8,6 +8,8 @@ import threading
8
8
  import time
9
9
  from typing import TextIO
10
10
 
11
+ from log_foundry.sinks.base import SinkDeliveryError
12
+
11
13
  __all__ = ["FileSink", "RotatingFileSink"]
12
14
 
13
15
  _WHEN_SECONDS = {
@@ -71,7 +73,13 @@ class FileSink:
71
73
  Raises:
72
74
  OSError: If the write or flush fails (FR-001).
73
75
  """
76
+ if not batch:
77
+ return
74
78
  with self._lock:
79
+ if self._closed:
80
+ raise SinkDeliveryError(
81
+ f"FileSink wrote none of {len(batch)} event(s): the sink is closed"
82
+ )
75
83
  for event in batch:
76
84
  self._stream.write(json.dumps(event) + "\n")
77
85
  self._stream.flush()
@@ -94,9 +102,9 @@ class FileSink:
94
102
  with self._lock:
95
103
  if self._closed:
96
104
  return
105
+ self._closed = True
97
106
  self._stream.flush()
98
107
  self._stream.close()
99
- self._closed = True
100
108
 
101
109
 
102
110
  class RotatingFileSink:
@@ -176,7 +184,13 @@ class RotatingFileSink:
176
184
  Raises:
177
185
  OSError: If a write, flush or rotation fails.
178
186
  """
187
+ if not batch:
188
+ return
179
189
  with self._lock:
190
+ if self._closed:
191
+ raise SinkDeliveryError(
192
+ f"RotatingFileSink wrote none of {len(batch)} event(s): the sink is closed"
193
+ )
180
194
  for event in batch:
181
195
  line = json.dumps(event) + "\n"
182
196
  data = len(line.encode(self._encoding))
@@ -205,9 +219,9 @@ class RotatingFileSink:
205
219
  with self._lock:
206
220
  if self._closed:
207
221
  return
222
+ self._closed = True
208
223
  self._stream.flush()
209
224
  self._stream.close()
210
- self._closed = True
211
225
 
212
226
  @staticmethod
213
227
  def _rollover_seconds(when: str | None, interval: int) -> float | None:
@@ -32,6 +32,10 @@ class FirehoseSink:
32
32
  the chunk sent, so none could be paired to an outcome. A non-zero count means those
33
33
  records were abandoned without the stream ever confirming them — treat it as loss, and
34
34
  as a sign the client is not AWS-shaped.
35
+
36
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
37
+ lock. ``boto3`` clients are documented thread-safe, this one is built once in
38
+ ``__init__``, and the sink rebinds nothing after construction.
35
39
  """
36
40
 
37
41
  MAX_RECORDS = 500
@@ -24,6 +24,10 @@ class KafkaSink:
24
24
  failed: Messages whose delivery callback reported an error.
25
25
  rejected: Messages ``produce()`` itself refused — a full local queue, a serialization fault
26
26
  — which never reached the producer's batch at all.
27
+
28
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
29
+ lock. ``confluent-kafka`` documents its ``Producer`` as thread-safe, and this sink adds no state
30
+ of its own to guard — ``produce()`` is a local hand-off into the client's internal queue.
27
31
  """
28
32
 
29
33
  def __init__(
@@ -32,6 +32,10 @@ class KinesisSink:
32
32
  describe the chunk sent, so none could be paired to an outcome. A non-zero count means
33
33
  those records were abandoned without the stream ever confirming them — treat it as loss,
34
34
  and as a sign the client is not AWS-shaped.
35
+
36
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
37
+ lock. ``boto3`` clients are documented thread-safe, this one is built once in
38
+ ``__init__``, and the sink rebinds nothing after construction.
35
39
  """
36
40
 
37
41
  MAX_RECORDS = 500
@@ -76,6 +76,7 @@ class MongoDBSink:
76
76
  self.dropped_oversized = 0
77
77
  self._closed = False
78
78
  self._counter_lock = threading.Lock()
79
+ self._close_lock = threading.Lock()
79
80
 
80
81
  def losses(self) -> SinkLosses:
81
82
  """Reports oversized drops and documents the server rejected or never took (FR-002).
@@ -109,12 +110,20 @@ class MongoDBSink:
109
110
  None.
110
111
 
111
112
  Raises:
112
- SinkDeliveryError: When nothing was inserted (SPEC-026 FR-001) a connection error
113
+ SinkDeliveryError: When the sink is already closed, since ``pymongo`` raises
114
+ ``InvalidOperation`` on any use of a closed client and the library has its own word
115
+ for "none of this was delivered". Also when nothing was inserted (SPEC-026 FR-001) — a connection error
113
116
  past the retry bound, or a bulk write every one of whose documents the server
114
117
  rejected. A bulk write that stored some is partial and never raises. A batch of
115
118
  nothing but oversized documents does not raise either: they can never fit, so there
116
119
  is nothing to retry, and they are reported through :meth:`losses`.
117
120
  """
121
+ if not batch:
122
+ return
123
+ if self._closed:
124
+ raise SinkDeliveryError(
125
+ f"MongoDBSink inserted none of {len(batch)} document(s): the sink is closed"
126
+ )
118
127
  documents = self._documents(batch)
119
128
  if not documents:
120
129
  return
@@ -157,7 +166,18 @@ class MongoDBSink:
157
166
  def close(self) -> None:
158
167
  """Closes the client only if the sink owns it (FR-005).
159
168
 
160
- Idempotent.
169
+ Idempotent, with the flag set under a lock so two concurrent ``close()`` calls cannot
170
+ both reach ``client.close()`` — ``atexit`` racing user code is the documented case.
171
+
172
+ The lock does **not** make a close wait for an in-flight ``insert_many``, and saying so
173
+ plainly matters: ``emit`` takes no lock here because ``pymongo``'s client is thread-safe,
174
+ and a close that waited would have to serialize against every insert, which is the
175
+ parallelism SPEC-028 FR-002 declines to remove. What covers the overlap instead is
176
+ :meth:`emit`'s own ``_closed`` check, which refuses the batch in the library's
177
+ vocabulary rather than letting it reach a closed client — pymongo raises
178
+ ``InvalidOperation`` on any use after close. An insert that passes the check and is
179
+ still in flight when the close lands can still see that error; it is counted as a failed
180
+ batch like any other, which is a reported loss at process exit rather than a silent one.
161
181
 
162
182
  Args:
163
183
  None.
@@ -168,11 +188,12 @@ class MongoDBSink:
168
188
  Raises:
169
189
  Exception: Whatever the client raises on close.
170
190
  """
171
- if self._closed:
172
- return
173
- if self._owns_client:
174
- self._client.close()
175
- self._closed = True
191
+ with self._close_lock:
192
+ if self._closed:
193
+ return
194
+ self._closed = True
195
+ if self._owns_client:
196
+ self._client.close()
176
197
 
177
198
  def _documents(self, batch: list[dict[str, object]]) -> list[dict[str, object]]:
178
199
  """Copies each event and drops any document too large to ever fit.
@@ -51,6 +51,7 @@ class NATSSink:
51
51
  self._loop = asyncio.new_event_loop()
52
52
  self.failed = 0
53
53
  self._counter_lock = threading.Lock()
54
+ self._lock = threading.Lock()
54
55
  if client is None:
55
56
  import nats # type: ignore[import-not-found]
56
57
 
@@ -62,6 +63,13 @@ class NATSSink:
62
63
  def emit(self, batch: list[dict[str, object]]) -> None:
63
64
  """Drives the async publishes to completion on the managed loop (FR-007).
64
65
 
66
+ The driver requirement satisfied (SPEC-028 FR-002): an ``asyncio`` event loop is
67
+ single-entry. A second thread calling ``run_until_complete`` on a loop that is already
68
+ running raises ``RuntimeError``, and can leave the loop's task machinery in a state where
69
+ a thread never returns from ``emit`` at all — measured as a permanently hung application
70
+ thread on the orphan path, which is the one outcome this library must never produce. The
71
+ lock makes the loop what the sink already assumed it was: entered by one caller at a time.
72
+
65
73
  Args:
66
74
  batch: The events to publish. An empty batch is a no-op.
67
75
 
@@ -73,7 +81,12 @@ class NATSSink:
73
81
  """
74
82
  if not batch:
75
83
  return
76
- self._loop.run_until_complete(self._publish_all(batch))
84
+ with self._lock:
85
+ if self._loop.is_closed():
86
+ raise SinkDeliveryError(
87
+ f"NATSSink published none of {len(batch)} event(s): the sink is closed"
88
+ )
89
+ self._loop.run_until_complete(self._publish_all(batch))
77
90
 
78
91
  def losses(self) -> SinkLosses:
79
92
  """Reports events whose publish raised (SPEC-026 FR-002).
@@ -102,12 +115,13 @@ class NATSSink:
102
115
  Raises:
103
116
  Exception: Whatever draining raises; the loop is closed regardless.
104
117
  """
105
- if self._loop.is_closed():
106
- return
107
- try:
108
- self._loop.run_until_complete(self._drain())
109
- finally:
110
- self._loop.close()
118
+ with self._lock:
119
+ if self._loop.is_closed():
120
+ return
121
+ try:
122
+ self._loop.run_until_complete(self._drain())
123
+ finally:
124
+ self._loop.close()
111
125
 
112
126
  async def _publish_all(self, batch: list[dict[str, object]]) -> None:
113
127
  """Publishes each event, isolating a per-event failure.
@@ -125,6 +125,10 @@ class PostgresSink:
125
125
  if not batch:
126
126
  return
127
127
  with self._lock:
128
+ if self._closed:
129
+ raise SinkDeliveryError(
130
+ f"PostgresSink inserted none of {len(batch)} event(s): the sink is closed"
131
+ )
128
132
  self._insert_batch(batch)
129
133
 
130
134
  def _insert_batch(self, batch: list[dict[str, object]]) -> None:
@@ -183,10 +187,10 @@ class PostgresSink:
183
187
  with self._lock:
184
188
  if self._closed:
185
189
  return
190
+ self._closed = True
186
191
  self._conn.commit()
187
192
  if self._owns_connection:
188
193
  self._conn.close()
189
- self._closed = True
190
194
 
191
195
  def _row(self, event: dict[str, object]) -> tuple[object, ...]:
192
196
  """Builds one row: the extracted columns, then the whole event as JSON.
@@ -18,6 +18,15 @@ class GooglePubSubSink:
18
18
  This is a durable-buffer sink on ``google-cloud-pubsub``, the optional ``gcp-pubsub`` extra,
19
19
  imported lazily. ``publish()`` returns a future that resolves asynchronously, so the sink
20
20
  accumulates the batch's futures and resolves them on :meth:`close`.
21
+
22
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport lock —
23
+ the publisher client owns its own batching and threading, and ``publish()`` is a local
24
+ hand-off. What it does hold is the pending-futures list, which is genuinely shared between
25
+ ``emit`` (appending) and ``close`` (resolving), so that list has its own small lock and
26
+ ``close`` swaps it out rather than iterating and clearing. Without the swap, a future
27
+ appended after the loop passed its index was dropped unresolved: an unconfirmed publish
28
+ never counted in ``failed`` and never reported by :meth:`losses`, which is precisely the
29
+ silent loss SPEC-026 exists to end.
21
30
  """
22
31
 
23
32
  def __init__(self, topic: str, *, client: Any = None) -> None:
@@ -42,6 +51,7 @@ class GooglePubSubSink:
42
51
  self.failed = 0
43
52
  self.rejected = 0
44
53
  self._counter_lock = threading.Lock()
54
+ self._futures_lock = threading.Lock()
45
55
  self._futures: list[Any] = []
46
56
 
47
57
  def losses(self) -> SinkLosses:
@@ -88,7 +98,8 @@ class GooglePubSubSink:
88
98
  self.rejected += 1
89
99
  _diag.lost("event", 1, f"GooglePubSubSink refused the publish, {type(err).__name__}")
90
100
  continue
91
- self._futures.append(future)
101
+ with self._futures_lock:
102
+ self._futures.append(future)
92
103
  published += 1
93
104
  if batch and not published:
94
105
  raise SinkDeliveryError(
@@ -98,7 +109,12 @@ class GooglePubSubSink:
98
109
  def close(self) -> None:
99
110
  """Resolves all pending publish futures, counting and logging errors (FR-008).
100
111
 
101
- Idempotent.
112
+ Idempotent. The pending list is swapped out under a lock rather than iterated and then
113
+ cleared (SPEC-028 FR-002): ``emit`` appends to it from any thread, so the old
114
+ iterate-then-``clear()`` discarded any future appended after the loop passed its index —
115
+ an unconfirmed publish whose ``result()`` was never called, never counted in ``failed``
116
+ and never reported by ``losses()``. That is the silent loss SPEC-026 exists to end,
117
+ reached through the one piece of shared state this sink has.
102
118
 
103
119
  Args:
104
120
  None.
@@ -110,7 +126,9 @@ class GooglePubSubSink:
110
126
  None. This is an isolation boundary: an unresolved future must never crash the worker
111
127
  (FR-011).
112
128
  """
113
- for future in self._futures:
129
+ with self._futures_lock:
130
+ pending, self._futures = self._futures, []
131
+ for future in pending:
114
132
  try:
115
133
  future.result()
116
134
  except Exception as err:
@@ -119,4 +137,3 @@ class GooglePubSubSink:
119
137
  _diag.lost(
120
138
  "event", 1, f"GooglePubSubSink publish unconfirmed, {type(err).__name__}"
121
139
  )
122
- self._futures.clear()
@@ -73,6 +73,8 @@ class RabbitMQSink:
73
73
  self._properties: Any = None
74
74
  self.failed = 0
75
75
  self._counter_lock = threading.Lock()
76
+ self._lock = threading.Lock()
77
+ self._closed = False
76
78
 
77
79
  def losses(self) -> SinkLosses:
78
80
  """Reports messages abandoned past the reconnect-retry bound (SPEC-026 FR-002).
@@ -95,6 +97,12 @@ class RabbitMQSink:
95
97
  A partial publish is counted and left alone: retrying it would re-publish the messages
96
98
  already on the exchange.
97
99
 
100
+ The driver requirement satisfied (SPEC-028 FR-002): ``pika`` states that one connection
101
+ must not be shared across threads, and this sink shares one connection, one channel, and
102
+ *rebinds* both — ``_reset`` closes and nulls the channel another thread may be mid-publish
103
+ on. It is structurally the same sink as ``SocketTransport``, one AMQP frame stream over
104
+ one socket, so it takes the same lock.
105
+
98
106
  Args:
99
107
  batch: The events to publish.
100
108
 
@@ -103,12 +111,23 @@ class RabbitMQSink:
103
111
 
104
112
  Raises:
105
113
  SinkDeliveryError: When no message reached the broker (SPEC-026 FR-001) — a down broker
106
- is the case the worker's retry and ``failed_batches`` exist for.
114
+ is the case the worker's retry and ``failed_batches`` exist for. Also when the sink
115
+ is already closed, which is not fussiness: ``_active_channel`` reopens a connection
116
+ whenever it finds none, and ``close`` is now idempotent, so an emit arriving after a
117
+ shutdown would open an AMQP connection that nothing will ever reap. One
118
+ ``log_foundry.info()`` after ``shutdown()`` reaches this on the caller's own thread.
107
119
  """
120
+ if not batch:
121
+ return
108
122
  published = 0
109
- for event in batch:
110
- if self._publish(json.dumps(event).encode("utf-8")):
111
- published += 1
123
+ with self._lock:
124
+ if self._closed:
125
+ raise SinkDeliveryError(
126
+ f"RabbitMQSink published none of {len(batch)} message(s): the sink is closed"
127
+ )
128
+ for event in batch:
129
+ if self._publish(json.dumps(event).encode("utf-8")):
130
+ published += 1
112
131
  if batch and not published:
113
132
  raise SinkDeliveryError(f"RabbitMQSink published none of {len(batch)} message(s)")
114
133
 
@@ -126,12 +145,16 @@ class RabbitMQSink:
126
145
  Raises:
127
146
  None.
128
147
  """
129
- if self._channel is not None:
130
- _safe_close(self._channel)
131
- self._channel = None
132
- if self._connection is not None:
133
- _safe_close(self._connection)
134
- self._connection = None
148
+ with self._lock:
149
+ if self._closed:
150
+ return
151
+ self._closed = True
152
+ if self._channel is not None:
153
+ _safe_close(self._channel)
154
+ self._channel = None
155
+ if self._connection is not None:
156
+ _safe_close(self._connection)
157
+ self._connection = None
135
158
 
136
159
  def _publish(self, body: bytes) -> bool:
137
160
  """Publishes one message within the retry bound.
@@ -21,6 +21,13 @@ class _RedisSink:
21
21
  The module is named ``redis`` to match the extra and imports the driver lazily, so it never
22
22
  shadows or requires the real package at import time. The worst-case delay (SPEC-027 FR-005)
23
23
  is ``max_retries`` interruptible waits per batch, 0.7 s at the defaults.
24
+
25
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
26
+ lock. ``redis-py`` documents that a client may be shared between threads — a
27
+ connection is taken from its pool only for the duration of a command, and command
28
+ execution never mutates the client. Its documented exceptions are ``PubSub`` and
29
+ ``Pipeline`` objects, which must not be passed between threads — which is why the pipeline
30
+ here is built and executed inside a single ``emit`` call and never stored on the instance.
24
31
  """
25
32
 
26
33
  def __init__(self, *, client: Any, url: str | None, max_retries: int) -> None:
@@ -33,6 +33,11 @@ class SentrySink:
33
33
  skipped: Events below the minimum level, or without a usable level, that were not sent.
34
34
  transport_errors: Events whose send raised something other than an already-counted
35
35
  abandonment — an SDK fault, or a response error ``HTTPSink`` does not retry.
36
+
37
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
38
+ lock. This sink owns no transport: it delegates to a ``sentry_sdk`` built for capture from any
39
+ thread, or to an ``HTTPSink`` that builds a fresh request per call and rebinds nothing.
40
+ Its counters take the counter lock like every other sink's.
36
41
  """
37
42
 
38
43
  def __init__(
@@ -25,6 +25,10 @@ class SNSSink:
25
25
 
26
26
  The worst-case delay (SPEC-027 FR-005) is ``max_retries`` waits per chunk, 0.7 s at the
27
27
  defaults. The waits are interruptible, so ``shutdown()`` cuts one short.
28
+
29
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
30
+ lock. ``boto3`` clients are documented thread-safe, this one is built once in
31
+ ``__init__``, and the sink rebinds nothing after construction.
28
32
  """
29
33
 
30
34
  MAX_BATCH = 10
@@ -7,6 +7,7 @@ import sqlite3
7
7
  import threading
8
8
 
9
9
  from log_foundry.sinks._chunk import valid_identifier
10
+ from log_foundry.sinks.base import SinkDeliveryError
10
11
 
11
12
  __all__ = ["SQLiteSink"]
12
13
 
@@ -78,8 +79,9 @@ class SQLiteSink:
78
79
 
79
80
  ``with connection`` opens a transaction and commits on success or rolls back on error,
80
81
  so the whole batch lands atomically. The lock holds that transaction to one thread
81
- (SPEC-028 FR-002); the row-building above it needs no protection, but sits inside for
82
- simplicity and costs nothing an uncontended lock does not.
82
+ (SPEC-028 FR-002). Row-building is deliberately *outside* it: those lines touch nothing
83
+ shared, so serializing the ``json.dumps`` of a whole batch would lengthen the critical
84
+ section for no gain.
83
85
 
84
86
  Args:
85
87
  batch: The events to insert.
@@ -89,13 +91,44 @@ class SQLiteSink:
89
91
 
90
92
  Raises:
91
93
  sqlite3.Error: If the insert fails.
94
+ SinkDeliveryError: If the sink is closed — whether the close landed while this call
95
+ was building its rows, or long before it. Either way the batch would otherwise reach
96
+ the driver as "Cannot operate on a closed database", and the library has a word of
97
+ its own for "none of this was delivered" (SPEC-026 FR-001). What the library should
98
+ *signal* about logging after a completed shutdown is a separate question, and
99
+ SPEC-030's; this only keeps driver internals out of the answer.
92
100
  """
101
+ if not batch:
102
+ return
93
103
  rows = [
94
104
  (*(event.get(col) for col in _COLUMNS), json.dumps(event)) for event in batch
95
105
  ]
96
106
  placeholders = ", ".join("?" * (len(_COLUMNS) + 1))
97
107
  columns = ", ".join((*_COLUMNS, "event"))
98
- with self._lock, self._conn:
108
+ with self._lock:
109
+ if self._closed:
110
+ raise SinkDeliveryError(
111
+ f"SQLiteSink inserted none of {len(batch)} event(s): the sink is closed"
112
+ )
113
+ self._insert(rows, columns, placeholders)
114
+
115
+ def _insert(
116
+ self, rows: list[tuple[object, ...]], columns: str, placeholders: str
117
+ ) -> None:
118
+ """Runs the insert transaction, with the emit lock already held.
119
+
120
+ Args:
121
+ rows: The prepared row tuples.
122
+ columns: The comma-separated column list.
123
+ placeholders: The matching parameter placeholders.
124
+
125
+ Returns:
126
+ None.
127
+
128
+ Raises:
129
+ sqlite3.Error: If the insert fails.
130
+ """
131
+ with self._conn:
99
132
  self._conn.executemany(
100
133
  f'INSERT INTO "{self._table}" ({columns}) VALUES ({placeholders})', rows
101
134
  )
@@ -118,10 +151,10 @@ class SQLiteSink:
118
151
  with self._lock:
119
152
  if self._closed:
120
153
  return
154
+ self._closed = True
121
155
  self._conn.commit()
122
156
  if self._owns_connection:
123
157
  self._conn.close()
124
- self._closed = True
125
158
 
126
159
  def _ensure_schema(self) -> None:
127
160
  """Idempotently creates the target table.
@@ -84,6 +84,13 @@ class SQSSink:
84
84
 
85
85
  The worst-case delay (SPEC-027 FR-005) is ``max_retries`` waits per chunk, 0.7 s at the
86
86
  defaults. The waits are interruptible, so ``shutdown()`` cuts one short.
87
+
88
+ The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
89
+ lock. ``boto3`` documents that clients — unlike Sessions and Resources — are
90
+ thread-safe, and this sink builds its client once in ``__init__`` rather than inside
91
+ ``emit`` — which is what keeps it clear of the documented caveat that calling
92
+ ``boto3.client()`` concurrently can produce response-ordering faults. Nothing else here is
93
+ rebound after construction.
87
94
  """
88
95
 
89
96
  MAX_BATCH = 10
@@ -395,9 +395,22 @@ class Worker:
395
395
  it, and ``atexit`` plus user code calling it at once is documented as normal.
396
396
 
397
397
  ``is_alive()`` is the safety condition rather than a heuristic: it reads ``False`` only
398
- after ``_run`` has returned, so the sink is provably out of use. The close itself runs
399
- outside the lock, because it can reach ``_diag`` and a wedged console must not stall a
400
- lock :meth:`submit` also takes.
398
+ after ``_run`` has returned, so the sink is provably out of use *by the worker*.
399
+
400
+ The close runs to completion, inline, and is deliberately **not** bounded — which leaves
401
+ one honest gap. SPEC-028 made ``close()`` take the sink's emit lock, so an application
402
+ thread on the orphan path can hold that lock inside a driver call with no timeout of its
403
+ own and delay this past ``shutdown``'s budget. Running the close on a joinable daemon
404
+ thread was tried and reverted: at interpreter exit the daemon is killed wherever it has
405
+ reached, which for ``SQLiteSink`` is between ``commit()`` and ``close()`` — turning the
406
+ leaked handle SPEC-027 FR-004 accepts into the partial write it was avoiding. It also
407
+ could not tell a slow-but-successful close from a stuck one, so it reported
408
+ ``ShutdownTimeout`` and "left open" for closes that had in fact completed, latching
409
+ SPEC-019's alert term on a healthy shutdown. A wrong signal is worse than a slow one.
410
+ The residual delay is recorded in ``architecture.md`` §13 rather than papered over.
411
+
412
+ The close runs outside the lock, because it can reach ``_diag`` and a wedged console must
413
+ not stall a lock :meth:`submit` also takes.
401
414
 
402
415
  Args:
403
416
  None.
@@ -406,7 +419,9 @@ class Worker:
406
419
  None.
407
420
 
408
421
  Raises:
409
- None.
422
+ BaseException: Whatever the sink's ``close`` raised that is not an ``Exception``.
423
+ ``_close_sink`` absorbs ``Exception`` but lets a ``KeyboardInterrupt`` or
424
+ ``SystemExit`` through to the caller (SPEC-025 FR-004).
410
425
  """
411
426
  with self._lock:
412
427
  if self._sink_closed or self._thread.is_alive():