log-foundry 0.10.2.dev24__tar.gz → 0.10.2.dev25__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/PKG-INFO +1 -1
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_socket.py +16 -2
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/clickhouse.py +10 -5
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/eventhubs.py +36 -1
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/file.py +16 -2
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/firehose.py +4 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/kafka.py +4 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/kinesis.py +4 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/mongodb.py +28 -7
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/nats.py +21 -7
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/postgres.py +5 -1
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/pubsub.py +21 -4
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/rabbitmq.py +33 -10
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/redis.py +7 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sentry.py +5 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sns.py +4 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sqlite.py +37 -4
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/sqs.py +7 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/worker.py +19 -4
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/LICENSE +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/README.md +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/__init__.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/config.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/decorator.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_retry.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/base.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/callback.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/datadog.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/elasticsearch.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/filtering.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/honeycomb.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/http.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/logging_sink.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/logstash.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/loki.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/multi.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/splunk.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/stdout.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/syslog.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/transform.py +0 -0
- {log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/util.py +0 -0
|
@@ -111,6 +111,7 @@ class SocketTransport:
|
|
|
111
111
|
self._counter_lock = threading.Lock()
|
|
112
112
|
self.stop_signal: threading.Event | None = None
|
|
113
113
|
self._lock = threading.Lock()
|
|
114
|
+
self._closed = False
|
|
114
115
|
|
|
115
116
|
def send_all(self, messages: list[bytes]) -> None:
|
|
116
117
|
"""Sends each pre-framed message, reconnecting on error (FR-005, FR-006).
|
|
@@ -119,6 +120,11 @@ class SocketTransport:
|
|
|
119
120
|
reconnect rebinds, so releasing between messages would let another thread send on a
|
|
120
121
|
socket this one is about to reset (SPEC-028 FR-002).
|
|
121
122
|
|
|
123
|
+
A closed transport refuses rather than reconnecting. ``_socket`` opens a connection
|
|
124
|
+
whenever it holds none and ``close`` only drops the one it has, so without this a single
|
|
125
|
+
``log_foundry.info()`` after ``shutdown()`` would open a TCP connection nothing will ever
|
|
126
|
+
reap — measured, and the same leak ``RabbitMQSink`` had (SPEC-028).
|
|
127
|
+
|
|
122
128
|
Args:
|
|
123
129
|
messages: The exact bytes to put on the wire, one call per message.
|
|
124
130
|
|
|
@@ -130,14 +136,21 @@ class SocketTransport:
|
|
|
130
136
|
this transport propagate a dead destination to the worker instead of reporting
|
|
131
137
|
success (SPEC-026 FR-001). A partial send does not raise, because the worker's retry
|
|
132
138
|
would re-send the messages that already landed, and an empty call is a no-op rather
|
|
133
|
-
than a total failure.
|
|
139
|
+
than a total failure. Also when the transport is already closed.
|
|
134
140
|
"""
|
|
141
|
+
if not messages:
|
|
142
|
+
return
|
|
135
143
|
with self._lock:
|
|
144
|
+
if self._closed:
|
|
145
|
+
raise SinkDeliveryError(
|
|
146
|
+
f"SocketTransport delivered none of {len(messages)} message(s): "
|
|
147
|
+
f"the transport is closed"
|
|
148
|
+
)
|
|
136
149
|
delivered = 0
|
|
137
150
|
for message in messages:
|
|
138
151
|
if self._send_one(message):
|
|
139
152
|
delivered += 1
|
|
140
|
-
if
|
|
153
|
+
if delivered == 0:
|
|
141
154
|
raise SinkDeliveryError(
|
|
142
155
|
f"SocketTransport delivered none of {len(messages)} message(s)"
|
|
143
156
|
)
|
|
@@ -175,6 +188,7 @@ class SocketTransport:
|
|
|
175
188
|
None.
|
|
176
189
|
"""
|
|
177
190
|
with self._lock:
|
|
191
|
+
self._closed = True
|
|
178
192
|
self._reset()
|
|
179
193
|
|
|
180
194
|
def _send_one(self, message: bytes) -> bool:
|
|
@@ -41,10 +41,11 @@ class ClickHouseSink:
|
|
|
41
41
|
sink is write-only, and the worst-case delay (SPEC-027 FR-005) is ``max_retries``
|
|
42
42
|
interruptible waits per chunk, 0.7 s at the defaults.
|
|
43
43
|
|
|
44
|
-
The driver requirement satisfied (SPEC-028 FR-002):
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
The driver requirement satisfied (SPEC-028 FR-002): ``clickhouse-connect`` states that a
|
|
45
|
+
client is **not** thread-safe when using session ids, and that concurrent queries in one
|
|
46
|
+
session raise ``ProgrammingError``. This sink calls ``get_client(dsn=...)``, which takes the
|
|
47
|
+
default auto-generated session, so it is squarely in that case and the lock is required
|
|
48
|
+
rather than merely prudent. One client per thread would be the alternative, and that is the
|
|
48
49
|
connection-pool design FR-002 puts out of scope.
|
|
49
50
|
"""
|
|
50
51
|
|
|
@@ -133,6 +134,10 @@ class ClickHouseSink:
|
|
|
133
134
|
return
|
|
134
135
|
chunks = inserted = 0
|
|
135
136
|
with self._lock:
|
|
137
|
+
if self._closed:
|
|
138
|
+
raise SinkDeliveryError(
|
|
139
|
+
f"ClickHouseSink inserted none of {len(batch)} event(s): the sink is closed"
|
|
140
|
+
)
|
|
136
141
|
for chunk in chunk_list(batch, self._chunk_size):
|
|
137
142
|
chunks += 1
|
|
138
143
|
inserted += self._insert([self._row(event) for event in chunk])
|
|
@@ -157,9 +162,9 @@ class ClickHouseSink:
|
|
|
157
162
|
with self._lock:
|
|
158
163
|
if self._closed:
|
|
159
164
|
return
|
|
165
|
+
self._closed = True
|
|
160
166
|
if self._owns_client:
|
|
161
167
|
self.client.close()
|
|
162
|
-
self._closed = True
|
|
163
168
|
|
|
164
169
|
def _row(self, event: dict[str, object]) -> list[object]:
|
|
165
170
|
"""Builds one row: the extracted columns, then the whole event as JSON.
|
|
@@ -68,6 +68,8 @@ class AzureEventHubsSink:
|
|
|
68
68
|
self.failed = 0
|
|
69
69
|
self.dropped_oversized = 0
|
|
70
70
|
self._counter_lock = threading.Lock()
|
|
71
|
+
self._lock = threading.Lock()
|
|
72
|
+
self._closed = False
|
|
71
73
|
|
|
72
74
|
def losses(self) -> SinkLosses:
|
|
73
75
|
"""Reports oversized drops and events in a batch abandoned past the bound (FR-002).
|
|
@@ -106,6 +108,32 @@ class AzureEventHubsSink:
|
|
|
106
108
|
"""
|
|
107
109
|
if not batch:
|
|
108
110
|
return
|
|
111
|
+
with self._lock:
|
|
112
|
+
if self._closed:
|
|
113
|
+
raise SinkDeliveryError(
|
|
114
|
+
f"AzureEventHubsSink sent none of {len(batch)} event(s): the sink is closed"
|
|
115
|
+
)
|
|
116
|
+
self._send_batch(batch)
|
|
117
|
+
|
|
118
|
+
def _send_batch(self, batch: list[dict[str, object]]) -> None:
|
|
119
|
+
"""Packs and sends the batch, with the emit lock already held.
|
|
120
|
+
|
|
121
|
+
Split out so the lock's extent is one line in :meth:`emit`. The driver requirement
|
|
122
|
+
satisfied (SPEC-028 FR-002): Microsoft affirmatively states that the
|
|
123
|
+
``EventHubProducerClient`` is not thread-safe and recommends guarding it with a
|
|
124
|
+
``threading.Lock``; the ``EventDataBatch`` this builds up across the loop is single-owner
|
|
125
|
+
state besides — two threads packing into batches from one producer interleave
|
|
126
|
+
``create_batch``/``add``/``send`` on it.
|
|
127
|
+
|
|
128
|
+
Args:
|
|
129
|
+
batch: The events to send, known non-empty.
|
|
130
|
+
|
|
131
|
+
Returns:
|
|
132
|
+
None.
|
|
133
|
+
|
|
134
|
+
Raises:
|
|
135
|
+
SinkDeliveryError: When every attempted send failed.
|
|
136
|
+
"""
|
|
109
137
|
event_data_cls = _event_data_cls()
|
|
110
138
|
current = self.producer.create_batch()
|
|
111
139
|
attempted = delivered = 0
|
|
@@ -132,6 +160,9 @@ class AzureEventHubsSink:
|
|
|
132
160
|
def close(self) -> None:
|
|
133
161
|
"""Closes the producer (FR-009).
|
|
134
162
|
|
|
163
|
+
Idempotent, and takes the emit lock so the producer is never closed out from under an
|
|
164
|
+
in-flight send (SPEC-028 FR-002).
|
|
165
|
+
|
|
135
166
|
Args:
|
|
136
167
|
None.
|
|
137
168
|
|
|
@@ -141,7 +172,11 @@ class AzureEventHubsSink:
|
|
|
141
172
|
Raises:
|
|
142
173
|
Exception: Whatever the producer raises on close.
|
|
143
174
|
"""
|
|
144
|
-
self.
|
|
175
|
+
with self._lock:
|
|
176
|
+
if self._closed:
|
|
177
|
+
return
|
|
178
|
+
self._closed = True
|
|
179
|
+
self.producer.close()
|
|
145
180
|
|
|
146
181
|
def _send(self, event_batch: Any) -> int:
|
|
147
182
|
"""Sends one ``EventDataBatch``, retrying failures (FR-009, FR-011).
|
|
@@ -8,6 +8,8 @@ import threading
|
|
|
8
8
|
import time
|
|
9
9
|
from typing import TextIO
|
|
10
10
|
|
|
11
|
+
from log_foundry.sinks.base import SinkDeliveryError
|
|
12
|
+
|
|
11
13
|
__all__ = ["FileSink", "RotatingFileSink"]
|
|
12
14
|
|
|
13
15
|
_WHEN_SECONDS = {
|
|
@@ -71,7 +73,13 @@ class FileSink:
|
|
|
71
73
|
Raises:
|
|
72
74
|
OSError: If the write or flush fails (FR-001).
|
|
73
75
|
"""
|
|
76
|
+
if not batch:
|
|
77
|
+
return
|
|
74
78
|
with self._lock:
|
|
79
|
+
if self._closed:
|
|
80
|
+
raise SinkDeliveryError(
|
|
81
|
+
f"FileSink wrote none of {len(batch)} event(s): the sink is closed"
|
|
82
|
+
)
|
|
75
83
|
for event in batch:
|
|
76
84
|
self._stream.write(json.dumps(event) + "\n")
|
|
77
85
|
self._stream.flush()
|
|
@@ -94,9 +102,9 @@ class FileSink:
|
|
|
94
102
|
with self._lock:
|
|
95
103
|
if self._closed:
|
|
96
104
|
return
|
|
105
|
+
self._closed = True
|
|
97
106
|
self._stream.flush()
|
|
98
107
|
self._stream.close()
|
|
99
|
-
self._closed = True
|
|
100
108
|
|
|
101
109
|
|
|
102
110
|
class RotatingFileSink:
|
|
@@ -176,7 +184,13 @@ class RotatingFileSink:
|
|
|
176
184
|
Raises:
|
|
177
185
|
OSError: If a write, flush or rotation fails.
|
|
178
186
|
"""
|
|
187
|
+
if not batch:
|
|
188
|
+
return
|
|
179
189
|
with self._lock:
|
|
190
|
+
if self._closed:
|
|
191
|
+
raise SinkDeliveryError(
|
|
192
|
+
f"RotatingFileSink wrote none of {len(batch)} event(s): the sink is closed"
|
|
193
|
+
)
|
|
180
194
|
for event in batch:
|
|
181
195
|
line = json.dumps(event) + "\n"
|
|
182
196
|
data = len(line.encode(self._encoding))
|
|
@@ -205,9 +219,9 @@ class RotatingFileSink:
|
|
|
205
219
|
with self._lock:
|
|
206
220
|
if self._closed:
|
|
207
221
|
return
|
|
222
|
+
self._closed = True
|
|
208
223
|
self._stream.flush()
|
|
209
224
|
self._stream.close()
|
|
210
|
-
self._closed = True
|
|
211
225
|
|
|
212
226
|
@staticmethod
|
|
213
227
|
def _rollover_seconds(when: str | None, interval: int) -> float | None:
|
|
@@ -32,6 +32,10 @@ class FirehoseSink:
|
|
|
32
32
|
the chunk sent, so none could be paired to an outcome. A non-zero count means those
|
|
33
33
|
records were abandoned without the stream ever confirming them — treat it as loss, and
|
|
34
34
|
as a sign the client is not AWS-shaped.
|
|
35
|
+
|
|
36
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
37
|
+
lock. ``boto3`` clients are documented thread-safe, this one is built once in
|
|
38
|
+
``__init__``, and the sink rebinds nothing after construction.
|
|
35
39
|
"""
|
|
36
40
|
|
|
37
41
|
MAX_RECORDS = 500
|
|
@@ -24,6 +24,10 @@ class KafkaSink:
|
|
|
24
24
|
failed: Messages whose delivery callback reported an error.
|
|
25
25
|
rejected: Messages ``produce()`` itself refused — a full local queue, a serialization fault
|
|
26
26
|
— which never reached the producer's batch at all.
|
|
27
|
+
|
|
28
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
29
|
+
lock. ``confluent-kafka`` documents its ``Producer`` as thread-safe, and this sink adds no state
|
|
30
|
+
of its own to guard — ``produce()`` is a local hand-off into the client's internal queue.
|
|
27
31
|
"""
|
|
28
32
|
|
|
29
33
|
def __init__(
|
|
@@ -32,6 +32,10 @@ class KinesisSink:
|
|
|
32
32
|
describe the chunk sent, so none could be paired to an outcome. A non-zero count means
|
|
33
33
|
those records were abandoned without the stream ever confirming them — treat it as loss,
|
|
34
34
|
and as a sign the client is not AWS-shaped.
|
|
35
|
+
|
|
36
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
37
|
+
lock. ``boto3`` clients are documented thread-safe, this one is built once in
|
|
38
|
+
``__init__``, and the sink rebinds nothing after construction.
|
|
35
39
|
"""
|
|
36
40
|
|
|
37
41
|
MAX_RECORDS = 500
|
|
@@ -76,6 +76,7 @@ class MongoDBSink:
|
|
|
76
76
|
self.dropped_oversized = 0
|
|
77
77
|
self._closed = False
|
|
78
78
|
self._counter_lock = threading.Lock()
|
|
79
|
+
self._close_lock = threading.Lock()
|
|
79
80
|
|
|
80
81
|
def losses(self) -> SinkLosses:
|
|
81
82
|
"""Reports oversized drops and documents the server rejected or never took (FR-002).
|
|
@@ -109,12 +110,20 @@ class MongoDBSink:
|
|
|
109
110
|
None.
|
|
110
111
|
|
|
111
112
|
Raises:
|
|
112
|
-
SinkDeliveryError: When
|
|
113
|
+
SinkDeliveryError: When the sink is already closed, since ``pymongo`` raises
|
|
114
|
+
``InvalidOperation`` on any use of a closed client and the library has its own word
|
|
115
|
+
for "none of this was delivered". Also when nothing was inserted (SPEC-026 FR-001) — a connection error
|
|
113
116
|
past the retry bound, or a bulk write every one of whose documents the server
|
|
114
117
|
rejected. A bulk write that stored some is partial and never raises. A batch of
|
|
115
118
|
nothing but oversized documents does not raise either: they can never fit, so there
|
|
116
119
|
is nothing to retry, and they are reported through :meth:`losses`.
|
|
117
120
|
"""
|
|
121
|
+
if not batch:
|
|
122
|
+
return
|
|
123
|
+
if self._closed:
|
|
124
|
+
raise SinkDeliveryError(
|
|
125
|
+
f"MongoDBSink inserted none of {len(batch)} document(s): the sink is closed"
|
|
126
|
+
)
|
|
118
127
|
documents = self._documents(batch)
|
|
119
128
|
if not documents:
|
|
120
129
|
return
|
|
@@ -157,7 +166,18 @@ class MongoDBSink:
|
|
|
157
166
|
def close(self) -> None:
|
|
158
167
|
"""Closes the client only if the sink owns it (FR-005).
|
|
159
168
|
|
|
160
|
-
Idempotent
|
|
169
|
+
Idempotent, with the flag set under a lock so two concurrent ``close()`` calls cannot
|
|
170
|
+
both reach ``client.close()`` — ``atexit`` racing user code is the documented case.
|
|
171
|
+
|
|
172
|
+
The lock does **not** make a close wait for an in-flight ``insert_many``, and saying so
|
|
173
|
+
plainly matters: ``emit`` takes no lock here because ``pymongo``'s client is thread-safe,
|
|
174
|
+
and a close that waited would have to serialize against every insert, which is the
|
|
175
|
+
parallelism SPEC-028 FR-002 declines to remove. What covers the overlap instead is
|
|
176
|
+
:meth:`emit`'s own ``_closed`` check, which refuses the batch in the library's
|
|
177
|
+
vocabulary rather than letting it reach a closed client — pymongo raises
|
|
178
|
+
``InvalidOperation`` on any use after close. An insert that passes the check and is
|
|
179
|
+
still in flight when the close lands can still see that error; it is counted as a failed
|
|
180
|
+
batch like any other, which is a reported loss at process exit rather than a silent one.
|
|
161
181
|
|
|
162
182
|
Args:
|
|
163
183
|
None.
|
|
@@ -168,11 +188,12 @@ class MongoDBSink:
|
|
|
168
188
|
Raises:
|
|
169
189
|
Exception: Whatever the client raises on close.
|
|
170
190
|
"""
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
self.
|
|
175
|
-
|
|
191
|
+
with self._close_lock:
|
|
192
|
+
if self._closed:
|
|
193
|
+
return
|
|
194
|
+
self._closed = True
|
|
195
|
+
if self._owns_client:
|
|
196
|
+
self._client.close()
|
|
176
197
|
|
|
177
198
|
def _documents(self, batch: list[dict[str, object]]) -> list[dict[str, object]]:
|
|
178
199
|
"""Copies each event and drops any document too large to ever fit.
|
|
@@ -51,6 +51,7 @@ class NATSSink:
|
|
|
51
51
|
self._loop = asyncio.new_event_loop()
|
|
52
52
|
self.failed = 0
|
|
53
53
|
self._counter_lock = threading.Lock()
|
|
54
|
+
self._lock = threading.Lock()
|
|
54
55
|
if client is None:
|
|
55
56
|
import nats # type: ignore[import-not-found]
|
|
56
57
|
|
|
@@ -62,6 +63,13 @@ class NATSSink:
|
|
|
62
63
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
63
64
|
"""Drives the async publishes to completion on the managed loop (FR-007).
|
|
64
65
|
|
|
66
|
+
The driver requirement satisfied (SPEC-028 FR-002): an ``asyncio`` event loop is
|
|
67
|
+
single-entry. A second thread calling ``run_until_complete`` on a loop that is already
|
|
68
|
+
running raises ``RuntimeError``, and can leave the loop's task machinery in a state where
|
|
69
|
+
a thread never returns from ``emit`` at all — measured as a permanently hung application
|
|
70
|
+
thread on the orphan path, which is the one outcome this library must never produce. The
|
|
71
|
+
lock makes the loop what the sink already assumed it was: entered by one caller at a time.
|
|
72
|
+
|
|
65
73
|
Args:
|
|
66
74
|
batch: The events to publish. An empty batch is a no-op.
|
|
67
75
|
|
|
@@ -73,7 +81,12 @@ class NATSSink:
|
|
|
73
81
|
"""
|
|
74
82
|
if not batch:
|
|
75
83
|
return
|
|
76
|
-
self.
|
|
84
|
+
with self._lock:
|
|
85
|
+
if self._loop.is_closed():
|
|
86
|
+
raise SinkDeliveryError(
|
|
87
|
+
f"NATSSink published none of {len(batch)} event(s): the sink is closed"
|
|
88
|
+
)
|
|
89
|
+
self._loop.run_until_complete(self._publish_all(batch))
|
|
77
90
|
|
|
78
91
|
def losses(self) -> SinkLosses:
|
|
79
92
|
"""Reports events whose publish raised (SPEC-026 FR-002).
|
|
@@ -102,12 +115,13 @@ class NATSSink:
|
|
|
102
115
|
Raises:
|
|
103
116
|
Exception: Whatever draining raises; the loop is closed regardless.
|
|
104
117
|
"""
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
118
|
+
with self._lock:
|
|
119
|
+
if self._loop.is_closed():
|
|
120
|
+
return
|
|
121
|
+
try:
|
|
122
|
+
self._loop.run_until_complete(self._drain())
|
|
123
|
+
finally:
|
|
124
|
+
self._loop.close()
|
|
111
125
|
|
|
112
126
|
async def _publish_all(self, batch: list[dict[str, object]]) -> None:
|
|
113
127
|
"""Publishes each event, isolating a per-event failure.
|
|
@@ -125,6 +125,10 @@ class PostgresSink:
|
|
|
125
125
|
if not batch:
|
|
126
126
|
return
|
|
127
127
|
with self._lock:
|
|
128
|
+
if self._closed:
|
|
129
|
+
raise SinkDeliveryError(
|
|
130
|
+
f"PostgresSink inserted none of {len(batch)} event(s): the sink is closed"
|
|
131
|
+
)
|
|
128
132
|
self._insert_batch(batch)
|
|
129
133
|
|
|
130
134
|
def _insert_batch(self, batch: list[dict[str, object]]) -> None:
|
|
@@ -183,10 +187,10 @@ class PostgresSink:
|
|
|
183
187
|
with self._lock:
|
|
184
188
|
if self._closed:
|
|
185
189
|
return
|
|
190
|
+
self._closed = True
|
|
186
191
|
self._conn.commit()
|
|
187
192
|
if self._owns_connection:
|
|
188
193
|
self._conn.close()
|
|
189
|
-
self._closed = True
|
|
190
194
|
|
|
191
195
|
def _row(self, event: dict[str, object]) -> tuple[object, ...]:
|
|
192
196
|
"""Builds one row: the extracted columns, then the whole event as JSON.
|
|
@@ -18,6 +18,15 @@ class GooglePubSubSink:
|
|
|
18
18
|
This is a durable-buffer sink on ``google-cloud-pubsub``, the optional ``gcp-pubsub`` extra,
|
|
19
19
|
imported lazily. ``publish()`` returns a future that resolves asynchronously, so the sink
|
|
20
20
|
accumulates the batch's futures and resolves them on :meth:`close`.
|
|
21
|
+
|
|
22
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport lock —
|
|
23
|
+
the publisher client owns its own batching and threading, and ``publish()`` is a local
|
|
24
|
+
hand-off. What it does hold is the pending-futures list, which is genuinely shared between
|
|
25
|
+
``emit`` (appending) and ``close`` (resolving), so that list has its own small lock and
|
|
26
|
+
``close`` swaps it out rather than iterating and clearing. Without the swap, a future
|
|
27
|
+
appended after the loop passed its index was dropped unresolved: an unconfirmed publish
|
|
28
|
+
never counted in ``failed`` and never reported by :meth:`losses`, which is precisely the
|
|
29
|
+
silent loss SPEC-026 exists to end.
|
|
21
30
|
"""
|
|
22
31
|
|
|
23
32
|
def __init__(self, topic: str, *, client: Any = None) -> None:
|
|
@@ -42,6 +51,7 @@ class GooglePubSubSink:
|
|
|
42
51
|
self.failed = 0
|
|
43
52
|
self.rejected = 0
|
|
44
53
|
self._counter_lock = threading.Lock()
|
|
54
|
+
self._futures_lock = threading.Lock()
|
|
45
55
|
self._futures: list[Any] = []
|
|
46
56
|
|
|
47
57
|
def losses(self) -> SinkLosses:
|
|
@@ -88,7 +98,8 @@ class GooglePubSubSink:
|
|
|
88
98
|
self.rejected += 1
|
|
89
99
|
_diag.lost("event", 1, f"GooglePubSubSink refused the publish, {type(err).__name__}")
|
|
90
100
|
continue
|
|
91
|
-
self.
|
|
101
|
+
with self._futures_lock:
|
|
102
|
+
self._futures.append(future)
|
|
92
103
|
published += 1
|
|
93
104
|
if batch and not published:
|
|
94
105
|
raise SinkDeliveryError(
|
|
@@ -98,7 +109,12 @@ class GooglePubSubSink:
|
|
|
98
109
|
def close(self) -> None:
|
|
99
110
|
"""Resolves all pending publish futures, counting and logging errors (FR-008).
|
|
100
111
|
|
|
101
|
-
Idempotent.
|
|
112
|
+
Idempotent. The pending list is swapped out under a lock rather than iterated and then
|
|
113
|
+
cleared (SPEC-028 FR-002): ``emit`` appends to it from any thread, so the old
|
|
114
|
+
iterate-then-``clear()`` discarded any future appended after the loop passed its index —
|
|
115
|
+
an unconfirmed publish whose ``result()`` was never called, never counted in ``failed``
|
|
116
|
+
and never reported by ``losses()``. That is the silent loss SPEC-026 exists to end,
|
|
117
|
+
reached through the one piece of shared state this sink has.
|
|
102
118
|
|
|
103
119
|
Args:
|
|
104
120
|
None.
|
|
@@ -110,7 +126,9 @@ class GooglePubSubSink:
|
|
|
110
126
|
None. This is an isolation boundary: an unresolved future must never crash the worker
|
|
111
127
|
(FR-011).
|
|
112
128
|
"""
|
|
113
|
-
|
|
129
|
+
with self._futures_lock:
|
|
130
|
+
pending, self._futures = self._futures, []
|
|
131
|
+
for future in pending:
|
|
114
132
|
try:
|
|
115
133
|
future.result()
|
|
116
134
|
except Exception as err:
|
|
@@ -119,4 +137,3 @@ class GooglePubSubSink:
|
|
|
119
137
|
_diag.lost(
|
|
120
138
|
"event", 1, f"GooglePubSubSink publish unconfirmed, {type(err).__name__}"
|
|
121
139
|
)
|
|
122
|
-
self._futures.clear()
|
|
@@ -73,6 +73,8 @@ class RabbitMQSink:
|
|
|
73
73
|
self._properties: Any = None
|
|
74
74
|
self.failed = 0
|
|
75
75
|
self._counter_lock = threading.Lock()
|
|
76
|
+
self._lock = threading.Lock()
|
|
77
|
+
self._closed = False
|
|
76
78
|
|
|
77
79
|
def losses(self) -> SinkLosses:
|
|
78
80
|
"""Reports messages abandoned past the reconnect-retry bound (SPEC-026 FR-002).
|
|
@@ -95,6 +97,12 @@ class RabbitMQSink:
|
|
|
95
97
|
A partial publish is counted and left alone: retrying it would re-publish the messages
|
|
96
98
|
already on the exchange.
|
|
97
99
|
|
|
100
|
+
The driver requirement satisfied (SPEC-028 FR-002): ``pika`` states that one connection
|
|
101
|
+
must not be shared across threads, and this sink shares one connection, one channel, and
|
|
102
|
+
*rebinds* both — ``_reset`` closes and nulls the channel another thread may be mid-publish
|
|
103
|
+
on. It is structurally the same sink as ``SocketTransport``, one AMQP frame stream over
|
|
104
|
+
one socket, so it takes the same lock.
|
|
105
|
+
|
|
98
106
|
Args:
|
|
99
107
|
batch: The events to publish.
|
|
100
108
|
|
|
@@ -103,12 +111,23 @@ class RabbitMQSink:
|
|
|
103
111
|
|
|
104
112
|
Raises:
|
|
105
113
|
SinkDeliveryError: When no message reached the broker (SPEC-026 FR-001) — a down broker
|
|
106
|
-
is the case the worker's retry and ``failed_batches`` exist for.
|
|
114
|
+
is the case the worker's retry and ``failed_batches`` exist for. Also when the sink
|
|
115
|
+
is already closed, which is not fussiness: ``_active_channel`` reopens a connection
|
|
116
|
+
whenever it finds none, and ``close`` is now idempotent, so an emit arriving after a
|
|
117
|
+
shutdown would open an AMQP connection that nothing will ever reap. One
|
|
118
|
+
``log_foundry.info()`` after ``shutdown()`` reaches this on the caller's own thread.
|
|
107
119
|
"""
|
|
120
|
+
if not batch:
|
|
121
|
+
return
|
|
108
122
|
published = 0
|
|
109
|
-
|
|
110
|
-
if self.
|
|
111
|
-
|
|
123
|
+
with self._lock:
|
|
124
|
+
if self._closed:
|
|
125
|
+
raise SinkDeliveryError(
|
|
126
|
+
f"RabbitMQSink published none of {len(batch)} message(s): the sink is closed"
|
|
127
|
+
)
|
|
128
|
+
for event in batch:
|
|
129
|
+
if self._publish(json.dumps(event).encode("utf-8")):
|
|
130
|
+
published += 1
|
|
112
131
|
if batch and not published:
|
|
113
132
|
raise SinkDeliveryError(f"RabbitMQSink published none of {len(batch)} message(s)")
|
|
114
133
|
|
|
@@ -126,12 +145,16 @@ class RabbitMQSink:
|
|
|
126
145
|
Raises:
|
|
127
146
|
None.
|
|
128
147
|
"""
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
148
|
+
with self._lock:
|
|
149
|
+
if self._closed:
|
|
150
|
+
return
|
|
151
|
+
self._closed = True
|
|
152
|
+
if self._channel is not None:
|
|
153
|
+
_safe_close(self._channel)
|
|
154
|
+
self._channel = None
|
|
155
|
+
if self._connection is not None:
|
|
156
|
+
_safe_close(self._connection)
|
|
157
|
+
self._connection = None
|
|
135
158
|
|
|
136
159
|
def _publish(self, body: bytes) -> bool:
|
|
137
160
|
"""Publishes one message within the retry bound.
|
|
@@ -21,6 +21,13 @@ class _RedisSink:
|
|
|
21
21
|
The module is named ``redis`` to match the extra and imports the driver lazily, so it never
|
|
22
22
|
shadows or requires the real package at import time. The worst-case delay (SPEC-027 FR-005)
|
|
23
23
|
is ``max_retries`` interruptible waits per batch, 0.7 s at the defaults.
|
|
24
|
+
|
|
25
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
26
|
+
lock. ``redis-py`` documents that a client may be shared between threads — a
|
|
27
|
+
connection is taken from its pool only for the duration of a command, and command
|
|
28
|
+
execution never mutates the client. Its documented exceptions are ``PubSub`` and
|
|
29
|
+
``Pipeline`` objects, which must not be passed between threads — which is why the pipeline
|
|
30
|
+
here is built and executed inside a single ``emit`` call and never stored on the instance.
|
|
24
31
|
"""
|
|
25
32
|
|
|
26
33
|
def __init__(self, *, client: Any, url: str | None, max_retries: int) -> None:
|
|
@@ -33,6 +33,11 @@ class SentrySink:
|
|
|
33
33
|
skipped: Events below the minimum level, or without a usable level, that were not sent.
|
|
34
34
|
transport_errors: Events whose send raised something other than an already-counted
|
|
35
35
|
abandonment — an SDK fault, or a response error ``HTTPSink`` does not retry.
|
|
36
|
+
|
|
37
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
38
|
+
lock. This sink owns no transport: it delegates to a ``sentry_sdk`` built for capture from any
|
|
39
|
+
thread, or to an ``HTTPSink`` that builds a fresh request per call and rebinds nothing.
|
|
40
|
+
Its counters take the counter lock like every other sink's.
|
|
36
41
|
"""
|
|
37
42
|
|
|
38
43
|
def __init__(
|
|
@@ -25,6 +25,10 @@ class SNSSink:
|
|
|
25
25
|
|
|
26
26
|
The worst-case delay (SPEC-027 FR-005) is ``max_retries`` waits per chunk, 0.7 s at the
|
|
27
27
|
defaults. The waits are interruptible, so ``shutdown()`` cuts one short.
|
|
28
|
+
|
|
29
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
30
|
+
lock. ``boto3`` clients are documented thread-safe, this one is built once in
|
|
31
|
+
``__init__``, and the sink rebinds nothing after construction.
|
|
28
32
|
"""
|
|
29
33
|
|
|
30
34
|
MAX_BATCH = 10
|
|
@@ -7,6 +7,7 @@ import sqlite3
|
|
|
7
7
|
import threading
|
|
8
8
|
|
|
9
9
|
from log_foundry.sinks._chunk import valid_identifier
|
|
10
|
+
from log_foundry.sinks.base import SinkDeliveryError
|
|
10
11
|
|
|
11
12
|
__all__ = ["SQLiteSink"]
|
|
12
13
|
|
|
@@ -78,8 +79,9 @@ class SQLiteSink:
|
|
|
78
79
|
|
|
79
80
|
``with connection`` opens a transaction and commits on success or rolls back on error,
|
|
80
81
|
so the whole batch lands atomically. The lock holds that transaction to one thread
|
|
81
|
-
(SPEC-028 FR-002)
|
|
82
|
-
|
|
82
|
+
(SPEC-028 FR-002). Row-building is deliberately *outside* it: those lines touch nothing
|
|
83
|
+
shared, so serializing the ``json.dumps`` of a whole batch would lengthen the critical
|
|
84
|
+
section for no gain.
|
|
83
85
|
|
|
84
86
|
Args:
|
|
85
87
|
batch: The events to insert.
|
|
@@ -89,13 +91,44 @@ class SQLiteSink:
|
|
|
89
91
|
|
|
90
92
|
Raises:
|
|
91
93
|
sqlite3.Error: If the insert fails.
|
|
94
|
+
SinkDeliveryError: If the sink is closed — whether the close landed while this call
|
|
95
|
+
was building its rows, or long before it. Either way the batch would otherwise reach
|
|
96
|
+
the driver as "Cannot operate on a closed database", and the library has a word of
|
|
97
|
+
its own for "none of this was delivered" (SPEC-026 FR-001). What the library should
|
|
98
|
+
*signal* about logging after a completed shutdown is a separate question, and
|
|
99
|
+
SPEC-030's; this only keeps driver internals out of the answer.
|
|
92
100
|
"""
|
|
101
|
+
if not batch:
|
|
102
|
+
return
|
|
93
103
|
rows = [
|
|
94
104
|
(*(event.get(col) for col in _COLUMNS), json.dumps(event)) for event in batch
|
|
95
105
|
]
|
|
96
106
|
placeholders = ", ".join("?" * (len(_COLUMNS) + 1))
|
|
97
107
|
columns = ", ".join((*_COLUMNS, "event"))
|
|
98
|
-
with self._lock
|
|
108
|
+
with self._lock:
|
|
109
|
+
if self._closed:
|
|
110
|
+
raise SinkDeliveryError(
|
|
111
|
+
f"SQLiteSink inserted none of {len(batch)} event(s): the sink is closed"
|
|
112
|
+
)
|
|
113
|
+
self._insert(rows, columns, placeholders)
|
|
114
|
+
|
|
115
|
+
def _insert(
|
|
116
|
+
self, rows: list[tuple[object, ...]], columns: str, placeholders: str
|
|
117
|
+
) -> None:
|
|
118
|
+
"""Runs the insert transaction, with the emit lock already held.
|
|
119
|
+
|
|
120
|
+
Args:
|
|
121
|
+
rows: The prepared row tuples.
|
|
122
|
+
columns: The comma-separated column list.
|
|
123
|
+
placeholders: The matching parameter placeholders.
|
|
124
|
+
|
|
125
|
+
Returns:
|
|
126
|
+
None.
|
|
127
|
+
|
|
128
|
+
Raises:
|
|
129
|
+
sqlite3.Error: If the insert fails.
|
|
130
|
+
"""
|
|
131
|
+
with self._conn:
|
|
99
132
|
self._conn.executemany(
|
|
100
133
|
f'INSERT INTO "{self._table}" ({columns}) VALUES ({placeholders})', rows
|
|
101
134
|
)
|
|
@@ -118,10 +151,10 @@ class SQLiteSink:
|
|
|
118
151
|
with self._lock:
|
|
119
152
|
if self._closed:
|
|
120
153
|
return
|
|
154
|
+
self._closed = True
|
|
121
155
|
self._conn.commit()
|
|
122
156
|
if self._owns_connection:
|
|
123
157
|
self._conn.close()
|
|
124
|
-
self._closed = True
|
|
125
158
|
|
|
126
159
|
def _ensure_schema(self) -> None:
|
|
127
160
|
"""Idempotently creates the target table.
|
|
@@ -84,6 +84,13 @@ class SQSSink:
|
|
|
84
84
|
|
|
85
85
|
The worst-case delay (SPEC-027 FR-005) is ``max_retries`` waits per chunk, 0.7 s at the
|
|
86
86
|
defaults. The waits are interruptible, so ``shutdown()`` cuts one short.
|
|
87
|
+
|
|
88
|
+
The driver requirement satisfied (SPEC-028 FR-002): this sink takes **no** transport
|
|
89
|
+
lock. ``boto3`` documents that clients — unlike Sessions and Resources — are
|
|
90
|
+
thread-safe, and this sink builds its client once in ``__init__`` rather than inside
|
|
91
|
+
``emit`` — which is what keeps it clear of the documented caveat that calling
|
|
92
|
+
``boto3.client()`` concurrently can produce response-ordering faults. Nothing else here is
|
|
93
|
+
rebound after construction.
|
|
87
94
|
"""
|
|
88
95
|
|
|
89
96
|
MAX_BATCH = 10
|
|
@@ -395,9 +395,22 @@ class Worker:
|
|
|
395
395
|
it, and ``atexit`` plus user code calling it at once is documented as normal.
|
|
396
396
|
|
|
397
397
|
``is_alive()`` is the safety condition rather than a heuristic: it reads ``False`` only
|
|
398
|
-
after ``_run`` has returned, so the sink is provably out of use
|
|
399
|
-
|
|
400
|
-
|
|
398
|
+
after ``_run`` has returned, so the sink is provably out of use *by the worker*.
|
|
399
|
+
|
|
400
|
+
The close runs to completion, inline, and is deliberately **not** bounded — which leaves
|
|
401
|
+
one honest gap. SPEC-028 made ``close()`` take the sink's emit lock, so an application
|
|
402
|
+
thread on the orphan path can hold that lock inside a driver call with no timeout of its
|
|
403
|
+
own and delay this past ``shutdown``'s budget. Running the close on a joinable daemon
|
|
404
|
+
thread was tried and reverted: at interpreter exit the daemon is killed wherever it has
|
|
405
|
+
reached, which for ``SQLiteSink`` is between ``commit()`` and ``close()`` — turning the
|
|
406
|
+
leaked handle SPEC-027 FR-004 accepts into the partial write it was avoiding. It also
|
|
407
|
+
could not tell a slow-but-successful close from a stuck one, so it reported
|
|
408
|
+
``ShutdownTimeout`` and "left open" for closes that had in fact completed, latching
|
|
409
|
+
SPEC-019's alert term on a healthy shutdown. A wrong signal is worse than a slow one.
|
|
410
|
+
The residual delay is recorded in ``architecture.md`` §13 rather than papered over.
|
|
411
|
+
|
|
412
|
+
The close runs outside the lock, because it can reach ``_diag`` and a wedged console must
|
|
413
|
+
not stall a lock :meth:`submit` also takes.
|
|
401
414
|
|
|
402
415
|
Args:
|
|
403
416
|
None.
|
|
@@ -406,7 +419,9 @@ class Worker:
|
|
|
406
419
|
None.
|
|
407
420
|
|
|
408
421
|
Raises:
|
|
409
|
-
|
|
422
|
+
BaseException: Whatever the sink's ``close`` raised that is not an ``Exception``.
|
|
423
|
+
``_close_sink`` absorbs ``Exception`` but lets a ``KeyboardInterrupt`` or
|
|
424
|
+
``SystemExit`` through to the caller (SPEC-025 FR-004).
|
|
410
425
|
"""
|
|
411
426
|
with self._lock:
|
|
412
427
|
if self._sink_closed or self._thread.is_alive():
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{log_foundry-0.10.2.dev24 → log_foundry-0.10.2.dev25}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|