log-foundry 0.10.2.dev22__tar.gz → 0.10.2.dev24__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/PKG-INFO +28 -10
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/README.md +27 -9
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_socket.py +30 -12
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/base.py +28 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/clickhouse.py +29 -15
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/elasticsearch.py +9 -6
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/eventhubs.py +9 -7
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/file.py +56 -23
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/firehose.py +14 -11
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/http.py +6 -3
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/kafka.py +8 -3
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/kinesis.py +14 -11
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/mongodb.py +21 -8
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/multi.py +6 -4
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/nats.py +6 -2
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/postgres.py +45 -13
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/pubsub.py +8 -3
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/rabbitmq.py +7 -6
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/redis.py +7 -6
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sentry.py +11 -8
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sns.py +9 -7
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sqlite.py +26 -13
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sqs.py +11 -8
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/util.py +4 -1
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/LICENSE +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/__init__.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/config.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/decorator.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_retry.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/callback.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/datadog.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/filtering.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/honeycomb.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/logging_sink.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/logstash.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/loki.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/splunk.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/stdout.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/syslog.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/transform.py +0 -0
- {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: log-foundry
|
|
3
|
-
Version: 0.10.2.
|
|
3
|
+
Version: 0.10.2.dev24
|
|
4
4
|
Summary: Generate logs for your console and JSON events for downstream consumption.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -669,7 +669,16 @@ in `CallbackSink`.
|
|
|
669
669
|
#### Writing your own sink
|
|
670
670
|
|
|
671
671
|
`Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
|
|
672
|
-
fails. They are not stylistic — the library's whole
|
|
672
|
+
fails and one about *when* it is called. They are not stylistic — the library's whole
|
|
673
|
+
loss-reporting apparatus is built on them:
|
|
674
|
+
|
|
675
|
+
- **Tolerate concurrent calls.** `emit` may run on more than one thread at once, and `close` may
|
|
676
|
+
be called while an `emit` is in flight. The worker drains on its own thread, but a level call
|
|
677
|
+
made with no active span emits synchronously on the *caller's* thread — which is any thread of
|
|
678
|
+
your application. If your sink holds mutable transport state (a stream it rebinds, a socket it
|
|
679
|
+
reuses, a connection with transaction scope), guard it with a `threading.Lock` held for the
|
|
680
|
+
whole operation that assumes exclusivity. If it holds none, you need do nothing. The library
|
|
681
|
+
cannot serialize this for you: it does not own the calling thread.
|
|
673
682
|
|
|
674
683
|
- **Raise when you delivered none of the batch**, after your own retries are spent. That is the
|
|
675
684
|
signal the worker's bounded retry and `health().failed_batches` depend on, and the one case where
|
|
@@ -685,29 +694,38 @@ never engages, `failed_batches` stays at zero, and `flush()` returns `True` whil
|
|
|
685
694
|
lost.
|
|
686
695
|
|
|
687
696
|
Optionally add `losses()` to report what you absorbed. It must never raise and must be safe to call
|
|
688
|
-
while `emit` is running (`health()` is a poll)
|
|
697
|
+
while `emit` is running (`health()` is a poll) — which means the counters need their own lock, kept
|
|
698
|
+
separate from the transport one so a poll never waits on an in-flight send:
|
|
689
699
|
|
|
690
700
|
```python
|
|
701
|
+
import threading
|
|
691
702
|
from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
692
703
|
|
|
693
704
|
class MySink:
|
|
694
705
|
def __init__(self) -> None:
|
|
695
706
|
self._dropped = self._failed = 0
|
|
707
|
+
self._lock = threading.Lock() # transport state
|
|
708
|
+
self._counter_lock = threading.Lock() # counters only, never held across I/O
|
|
696
709
|
|
|
697
710
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
698
711
|
delivered = 0
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
712
|
+
with self._lock: # your connection, socket or stream
|
|
713
|
+
for chunk in self._chunks(batch):
|
|
714
|
+
if self._send(chunk): # your own bounded retry
|
|
715
|
+
delivered += len(chunk)
|
|
716
|
+
else:
|
|
717
|
+
with self._counter_lock: # transport -> counter, never the reverse
|
|
718
|
+
self._failed += len(chunk)
|
|
704
719
|
if batch and not delivered:
|
|
705
720
|
raise SinkDeliveryError(f"MySink delivered none of {len(batch)} event(s)")
|
|
706
721
|
|
|
707
722
|
def losses(self) -> SinkLosses:
|
|
708
|
-
|
|
723
|
+
with self._counter_lock: # both fields from one instant
|
|
724
|
+
return SinkLosses(dropped=self._dropped, failed=self._failed)
|
|
709
725
|
|
|
710
|
-
def close(self) -> None:
|
|
726
|
+
def close(self) -> None:
|
|
727
|
+
with self._lock: # never release under an active writer
|
|
728
|
+
...
|
|
711
729
|
```
|
|
712
730
|
|
|
713
731
|
`losses()` is optional and probed by name, so a sink written before it existed keeps working and
|
|
@@ -633,7 +633,16 @@ in `CallbackSink`.
|
|
|
633
633
|
#### Writing your own sink
|
|
634
634
|
|
|
635
635
|
`Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
|
|
636
|
-
fails. They are not stylistic — the library's whole
|
|
636
|
+
fails and one about *when* it is called. They are not stylistic — the library's whole
|
|
637
|
+
loss-reporting apparatus is built on them:
|
|
638
|
+
|
|
639
|
+
- **Tolerate concurrent calls.** `emit` may run on more than one thread at once, and `close` may
|
|
640
|
+
be called while an `emit` is in flight. The worker drains on its own thread, but a level call
|
|
641
|
+
made with no active span emits synchronously on the *caller's* thread — which is any thread of
|
|
642
|
+
your application. If your sink holds mutable transport state (a stream it rebinds, a socket it
|
|
643
|
+
reuses, a connection with transaction scope), guard it with a `threading.Lock` held for the
|
|
644
|
+
whole operation that assumes exclusivity. If it holds none, you need do nothing. The library
|
|
645
|
+
cannot serialize this for you: it does not own the calling thread.
|
|
637
646
|
|
|
638
647
|
- **Raise when you delivered none of the batch**, after your own retries are spent. That is the
|
|
639
648
|
signal the worker's bounded retry and `health().failed_batches` depend on, and the one case where
|
|
@@ -649,29 +658,38 @@ never engages, `failed_batches` stays at zero, and `flush()` returns `True` whil
|
|
|
649
658
|
lost.
|
|
650
659
|
|
|
651
660
|
Optionally add `losses()` to report what you absorbed. It must never raise and must be safe to call
|
|
652
|
-
while `emit` is running (`health()` is a poll)
|
|
661
|
+
while `emit` is running (`health()` is a poll) — which means the counters need their own lock, kept
|
|
662
|
+
separate from the transport one so a poll never waits on an in-flight send:
|
|
653
663
|
|
|
654
664
|
```python
|
|
665
|
+
import threading
|
|
655
666
|
from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
656
667
|
|
|
657
668
|
class MySink:
|
|
658
669
|
def __init__(self) -> None:
|
|
659
670
|
self._dropped = self._failed = 0
|
|
671
|
+
self._lock = threading.Lock() # transport state
|
|
672
|
+
self._counter_lock = threading.Lock() # counters only, never held across I/O
|
|
660
673
|
|
|
661
674
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
662
675
|
delivered = 0
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
676
|
+
with self._lock: # your connection, socket or stream
|
|
677
|
+
for chunk in self._chunks(batch):
|
|
678
|
+
if self._send(chunk): # your own bounded retry
|
|
679
|
+
delivered += len(chunk)
|
|
680
|
+
else:
|
|
681
|
+
with self._counter_lock: # transport -> counter, never the reverse
|
|
682
|
+
self._failed += len(chunk)
|
|
668
683
|
if batch and not delivered:
|
|
669
684
|
raise SinkDeliveryError(f"MySink delivered none of {len(batch)} event(s)")
|
|
670
685
|
|
|
671
686
|
def losses(self) -> SinkLosses:
|
|
672
|
-
|
|
687
|
+
with self._counter_lock: # both fields from one instant
|
|
688
|
+
return SinkLosses(dropped=self._dropped, failed=self._failed)
|
|
673
689
|
|
|
674
|
-
def close(self) -> None:
|
|
690
|
+
def close(self) -> None:
|
|
691
|
+
with self._lock: # never release under an active writer
|
|
692
|
+
...
|
|
675
693
|
```
|
|
676
694
|
|
|
677
695
|
`losses()` is optional and probed by name, so a sink written before it existed keeps working and
|
|
@@ -3,10 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import socket
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
if TYPE_CHECKING:
|
|
9
|
-
import threading
|
|
6
|
+
import threading
|
|
10
7
|
|
|
11
8
|
from log_foundry import _diag
|
|
12
9
|
from log_foundry.sinks._retry import wait
|
|
@@ -64,6 +61,14 @@ class SocketTransport:
|
|
|
64
61
|
100-message batch against a dead destination is roughly 70 s of backoff on the single drain
|
|
65
62
|
thread at the defaults. The wait is interruptible, so ``shutdown()`` cuts it short.
|
|
66
63
|
|
|
64
|
+
Sends are serialized on a lock (SPEC-028 FR-002). One TCP connection is shared by every
|
|
65
|
+
caller, and ``sendall`` gives no atomicity against a concurrent one: two interleaved calls
|
|
66
|
+
splice their bytes into the stream, which turns octet-counted syslog framing into a sequence
|
|
67
|
+
the receiver cannot resynchronize — it reads the next frame's length from the middle of the
|
|
68
|
+
previous frame's payload and is lost for the life of the connection. That the lock is held
|
|
69
|
+
across the backoff waits is deliberate; they are interruptible, so a ``shutdown()`` releases
|
|
70
|
+
it promptly.
|
|
71
|
+
|
|
67
72
|
Attributes:
|
|
68
73
|
failed: Messages abandoned past the reconnect-retry bound.
|
|
69
74
|
"""
|
|
@@ -103,11 +108,17 @@ class SocketTransport:
|
|
|
103
108
|
self._max_retries = max(max_retries, 0)
|
|
104
109
|
self._sock: socket.socket | None = None
|
|
105
110
|
self.failed = 0
|
|
111
|
+
self._counter_lock = threading.Lock()
|
|
106
112
|
self.stop_signal: threading.Event | None = None
|
|
113
|
+
self._lock = threading.Lock()
|
|
107
114
|
|
|
108
115
|
def send_all(self, messages: list[bytes]) -> None:
|
|
109
116
|
"""Sends each pre-framed message, reconnecting on error (FR-005, FR-006).
|
|
110
117
|
|
|
118
|
+
The lock spans the whole call, not each message: it also guards ``_sock``, which a
|
|
119
|
+
reconnect rebinds, so releasing between messages would let another thread send on a
|
|
120
|
+
socket this one is about to reset (SPEC-028 FR-002).
|
|
121
|
+
|
|
111
122
|
Args:
|
|
112
123
|
messages: The exact bytes to put on the wire, one call per message.
|
|
113
124
|
|
|
@@ -121,10 +132,11 @@ class SocketTransport:
|
|
|
121
132
|
would re-send the messages that already landed, and an empty call is a no-op rather
|
|
122
133
|
than a total failure.
|
|
123
134
|
"""
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
135
|
+
with self._lock:
|
|
136
|
+
delivered = 0
|
|
137
|
+
for message in messages:
|
|
138
|
+
if self._send_one(message):
|
|
139
|
+
delivered += 1
|
|
128
140
|
if messages and delivered == 0:
|
|
129
141
|
raise SinkDeliveryError(
|
|
130
142
|
f"SocketTransport delivered none of {len(messages)} message(s)"
|
|
@@ -142,12 +154,16 @@ class SocketTransport:
|
|
|
142
154
|
Raises:
|
|
143
155
|
None.
|
|
144
156
|
"""
|
|
145
|
-
|
|
157
|
+
with self._counter_lock:
|
|
158
|
+
return SinkLosses(dropped=0, failed=self.failed)
|
|
146
159
|
|
|
147
160
|
def close(self) -> None:
|
|
148
161
|
"""Closes the held socket, if any (FR-005, FR-012).
|
|
149
162
|
|
|
150
|
-
Idempotent
|
|
163
|
+
Idempotent, and takes the send lock so it never closes the socket out from under an
|
|
164
|
+
in-flight ``send_all`` (SPEC-028 FR-002). ``_reset`` does not take the lock itself,
|
|
165
|
+
because ``_send_one`` calls it while ``send_all`` already holds one — the lock is
|
|
166
|
+
deliberately not re-entrant, so that path must stay the only unlocked caller.
|
|
151
167
|
|
|
152
168
|
Args:
|
|
153
169
|
None.
|
|
@@ -158,7 +174,8 @@ class SocketTransport:
|
|
|
158
174
|
Raises:
|
|
159
175
|
None.
|
|
160
176
|
"""
|
|
161
|
-
self.
|
|
177
|
+
with self._lock:
|
|
178
|
+
self._reset()
|
|
162
179
|
|
|
163
180
|
def _send_one(self, message: bytes) -> bool:
|
|
164
181
|
"""Sends one message within the retry bound.
|
|
@@ -190,7 +207,8 @@ class SocketTransport:
|
|
|
190
207
|
if attempt < self._max_retries:
|
|
191
208
|
wait(_BACKOFF_BASE * (2**attempt), self.stop_signal)
|
|
192
209
|
continue
|
|
193
|
-
self.
|
|
210
|
+
with self._counter_lock:
|
|
211
|
+
self.failed += 1
|
|
194
212
|
_diag.lost(
|
|
195
213
|
"message",
|
|
196
214
|
1,
|
|
@@ -57,11 +57,32 @@ class Sink(Protocol):
|
|
|
57
57
|
``None`` is the same answer as having no method at all, which is what keeps a third-party
|
|
58
58
|
sink written against the pre-SPEC-026 interface satisfying this one. :func:`read_losses` is
|
|
59
59
|
the probe.
|
|
60
|
+
|
|
61
|
+
"Safe to call during an emit" is a concurrency requirement once :meth:`emit` is (SPEC-028
|
|
62
|
+
FR-003). The shipped sinks keep their loss counters under a **dedicated** lock, separate
|
|
63
|
+
from whatever guards their transport: an increment is a read-modify-write that Python does
|
|
64
|
+
not promise is atomic, and the pair read must come from one instant rather than straddling
|
|
65
|
+
another thread's increments. It is deliberately not the transport lock — ``health()`` is
|
|
66
|
+
the call an operator makes when a destination is already hanging, and sharing one lock
|
|
67
|
+
would make that poll wait for an in-flight emit and its retry backoff. Where a sink holds
|
|
68
|
+
both, the order is always transport then counter, never the reverse.
|
|
60
69
|
"""
|
|
61
70
|
|
|
62
71
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
63
72
|
"""Ships a batch of serialized event dicts.
|
|
64
73
|
|
|
74
|
+
**May be called concurrently from more than one thread, and must tolerate it**
|
|
75
|
+
(SPEC-028 FR-001). The background worker drains on its own thread, and a level call
|
|
76
|
+
made with no active span emits synchronously on the *caller's* thread (arch §12), which
|
|
77
|
+
may be any of the application's — an audit observed one sink object entered by two
|
|
78
|
+
application threads and the worker at once. An implementation holding mutable transport
|
|
79
|
+
state, such as a stream it rebinds, a reused socket or a connection with transaction
|
|
80
|
+
scope, must serialize access to it for the whole span of the operation that assumes
|
|
81
|
+
exclusivity. One that holds no such state need do nothing.
|
|
82
|
+
|
|
83
|
+
This is a requirement on implementations, not a promise the library serializes on their
|
|
84
|
+
behalf. It cannot: the orphan path runs on a thread the library does not own.
|
|
85
|
+
|
|
65
86
|
Raise when the batch delivered nothing and it was non-empty — the worker's bounded
|
|
66
87
|
retry and ``health().failed_batches`` depend on that signal, and a retry there cannot
|
|
67
88
|
duplicate anything (SPEC-026 FR-001). Raise after the sink's own retries are spent, so
|
|
@@ -88,6 +109,13 @@ class Sink(Protocol):
|
|
|
88
109
|
def close(self) -> None:
|
|
89
110
|
"""Flushes and releases any resources.
|
|
90
111
|
|
|
112
|
+
**May be called while an ``emit`` is in flight on another thread** (SPEC-028 FR-001).
|
|
113
|
+
It must therefore either wait for that emit or become a no-op, and must never release a
|
|
114
|
+
resource a concurrent ``emit`` is about to use — a half-released transport is worse than
|
|
115
|
+
an unreleased one, because the emit then fails against a closed handle instead of
|
|
116
|
+
succeeding. Taking the same lock ``emit`` takes satisfies this; so does an idempotent
|
|
117
|
+
guard checked under that lock.
|
|
118
|
+
|
|
91
119
|
Args:
|
|
92
120
|
None.
|
|
93
121
|
|
|
@@ -3,10 +3,8 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
if TYPE_CHECKING:
|
|
9
|
-
import threading
|
|
6
|
+
import threading
|
|
7
|
+
from typing import Any
|
|
10
8
|
|
|
11
9
|
from log_foundry import _diag
|
|
12
10
|
from log_foundry.sinks._chunk import chunk_list, valid_identifier
|
|
@@ -42,6 +40,12 @@ class ClickHouseSink:
|
|
|
42
40
|
columns plus the full event as a ``String`` column, inserted in a single call per chunk. The
|
|
43
41
|
sink is write-only, and the worst-case delay (SPEC-027 FR-005) is ``max_retries``
|
|
44
42
|
interruptible waits per chunk, 0.7 s at the defaults.
|
|
43
|
+
|
|
44
|
+
The driver requirement satisfied (SPEC-028 FR-002): a ``clickhouse-connect`` client holds
|
|
45
|
+
per-session state across an insert and the project does not publish it as safe to share
|
|
46
|
+
between threads, so this sink serializes its use rather than assuming otherwise. A lock is
|
|
47
|
+
the conservative reading — one client per thread would be the alternative, and that is the
|
|
48
|
+
connection-pool design FR-002 puts out of scope.
|
|
45
49
|
"""
|
|
46
50
|
|
|
47
51
|
def __init__(
|
|
@@ -81,6 +85,8 @@ class ClickHouseSink:
|
|
|
81
85
|
self.stop_signal: threading.Event | None = None
|
|
82
86
|
self.failed = 0
|
|
83
87
|
self._closed = False
|
|
88
|
+
self._lock = threading.Lock()
|
|
89
|
+
self._counter_lock = threading.Lock()
|
|
84
90
|
self._owns_client = client is None
|
|
85
91
|
if client is None:
|
|
86
92
|
import clickhouse_connect # type: ignore[import-not-found]
|
|
@@ -93,6 +99,9 @@ class ClickHouseSink:
|
|
|
93
99
|
def losses(self) -> SinkLosses:
|
|
94
100
|
"""Reports rows in a chunk abandoned past the retry bound (SPEC-026 FR-002).
|
|
95
101
|
|
|
102
|
+
Reads under the counter lock rather than the emit lock (SPEC-028 FR-003), so a poll
|
|
103
|
+
never waits on an in-flight insert and its backoff.
|
|
104
|
+
|
|
96
105
|
Args:
|
|
97
106
|
None.
|
|
98
107
|
|
|
@@ -102,7 +111,8 @@ class ClickHouseSink:
|
|
|
102
111
|
Raises:
|
|
103
112
|
None.
|
|
104
113
|
"""
|
|
105
|
-
|
|
114
|
+
with self._counter_lock:
|
|
115
|
+
return SinkLosses(dropped=0, failed=self.failed)
|
|
106
116
|
|
|
107
117
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
108
118
|
"""Inserts each chunk as one columnar call, retrying on failure (FR-002).
|
|
@@ -122,16 +132,18 @@ class ClickHouseSink:
|
|
|
122
132
|
if not batch:
|
|
123
133
|
return
|
|
124
134
|
chunks = inserted = 0
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
135
|
+
with self._lock:
|
|
136
|
+
for chunk in chunk_list(batch, self._chunk_size):
|
|
137
|
+
chunks += 1
|
|
138
|
+
inserted += self._insert([self._row(event) for event in chunk])
|
|
128
139
|
if chunks and not inserted:
|
|
129
140
|
raise SinkDeliveryError(f"ClickHouseSink inserted none of {chunks} chunk(s)")
|
|
130
141
|
|
|
131
142
|
def close(self) -> None:
|
|
132
143
|
"""Closes the client only if the sink owns it (FR-005).
|
|
133
144
|
|
|
134
|
-
Idempotent
|
|
145
|
+
Idempotent, and takes the emit lock so the client is never closed mid-insert
|
|
146
|
+
(SPEC-028 FR-002).
|
|
135
147
|
|
|
136
148
|
Args:
|
|
137
149
|
None.
|
|
@@ -142,11 +154,12 @@ class ClickHouseSink:
|
|
|
142
154
|
Raises:
|
|
143
155
|
Exception: Whatever the client raises on close.
|
|
144
156
|
"""
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
self.
|
|
149
|
-
|
|
157
|
+
with self._lock:
|
|
158
|
+
if self._closed:
|
|
159
|
+
return
|
|
160
|
+
if self._owns_client:
|
|
161
|
+
self.client.close()
|
|
162
|
+
self._closed = True
|
|
150
163
|
|
|
151
164
|
def _row(self, event: dict[str, object]) -> list[object]:
|
|
152
165
|
"""Builds one row: the extracted columns, then the whole event as JSON.
|
|
@@ -182,7 +195,8 @@ class ClickHouseSink:
|
|
|
182
195
|
if attempt < self.max_retries:
|
|
183
196
|
wait(_BACKOFF_BASE * (2**attempt), self.stop_signal)
|
|
184
197
|
continue
|
|
185
|
-
self.
|
|
198
|
+
with self._counter_lock:
|
|
199
|
+
self.failed += len(rows)
|
|
186
200
|
_diag.lost(
|
|
187
201
|
"row",
|
|
188
202
|
len(rows),
|
{log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
@@ -98,10 +98,11 @@ class ElasticsearchSink(HTTPSink):
|
|
|
98
98
|
Raises:
|
|
99
99
|
None.
|
|
100
100
|
"""
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
101
|
+
with self._counter_lock:
|
|
102
|
+
return SinkLosses(
|
|
103
|
+
dropped=self.dropped_oversized,
|
|
104
|
+
failed=self.failed + self.item_errors + self.dropped_unadjudicated,
|
|
105
|
+
)
|
|
105
106
|
|
|
106
107
|
def _parse_bulk_response(self, payload: bytes, sent: int) -> bool:
|
|
107
108
|
"""Counts rejected items and reports whether the response proves nothing was indexed.
|
|
@@ -139,7 +140,8 @@ class ElasticsearchSink(HTTPSink):
|
|
|
139
140
|
items = usable_results(data.get("items"))
|
|
140
141
|
errors = sum(1 for item in items if _has_error(item))
|
|
141
142
|
if len(items) != sent or not errors:
|
|
142
|
-
self.
|
|
143
|
+
with self._counter_lock:
|
|
144
|
+
self.dropped_unadjudicated += sent
|
|
143
145
|
_diag.lost(
|
|
144
146
|
"event",
|
|
145
147
|
sent,
|
|
@@ -148,7 +150,8 @@ class ElasticsearchSink(HTTPSink):
|
|
|
148
150
|
f"not retried",
|
|
149
151
|
)
|
|
150
152
|
return False
|
|
151
|
-
self.
|
|
153
|
+
with self._counter_lock:
|
|
154
|
+
self.item_errors += errors
|
|
152
155
|
_diag.lost("bulk item", errors, f"{type(self).__name__}, rejected by the server")
|
|
153
156
|
return errors == sent
|
|
154
157
|
|
|
@@ -3,10 +3,8 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
if TYPE_CHECKING:
|
|
9
|
-
import threading
|
|
6
|
+
import threading
|
|
7
|
+
from typing import Any
|
|
10
8
|
|
|
11
9
|
from log_foundry import _diag
|
|
12
10
|
from log_foundry.sinks._retry import wait
|
|
@@ -69,6 +67,7 @@ class AzureEventHubsSink:
|
|
|
69
67
|
self.stop_signal: threading.Event | None = None
|
|
70
68
|
self.failed = 0
|
|
71
69
|
self.dropped_oversized = 0
|
|
70
|
+
self._counter_lock = threading.Lock()
|
|
72
71
|
|
|
73
72
|
def losses(self) -> SinkLosses:
|
|
74
73
|
"""Reports oversized drops and events in a batch abandoned past the bound (FR-002).
|
|
@@ -82,7 +81,8 @@ class AzureEventHubsSink:
|
|
|
82
81
|
Raises:
|
|
83
82
|
None.
|
|
84
83
|
"""
|
|
85
|
-
|
|
84
|
+
with self._counter_lock:
|
|
85
|
+
return SinkLosses(dropped=self.dropped_oversized, failed=self.failed)
|
|
86
86
|
|
|
87
87
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
88
88
|
"""Packs events into batches within the size limit and sends each (FR-009).
|
|
@@ -118,7 +118,8 @@ class AzureEventHubsSink:
|
|
|
118
118
|
delivered += self._send(current)
|
|
119
119
|
current = self.producer.create_batch()
|
|
120
120
|
if not _try_add(current, data):
|
|
121
|
-
self.
|
|
121
|
+
with self._counter_lock:
|
|
122
|
+
self.dropped_oversized += 1
|
|
122
123
|
_diag.lost("event", 1, "AzureEventHubsSink, too large for an empty 1 MB batch")
|
|
123
124
|
if len(current) > 0:
|
|
124
125
|
attempted += 1
|
|
@@ -167,7 +168,8 @@ class AzureEventHubsSink:
|
|
|
167
168
|
if attempt < self.max_retries:
|
|
168
169
|
wait(_BACKOFF_BASE * (2**attempt), self.stop_signal)
|
|
169
170
|
continue
|
|
170
|
-
self.
|
|
171
|
+
with self._counter_lock:
|
|
172
|
+
self.failed += len(event_batch)
|
|
171
173
|
_diag.lost(
|
|
172
174
|
"event",
|
|
173
175
|
len(event_batch),
|
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import os
|
|
7
|
+
import threading
|
|
7
8
|
import time
|
|
8
9
|
from typing import TextIO
|
|
9
10
|
|
|
@@ -22,8 +23,12 @@ class FileSink:
|
|
|
22
23
|
|
|
23
24
|
Not every deployment ships to a cloud queue; local dev, debugging, air-gapped hosts and
|
|
24
25
|
simple archival just want events on the local disk. Writes are synchronous stdlib calls
|
|
25
|
-
only
|
|
26
|
-
|
|
26
|
+
only.
|
|
27
|
+
|
|
28
|
+
Writers within the process are serialized on a lock, because ``emit`` may be called
|
|
29
|
+
concurrently (SPEC-028 FR-002) — this module claimed a single worker thread until that spec
|
|
30
|
+
measured the orphan path emitting on application threads at the same time. Cross-*process*
|
|
31
|
+
coordination remains out of scope: two processes appending to one path are on their own.
|
|
27
32
|
"""
|
|
28
33
|
|
|
29
34
|
def __init__(self, path: str, *, encoding: str = "utf-8") -> None:
|
|
@@ -47,10 +52,16 @@ class FileSink:
|
|
|
47
52
|
self._encoding = encoding
|
|
48
53
|
self._stream: TextIO = open(path, "a", encoding=encoding)
|
|
49
54
|
self._closed = False
|
|
55
|
+
self._lock = threading.Lock()
|
|
50
56
|
|
|
51
57
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
52
58
|
"""Writes every event as one newline-terminated ``json.dumps`` line, then flushes.
|
|
53
59
|
|
|
60
|
+
The lock covers the whole batch rather than each line (SPEC-028 FR-002). A text stream
|
|
61
|
+
does not promise that one ``write`` is atomic against another, so per-line locking could
|
|
62
|
+
still interleave two events' bytes; batch-wide locking also keeps a batch contiguous in
|
|
63
|
+
the file, which is what makes the output readable.
|
|
64
|
+
|
|
54
65
|
Args:
|
|
55
66
|
batch: The events to write.
|
|
56
67
|
|
|
@@ -60,13 +71,17 @@ class FileSink:
|
|
|
60
71
|
Raises:
|
|
61
72
|
OSError: If the write or flush fails (FR-001).
|
|
62
73
|
"""
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
74
|
+
with self._lock:
|
|
75
|
+
for event in batch:
|
|
76
|
+
self._stream.write(json.dumps(event) + "\n")
|
|
77
|
+
self._stream.flush()
|
|
66
78
|
|
|
67
79
|
def close(self) -> None:
|
|
68
80
|
"""Flushes and closes the file handle, with a second call a no-op (FR-001).
|
|
69
81
|
|
|
82
|
+
Taking the same lock ``emit`` takes means a close waits for an in-flight write rather
|
|
83
|
+
than pulling the stream out from under it (SPEC-028 FR-002).
|
|
84
|
+
|
|
70
85
|
Args:
|
|
71
86
|
None.
|
|
72
87
|
|
|
@@ -76,11 +91,12 @@ class FileSink:
|
|
|
76
91
|
Raises:
|
|
77
92
|
OSError: If the flush or close fails.
|
|
78
93
|
"""
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
94
|
+
with self._lock:
|
|
95
|
+
if self._closed:
|
|
96
|
+
return
|
|
97
|
+
self._stream.flush()
|
|
98
|
+
self._stream.close()
|
|
99
|
+
self._closed = True
|
|
84
100
|
|
|
85
101
|
|
|
86
102
|
class RotatingFileSink:
|
|
@@ -95,6 +111,11 @@ class RotatingFileSink:
|
|
|
95
111
|
count, and opens a fresh active file — a backup count of zero keeps none, simply replacing
|
|
96
112
|
the active file. No event is lost across a rotation, because the rotate happens before the
|
|
97
113
|
pending event is written and the event lands in the fresh file.
|
|
114
|
+
|
|
115
|
+
A rotation rebinds the active stream, so it is the sink where concurrent writers did real
|
|
116
|
+
damage: a second thread mid-``emit`` could write to the handle rotation had just closed, or
|
|
117
|
+
to the pre-rotation file it had already renamed away. Both are serialized on a lock
|
|
118
|
+
(SPEC-028 FR-002).
|
|
98
119
|
"""
|
|
99
120
|
|
|
100
121
|
def __init__(
|
|
@@ -136,10 +157,16 @@ class RotatingFileSink:
|
|
|
136
157
|
self._size = os.path.getsize(path) if os.path.exists(path) else 0
|
|
137
158
|
self._next_rollover = self._schedule_next()
|
|
138
159
|
self._closed = False
|
|
160
|
+
self._lock = threading.Lock()
|
|
139
161
|
|
|
140
162
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
141
163
|
"""Appends each event, rotating first whenever a size or time trigger fires (FR-002).
|
|
142
164
|
|
|
165
|
+
The lock spans the whole batch, so the decide-rotate-write-account sequence is
|
|
166
|
+
indivisible (SPEC-028 FR-002). Guarding only ``_rotate`` would not be enough: the
|
|
167
|
+
``_should_rotate`` check and the write that follows it must see the same stream, or a
|
|
168
|
+
rotation between them sends the line to a closed handle.
|
|
169
|
+
|
|
143
170
|
Args:
|
|
144
171
|
batch: The events to write.
|
|
145
172
|
|
|
@@ -149,18 +176,23 @@ class RotatingFileSink:
|
|
|
149
176
|
Raises:
|
|
150
177
|
OSError: If a write, flush or rotation fails.
|
|
151
178
|
"""
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
self.
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
179
|
+
with self._lock:
|
|
180
|
+
for event in batch:
|
|
181
|
+
line = json.dumps(event) + "\n"
|
|
182
|
+
data = len(line.encode(self._encoding))
|
|
183
|
+
if self._should_rotate(data):
|
|
184
|
+
self._rotate()
|
|
185
|
+
self._stream.write(line)
|
|
186
|
+
self._size += data
|
|
187
|
+
self._stream.flush()
|
|
160
188
|
|
|
161
189
|
def close(self) -> None:
|
|
162
190
|
"""Flushes and closes the active handle, with a second call a no-op (FR-002).
|
|
163
191
|
|
|
192
|
+
Taking the same lock ``emit`` takes means a close waits for an in-flight write, and in
|
|
193
|
+
particular never lands between a rotation and the write it was making room for
|
|
194
|
+
(SPEC-028 FR-002).
|
|
195
|
+
|
|
164
196
|
Args:
|
|
165
197
|
None.
|
|
166
198
|
|
|
@@ -170,11 +202,12 @@ class RotatingFileSink:
|
|
|
170
202
|
Raises:
|
|
171
203
|
OSError: If the flush or close fails.
|
|
172
204
|
"""
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
205
|
+
with self._lock:
|
|
206
|
+
if self._closed:
|
|
207
|
+
return
|
|
208
|
+
self._stream.flush()
|
|
209
|
+
self._stream.close()
|
|
210
|
+
self._closed = True
|
|
178
211
|
|
|
179
212
|
@staticmethod
|
|
180
213
|
def _rollover_seconds(when: str | None, interval: int) -> float | None:
|