log-foundry 0.10.2.dev22__tar.gz → 0.10.2.dev24__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/PKG-INFO +28 -10
  2. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/README.md +27 -9
  3. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/pyproject.toml +1 -1
  4. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_socket.py +30 -12
  5. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/base.py +28 -0
  6. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/clickhouse.py +29 -15
  7. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/elasticsearch.py +9 -6
  8. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/eventhubs.py +9 -7
  9. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/file.py +56 -23
  10. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/firehose.py +14 -11
  11. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/http.py +6 -3
  12. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/kafka.py +8 -3
  13. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/kinesis.py +14 -11
  14. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/mongodb.py +21 -8
  15. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/multi.py +6 -4
  16. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/nats.py +6 -2
  17. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/postgres.py +45 -13
  18. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/pubsub.py +8 -3
  19. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/rabbitmq.py +7 -6
  20. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/redis.py +7 -6
  21. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sentry.py +11 -8
  22. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sns.py +9 -7
  23. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sqlite.py +26 -13
  24. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/sqs.py +11 -8
  25. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/util.py +4 -1
  26. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/LICENSE +0 -0
  27. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/__init__.py +0 -0
  28. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/_diag.py +0 -0
  29. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/api.py +0 -0
  30. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/config.py +0 -0
  31. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/console.py +0 -0
  32. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/context.py +0 -0
  33. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/decorator.py +0 -0
  34. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/ids.py +0 -0
  35. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/model.py +0 -0
  36. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/py.typed +0 -0
  37. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sanitize.py +0 -0
  38. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/__init__.py +0 -0
  39. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_batch.py +0 -0
  40. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_chunk.py +0 -0
  41. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_retry.py +0 -0
  42. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/_time.py +0 -0
  43. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/callback.py +0 -0
  44. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/datadog.py +0 -0
  45. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/filtering.py +0 -0
  46. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/honeycomb.py +0 -0
  47. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/logging_sink.py +0 -0
  48. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/logstash.py +0 -0
  49. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/loki.py +0 -0
  50. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/newrelic.py +0 -0
  51. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/splunk.py +0 -0
  52. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/stdout.py +0 -0
  53. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/syslog.py +0 -0
  54. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/sinks/transform.py +0 -0
  55. {log_foundry-0.10.2.dev22 → log_foundry-0.10.2.dev24}/src/log_foundry/worker.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev22
3
+ Version: 0.10.2.dev24
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -669,7 +669,16 @@ in `CallbackSink`.
669
669
  #### Writing your own sink
670
670
 
671
671
  `Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
672
- fails. They are not stylistic — the library's whole loss-reporting apparatus is built on them:
672
+ fails and one about *when* it is called. They are not stylistic — the library's whole
673
+ loss-reporting apparatus is built on them:
674
+
675
+ - **Tolerate concurrent calls.** `emit` may run on more than one thread at once, and `close` may
676
+ be called while an `emit` is in flight. The worker drains on its own thread, but a level call
677
+ made with no active span emits synchronously on the *caller's* thread — which is any thread of
678
+ your application. If your sink holds mutable transport state (a stream it rebinds, a socket it
679
+ reuses, a connection with transaction scope), guard it with a `threading.Lock` held for the
680
+ whole operation that assumes exclusivity. If it holds none, you need do nothing. The library
681
+ cannot serialize this for you: it does not own the calling thread.
673
682
 
674
683
  - **Raise when you delivered none of the batch**, after your own retries are spent. That is the
675
684
  signal the worker's bounded retry and `health().failed_batches` depend on, and the one case where
@@ -685,29 +694,38 @@ never engages, `failed_batches` stays at zero, and `flush()` returns `True` whil
685
694
  lost.
686
695
 
687
696
  Optionally add `losses()` to report what you absorbed. It must never raise and must be safe to call
688
- while `emit` is running (`health()` is a poll):
697
+ while `emit` is running (`health()` is a poll) — which means the counters need their own lock, kept
698
+ separate from the transport one so a poll never waits on an in-flight send:
689
699
 
690
700
  ```python
701
+ import threading
691
702
  from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
692
703
 
693
704
  class MySink:
694
705
  def __init__(self) -> None:
695
706
  self._dropped = self._failed = 0
707
+ self._lock = threading.Lock() # transport state
708
+ self._counter_lock = threading.Lock() # counters only, never held across I/O
696
709
 
697
710
  def emit(self, batch: list[dict[str, object]]) -> None:
698
711
  delivered = 0
699
- for chunk in self._chunks(batch):
700
- if self._send(chunk): # your own bounded retry
701
- delivered += len(chunk)
702
- else:
703
- self._failed += len(chunk)
712
+ with self._lock: # your connection, socket or stream
713
+ for chunk in self._chunks(batch):
714
+ if self._send(chunk): # your own bounded retry
715
+ delivered += len(chunk)
716
+ else:
717
+ with self._counter_lock: # transport -> counter, never the reverse
718
+ self._failed += len(chunk)
704
719
  if batch and not delivered:
705
720
  raise SinkDeliveryError(f"MySink delivered none of {len(batch)} event(s)")
706
721
 
707
722
  def losses(self) -> SinkLosses:
708
- return SinkLosses(dropped=self._dropped, failed=self._failed)
723
+ with self._counter_lock: # both fields from one instant
724
+ return SinkLosses(dropped=self._dropped, failed=self._failed)
709
725
 
710
- def close(self) -> None: ...
726
+ def close(self) -> None:
727
+ with self._lock: # never release under an active writer
728
+ ...
711
729
  ```
712
730
 
713
731
  `losses()` is optional and probed by name, so a sink written before it existed keeps working and
@@ -633,7 +633,16 @@ in `CallbackSink`.
633
633
  #### Writing your own sink
634
634
 
635
635
  `Sink` is two required methods, `emit(batch)` and `close()`, plus two rules about *how* `emit`
636
- fails. They are not stylistic — the library's whole loss-reporting apparatus is built on them:
636
+ fails and one about *when* it is called. They are not stylistic — the library's whole
637
+ loss-reporting apparatus is built on them:
638
+
639
+ - **Tolerate concurrent calls.** `emit` may run on more than one thread at once, and `close` may
640
+ be called while an `emit` is in flight. The worker drains on its own thread, but a level call
641
+ made with no active span emits synchronously on the *caller's* thread — which is any thread of
642
+ your application. If your sink holds mutable transport state (a stream it rebinds, a socket it
643
+ reuses, a connection with transaction scope), guard it with a `threading.Lock` held for the
644
+ whole operation that assumes exclusivity. If it holds none, you need do nothing. The library
645
+ cannot serialize this for you: it does not own the calling thread.
637
646
 
638
647
  - **Raise when you delivered none of the batch**, after your own retries are spent. That is the
639
648
  signal the worker's bounded retry and `health().failed_batches` depend on, and the one case where
@@ -649,29 +658,38 @@ never engages, `failed_batches` stays at zero, and `flush()` returns `True` whil
649
658
  lost.
650
659
 
651
660
  Optionally add `losses()` to report what you absorbed. It must never raise and must be safe to call
652
- while `emit` is running (`health()` is a poll):
661
+ while `emit` is running (`health()` is a poll) — which means the counters need their own lock, kept
662
+ separate from the transport one so a poll never waits on an in-flight send:
653
663
 
654
664
  ```python
665
+ import threading
655
666
  from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
656
667
 
657
668
  class MySink:
658
669
  def __init__(self) -> None:
659
670
  self._dropped = self._failed = 0
671
+ self._lock = threading.Lock() # transport state
672
+ self._counter_lock = threading.Lock() # counters only, never held across I/O
660
673
 
661
674
  def emit(self, batch: list[dict[str, object]]) -> None:
662
675
  delivered = 0
663
- for chunk in self._chunks(batch):
664
- if self._send(chunk): # your own bounded retry
665
- delivered += len(chunk)
666
- else:
667
- self._failed += len(chunk)
676
+ with self._lock: # your connection, socket or stream
677
+ for chunk in self._chunks(batch):
678
+ if self._send(chunk): # your own bounded retry
679
+ delivered += len(chunk)
680
+ else:
681
+ with self._counter_lock: # transport -> counter, never the reverse
682
+ self._failed += len(chunk)
668
683
  if batch and not delivered:
669
684
  raise SinkDeliveryError(f"MySink delivered none of {len(batch)} event(s)")
670
685
 
671
686
  def losses(self) -> SinkLosses:
672
- return SinkLosses(dropped=self._dropped, failed=self._failed)
687
+ with self._counter_lock: # both fields from one instant
688
+ return SinkLosses(dropped=self._dropped, failed=self._failed)
673
689
 
674
- def close(self) -> None: ...
690
+ def close(self) -> None:
691
+ with self._lock: # never release under an active writer
692
+ ...
675
693
  ```
676
694
 
677
695
  `losses()` is optional and probed by name, so a sink written before it existed keeps working and
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev22"
23
+ version = "0.10.2.dev24"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -3,10 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import socket
6
- from typing import TYPE_CHECKING
7
-
8
- if TYPE_CHECKING:
9
- import threading
6
+ import threading
10
7
 
11
8
  from log_foundry import _diag
12
9
  from log_foundry.sinks._retry import wait
@@ -64,6 +61,14 @@ class SocketTransport:
64
61
  100-message batch against a dead destination is roughly 70 s of backoff on the single drain
65
62
  thread at the defaults. The wait is interruptible, so ``shutdown()`` cuts it short.
66
63
 
64
+ Sends are serialized on a lock (SPEC-028 FR-002). One TCP connection is shared by every
65
+ caller, and ``sendall`` gives no atomicity against a concurrent one: two interleaved calls
66
+ splice their bytes into the stream, which turns octet-counted syslog framing into a sequence
67
+ the receiver cannot resynchronize — it reads the next frame's length from the middle of the
68
+ previous frame's payload and is lost for the life of the connection. That the lock is held
69
+ across the backoff waits is deliberate; they are interruptible, so a ``shutdown()`` releases
70
+ it promptly.
71
+
67
72
  Attributes:
68
73
  failed: Messages abandoned past the reconnect-retry bound.
69
74
  """
@@ -103,11 +108,17 @@ class SocketTransport:
103
108
  self._max_retries = max(max_retries, 0)
104
109
  self._sock: socket.socket | None = None
105
110
  self.failed = 0
111
+ self._counter_lock = threading.Lock()
106
112
  self.stop_signal: threading.Event | None = None
113
+ self._lock = threading.Lock()
107
114
 
108
115
  def send_all(self, messages: list[bytes]) -> None:
109
116
  """Sends each pre-framed message, reconnecting on error (FR-005, FR-006).
110
117
 
118
+ The lock spans the whole call, not each message: it also guards ``_sock``, which a
119
+ reconnect rebinds, so releasing between messages would let another thread send on a
120
+ socket this one is about to reset (SPEC-028 FR-002).
121
+
111
122
  Args:
112
123
  messages: The exact bytes to put on the wire, one call per message.
113
124
 
@@ -121,10 +132,11 @@ class SocketTransport:
121
132
  would re-send the messages that already landed, and an empty call is a no-op rather
122
133
  than a total failure.
123
134
  """
124
- delivered = 0
125
- for message in messages:
126
- if self._send_one(message):
127
- delivered += 1
135
+ with self._lock:
136
+ delivered = 0
137
+ for message in messages:
138
+ if self._send_one(message):
139
+ delivered += 1
128
140
  if messages and delivered == 0:
129
141
  raise SinkDeliveryError(
130
142
  f"SocketTransport delivered none of {len(messages)} message(s)"
@@ -142,12 +154,16 @@ class SocketTransport:
142
154
  Raises:
143
155
  None.
144
156
  """
145
- return SinkLosses(dropped=0, failed=self.failed)
157
+ with self._counter_lock:
158
+ return SinkLosses(dropped=0, failed=self.failed)
146
159
 
147
160
  def close(self) -> None:
148
161
  """Closes the held socket, if any (FR-005, FR-012).
149
162
 
150
- Idempotent.
163
+ Idempotent, and takes the send lock so it never closes the socket out from under an
164
+ in-flight ``send_all`` (SPEC-028 FR-002). ``_reset`` does not take the lock itself,
165
+ because ``_send_one`` calls it while ``send_all`` already holds one — the lock is
166
+ deliberately not re-entrant, so that path must stay the only unlocked caller.
151
167
 
152
168
  Args:
153
169
  None.
@@ -158,7 +174,8 @@ class SocketTransport:
158
174
  Raises:
159
175
  None.
160
176
  """
161
- self._reset()
177
+ with self._lock:
178
+ self._reset()
162
179
 
163
180
  def _send_one(self, message: bytes) -> bool:
164
181
  """Sends one message within the retry bound.
@@ -190,7 +207,8 @@ class SocketTransport:
190
207
  if attempt < self._max_retries:
191
208
  wait(_BACKOFF_BASE * (2**attempt), self.stop_signal)
192
209
  continue
193
- self.failed += 1
210
+ with self._counter_lock:
211
+ self.failed += 1
194
212
  _diag.lost(
195
213
  "message",
196
214
  1,
@@ -57,11 +57,32 @@ class Sink(Protocol):
57
57
  ``None`` is the same answer as having no method at all, which is what keeps a third-party
58
58
  sink written against the pre-SPEC-026 interface satisfying this one. :func:`read_losses` is
59
59
  the probe.
60
+
61
+ "Safe to call during an emit" is a concurrency requirement once :meth:`emit` is (SPEC-028
62
+ FR-003). The shipped sinks keep their loss counters under a **dedicated** lock, separate
63
+ from whatever guards their transport: an increment is a read-modify-write that Python does
64
+ not promise is atomic, and the pair read must come from one instant rather than straddling
65
+ another thread's increments. It is deliberately not the transport lock — ``health()`` is
66
+ the call an operator makes when a destination is already hanging, and sharing one lock
67
+ would make that poll wait for an in-flight emit and its retry backoff. Where a sink holds
68
+ both, the order is always transport then counter, never the reverse.
60
69
  """
61
70
 
62
71
  def emit(self, batch: list[dict[str, object]]) -> None:
63
72
  """Ships a batch of serialized event dicts.
64
73
 
74
+ **May be called concurrently from more than one thread, and must tolerate it**
75
+ (SPEC-028 FR-001). The background worker drains on its own thread, and a level call
76
+ made with no active span emits synchronously on the *caller's* thread (arch §12), which
77
+ may be any of the application's — an audit observed one sink object entered by two
78
+ application threads and the worker at once. An implementation holding mutable transport
79
+ state, such as a stream it rebinds, a reused socket or a connection with transaction
80
+ scope, must serialize access to it for the whole span of the operation that assumes
81
+ exclusivity. One that holds no such state need do nothing.
82
+
83
+ This is a requirement on implementations, not a promise the library serializes on their
84
+ behalf. It cannot: the orphan path runs on a thread the library does not own.
85
+
65
86
  Raise when the batch delivered nothing and it was non-empty — the worker's bounded
66
87
  retry and ``health().failed_batches`` depend on that signal, and a retry there cannot
67
88
  duplicate anything (SPEC-026 FR-001). Raise after the sink's own retries are spent, so
@@ -88,6 +109,13 @@ class Sink(Protocol):
88
109
  def close(self) -> None:
89
110
  """Flushes and releases any resources.
90
111
 
112
+ **May be called while an ``emit`` is in flight on another thread** (SPEC-028 FR-001).
113
+ It must therefore either wait for that emit or become a no-op, and must never release a
114
+ resource a concurrent ``emit`` is about to use — a half-released transport is worse than
115
+ an unreleased one, because the emit then fails against a closed handle instead of
116
+ succeeding. Taking the same lock ``emit`` takes satisfies this; so does an idempotent
117
+ guard checked under that lock.
118
+
91
119
  Args:
92
120
  None.
93
121
 
@@ -3,10 +3,8 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import json
6
- from typing import TYPE_CHECKING, Any
7
-
8
- if TYPE_CHECKING:
9
- import threading
6
+ import threading
7
+ from typing import Any
10
8
 
11
9
  from log_foundry import _diag
12
10
  from log_foundry.sinks._chunk import chunk_list, valid_identifier
@@ -42,6 +40,12 @@ class ClickHouseSink:
42
40
  columns plus the full event as a ``String`` column, inserted in a single call per chunk. The
43
41
  sink is write-only, and the worst-case delay (SPEC-027 FR-005) is ``max_retries``
44
42
  interruptible waits per chunk, 0.7 s at the defaults.
43
+
44
+ The driver requirement satisfied (SPEC-028 FR-002): a ``clickhouse-connect`` client holds
45
+ per-session state across an insert and the project does not publish it as safe to share
46
+ between threads, so this sink serializes its use rather than assuming otherwise. A lock is
47
+ the conservative reading — one client per thread would be the alternative, and that is the
48
+ connection-pool design FR-002 puts out of scope.
45
49
  """
46
50
 
47
51
  def __init__(
@@ -81,6 +85,8 @@ class ClickHouseSink:
81
85
  self.stop_signal: threading.Event | None = None
82
86
  self.failed = 0
83
87
  self._closed = False
88
+ self._lock = threading.Lock()
89
+ self._counter_lock = threading.Lock()
84
90
  self._owns_client = client is None
85
91
  if client is None:
86
92
  import clickhouse_connect # type: ignore[import-not-found]
@@ -93,6 +99,9 @@ class ClickHouseSink:
93
99
  def losses(self) -> SinkLosses:
94
100
  """Reports rows in a chunk abandoned past the retry bound (SPEC-026 FR-002).
95
101
 
102
+ Reads under the counter lock rather than the emit lock (SPEC-028 FR-003), so a poll
103
+ never waits on an in-flight insert and its backoff.
104
+
96
105
  Args:
97
106
  None.
98
107
 
@@ -102,7 +111,8 @@ class ClickHouseSink:
102
111
  Raises:
103
112
  None.
104
113
  """
105
- return SinkLosses(dropped=0, failed=self.failed)
114
+ with self._counter_lock:
115
+ return SinkLosses(dropped=0, failed=self.failed)
106
116
 
107
117
  def emit(self, batch: list[dict[str, object]]) -> None:
108
118
  """Inserts each chunk as one columnar call, retrying on failure (FR-002).
@@ -122,16 +132,18 @@ class ClickHouseSink:
122
132
  if not batch:
123
133
  return
124
134
  chunks = inserted = 0
125
- for chunk in chunk_list(batch, self._chunk_size):
126
- chunks += 1
127
- inserted += self._insert([self._row(event) for event in chunk])
135
+ with self._lock:
136
+ for chunk in chunk_list(batch, self._chunk_size):
137
+ chunks += 1
138
+ inserted += self._insert([self._row(event) for event in chunk])
128
139
  if chunks and not inserted:
129
140
  raise SinkDeliveryError(f"ClickHouseSink inserted none of {chunks} chunk(s)")
130
141
 
131
142
  def close(self) -> None:
132
143
  """Closes the client only if the sink owns it (FR-005).
133
144
 
134
- Idempotent.
145
+ Idempotent, and takes the emit lock so the client is never closed mid-insert
146
+ (SPEC-028 FR-002).
135
147
 
136
148
  Args:
137
149
  None.
@@ -142,11 +154,12 @@ class ClickHouseSink:
142
154
  Raises:
143
155
  Exception: Whatever the client raises on close.
144
156
  """
145
- if self._closed:
146
- return
147
- if self._owns_client:
148
- self.client.close()
149
- self._closed = True
157
+ with self._lock:
158
+ if self._closed:
159
+ return
160
+ if self._owns_client:
161
+ self.client.close()
162
+ self._closed = True
150
163
 
151
164
  def _row(self, event: dict[str, object]) -> list[object]:
152
165
  """Builds one row: the extracted columns, then the whole event as JSON.
@@ -182,7 +195,8 @@ class ClickHouseSink:
182
195
  if attempt < self.max_retries:
183
196
  wait(_BACKOFF_BASE * (2**attempt), self.stop_signal)
184
197
  continue
185
- self.failed += len(rows)
198
+ with self._counter_lock:
199
+ self.failed += len(rows)
186
200
  _diag.lost(
187
201
  "row",
188
202
  len(rows),
@@ -98,10 +98,11 @@ class ElasticsearchSink(HTTPSink):
98
98
  Raises:
99
99
  None.
100
100
  """
101
- return SinkLosses(
102
- dropped=self.dropped_oversized,
103
- failed=self.failed + self.item_errors + self.dropped_unadjudicated,
104
- )
101
+ with self._counter_lock:
102
+ return SinkLosses(
103
+ dropped=self.dropped_oversized,
104
+ failed=self.failed + self.item_errors + self.dropped_unadjudicated,
105
+ )
105
106
 
106
107
  def _parse_bulk_response(self, payload: bytes, sent: int) -> bool:
107
108
  """Counts rejected items and reports whether the response proves nothing was indexed.
@@ -139,7 +140,8 @@ class ElasticsearchSink(HTTPSink):
139
140
  items = usable_results(data.get("items"))
140
141
  errors = sum(1 for item in items if _has_error(item))
141
142
  if len(items) != sent or not errors:
142
- self.dropped_unadjudicated += sent
143
+ with self._counter_lock:
144
+ self.dropped_unadjudicated += sent
143
145
  _diag.lost(
144
146
  "event",
145
147
  sent,
@@ -148,7 +150,8 @@ class ElasticsearchSink(HTTPSink):
148
150
  f"not retried",
149
151
  )
150
152
  return False
151
- self.item_errors += errors
153
+ with self._counter_lock:
154
+ self.item_errors += errors
152
155
  _diag.lost("bulk item", errors, f"{type(self).__name__}, rejected by the server")
153
156
  return errors == sent
154
157
 
@@ -3,10 +3,8 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  import json
6
- from typing import TYPE_CHECKING, Any
7
-
8
- if TYPE_CHECKING:
9
- import threading
6
+ import threading
7
+ from typing import Any
10
8
 
11
9
  from log_foundry import _diag
12
10
  from log_foundry.sinks._retry import wait
@@ -69,6 +67,7 @@ class AzureEventHubsSink:
69
67
  self.stop_signal: threading.Event | None = None
70
68
  self.failed = 0
71
69
  self.dropped_oversized = 0
70
+ self._counter_lock = threading.Lock()
72
71
 
73
72
  def losses(self) -> SinkLosses:
74
73
  """Reports oversized drops and events in a batch abandoned past the bound (FR-002).
@@ -82,7 +81,8 @@ class AzureEventHubsSink:
82
81
  Raises:
83
82
  None.
84
83
  """
85
- return SinkLosses(dropped=self.dropped_oversized, failed=self.failed)
84
+ with self._counter_lock:
85
+ return SinkLosses(dropped=self.dropped_oversized, failed=self.failed)
86
86
 
87
87
  def emit(self, batch: list[dict[str, object]]) -> None:
88
88
  """Packs events into batches within the size limit and sends each (FR-009).
@@ -118,7 +118,8 @@ class AzureEventHubsSink:
118
118
  delivered += self._send(current)
119
119
  current = self.producer.create_batch()
120
120
  if not _try_add(current, data):
121
- self.dropped_oversized += 1
121
+ with self._counter_lock:
122
+ self.dropped_oversized += 1
122
123
  _diag.lost("event", 1, "AzureEventHubsSink, too large for an empty 1 MB batch")
123
124
  if len(current) > 0:
124
125
  attempted += 1
@@ -167,7 +168,8 @@ class AzureEventHubsSink:
167
168
  if attempt < self.max_retries:
168
169
  wait(_BACKOFF_BASE * (2**attempt), self.stop_signal)
169
170
  continue
170
- self.failed += len(event_batch)
171
+ with self._counter_lock:
172
+ self.failed += len(event_batch)
171
173
  _diag.lost(
172
174
  "event",
173
175
  len(event_batch),
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  import json
6
6
  import os
7
+ import threading
7
8
  import time
8
9
  from typing import TextIO
9
10
 
@@ -22,8 +23,12 @@ class FileSink:
22
23
 
23
24
  Not every deployment ships to a cloud queue; local dev, debugging, air-gapped hosts and
24
25
  simple archival just want events on the local disk. Writes are synchronous stdlib calls
25
- only, and a single-process, single-worker-thread writer is assumed (arch §9) — cross-process
26
- coordination is out of scope.
26
+ only.
27
+
28
+ Writers within the process are serialized on a lock, because ``emit`` may be called
29
+ concurrently (SPEC-028 FR-002) — this module claimed a single worker thread until that spec
30
+ measured the orphan path emitting on application threads at the same time. Cross-*process*
31
+ coordination remains out of scope: two processes appending to one path are on their own.
27
32
  """
28
33
 
29
34
  def __init__(self, path: str, *, encoding: str = "utf-8") -> None:
@@ -47,10 +52,16 @@ class FileSink:
47
52
  self._encoding = encoding
48
53
  self._stream: TextIO = open(path, "a", encoding=encoding)
49
54
  self._closed = False
55
+ self._lock = threading.Lock()
50
56
 
51
57
  def emit(self, batch: list[dict[str, object]]) -> None:
52
58
  """Writes every event as one newline-terminated ``json.dumps`` line, then flushes.
53
59
 
60
+ The lock covers the whole batch rather than each line (SPEC-028 FR-002). A text stream
61
+ does not promise that one ``write`` is atomic against another, so per-line locking could
62
+ still interleave two events' bytes; batch-wide locking also keeps a batch contiguous in
63
+ the file, which is what makes the output readable.
64
+
54
65
  Args:
55
66
  batch: The events to write.
56
67
 
@@ -60,13 +71,17 @@ class FileSink:
60
71
  Raises:
61
72
  OSError: If the write or flush fails (FR-001).
62
73
  """
63
- for event in batch:
64
- self._stream.write(json.dumps(event) + "\n")
65
- self._stream.flush()
74
+ with self._lock:
75
+ for event in batch:
76
+ self._stream.write(json.dumps(event) + "\n")
77
+ self._stream.flush()
66
78
 
67
79
  def close(self) -> None:
68
80
  """Flushes and closes the file handle, with a second call a no-op (FR-001).
69
81
 
82
+ Taking the same lock ``emit`` takes means a close waits for an in-flight write rather
83
+ than pulling the stream out from under it (SPEC-028 FR-002).
84
+
70
85
  Args:
71
86
  None.
72
87
 
@@ -76,11 +91,12 @@ class FileSink:
76
91
  Raises:
77
92
  OSError: If the flush or close fails.
78
93
  """
79
- if self._closed:
80
- return
81
- self._stream.flush()
82
- self._stream.close()
83
- self._closed = True
94
+ with self._lock:
95
+ if self._closed:
96
+ return
97
+ self._stream.flush()
98
+ self._stream.close()
99
+ self._closed = True
84
100
 
85
101
 
86
102
  class RotatingFileSink:
@@ -95,6 +111,11 @@ class RotatingFileSink:
95
111
  count, and opens a fresh active file — a backup count of zero keeps none, simply replacing
96
112
  the active file. No event is lost across a rotation, because the rotate happens before the
97
113
  pending event is written and the event lands in the fresh file.
114
+
115
+ A rotation rebinds the active stream, so it is the sink where concurrent writers did real
116
+ damage: a second thread mid-``emit`` could write to the handle rotation had just closed, or
117
+ to the pre-rotation file it had already renamed away. Both are serialized on a lock
118
+ (SPEC-028 FR-002).
98
119
  """
99
120
 
100
121
  def __init__(
@@ -136,10 +157,16 @@ class RotatingFileSink:
136
157
  self._size = os.path.getsize(path) if os.path.exists(path) else 0
137
158
  self._next_rollover = self._schedule_next()
138
159
  self._closed = False
160
+ self._lock = threading.Lock()
139
161
 
140
162
  def emit(self, batch: list[dict[str, object]]) -> None:
141
163
  """Appends each event, rotating first whenever a size or time trigger fires (FR-002).
142
164
 
165
+ The lock spans the whole batch, so the decide-rotate-write-account sequence is
166
+ indivisible (SPEC-028 FR-002). Guarding only ``_rotate`` would not be enough: the
167
+ ``_should_rotate`` check and the write that follows it must see the same stream, or a
168
+ rotation between them sends the line to a closed handle.
169
+
143
170
  Args:
144
171
  batch: The events to write.
145
172
 
@@ -149,18 +176,23 @@ class RotatingFileSink:
149
176
  Raises:
150
177
  OSError: If a write, flush or rotation fails.
151
178
  """
152
- for event in batch:
153
- line = json.dumps(event) + "\n"
154
- data = len(line.encode(self._encoding))
155
- if self._should_rotate(data):
156
- self._rotate()
157
- self._stream.write(line)
158
- self._size += data
159
- self._stream.flush()
179
+ with self._lock:
180
+ for event in batch:
181
+ line = json.dumps(event) + "\n"
182
+ data = len(line.encode(self._encoding))
183
+ if self._should_rotate(data):
184
+ self._rotate()
185
+ self._stream.write(line)
186
+ self._size += data
187
+ self._stream.flush()
160
188
 
161
189
  def close(self) -> None:
162
190
  """Flushes and closes the active handle, with a second call a no-op (FR-002).
163
191
 
192
+ Taking the same lock ``emit`` takes means a close waits for an in-flight write, and in
193
+ particular never lands between a rotation and the write it was making room for
194
+ (SPEC-028 FR-002).
195
+
164
196
  Args:
165
197
  None.
166
198
 
@@ -170,11 +202,12 @@ class RotatingFileSink:
170
202
  Raises:
171
203
  OSError: If the flush or close fails.
172
204
  """
173
- if self._closed:
174
- return
175
- self._stream.flush()
176
- self._stream.close()
177
- self._closed = True
205
+ with self._lock:
206
+ if self._closed:
207
+ return
208
+ self._stream.flush()
209
+ self._stream.close()
210
+ self._closed = True
178
211
 
179
212
  @staticmethod
180
213
  def _rollover_seconds(when: str | None, interval: int) -> float | None: