log-foundry 0.10.2.dev16__tar.gz → 0.10.2.dev18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/PKG-INFO +1 -1
  2. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/pyproject.toml +1 -1
  3. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_socket.py +30 -6
  4. log_foundry-0.10.2.dev18/src/log_foundry/sinks/base.py +119 -0
  5. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/clickhouse.py +24 -7
  6. log_foundry-0.10.2.dev18/src/log_foundry/sinks/elasticsearch.py +141 -0
  7. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/eventhubs.py +42 -11
  8. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/filtering.py +17 -1
  9. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/firehose.py +61 -9
  10. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/http.py +41 -14
  11. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/kafka.py +36 -2
  12. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/kinesis.py +60 -9
  13. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/logstash.py +13 -0
  14. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/mongodb.py +30 -4
  15. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/multi.py +27 -0
  16. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/nats.py +16 -1
  17. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/postgres.py +17 -3
  18. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/pubsub.py +31 -2
  19. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/rabbitmq.py +24 -6
  20. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/redis.py +17 -3
  21. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sentry.py +64 -5
  22. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sns.py +30 -7
  23. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sqs.py +65 -7
  24. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/syslog.py +13 -0
  25. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/transform.py +17 -1
  26. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/util.py +4 -0
  27. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/worker.py +21 -1
  28. log_foundry-0.10.2.dev16/src/log_foundry/sinks/base.py +0 -22
  29. log_foundry-0.10.2.dev16/src/log_foundry/sinks/elasticsearch.py +0 -68
  30. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/LICENSE +0 -0
  31. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/README.md +0 -0
  32. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/__init__.py +0 -0
  33. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/_diag.py +0 -0
  34. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/api.py +0 -0
  35. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/config.py +0 -0
  36. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/console.py +0 -0
  37. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/context.py +0 -0
  38. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/decorator.py +0 -0
  39. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/ids.py +0 -0
  40. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/model.py +0 -0
  41. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/py.typed +0 -0
  42. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sanitize.py +0 -0
  43. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/__init__.py +0 -0
  44. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_batch.py +0 -0
  45. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_chunk.py +0 -0
  46. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_time.py +0 -0
  47. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/callback.py +0 -0
  48. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/datadog.py +0 -0
  49. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/file.py +0 -0
  50. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/honeycomb.py +0 -0
  51. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/logging_sink.py +0 -0
  52. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/loki.py +0 -0
  53. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/newrelic.py +0 -0
  54. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/splunk.py +0 -0
  55. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sqlite.py +0 -0
  56. {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/stdout.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev16
3
+ Version: 0.10.2.dev18
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev16"
23
+ version = "0.10.2.dev18"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -15,6 +15,7 @@ import socket
15
15
  import time
16
16
 
17
17
  from log_foundry import _diag
18
+ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
18
19
 
19
20
  __all__ = ["SocketTransport"]
20
21
 
@@ -53,14 +54,35 @@ class SocketTransport:
53
54
  self._port = port
54
55
  self._transport = transport
55
56
  self._timeout = timeout
56
- self._max_retries = max_retries
57
+ # Floored for the reason ``Worker._emit`` floors its own (SPEC-021): a negative value
58
+ # made no attempt at all and abandoned the message without moving ``failed``.
59
+ self._max_retries = max(max_retries, 0)
57
60
  self._sock: socket.socket | None = None
58
61
  self.failed = 0
59
62
 
60
63
  def send_all(self, messages: list[bytes]) -> None:
61
- """Send each pre-framed message, reconnecting on error (FR-005, FR-006)."""
64
+ """Send each pre-framed message, reconnecting on error (FR-005, FR-006).
65
+
66
+ Raises :class:`~log_foundry.sinks.base.SinkDeliveryError` when **none** of the messages
67
+ reached the socket, so the sinks built on this transport propagate a dead destination to
68
+ the worker instead of reporting success (SPEC-026 FR-001). A partial send does not raise:
69
+ the worker's retry would re-send the messages that already landed.
70
+
71
+ An empty call is a no-op, not a total failure — ``delivered == 0`` there means there was
72
+ nothing to deliver.
73
+ """
74
+ delivered = 0
62
75
  for message in messages:
63
- self._send_one(message)
76
+ if self._send_one(message):
77
+ delivered += 1
78
+ if messages and delivered == 0:
79
+ raise SinkDeliveryError(
80
+ f"SocketTransport delivered none of {len(messages)} message(s)"
81
+ )
82
+
83
+ def losses(self) -> SinkLosses:
84
+ """Messages abandoned past the reconnect-retry bound (SPEC-026 FR-002). Never raises."""
85
+ return SinkLosses(dropped=0, failed=self.failed)
64
86
 
65
87
  def close(self) -> None:
66
88
  """Close the held socket, if any; idempotent (FR-005, FR-012)."""
@@ -68,14 +90,15 @@ class SocketTransport:
68
90
 
69
91
  # -- internals ----------------------------------------------------------------------
70
92
 
71
- def _send_one(self, message: bytes) -> None:
93
+ def _send_one(self, message: bytes) -> bool:
94
+ """Send one message within the retry bound; ``False`` once it is abandoned."""
72
95
  for attempt in range(self._max_retries + 1):
73
96
  try:
74
97
  if self._transport == "udp":
75
98
  self._socket().sendto(message, (self._host, self._port))
76
99
  else:
77
100
  self._socket().sendall(message)
78
- return
101
+ return True
79
102
  except OSError as err:
80
103
  self._reset() # force a fresh connection on the next attempt
81
104
  if attempt < self._max_retries:
@@ -92,7 +115,8 @@ class SocketTransport:
92
115
  f"SocketTransport, {self._max_retries + 1} attempt(s), "
93
116
  f"{type(err).__name__} {_diag.errno_of(err)}".rstrip(),
94
117
  )
95
- return
118
+ return False
119
+ return False # unreachable: the loop returns on every path (mypy needs the exit)
96
120
 
97
121
  def _socket(self) -> socket.socket:
98
122
  if self._sock is None:
@@ -0,0 +1,119 @@
1
+ """The sink interface (arch §8).
2
+
3
+ A sink is the swappable output transport. It receives *already-built, batched* event
4
+ dicts from the worker and knows nothing about spans or context — that dumbness is what
5
+ makes sinks trivially interchangeable (StdoutSink, SQSSink, …). Only the Protocol lives
6
+ here; concrete sinks arrive in later phases.
7
+
8
+ The interface carries two obligations beyond "put these somewhere" (SPEC-026), because the
9
+ library's whole loss-reporting apparatus is built on them:
10
+
11
+ * **Total failure raises.** A sink that delivered *none* of a batch must let something
12
+ propagate out of ``emit``. That is the signal the worker's bounded retry and
13
+ ``health().failed_batches`` are built on, and the one case where a retry cannot create
14
+ duplicates — nothing landed, so there is nothing downstream to duplicate.
15
+ * **Partial failure does not.** A batch where some records landed must not be retried
16
+ wholesale; it is counted and made readable through the optional ``losses()``.
17
+
18
+ A sink that absorbs a total failure and returns normally is a sink the worker believes:
19
+ the retry never engages, ``failed_batches`` stays at zero and ``flush()`` returns ``True``
20
+ while every event is lost. That reading is precisely what SPEC-017 existed to make
21
+ impossible, and what SPEC-026 generalized from ``MultiSink`` to the whole family.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ from typing import NamedTuple, Protocol, runtime_checkable
27
+
28
+ __all__ = ["Sink", "SinkDeliveryError", "SinkLosses", "read_losses"]
29
+
30
+
31
+ class SinkDeliveryError(Exception):
32
+ """Raised by a sink whose ``emit`` delivered none of the batch (SPEC-026 FR-001).
33
+
34
+ A distinct type so an operator reading a ``stopped_reason`` or a diagnostic line can tell
35
+ "the destination refused everything" from "the sink itself has a bug". Sinks that already
36
+ have a natural exception to re-raise — a driver error, a ``MultiSink`` child's — re-raise
37
+ that instead: the contract is that *something* propagates, not that it must be this type.
38
+ """
39
+
40
+
41
+ class SinkLosses(NamedTuple):
42
+ """What a sink discarded or could not confirm, cumulative for its lifetime (SPEC-026 FR-002).
43
+
44
+ Two fields rather than one because the remedies differ: ``dropped`` means the destination
45
+ could never have accepted the event as built (an oversized record), so the fix is upstream in
46
+ what the application logs; ``failed`` means delivery was attempted and the destination did not
47
+ confirm it, so the fix is the destination or the network.
48
+
49
+ ``failed`` is an **upper bound** on loss, not a count of it. A sink that also raises on total
50
+ failure counts the attempt here *and* hands the batch back to the worker, whose retry may
51
+ then deliver it — so a transient outage leaves ``failed`` non-zero with nothing actually
52
+ lost. ``health().failed_batches`` is the worker-level record of a batch given up on for good;
53
+ this is the sink-level record of everything that did not go through first time.
54
+ """
55
+
56
+ dropped: int
57
+ failed: int
58
+
59
+
60
+ @runtime_checkable
61
+ class Sink(Protocol):
62
+ def emit(self, batch: list[dict[str, object]]) -> None:
63
+ """Ship a batch of serialized event dicts.
64
+
65
+ **Raise when the batch delivered nothing** and it was non-empty — the worker's bounded
66
+ retry and ``health().failed_batches`` depend on that signal, and a retry there cannot
67
+ duplicate anything (SPEC-026 FR-001). Raise *after* the sink's own retries are spent, so
68
+ the worker's retry composes on top rather than replacing it; a sink whose own budget
69
+ makes the worker's redundant should say so in its docstring rather than absorb silently.
70
+
71
+ **Do not raise on partial failure.** A batch where some records landed would be
72
+ re-delivered wholesale by the worker's retry, and duplicates downstream are worse than
73
+ the counted loss (SPEC-017 FR-004, SPEC-018). Report that through ``losses()`` instead.
74
+
75
+ ``emit([])`` is a no-op and never raises: an empty batch has not failed to deliver.
76
+ """
77
+ ...
78
+
79
+ def close(self) -> None:
80
+ """Flush and release any resources."""
81
+ ...
82
+
83
+ # ``losses()`` is optional, and deliberately *not* declared here: ``Sink`` is structural, and
84
+ # a third-party sink written against the pre-SPEC-026 interface must keep satisfying it.
85
+ # ``read_losses`` below is the probe; an absent method reads as "reports nothing".
86
+ #
87
+ # def losses(self) -> SinkLosses | None:
88
+ # """Cumulative loss this sink absorbed. Never raises; safe to call during emit.
89
+ #
90
+ # ``None`` is the same answer as having no method at all — "this sink reports
91
+ # nothing". A wrapper returns it when what it wraps reports nothing.
92
+ # """
93
+ # return SinkLosses(dropped=self._dropped, failed=self._failed)
94
+
95
+
96
+ def read_losses(sink: object) -> SinkLosses | None:
97
+ """Read a sink's optional ``losses()``; ``None`` when it reports nothing.
98
+
99
+ ``None`` covers four cases deliberately treated alike: no ``losses`` attribute, one that is
100
+ not callable, one that raised, and one that returned anything other than a ``SinkLosses`` —
101
+ including ``None`` itself, which the shipped wrapper sinks return when what they wrap
102
+ reports nothing.
103
+
104
+ The single reader for the optional half of the protocol (SPEC-026 FR-002), shared by
105
+ ``Worker.health`` and ``MultiSink.losses`` so the probe and its guarantees are written once.
106
+
107
+ Total by design. ``health()`` is documented "Never raises" and is the call an operator makes
108
+ when things are *already* going wrong, so a third-party sink with a broken accessor must not
109
+ be able to take the snapshot down with it. The returned shape is checked as well as the call:
110
+ a ``losses()`` returning something else would otherwise put an arbitrary object where callers
111
+ read two integers. Nothing is written to stderr — a poll happens as often as the caller likes,
112
+ and a broken accessor is a sink bug rather than a loss.
113
+ """
114
+ try:
115
+ accessor = getattr(sink, "losses", None)
116
+ losses = accessor() if callable(accessor) else None
117
+ except Exception:
118
+ return None
119
+ return losses if isinstance(losses, SinkLosses) else None
@@ -14,6 +14,7 @@ from typing import Any
14
14
 
15
15
  from log_foundry import _diag
16
16
  from log_foundry.sinks._chunk import chunk_list, valid_identifier
17
+ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
17
18
 
18
19
  __all__ = ["ClickHouseSink"]
19
20
 
@@ -54,7 +55,9 @@ class ClickHouseSink:
54
55
  ) -> None:
55
56
  self._table = valid_identifier(table)
56
57
  self._chunk_size = chunk_size
57
- self.max_retries = max_retries
58
+ # Floored as ``Worker._emit`` floors its own (SPEC-021): a negative value returned
59
+ # from ``_insert`` having attempted nothing, and reported success.
60
+ self.max_retries = max(max_retries, 0)
58
61
  self.failed = 0
59
62
  self._closed = False
60
63
  self._owns_client = client is None
@@ -66,13 +69,25 @@ class ClickHouseSink:
66
69
  if create_table:
67
70
  self._ensure_schema()
68
71
 
72
+ def losses(self) -> SinkLosses:
73
+ """Rows in a chunk abandoned past the retry bound (SPEC-026 FR-002). Never raises."""
74
+ return SinkLosses(dropped=0, failed=self.failed)
75
+
69
76
  def emit(self, batch: list[dict[str, object]]) -> None:
70
- """Insert each chunk as one columnar ``insert`` call, retrying on failure (FR-002)."""
77
+ """Insert each chunk as one columnar ``insert`` call, retrying on failure (FR-002).
78
+
79
+ Raises when every chunk failed (SPEC-026 FR-001) — which is the whole batch for any
80
+ batch that fits one chunk, the ordinary case. A partially-inserted batch does not raise:
81
+ the chunks that landed are committed, and the worker's retry would duplicate them.
82
+ """
71
83
  if not batch:
72
84
  return
85
+ chunks = inserted = 0
73
86
  for chunk in chunk_list(batch, self._chunk_size):
74
- rows = [self._row(event) for event in chunk]
75
- self._insert(rows)
87
+ chunks += 1
88
+ inserted += self._insert([self._row(event) for event in chunk])
89
+ if chunks and not inserted:
90
+ raise SinkDeliveryError(f"ClickHouseSink inserted none of {chunks} chunk(s)")
76
91
 
77
92
  def close(self) -> None:
78
93
  """Close the client only if the sink owns it; idempotent (FR-005)."""
@@ -87,11 +102,12 @@ class ClickHouseSink:
87
102
  def _row(self, event: dict[str, object]) -> list[object]:
88
103
  return [*(event.get(col) for col in _COLUMNS), json.dumps(event)]
89
104
 
90
- def _insert(self, rows: list[list[object]]) -> None:
105
+ def _insert(self, rows: list[list[object]]) -> int:
106
+ """Insert one chunk within the retry bound; ``1`` if it landed, ``0`` once abandoned."""
91
107
  for attempt in range(self.max_retries + 1):
92
108
  try:
93
109
  self.client.insert(self._table, data=rows, column_names=_COLUMN_NAMES)
94
- return
110
+ return 1
95
111
  except Exception as err: # isolation boundary: never crash the worker (FR-006)
96
112
  if attempt < self.max_retries:
97
113
  time.sleep(_BACKOFF_BASE * (2**attempt))
@@ -102,7 +118,8 @@ class ClickHouseSink:
102
118
  len(rows),
103
119
  f"ClickHouseSink, {self.max_retries + 1} attempts, {type(err).__name__}",
104
120
  )
105
- return
121
+ return 0
122
+ return 0 # unreachable: the loop returns on every path (mypy needs the exit)
106
123
 
107
124
  def _ensure_schema(self) -> None:
108
125
  columns = ", ".join(f"{col} {_COLUMN_TYPES[col]}" for col in _COLUMNS)
@@ -0,0 +1,141 @@
1
+ """ElasticsearchSink / OpenSearchSink — index events via the ``_bulk`` API (arch §8, SPEC-009).
2
+
3
+ A thin specialization of :class:`~log_foundry.sinks.http.HTTPSink`: each event becomes an action line
4
+ (``{"index": {"_index": target}}``) followed by its source line, POSTed to ``_bulk`` as newline-
5
+ delimited JSON. The bulk response is inspected per item so a partial failure is counted and logged
6
+ without discarding the successfully-indexed items. OpenSearch speaks the same bulk protocol, so
7
+ ``OpenSearchSink`` is a straight reuse (endpoint/auth differ only by configuration).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+
14
+ from log_foundry import _diag
15
+ from log_foundry.sinks._batch import usable_results
16
+ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
17
+ from log_foundry.sinks.http import HTTPSink
18
+
19
+ __all__ = ["ElasticsearchSink", "OpenSearchSink"]
20
+
21
+
22
+ class ElasticsearchSink(HTTPSink):
23
+ """POST events to an Elasticsearch ``_bulk`` endpoint, parsing per-item errors (FR-003).
24
+
25
+ Attributes:
26
+ item_errors: Count of bulk items the server reported as failed (distinct from ``failed``,
27
+ which counts whole requests abandoned past the retry bound).
28
+ dropped_unadjudicated: Events whose outcome a ``_bulk`` response did not describe — its
29
+ ``items`` array did not line up with the batch sent, so no event in it could be
30
+ paired to an outcome. Abandoned rather than retried, for SPEC-018's reason: the
31
+ request succeeded, so re-sending would duplicate whatever landed.
32
+ """
33
+
34
+ def __init__(self, url: str, *, index: str, auth: str | tuple[str, str] | None = None,
35
+ **http_kwargs: object) -> None:
36
+ self._index = index
37
+ super().__init__(
38
+ url.rstrip("/") + "/_bulk", auth=auth, body_format="ndjson", **http_kwargs # type: ignore[arg-type]
39
+ )
40
+ self.item_errors = 0
41
+ self.dropped_unadjudicated = 0
42
+
43
+ def emit(self, batch: list[dict[str, object]]) -> None:
44
+ """Build the ``_bulk`` NDJSON payload, POST it, and parse the response items (FR-003)."""
45
+ if not batch:
46
+ return
47
+ lines: list[str] = []
48
+ for event in batch:
49
+ lines.append(json.dumps({"index": {"_index": self._index}}))
50
+ lines.append(json.dumps(event))
51
+ body = ("\n".join(lines) + "\n").encode("utf-8") # bulk must be newline-terminated
52
+ # An abandoned request raises out of ``_send`` now (SPEC-026 FR-001) — nothing was
53
+ # indexed, so there is no response to parse and nothing downstream to duplicate.
54
+ payload = self._send(body, content_type="application/x-ndjson")
55
+ if self._parse_bulk_response(payload, len(batch)):
56
+ # Every item carried an error, so the 200 indexed nothing: a total failure like any
57
+ # other, and it must reach the worker (FR-001). A retry cannot duplicate, and where
58
+ # the cause is permanent (a mapping conflict) the worker abandons the batch after
59
+ # its bound and records it — which beats a silent success. A response rejecting
60
+ # *some* items stays partial and is reported through ``losses()``.
61
+ raise SinkDeliveryError(
62
+ f"{type(self).__name__} indexed none of {len(batch)} event(s)"
63
+ )
64
+
65
+ def losses(self) -> SinkLosses:
66
+ """Abandoned requests plus server-rejected bulk items (SPEC-026 FR-002). Never raises.
67
+
68
+ All three are summed into ``failed`` because each is an event the server did not
69
+ confirm; they stay apart on the instance (``failed`` / ``item_errors`` /
70
+ ``dropped_unadjudicated``) for anyone who needs to tell "the request never landed" from
71
+ "the request landed and these items bounced" from "the response did not say".
72
+ """
73
+ return SinkLosses(
74
+ dropped=self.dropped_oversized,
75
+ failed=self.failed + self.item_errors + self.dropped_unadjudicated,
76
+ )
77
+
78
+ def _parse_bulk_response(self, payload: bytes, sent: int) -> bool:
79
+ """Count rejected items; return whether the response proves *nothing* was indexed.
80
+
81
+ The ``items`` array is positional — entry *i* describes event *i* — so the rule
82
+ ``sinks/_batch.py`` states applies here too (SPEC-018): a length disagreement is
83
+ evidence the arrays do not describe each other, not an invitation to read the overlap.
84
+ Counting errors does not depend on position, but concluding "all of them failed" does,
85
+ so that conclusion is gated on ``len(items) == sent``. Without the gate both directions
86
+ are wrong and both are reachable: a longer ``items`` array could make a partial success
87
+ raise (and the worker's retry would duplicate what landed), and a shorter one could
88
+ report a total failure as success.
89
+
90
+ Shape is adjudicated as well as length, through ``_batch.usable_results`` — the same
91
+ helper and the same reason: a list of non-mappings carries no per-record outcome that
92
+ can be read, so it describes nothing rather than describing success. And a body that
93
+ says ``errors: true`` while no entry yields a readable one contradicts itself, which is
94
+ the same "cannot tell" as a short array; reading it as "no items failed" is how a total
95
+ failure came back as success one level below the length check.
96
+
97
+ An array that cannot be adjudicated is counted on ``dropped_unadjudicated`` and left
98
+ alone — like ``KinesisSink``'s, the request itself succeeded, so re-sending it would
99
+ duplicate whatever did land. An unparseable or errors-free body returns ``False``: the
100
+ request succeeded, and a body this sink cannot read is not evidence against that.
101
+ """
102
+ try:
103
+ data = json.loads(payload)
104
+ except (ValueError, TypeError):
105
+ return False
106
+ if not isinstance(data, dict) or not data.get("errors"):
107
+ return False
108
+ items = usable_results(data.get("items"))
109
+ errors = sum(1 for item in items if _has_error(item))
110
+ if len(items) != sent or not errors:
111
+ # ``not errors`` is unadjudicable rather than clean *because* the server said
112
+ # ``errors: true``: the two disagree, and this sink is the one that cannot read the
113
+ # answer. An errors-free response never reaches here — it returned above.
114
+ self.dropped_unadjudicated += sent
115
+ _diag.lost(
116
+ "event",
117
+ sent,
118
+ f"{type(self).__name__} could not adjudicate a _bulk response "
119
+ f"({sent} event(s) sent, {len(items)} readable item(s), {errors} error(s)); "
120
+ f"not retried",
121
+ )
122
+ return False
123
+ # Unconditional only because the gate above already returned on ``not errors``; a
124
+ # loosened gate would start writing "lost 0 bulk item(s)" here.
125
+ self.item_errors += errors
126
+ _diag.lost("bulk item", errors, f"{type(self).__name__}, rejected by the server")
127
+ return errors == sent
128
+
129
+
130
+ def _has_error(item: dict[str, object]) -> bool:
131
+ """Whether a ``_bulk`` item reports a failure under *any* of its action keys.
132
+
133
+ Every value is examined, not just the first: a real response carries one key per item, but
134
+ reading only ``next(iter(...))`` would score an item whose error sits under a second key as
135
+ a success — and a miscounted success is what turns a total failure into a partial one.
136
+ """
137
+ return any(isinstance(result, dict) and result.get("error") for result in item.values())
138
+
139
+
140
+ class OpenSearchSink(ElasticsearchSink):
141
+ """OpenSearch reuses the Elasticsearch ``_bulk`` protocol verbatim (FR-003)."""
@@ -14,6 +14,7 @@ import time
14
14
  from typing import Any
15
15
 
16
16
  from log_foundry import _diag
17
+ from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
17
18
 
18
19
  __all__ = ["AzureEventHubsSink"]
19
20
 
@@ -44,27 +45,52 @@ class AzureEventHubsSink:
44
45
  connection_str, eventhub_name=eventhub
45
46
  )
46
47
  self.producer = producer
47
- self.max_retries = max_retries
48
+ # Floored as ``Worker._emit`` floors its own (SPEC-021): a negative value returned
49
+ # from ``_send`` having attempted nothing, and reported success.
50
+ self.max_retries = max(max_retries, 0)
48
51
  self.failed = 0
49
52
  self.dropped_oversized = 0
50
53
 
54
+ def losses(self) -> SinkLosses:
55
+ """Oversized drops and events in a batch abandoned past the retry bound (FR-002)."""
56
+ return SinkLosses(dropped=self.dropped_oversized, failed=self.failed)
57
+
51
58
  def emit(self, batch: list[dict[str, object]]) -> None:
52
- """Pack events into ≤ 1 MB EventDataBatches and send each; drop oversized events (FR-009)."""
59
+ """Pack events into ≤ 1 MB EventDataBatches and send each; drop oversized events (FR-009).
60
+
61
+ Raises when every ``EventDataBatch`` failed to send and at least one was attempted
62
+ (SPEC-026 FR-001). An event dropped for being too large is not a send failure — it can
63
+ never fit, so a batch of nothing but oversized events has nothing to retry and does not
64
+ raise; it is reported through ``losses().dropped`` instead.
65
+ """
53
66
  if not batch:
54
67
  return
55
68
  event_data_cls = _event_data_cls()
56
69
  current = self.producer.create_batch()
70
+ attempted = delivered = 0
57
71
  for event in batch:
58
72
  data = event_data_cls(json.dumps(event).encode("utf-8"))
59
73
  if _try_add(current, data):
60
74
  continue
61
- # current batch is full: send it and start a fresh one for this event.
62
- self._send(current)
75
+ # current batch is full: send it and start a fresh one for this event. Guarded on
76
+ # emptiness because ``_try_add`` also fails against a *fresh* batch when the event
77
+ # is oversized, and the SDK short-circuits ``send_batch`` on an empty batch and
78
+ # returns — a phantom success that counted as delivery and suppressed the raise for
79
+ # everything else in the emit.
80
+ if len(current) > 0:
81
+ attempted += 1
82
+ delivered += self._send(current)
63
83
  current = self.producer.create_batch()
64
84
  if not _try_add(current, data):
65
85
  self.dropped_oversized += 1
66
86
  _diag.lost("event", 1, "AzureEventHubsSink, too large for an empty 1 MB batch")
67
- self._send(current)
87
+ if len(current) > 0:
88
+ attempted += 1
89
+ delivered += self._send(current)
90
+ if attempted and not delivered:
91
+ raise SinkDeliveryError(
92
+ f"AzureEventHubsSink sent none of {attempted} EventDataBatch(es)"
93
+ )
68
94
 
69
95
  def close(self) -> None:
70
96
  """Close the producer (FR-009)."""
@@ -72,14 +98,18 @@ class AzureEventHubsSink:
72
98
 
73
99
  # -- internals ----------------------------------------------------------------------
74
100
 
75
- def _send(self, event_batch: Any) -> None:
76
- """Send one EventDataBatch (skipping an empty one), retrying failures (FR-009, FR-011)."""
77
- if len(event_batch) == 0:
78
- return
101
+ def _send(self, event_batch: Any) -> int:
102
+ """Send one EventDataBatch, retrying failures; ``1`` if it landed (FR-009, FR-011).
103
+
104
+ Callers only reach this with a non-empty batch — both call sites check — and count the
105
+ result: the "did anything land" question in ``emit`` cannot be answered by a method that
106
+ returns nothing. An empty one must never be sent: the SDK returns immediately without
107
+ contacting the hub, which would score as a delivery nobody made.
108
+ """
79
109
  for attempt in range(self.max_retries + 1):
80
110
  try:
81
111
  self.producer.send_batch(event_batch)
82
- return
112
+ return 1
83
113
  except Exception as err: # isolation boundary: never crash the worker (FR-011)
84
114
  if attempt < self.max_retries:
85
115
  time.sleep(_BACKOFF_BASE * (2**attempt))
@@ -91,7 +121,8 @@ class AzureEventHubsSink:
91
121
  f"AzureEventHubsSink, one batch, {self.max_retries + 1} attempts, "
92
122
  f"{type(err).__name__}",
93
123
  )
94
- return
124
+ return 0
125
+ return 0 # unreachable: the loop returns on every path (mypy needs the exit)
95
126
 
96
127
 
97
128
  def _try_add(event_batch: Any, data: Any) -> bool:
@@ -10,10 +10,12 @@ from __future__ import annotations
10
10
 
11
11
  from typing import TYPE_CHECKING
12
12
 
13
+ from log_foundry.sinks.base import read_losses
14
+
13
15
  if TYPE_CHECKING:
14
16
  from collections.abc import Callable
15
17
 
16
- from log_foundry.sinks.base import Sink
18
+ from log_foundry.sinks.base import Sink, SinkLosses
17
19
 
18
20
  __all__ = ["FilteringSink"]
19
21
 
@@ -69,6 +71,20 @@ class FilteringSink:
69
71
  return False
70
72
  return True
71
73
 
74
+ def losses(self) -> SinkLosses | None:
75
+ """Report the inner sink's losses (SPEC-026 FR-002). Never raises.
76
+
77
+ A wrapper that reported nothing would hide the destination it wraps: ``health().sink``
78
+ would read ``None`` for a ``FilteringSink`` in front of a sink that counts perfectly well.
79
+ Events this sink itself declines to forward are not loss — they are the configuration
80
+ working, the same reason ``NullSink`` reports nothing.
81
+
82
+ ``None`` passes through unchanged rather than becoming ``SinkLosses(0, 0)``: FR-003
83
+ distinguishes "the sink reports nothing" from "the sink reports no loss", and a wrapper
84
+ that flattened the two would claim a clean bill of health on a sink that never gave one.
85
+ """
86
+ return read_losses(self._inner)
87
+
72
88
  def close(self) -> None:
73
89
  """Close the inner sink (FR-003)."""
74
90
  self._inner.close()