log-foundry 0.10.2.dev16__tar.gz → 0.10.2.dev18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/PKG-INFO +1 -1
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_socket.py +30 -6
- log_foundry-0.10.2.dev18/src/log_foundry/sinks/base.py +119 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/clickhouse.py +24 -7
- log_foundry-0.10.2.dev18/src/log_foundry/sinks/elasticsearch.py +141 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/eventhubs.py +42 -11
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/filtering.py +17 -1
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/firehose.py +61 -9
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/http.py +41 -14
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/kafka.py +36 -2
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/kinesis.py +60 -9
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/logstash.py +13 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/mongodb.py +30 -4
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/multi.py +27 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/nats.py +16 -1
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/postgres.py +17 -3
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/pubsub.py +31 -2
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/rabbitmq.py +24 -6
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/redis.py +17 -3
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sentry.py +64 -5
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sns.py +30 -7
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sqs.py +65 -7
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/syslog.py +13 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/transform.py +17 -1
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/util.py +4 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/worker.py +21 -1
- log_foundry-0.10.2.dev16/src/log_foundry/sinks/base.py +0 -22
- log_foundry-0.10.2.dev16/src/log_foundry/sinks/elasticsearch.py +0 -68
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/LICENSE +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/README.md +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/__init__.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/config.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/decorator.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/callback.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/datadog.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/file.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/honeycomb.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/logging_sink.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/loki.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/splunk.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/sqlite.py +0 -0
- {log_foundry-0.10.2.dev16 → log_foundry-0.10.2.dev18}/src/log_foundry/sinks/stdout.py +0 -0
|
@@ -15,6 +15,7 @@ import socket
|
|
|
15
15
|
import time
|
|
16
16
|
|
|
17
17
|
from log_foundry import _diag
|
|
18
|
+
from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
18
19
|
|
|
19
20
|
__all__ = ["SocketTransport"]
|
|
20
21
|
|
|
@@ -53,14 +54,35 @@ class SocketTransport:
|
|
|
53
54
|
self._port = port
|
|
54
55
|
self._transport = transport
|
|
55
56
|
self._timeout = timeout
|
|
56
|
-
|
|
57
|
+
# Floored for the reason ``Worker._emit`` floors its own (SPEC-021): a negative value
|
|
58
|
+
# made no attempt at all and abandoned the message without moving ``failed``.
|
|
59
|
+
self._max_retries = max(max_retries, 0)
|
|
57
60
|
self._sock: socket.socket | None = None
|
|
58
61
|
self.failed = 0
|
|
59
62
|
|
|
60
63
|
def send_all(self, messages: list[bytes]) -> None:
|
|
61
|
-
"""Send each pre-framed message, reconnecting on error (FR-005, FR-006).
|
|
64
|
+
"""Send each pre-framed message, reconnecting on error (FR-005, FR-006).
|
|
65
|
+
|
|
66
|
+
Raises :class:`~log_foundry.sinks.base.SinkDeliveryError` when **none** of the messages
|
|
67
|
+
reached the socket, so the sinks built on this transport propagate a dead destination to
|
|
68
|
+
the worker instead of reporting success (SPEC-026 FR-001). A partial send does not raise:
|
|
69
|
+
the worker's retry would re-send the messages that already landed.
|
|
70
|
+
|
|
71
|
+
An empty call is a no-op, not a total failure — ``delivered == 0`` there means there was
|
|
72
|
+
nothing to deliver.
|
|
73
|
+
"""
|
|
74
|
+
delivered = 0
|
|
62
75
|
for message in messages:
|
|
63
|
-
self._send_one(message)
|
|
76
|
+
if self._send_one(message):
|
|
77
|
+
delivered += 1
|
|
78
|
+
if messages and delivered == 0:
|
|
79
|
+
raise SinkDeliveryError(
|
|
80
|
+
f"SocketTransport delivered none of {len(messages)} message(s)"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
def losses(self) -> SinkLosses:
|
|
84
|
+
"""Messages abandoned past the reconnect-retry bound (SPEC-026 FR-002). Never raises."""
|
|
85
|
+
return SinkLosses(dropped=0, failed=self.failed)
|
|
64
86
|
|
|
65
87
|
def close(self) -> None:
|
|
66
88
|
"""Close the held socket, if any; idempotent (FR-005, FR-012)."""
|
|
@@ -68,14 +90,15 @@ class SocketTransport:
|
|
|
68
90
|
|
|
69
91
|
# -- internals ----------------------------------------------------------------------
|
|
70
92
|
|
|
71
|
-
def _send_one(self, message: bytes) ->
|
|
93
|
+
def _send_one(self, message: bytes) -> bool:
|
|
94
|
+
"""Send one message within the retry bound; ``False`` once it is abandoned."""
|
|
72
95
|
for attempt in range(self._max_retries + 1):
|
|
73
96
|
try:
|
|
74
97
|
if self._transport == "udp":
|
|
75
98
|
self._socket().sendto(message, (self._host, self._port))
|
|
76
99
|
else:
|
|
77
100
|
self._socket().sendall(message)
|
|
78
|
-
return
|
|
101
|
+
return True
|
|
79
102
|
except OSError as err:
|
|
80
103
|
self._reset() # force a fresh connection on the next attempt
|
|
81
104
|
if attempt < self._max_retries:
|
|
@@ -92,7 +115,8 @@ class SocketTransport:
|
|
|
92
115
|
f"SocketTransport, {self._max_retries + 1} attempt(s), "
|
|
93
116
|
f"{type(err).__name__} {_diag.errno_of(err)}".rstrip(),
|
|
94
117
|
)
|
|
95
|
-
return
|
|
118
|
+
return False
|
|
119
|
+
return False # unreachable: the loop returns on every path (mypy needs the exit)
|
|
96
120
|
|
|
97
121
|
def _socket(self) -> socket.socket:
|
|
98
122
|
if self._sock is None:
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""The sink interface (arch §8).
|
|
2
|
+
|
|
3
|
+
A sink is the swappable output transport. It receives *already-built, batched* event
|
|
4
|
+
dicts from the worker and knows nothing about spans or context — that dumbness is what
|
|
5
|
+
makes sinks trivially interchangeable (StdoutSink, SQSSink, …). Only the Protocol lives
|
|
6
|
+
here; concrete sinks arrive in later phases.
|
|
7
|
+
|
|
8
|
+
The interface carries two obligations beyond "put these somewhere" (SPEC-026), because the
|
|
9
|
+
library's whole loss-reporting apparatus is built on them:
|
|
10
|
+
|
|
11
|
+
* **Total failure raises.** A sink that delivered *none* of a batch must let something
|
|
12
|
+
propagate out of ``emit``. That is the signal the worker's bounded retry and
|
|
13
|
+
``health().failed_batches`` are built on, and the one case where a retry cannot create
|
|
14
|
+
duplicates — nothing landed, so there is nothing downstream to duplicate.
|
|
15
|
+
* **Partial failure does not.** A batch where some records landed must not be retried
|
|
16
|
+
wholesale; it is counted and made readable through the optional ``losses()``.
|
|
17
|
+
|
|
18
|
+
A sink that absorbs a total failure and returns normally is a sink the worker believes:
|
|
19
|
+
the retry never engages, ``failed_batches`` stays at zero and ``flush()`` returns ``True``
|
|
20
|
+
while every event is lost. That reading is precisely what SPEC-017 existed to make
|
|
21
|
+
impossible, and what SPEC-026 generalized from ``MultiSink`` to the whole family.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from typing import NamedTuple, Protocol, runtime_checkable
|
|
27
|
+
|
|
28
|
+
__all__ = ["Sink", "SinkDeliveryError", "SinkLosses", "read_losses"]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SinkDeliveryError(Exception):
|
|
32
|
+
"""Raised by a sink whose ``emit`` delivered none of the batch (SPEC-026 FR-001).
|
|
33
|
+
|
|
34
|
+
A distinct type so an operator reading a ``stopped_reason`` or a diagnostic line can tell
|
|
35
|
+
"the destination refused everything" from "the sink itself has a bug". Sinks that already
|
|
36
|
+
have a natural exception to re-raise — a driver error, a ``MultiSink`` child's — re-raise
|
|
37
|
+
that instead: the contract is that *something* propagates, not that it must be this type.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SinkLosses(NamedTuple):
|
|
42
|
+
"""What a sink discarded or could not confirm, cumulative for its lifetime (SPEC-026 FR-002).
|
|
43
|
+
|
|
44
|
+
Two fields rather than one because the remedies differ: ``dropped`` means the destination
|
|
45
|
+
could never have accepted the event as built (an oversized record), so the fix is upstream in
|
|
46
|
+
what the application logs; ``failed`` means delivery was attempted and the destination did not
|
|
47
|
+
confirm it, so the fix is the destination or the network.
|
|
48
|
+
|
|
49
|
+
``failed`` is an **upper bound** on loss, not a count of it. A sink that also raises on total
|
|
50
|
+
failure counts the attempt here *and* hands the batch back to the worker, whose retry may
|
|
51
|
+
then deliver it — so a transient outage leaves ``failed`` non-zero with nothing actually
|
|
52
|
+
lost. ``health().failed_batches`` is the worker-level record of a batch given up on for good;
|
|
53
|
+
this is the sink-level record of everything that did not go through first time.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
dropped: int
|
|
57
|
+
failed: int
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@runtime_checkable
|
|
61
|
+
class Sink(Protocol):
|
|
62
|
+
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
63
|
+
"""Ship a batch of serialized event dicts.
|
|
64
|
+
|
|
65
|
+
**Raise when the batch delivered nothing** and it was non-empty — the worker's bounded
|
|
66
|
+
retry and ``health().failed_batches`` depend on that signal, and a retry there cannot
|
|
67
|
+
duplicate anything (SPEC-026 FR-001). Raise *after* the sink's own retries are spent, so
|
|
68
|
+
the worker's retry composes on top rather than replacing it; a sink whose own budget
|
|
69
|
+
makes the worker's redundant should say so in its docstring rather than absorb silently.
|
|
70
|
+
|
|
71
|
+
**Do not raise on partial failure.** A batch where some records landed would be
|
|
72
|
+
re-delivered wholesale by the worker's retry, and duplicates downstream are worse than
|
|
73
|
+
the counted loss (SPEC-017 FR-004, SPEC-018). Report that through ``losses()`` instead.
|
|
74
|
+
|
|
75
|
+
``emit([])`` is a no-op and never raises: an empty batch has not failed to deliver.
|
|
76
|
+
"""
|
|
77
|
+
...
|
|
78
|
+
|
|
79
|
+
def close(self) -> None:
|
|
80
|
+
"""Flush and release any resources."""
|
|
81
|
+
...
|
|
82
|
+
|
|
83
|
+
# ``losses()`` is optional, and deliberately *not* declared here: ``Sink`` is structural, and
|
|
84
|
+
# a third-party sink written against the pre-SPEC-026 interface must keep satisfying it.
|
|
85
|
+
# ``read_losses`` below is the probe; an absent method reads as "reports nothing".
|
|
86
|
+
#
|
|
87
|
+
# def losses(self) -> SinkLosses | None:
|
|
88
|
+
# """Cumulative loss this sink absorbed. Never raises; safe to call during emit.
|
|
89
|
+
#
|
|
90
|
+
# ``None`` is the same answer as having no method at all — "this sink reports
|
|
91
|
+
# nothing". A wrapper returns it when what it wraps reports nothing.
|
|
92
|
+
# """
|
|
93
|
+
# return SinkLosses(dropped=self._dropped, failed=self._failed)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def read_losses(sink: object) -> SinkLosses | None:
|
|
97
|
+
"""Read a sink's optional ``losses()``; ``None`` when it reports nothing.
|
|
98
|
+
|
|
99
|
+
``None`` covers four cases deliberately treated alike: no ``losses`` attribute, one that is
|
|
100
|
+
not callable, one that raised, and one that returned anything other than a ``SinkLosses`` —
|
|
101
|
+
including ``None`` itself, which the shipped wrapper sinks return when what they wrap
|
|
102
|
+
reports nothing.
|
|
103
|
+
|
|
104
|
+
The single reader for the optional half of the protocol (SPEC-026 FR-002), shared by
|
|
105
|
+
``Worker.health`` and ``MultiSink.losses`` so the probe and its guarantees are written once.
|
|
106
|
+
|
|
107
|
+
Total by design. ``health()`` is documented "Never raises" and is the call an operator makes
|
|
108
|
+
when things are *already* going wrong, so a third-party sink with a broken accessor must not
|
|
109
|
+
be able to take the snapshot down with it. The returned shape is checked as well as the call:
|
|
110
|
+
a ``losses()`` returning something else would otherwise put an arbitrary object where callers
|
|
111
|
+
read two integers. Nothing is written to stderr — a poll happens as often as the caller likes,
|
|
112
|
+
and a broken accessor is a sink bug rather than a loss.
|
|
113
|
+
"""
|
|
114
|
+
try:
|
|
115
|
+
accessor = getattr(sink, "losses", None)
|
|
116
|
+
losses = accessor() if callable(accessor) else None
|
|
117
|
+
except Exception:
|
|
118
|
+
return None
|
|
119
|
+
return losses if isinstance(losses, SinkLosses) else None
|
|
@@ -14,6 +14,7 @@ from typing import Any
|
|
|
14
14
|
|
|
15
15
|
from log_foundry import _diag
|
|
16
16
|
from log_foundry.sinks._chunk import chunk_list, valid_identifier
|
|
17
|
+
from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
17
18
|
|
|
18
19
|
__all__ = ["ClickHouseSink"]
|
|
19
20
|
|
|
@@ -54,7 +55,9 @@ class ClickHouseSink:
|
|
|
54
55
|
) -> None:
|
|
55
56
|
self._table = valid_identifier(table)
|
|
56
57
|
self._chunk_size = chunk_size
|
|
57
|
-
|
|
58
|
+
# Floored as ``Worker._emit`` floors its own (SPEC-021): a negative value returned
|
|
59
|
+
# from ``_insert`` having attempted nothing, and reported success.
|
|
60
|
+
self.max_retries = max(max_retries, 0)
|
|
58
61
|
self.failed = 0
|
|
59
62
|
self._closed = False
|
|
60
63
|
self._owns_client = client is None
|
|
@@ -66,13 +69,25 @@ class ClickHouseSink:
|
|
|
66
69
|
if create_table:
|
|
67
70
|
self._ensure_schema()
|
|
68
71
|
|
|
72
|
+
def losses(self) -> SinkLosses:
|
|
73
|
+
"""Rows in a chunk abandoned past the retry bound (SPEC-026 FR-002). Never raises."""
|
|
74
|
+
return SinkLosses(dropped=0, failed=self.failed)
|
|
75
|
+
|
|
69
76
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
70
|
-
"""Insert each chunk as one columnar ``insert`` call, retrying on failure (FR-002).
|
|
77
|
+
"""Insert each chunk as one columnar ``insert`` call, retrying on failure (FR-002).
|
|
78
|
+
|
|
79
|
+
Raises when every chunk failed (SPEC-026 FR-001) — which is the whole batch for any
|
|
80
|
+
batch that fits one chunk, the ordinary case. A partially-inserted batch does not raise:
|
|
81
|
+
the chunks that landed are committed, and the worker's retry would duplicate them.
|
|
82
|
+
"""
|
|
71
83
|
if not batch:
|
|
72
84
|
return
|
|
85
|
+
chunks = inserted = 0
|
|
73
86
|
for chunk in chunk_list(batch, self._chunk_size):
|
|
74
|
-
|
|
75
|
-
self._insert(
|
|
87
|
+
chunks += 1
|
|
88
|
+
inserted += self._insert([self._row(event) for event in chunk])
|
|
89
|
+
if chunks and not inserted:
|
|
90
|
+
raise SinkDeliveryError(f"ClickHouseSink inserted none of {chunks} chunk(s)")
|
|
76
91
|
|
|
77
92
|
def close(self) -> None:
|
|
78
93
|
"""Close the client only if the sink owns it; idempotent (FR-005)."""
|
|
@@ -87,11 +102,12 @@ class ClickHouseSink:
|
|
|
87
102
|
def _row(self, event: dict[str, object]) -> list[object]:
|
|
88
103
|
return [*(event.get(col) for col in _COLUMNS), json.dumps(event)]
|
|
89
104
|
|
|
90
|
-
def _insert(self, rows: list[list[object]]) ->
|
|
105
|
+
def _insert(self, rows: list[list[object]]) -> int:
|
|
106
|
+
"""Insert one chunk within the retry bound; ``1`` if it landed, ``0`` once abandoned."""
|
|
91
107
|
for attempt in range(self.max_retries + 1):
|
|
92
108
|
try:
|
|
93
109
|
self.client.insert(self._table, data=rows, column_names=_COLUMN_NAMES)
|
|
94
|
-
return
|
|
110
|
+
return 1
|
|
95
111
|
except Exception as err: # isolation boundary: never crash the worker (FR-006)
|
|
96
112
|
if attempt < self.max_retries:
|
|
97
113
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
@@ -102,7 +118,8 @@ class ClickHouseSink:
|
|
|
102
118
|
len(rows),
|
|
103
119
|
f"ClickHouseSink, {self.max_retries + 1} attempts, {type(err).__name__}",
|
|
104
120
|
)
|
|
105
|
-
return
|
|
121
|
+
return 0
|
|
122
|
+
return 0 # unreachable: the loop returns on every path (mypy needs the exit)
|
|
106
123
|
|
|
107
124
|
def _ensure_schema(self) -> None:
|
|
108
125
|
columns = ", ".join(f"{col} {_COLUMN_TYPES[col]}" for col in _COLUMNS)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""ElasticsearchSink / OpenSearchSink — index events via the ``_bulk`` API (arch §8, SPEC-009).
|
|
2
|
+
|
|
3
|
+
A thin specialization of :class:`~log_foundry.sinks.http.HTTPSink`: each event becomes an action line
|
|
4
|
+
(``{"index": {"_index": target}}``) followed by its source line, POSTed to ``_bulk`` as newline-
|
|
5
|
+
delimited JSON. The bulk response is inspected per item so a partial failure is counted and logged
|
|
6
|
+
without discarding the successfully-indexed items. OpenSearch speaks the same bulk protocol, so
|
|
7
|
+
``OpenSearchSink`` is a straight reuse (endpoint/auth differ only by configuration).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
|
|
14
|
+
from log_foundry import _diag
|
|
15
|
+
from log_foundry.sinks._batch import usable_results
|
|
16
|
+
from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
17
|
+
from log_foundry.sinks.http import HTTPSink
|
|
18
|
+
|
|
19
|
+
__all__ = ["ElasticsearchSink", "OpenSearchSink"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ElasticsearchSink(HTTPSink):
|
|
23
|
+
"""POST events to an Elasticsearch ``_bulk`` endpoint, parsing per-item errors (FR-003).
|
|
24
|
+
|
|
25
|
+
Attributes:
|
|
26
|
+
item_errors: Count of bulk items the server reported as failed (distinct from ``failed``,
|
|
27
|
+
which counts whole requests abandoned past the retry bound).
|
|
28
|
+
dropped_unadjudicated: Events whose outcome a ``_bulk`` response did not describe — its
|
|
29
|
+
``items`` array did not line up with the batch sent, so no event in it could be
|
|
30
|
+
paired to an outcome. Abandoned rather than retried, for SPEC-018's reason: the
|
|
31
|
+
request succeeded, so re-sending would duplicate whatever landed.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, url: str, *, index: str, auth: str | tuple[str, str] | None = None,
|
|
35
|
+
**http_kwargs: object) -> None:
|
|
36
|
+
self._index = index
|
|
37
|
+
super().__init__(
|
|
38
|
+
url.rstrip("/") + "/_bulk", auth=auth, body_format="ndjson", **http_kwargs # type: ignore[arg-type]
|
|
39
|
+
)
|
|
40
|
+
self.item_errors = 0
|
|
41
|
+
self.dropped_unadjudicated = 0
|
|
42
|
+
|
|
43
|
+
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
44
|
+
"""Build the ``_bulk`` NDJSON payload, POST it, and parse the response items (FR-003)."""
|
|
45
|
+
if not batch:
|
|
46
|
+
return
|
|
47
|
+
lines: list[str] = []
|
|
48
|
+
for event in batch:
|
|
49
|
+
lines.append(json.dumps({"index": {"_index": self._index}}))
|
|
50
|
+
lines.append(json.dumps(event))
|
|
51
|
+
body = ("\n".join(lines) + "\n").encode("utf-8") # bulk must be newline-terminated
|
|
52
|
+
# An abandoned request raises out of ``_send`` now (SPEC-026 FR-001) — nothing was
|
|
53
|
+
# indexed, so there is no response to parse and nothing downstream to duplicate.
|
|
54
|
+
payload = self._send(body, content_type="application/x-ndjson")
|
|
55
|
+
if self._parse_bulk_response(payload, len(batch)):
|
|
56
|
+
# Every item carried an error, so the 200 indexed nothing: a total failure like any
|
|
57
|
+
# other, and it must reach the worker (FR-001). A retry cannot duplicate, and where
|
|
58
|
+
# the cause is permanent (a mapping conflict) the worker abandons the batch after
|
|
59
|
+
# its bound and records it — which beats a silent success. A response rejecting
|
|
60
|
+
# *some* items stays partial and is reported through ``losses()``.
|
|
61
|
+
raise SinkDeliveryError(
|
|
62
|
+
f"{type(self).__name__} indexed none of {len(batch)} event(s)"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def losses(self) -> SinkLosses:
|
|
66
|
+
"""Abandoned requests plus server-rejected bulk items (SPEC-026 FR-002). Never raises.
|
|
67
|
+
|
|
68
|
+
All three are summed into ``failed`` because each is an event the server did not
|
|
69
|
+
confirm; they stay apart on the instance (``failed`` / ``item_errors`` /
|
|
70
|
+
``dropped_unadjudicated``) for anyone who needs to tell "the request never landed" from
|
|
71
|
+
"the request landed and these items bounced" from "the response did not say".
|
|
72
|
+
"""
|
|
73
|
+
return SinkLosses(
|
|
74
|
+
dropped=self.dropped_oversized,
|
|
75
|
+
failed=self.failed + self.item_errors + self.dropped_unadjudicated,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def _parse_bulk_response(self, payload: bytes, sent: int) -> bool:
|
|
79
|
+
"""Count rejected items; return whether the response proves *nothing* was indexed.
|
|
80
|
+
|
|
81
|
+
The ``items`` array is positional — entry *i* describes event *i* — so the rule
|
|
82
|
+
``sinks/_batch.py`` states applies here too (SPEC-018): a length disagreement is
|
|
83
|
+
evidence the arrays do not describe each other, not an invitation to read the overlap.
|
|
84
|
+
Counting errors does not depend on position, but concluding "all of them failed" does,
|
|
85
|
+
so that conclusion is gated on ``len(items) == sent``. Without the gate both directions
|
|
86
|
+
are wrong and both are reachable: a longer ``items`` array could make a partial success
|
|
87
|
+
raise (and the worker's retry would duplicate what landed), and a shorter one could
|
|
88
|
+
report a total failure as success.
|
|
89
|
+
|
|
90
|
+
Shape is adjudicated as well as length, through ``_batch.usable_results`` — the same
|
|
91
|
+
helper and the same reason: a list of non-mappings carries no per-record outcome that
|
|
92
|
+
can be read, so it describes nothing rather than describing success. And a body that
|
|
93
|
+
says ``errors: true`` while no entry yields a readable one contradicts itself, which is
|
|
94
|
+
the same "cannot tell" as a short array; reading it as "no items failed" is how a total
|
|
95
|
+
failure came back as success one level below the length check.
|
|
96
|
+
|
|
97
|
+
An array that cannot be adjudicated is counted on ``dropped_unadjudicated`` and left
|
|
98
|
+
alone — like ``KinesisSink``'s, the request itself succeeded, so re-sending it would
|
|
99
|
+
duplicate whatever did land. An unparseable or errors-free body returns ``False``: the
|
|
100
|
+
request succeeded, and a body this sink cannot read is not evidence against that.
|
|
101
|
+
"""
|
|
102
|
+
try:
|
|
103
|
+
data = json.loads(payload)
|
|
104
|
+
except (ValueError, TypeError):
|
|
105
|
+
return False
|
|
106
|
+
if not isinstance(data, dict) or not data.get("errors"):
|
|
107
|
+
return False
|
|
108
|
+
items = usable_results(data.get("items"))
|
|
109
|
+
errors = sum(1 for item in items if _has_error(item))
|
|
110
|
+
if len(items) != sent or not errors:
|
|
111
|
+
# ``not errors`` is unadjudicable rather than clean *because* the server said
|
|
112
|
+
# ``errors: true``: the two disagree, and this sink is the one that cannot read the
|
|
113
|
+
# answer. An errors-free response never reaches here — it returned above.
|
|
114
|
+
self.dropped_unadjudicated += sent
|
|
115
|
+
_diag.lost(
|
|
116
|
+
"event",
|
|
117
|
+
sent,
|
|
118
|
+
f"{type(self).__name__} could not adjudicate a _bulk response "
|
|
119
|
+
f"({sent} event(s) sent, {len(items)} readable item(s), {errors} error(s)); "
|
|
120
|
+
f"not retried",
|
|
121
|
+
)
|
|
122
|
+
return False
|
|
123
|
+
# Unconditional only because the gate above already returned on ``not errors``; a
|
|
124
|
+
# loosened gate would start writing "lost 0 bulk item(s)" here.
|
|
125
|
+
self.item_errors += errors
|
|
126
|
+
_diag.lost("bulk item", errors, f"{type(self).__name__}, rejected by the server")
|
|
127
|
+
return errors == sent
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _has_error(item: dict[str, object]) -> bool:
|
|
131
|
+
"""Whether a ``_bulk`` item reports a failure under *any* of its action keys.
|
|
132
|
+
|
|
133
|
+
Every value is examined, not just the first: a real response carries one key per item, but
|
|
134
|
+
reading only ``next(iter(...))`` would score an item whose error sits under a second key as
|
|
135
|
+
a success — and a miscounted success is what turns a total failure into a partial one.
|
|
136
|
+
"""
|
|
137
|
+
return any(isinstance(result, dict) and result.get("error") for result in item.values())
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class OpenSearchSink(ElasticsearchSink):
|
|
141
|
+
"""OpenSearch reuses the Elasticsearch ``_bulk`` protocol verbatim (FR-003)."""
|
|
@@ -14,6 +14,7 @@ import time
|
|
|
14
14
|
from typing import Any
|
|
15
15
|
|
|
16
16
|
from log_foundry import _diag
|
|
17
|
+
from log_foundry.sinks.base import SinkDeliveryError, SinkLosses
|
|
17
18
|
|
|
18
19
|
__all__ = ["AzureEventHubsSink"]
|
|
19
20
|
|
|
@@ -44,27 +45,52 @@ class AzureEventHubsSink:
|
|
|
44
45
|
connection_str, eventhub_name=eventhub
|
|
45
46
|
)
|
|
46
47
|
self.producer = producer
|
|
47
|
-
|
|
48
|
+
# Floored as ``Worker._emit`` floors its own (SPEC-021): a negative value returned
|
|
49
|
+
# from ``_send`` having attempted nothing, and reported success.
|
|
50
|
+
self.max_retries = max(max_retries, 0)
|
|
48
51
|
self.failed = 0
|
|
49
52
|
self.dropped_oversized = 0
|
|
50
53
|
|
|
54
|
+
def losses(self) -> SinkLosses:
|
|
55
|
+
"""Oversized drops and events in a batch abandoned past the retry bound (FR-002)."""
|
|
56
|
+
return SinkLosses(dropped=self.dropped_oversized, failed=self.failed)
|
|
57
|
+
|
|
51
58
|
def emit(self, batch: list[dict[str, object]]) -> None:
|
|
52
|
-
"""Pack events into ≤ 1 MB EventDataBatches and send each; drop oversized events (FR-009).
|
|
59
|
+
"""Pack events into ≤ 1 MB EventDataBatches and send each; drop oversized events (FR-009).
|
|
60
|
+
|
|
61
|
+
Raises when every ``EventDataBatch`` failed to send and at least one was attempted
|
|
62
|
+
(SPEC-026 FR-001). An event dropped for being too large is not a send failure — it can
|
|
63
|
+
never fit, so a batch of nothing but oversized events has nothing to retry and does not
|
|
64
|
+
raise; it is reported through ``losses().dropped`` instead.
|
|
65
|
+
"""
|
|
53
66
|
if not batch:
|
|
54
67
|
return
|
|
55
68
|
event_data_cls = _event_data_cls()
|
|
56
69
|
current = self.producer.create_batch()
|
|
70
|
+
attempted = delivered = 0
|
|
57
71
|
for event in batch:
|
|
58
72
|
data = event_data_cls(json.dumps(event).encode("utf-8"))
|
|
59
73
|
if _try_add(current, data):
|
|
60
74
|
continue
|
|
61
|
-
# current batch is full: send it and start a fresh one for this event.
|
|
62
|
-
|
|
75
|
+
# current batch is full: send it and start a fresh one for this event. Guarded on
|
|
76
|
+
# emptiness because ``_try_add`` also fails against a *fresh* batch when the event
|
|
77
|
+
# is oversized, and the SDK short-circuits ``send_batch`` on an empty batch and
|
|
78
|
+
# returns — a phantom success that counted as delivery and suppressed the raise for
|
|
79
|
+
# everything else in the emit.
|
|
80
|
+
if len(current) > 0:
|
|
81
|
+
attempted += 1
|
|
82
|
+
delivered += self._send(current)
|
|
63
83
|
current = self.producer.create_batch()
|
|
64
84
|
if not _try_add(current, data):
|
|
65
85
|
self.dropped_oversized += 1
|
|
66
86
|
_diag.lost("event", 1, "AzureEventHubsSink, too large for an empty 1 MB batch")
|
|
67
|
-
|
|
87
|
+
if len(current) > 0:
|
|
88
|
+
attempted += 1
|
|
89
|
+
delivered += self._send(current)
|
|
90
|
+
if attempted and not delivered:
|
|
91
|
+
raise SinkDeliveryError(
|
|
92
|
+
f"AzureEventHubsSink sent none of {attempted} EventDataBatch(es)"
|
|
93
|
+
)
|
|
68
94
|
|
|
69
95
|
def close(self) -> None:
|
|
70
96
|
"""Close the producer (FR-009)."""
|
|
@@ -72,14 +98,18 @@ class AzureEventHubsSink:
|
|
|
72
98
|
|
|
73
99
|
# -- internals ----------------------------------------------------------------------
|
|
74
100
|
|
|
75
|
-
def _send(self, event_batch: Any) ->
|
|
76
|
-
"""Send one EventDataBatch
|
|
77
|
-
|
|
78
|
-
|
|
101
|
+
def _send(self, event_batch: Any) -> int:
|
|
102
|
+
"""Send one EventDataBatch, retrying failures; ``1`` if it landed (FR-009, FR-011).
|
|
103
|
+
|
|
104
|
+
Callers only reach this with a non-empty batch — both call sites check — and count the
|
|
105
|
+
result: the "did anything land" question in ``emit`` cannot be answered by a method that
|
|
106
|
+
returns nothing. An empty one must never be sent: the SDK returns immediately without
|
|
107
|
+
contacting the hub, which would score as a delivery nobody made.
|
|
108
|
+
"""
|
|
79
109
|
for attempt in range(self.max_retries + 1):
|
|
80
110
|
try:
|
|
81
111
|
self.producer.send_batch(event_batch)
|
|
82
|
-
return
|
|
112
|
+
return 1
|
|
83
113
|
except Exception as err: # isolation boundary: never crash the worker (FR-011)
|
|
84
114
|
if attempt < self.max_retries:
|
|
85
115
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
@@ -91,7 +121,8 @@ class AzureEventHubsSink:
|
|
|
91
121
|
f"AzureEventHubsSink, one batch, {self.max_retries + 1} attempts, "
|
|
92
122
|
f"{type(err).__name__}",
|
|
93
123
|
)
|
|
94
|
-
return
|
|
124
|
+
return 0
|
|
125
|
+
return 0 # unreachable: the loop returns on every path (mypy needs the exit)
|
|
95
126
|
|
|
96
127
|
|
|
97
128
|
def _try_add(event_batch: Any, data: Any) -> bool:
|
|
@@ -10,10 +10,12 @@ from __future__ import annotations
|
|
|
10
10
|
|
|
11
11
|
from typing import TYPE_CHECKING
|
|
12
12
|
|
|
13
|
+
from log_foundry.sinks.base import read_losses
|
|
14
|
+
|
|
13
15
|
if TYPE_CHECKING:
|
|
14
16
|
from collections.abc import Callable
|
|
15
17
|
|
|
16
|
-
from log_foundry.sinks.base import Sink
|
|
18
|
+
from log_foundry.sinks.base import Sink, SinkLosses
|
|
17
19
|
|
|
18
20
|
__all__ = ["FilteringSink"]
|
|
19
21
|
|
|
@@ -69,6 +71,20 @@ class FilteringSink:
|
|
|
69
71
|
return False
|
|
70
72
|
return True
|
|
71
73
|
|
|
74
|
+
def losses(self) -> SinkLosses | None:
|
|
75
|
+
"""Report the inner sink's losses (SPEC-026 FR-002). Never raises.
|
|
76
|
+
|
|
77
|
+
A wrapper that reported nothing would hide the destination it wraps: ``health().sink``
|
|
78
|
+
would read ``None`` for a ``FilteringSink`` in front of a sink that counts perfectly well.
|
|
79
|
+
Events this sink itself declines to forward are not loss — they are the configuration
|
|
80
|
+
working, the same reason ``NullSink`` reports nothing.
|
|
81
|
+
|
|
82
|
+
``None`` passes through unchanged rather than becoming ``SinkLosses(0, 0)``: FR-003
|
|
83
|
+
distinguishes "the sink reports nothing" from "the sink reports no loss", and a wrapper
|
|
84
|
+
that flattened the two would claim a clean bill of health on a sink that never gave one.
|
|
85
|
+
"""
|
|
86
|
+
return read_losses(self._inner)
|
|
87
|
+
|
|
72
88
|
def close(self) -> None:
|
|
73
89
|
"""Close the inner sink (FR-003)."""
|
|
74
90
|
self._inner.close()
|