log-foundry 0.10.2.dev13__tar.gz → 0.10.2.dev14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/PKG-INFO +1 -1
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/pyproject.toml +1 -1
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/_socket.py +11 -4
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/clickhouse.py +5 -4
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/elasticsearch.py +2 -2
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/eventhubs.py +8 -8
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/firehose.py +15 -10
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/http.py +14 -6
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/kafka.py +31 -3
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/kinesis.py +15 -11
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/mongodb.py +9 -10
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/multi.py +7 -8
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/nats.py +3 -2
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/postgres.py +9 -4
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/pubsub.py +3 -2
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/rabbitmq.py +6 -4
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/redis.py +6 -4
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/sns.py +6 -8
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/sqs.py +18 -10
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/LICENSE +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/README.md +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/__init__.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/_diag.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/api.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/config.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/console.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/context.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/decorator.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/ids.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/model.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/py.typed +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sanitize.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/__init__.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/_batch.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/_chunk.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/_time.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/base.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/callback.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/datadog.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/file.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/filtering.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/honeycomb.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/logging_sink.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/logstash.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/loki.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/newrelic.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/sentry.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/splunk.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/sqlite.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/stdout.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/syslog.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/transform.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/util.py +0 -0
- {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/worker.py +0 -0
|
@@ -12,9 +12,10 @@ substitute a fake socket without any network access.
|
|
|
12
12
|
from __future__ import annotations
|
|
13
13
|
|
|
14
14
|
import socket
|
|
15
|
-
import sys
|
|
16
15
|
import time
|
|
17
16
|
|
|
17
|
+
from log_foundry import _diag
|
|
18
|
+
|
|
18
19
|
__all__ = ["SocketTransport"]
|
|
19
20
|
|
|
20
21
|
_BACKOFF_BASE = 0.1 # seconds; delay for retry attempt n is _BACKOFF_BASE * 2**n
|
|
@@ -81,9 +82,15 @@ class SocketTransport:
|
|
|
81
82
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
82
83
|
continue
|
|
83
84
|
self.failed += 1
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
85
|
+
# Guarded now (SPEC-029 FR-003): this runs on the worker thread, and the bare
|
|
86
|
+
# ``sys.stderr.write`` this replaced could end delivery for good. ``errno``
|
|
87
|
+
# because an ``OSError`` type name alone does not tell "connection refused" from
|
|
88
|
+
# "host unknown", and the code is an integer from the OS — not caller data.
|
|
89
|
+
_diag.lost(
|
|
90
|
+
"message",
|
|
91
|
+
1,
|
|
92
|
+
f"SocketTransport, {self._max_retries + 1} attempt(s), "
|
|
93
|
+
f"{type(err).__name__} {_diag.errno_of(err)}".rstrip(),
|
|
87
94
|
)
|
|
88
95
|
return
|
|
89
96
|
|
|
@@ -9,10 +9,10 @@ idempotent ``MergeTree`` ``create_table`` convenience is off by default. Write-o
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
-
import sys
|
|
13
12
|
import time
|
|
14
13
|
from typing import Any
|
|
15
14
|
|
|
15
|
+
from log_foundry import _diag
|
|
16
16
|
from log_foundry.sinks._chunk import chunk_list, valid_identifier
|
|
17
17
|
|
|
18
18
|
__all__ = ["ClickHouseSink"]
|
|
@@ -97,9 +97,10 @@ class ClickHouseSink:
|
|
|
97
97
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
98
98
|
continue
|
|
99
99
|
self.failed += len(rows)
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
_diag.lost(
|
|
101
|
+
"row",
|
|
102
|
+
len(rows),
|
|
103
|
+
f"ClickHouseSink, {self.max_retries + 1} attempts, {type(err).__name__}",
|
|
103
104
|
)
|
|
104
105
|
return
|
|
105
106
|
|
{log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev14}/src/log_foundry/sinks/elasticsearch.py
RENAMED
|
@@ -10,8 +10,8 @@ without discarding the successfully-indexed items. OpenSearch speaks the same bu
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
|
-
import sys
|
|
14
13
|
|
|
14
|
+
from log_foundry import _diag
|
|
15
15
|
from log_foundry.sinks.http import HTTPSink
|
|
16
16
|
|
|
17
17
|
__all__ = ["ElasticsearchSink", "OpenSearchSink"]
|
|
@@ -61,7 +61,7 @@ class ElasticsearchSink(HTTPSink):
|
|
|
61
61
|
errors += 1
|
|
62
62
|
if errors:
|
|
63
63
|
self.item_errors += errors
|
|
64
|
-
|
|
64
|
+
_diag.lost("bulk item", errors, f"{type(self).__name__}, rejected by the server")
|
|
65
65
|
|
|
66
66
|
|
|
67
67
|
class OpenSearchSink(ElasticsearchSink):
|
|
@@ -10,10 +10,11 @@ send errors are retried within a bound. ``close()`` closes the producer.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
|
-
import sys
|
|
14
13
|
import time
|
|
15
14
|
from typing import Any
|
|
16
15
|
|
|
16
|
+
from log_foundry import _diag
|
|
17
|
+
|
|
17
18
|
__all__ = ["AzureEventHubsSink"]
|
|
18
19
|
|
|
19
20
|
_BACKOFF_BASE = 0.1
|
|
@@ -62,10 +63,7 @@ class AzureEventHubsSink:
|
|
|
62
63
|
current = self.producer.create_batch()
|
|
63
64
|
if not _try_add(current, data):
|
|
64
65
|
self.dropped_oversized += 1
|
|
65
|
-
|
|
66
|
-
"log-foundry: AzureEventHubsSink dropped an event too large for an "
|
|
67
|
-
"empty 1 MB batch\n"
|
|
68
|
-
)
|
|
66
|
+
_diag.lost("event", 1, "AzureEventHubsSink, too large for an empty 1 MB batch")
|
|
69
67
|
self._send(current)
|
|
70
68
|
|
|
71
69
|
def close(self) -> None:
|
|
@@ -87,9 +85,11 @@ class AzureEventHubsSink:
|
|
|
87
85
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
88
86
|
continue
|
|
89
87
|
self.failed += len(event_batch)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
88
|
+
_diag.lost(
|
|
89
|
+
"event",
|
|
90
|
+
len(event_batch),
|
|
91
|
+
f"AzureEventHubsSink, one batch, {self.max_retries + 1} attempts, "
|
|
92
|
+
f"{type(err).__name__}",
|
|
93
93
|
)
|
|
94
94
|
return
|
|
95
95
|
|
|
@@ -9,9 +9,9 @@ retried within a bounded count.
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
-
import sys
|
|
13
12
|
from typing import Any
|
|
14
13
|
|
|
14
|
+
from log_foundry import _diag
|
|
15
15
|
from log_foundry.sinks._batch import adjudicate_positional, usable_results
|
|
16
16
|
from log_foundry.sinks._chunk import chunk_items
|
|
17
17
|
|
|
@@ -69,9 +69,11 @@ class FirehoseSink:
|
|
|
69
69
|
data = json.dumps(event).encode("utf-8")
|
|
70
70
|
if len(data) > self.MAX_RECORD_BYTES:
|
|
71
71
|
self.dropped_oversized += 1
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
_diag.lost(
|
|
73
|
+
"event",
|
|
74
|
+
1,
|
|
75
|
+
f"FirehoseSink, {len(data)} bytes exceeds the "
|
|
76
|
+
f"{self.MAX_RECORD_BYTES}-byte per-record limit",
|
|
75
77
|
)
|
|
76
78
|
continue
|
|
77
79
|
records.append({"Data": data})
|
|
@@ -89,10 +91,12 @@ class FirehoseSink:
|
|
|
89
91
|
verdict = adjudicate_positional(records, results)
|
|
90
92
|
if verdict.unadjudicated:
|
|
91
93
|
self.dropped_unadjudicated += verdict.unadjudicated
|
|
92
|
-
|
|
93
|
-
|
|
94
|
+
_diag.lost(
|
|
95
|
+
"record",
|
|
96
|
+
verdict.unadjudicated,
|
|
97
|
+
f"FirehoseSink could not adjudicate a put_record_batch response "
|
|
94
98
|
f"({len(records)} record(s) sent, {len(results)} result(s) returned); "
|
|
95
|
-
f"
|
|
99
|
+
f"abandoned, not retried",
|
|
96
100
|
)
|
|
97
101
|
return
|
|
98
102
|
records = verdict.retry
|
|
@@ -100,8 +104,9 @@ class FirehoseSink:
|
|
|
100
104
|
return
|
|
101
105
|
if attempt >= self.max_retries:
|
|
102
106
|
self.failed += len(records)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
107
|
+
_diag.lost(
|
|
108
|
+
"record",
|
|
109
|
+
len(records),
|
|
110
|
+
f"FirehoseSink, still failing after {self.max_retries + 1} attempts; abandoned",
|
|
106
111
|
)
|
|
107
112
|
return
|
|
@@ -15,13 +15,14 @@ from __future__ import annotations
|
|
|
15
15
|
|
|
16
16
|
import gzip as _gzip
|
|
17
17
|
import json
|
|
18
|
-
import sys
|
|
19
18
|
import time
|
|
20
19
|
import urllib.error
|
|
21
20
|
import urllib.request
|
|
22
21
|
from base64 import b64encode
|
|
23
22
|
from typing import TYPE_CHECKING, Any
|
|
24
23
|
|
|
24
|
+
from log_foundry import _diag
|
|
25
|
+
|
|
25
26
|
if TYPE_CHECKING:
|
|
26
27
|
from collections.abc import Callable
|
|
27
28
|
|
|
@@ -128,7 +129,11 @@ class HTTPSink:
|
|
|
128
129
|
if attempt < self.max_retries:
|
|
129
130
|
self._sleep_backoff(attempt, None)
|
|
130
131
|
continue
|
|
131
|
-
|
|
132
|
+
# The type plus the OS code, never the message: a ``URLError``'s text embeds the
|
|
133
|
+
# URL and whatever the resolver said (SPEC-029 FR-002).
|
|
134
|
+
self._abandon(
|
|
135
|
+
f"connection error, {type(err).__name__} {_diag.errno_of(err)}".rstrip()
|
|
136
|
+
)
|
|
132
137
|
return None
|
|
133
138
|
if 200 <= status < 300:
|
|
134
139
|
return payload
|
|
@@ -198,11 +203,14 @@ class HTTPSink:
|
|
|
198
203
|
time.sleep(delay)
|
|
199
204
|
|
|
200
205
|
def _abandon(self, reason: str) -> None:
|
|
201
|
-
"""Count and log a request abandoned past the retry bound (FR-012).
|
|
206
|
+
"""Count and log a request abandoned past the retry bound (FR-012).
|
|
207
|
+
|
|
208
|
+
``reason`` carries only library-controlled values — an HTTP status, an exception type, an
|
|
209
|
+
``errno`` — never a server-supplied body or an exception's text (SPEC-029 FR-002).
|
|
210
|
+
"""
|
|
202
211
|
self.failed += 1
|
|
203
|
-
|
|
204
|
-
f"
|
|
205
|
-
f"{self.max_retries + 1} attempt(s) ({reason})\n"
|
|
212
|
+
_diag.lost(
|
|
213
|
+
"request", 1, f"{type(self).__name__}, {self.max_retries + 1} attempt(s), {reason}"
|
|
206
214
|
)
|
|
207
215
|
|
|
208
216
|
|
|
@@ -10,9 +10,10 @@ raised out of ``emit``.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
|
-
import sys
|
|
14
13
|
from typing import Any
|
|
15
14
|
|
|
15
|
+
from log_foundry import _diag
|
|
16
|
+
|
|
16
17
|
__all__ = ["KafkaSink"]
|
|
17
18
|
|
|
18
19
|
|
|
@@ -65,7 +66,34 @@ class KafkaSink:
|
|
|
65
66
|
return str(value).encode("utf-8") if value is not None else None
|
|
66
67
|
|
|
67
68
|
def _on_delivery(self, err: object, msg: object) -> None:
|
|
68
|
-
"""confluent-kafka delivery callback: count and log failures (FR-002).
|
|
69
|
+
"""confluent-kafka delivery callback: count and log failures (FR-002).
|
|
70
|
+
|
|
71
|
+
``err`` is a ``KafkaError``, not an exception, so it goes to :func:`~_diag.lost` rather
|
|
72
|
+
than :func:`~_diag.absorbed`. Its ``str`` is a human message that can quote the record, so
|
|
73
|
+
only the type and the numeric ``code()`` are written — the code is librdkafka's own
|
|
74
|
+
enumeration and is what makes a delivery failure diagnosable (SPEC-029 FR-002).
|
|
75
|
+
"""
|
|
69
76
|
if err is not None:
|
|
70
77
|
self.failed += 1
|
|
71
|
-
|
|
78
|
+
_diag.lost("message", 1, f"KafkaSink delivery, {type(err).__name__}{_code(err)}")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _code(err: object) -> str:
|
|
82
|
+
"""Render a ``KafkaError``'s numeric code as ``" code=N"``, or ``""`` if it has none.
|
|
83
|
+
|
|
84
|
+
``code()`` is a method on confluent-kafka's ``KafkaError``, returns one of librdkafka's
|
|
85
|
+
integer constants, and carries no caller data. Deliberately narrow: only an ``int`` is written,
|
|
86
|
+
so a stub or a future version returning something else contributes nothing rather than
|
|
87
|
+
smuggling text past the type-name rule.
|
|
88
|
+
|
|
89
|
+
Total, and the ``int()`` is inside the guard for the same reason ``_diag.errno_of`` puts it
|
|
90
|
+
there: ``isinstance(code, int)`` admits a *subclass*, whose ``__int__`` is Python that can
|
|
91
|
+
raise — and this runs in a delivery callback, so an escaping exception would surface from
|
|
92
|
+
``emit`` and cost the whole batch. A diagnostic can never be the failure (SPEC-029 FR-003).
|
|
93
|
+
"""
|
|
94
|
+
try:
|
|
95
|
+
code = getattr(err, "code", None)
|
|
96
|
+
value = code() if callable(code) else None
|
|
97
|
+
return f" code={int(value)}" if isinstance(value, int) else ""
|
|
98
|
+
except Exception:
|
|
99
|
+
return ""
|
|
@@ -10,9 +10,9 @@ unless a sink is built without an injected client. Each incoming batch is re-chu
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
|
-
import sys
|
|
14
13
|
from typing import Any
|
|
15
14
|
|
|
15
|
+
from log_foundry import _diag
|
|
16
16
|
from log_foundry.sinks._batch import adjudicate_positional, usable_results
|
|
17
17
|
from log_foundry.sinks._chunk import chunk_items
|
|
18
18
|
|
|
@@ -77,9 +77,11 @@ class KinesisSink:
|
|
|
77
77
|
data = json.dumps(event).encode("utf-8")
|
|
78
78
|
if len(data) > self.MAX_RECORD_BYTES:
|
|
79
79
|
self.dropped_oversized += 1
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
80
|
+
_diag.lost(
|
|
81
|
+
"event",
|
|
82
|
+
1,
|
|
83
|
+
f"KinesisSink, {len(data)} bytes exceeds the "
|
|
84
|
+
f"{self.MAX_RECORD_BYTES}-byte per-record limit",
|
|
83
85
|
)
|
|
84
86
|
continue
|
|
85
87
|
key = str(event.get(self.partition_key_field) or "log-foundry")[:256]
|
|
@@ -96,10 +98,11 @@ class KinesisSink:
|
|
|
96
98
|
verdict = adjudicate_positional(records, results)
|
|
97
99
|
if verdict.unadjudicated:
|
|
98
100
|
self.dropped_unadjudicated += verdict.unadjudicated
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
f"
|
|
101
|
+
_diag.lost(
|
|
102
|
+
"record",
|
|
103
|
+
verdict.unadjudicated,
|
|
104
|
+
f"KinesisSink could not adjudicate a put_records response ({len(records)} "
|
|
105
|
+
f"record(s) sent, {len(results)} result(s) returned); abandoned, not retried",
|
|
103
106
|
)
|
|
104
107
|
return
|
|
105
108
|
records = verdict.retry
|
|
@@ -107,8 +110,9 @@ class KinesisSink:
|
|
|
107
110
|
return
|
|
108
111
|
if attempt >= self.max_retries:
|
|
109
112
|
self.failed += len(records)
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
+
_diag.lost(
|
|
114
|
+
"record",
|
|
115
|
+
len(records),
|
|
116
|
+
f"KinesisSink, still failing after {self.max_retries + 1} attempts; abandoned",
|
|
113
117
|
)
|
|
114
118
|
return
|
|
@@ -10,10 +10,11 @@ documents retained. Write-only — querying is the downstream tool's job.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
|
-
import sys
|
|
14
13
|
import time
|
|
15
14
|
from typing import Any
|
|
16
15
|
|
|
16
|
+
from log_foundry import _diag
|
|
17
|
+
|
|
17
18
|
__all__ = ["MongoDBSink"]
|
|
18
19
|
|
|
19
20
|
_BACKOFF_BASE = 0.1
|
|
@@ -66,18 +67,18 @@ class MongoDBSink:
|
|
|
66
67
|
# do not retry (successes are already in, a retry would duplicate/re-error).
|
|
67
68
|
rejects = len(details["writeErrors"])
|
|
68
69
|
self.failed += rejects
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
f"write; the rest were inserted\n"
|
|
70
|
+
_diag.lost(
|
|
71
|
+
"document", rejects, "MongoDBSink bulk write; the rest were inserted"
|
|
72
72
|
)
|
|
73
73
|
return
|
|
74
74
|
if attempt < self.max_retries:
|
|
75
75
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
76
76
|
continue
|
|
77
77
|
self.failed += len(documents)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
78
|
+
_diag.lost(
|
|
79
|
+
"document",
|
|
80
|
+
len(documents),
|
|
81
|
+
f"MongoDBSink, {self.max_retries + 1} attempts, {type(err).__name__}",
|
|
81
82
|
)
|
|
82
83
|
return
|
|
83
84
|
|
|
@@ -97,9 +98,7 @@ class MongoDBSink:
|
|
|
97
98
|
for event in batch:
|
|
98
99
|
if len(json.dumps(event).encode("utf-8")) > _MAX_DOC_BYTES:
|
|
99
100
|
self.dropped_oversized += 1
|
|
100
|
-
|
|
101
|
-
"log-foundry: MongoDBSink dropped a document exceeding the 16 MB limit\n"
|
|
102
|
-
)
|
|
101
|
+
_diag.lost("document", 1, "MongoDBSink, exceeds the 16 MB limit")
|
|
103
102
|
continue
|
|
104
103
|
documents.append(dict(event))
|
|
105
104
|
return documents
|
|
@@ -16,9 +16,10 @@ has nothing to retry.
|
|
|
16
16
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
-
import sys
|
|
20
19
|
from typing import TYPE_CHECKING
|
|
21
20
|
|
|
21
|
+
from log_foundry import _diag
|
|
22
|
+
|
|
22
23
|
if TYPE_CHECKING:
|
|
23
24
|
from log_foundry.sinks.base import Sink
|
|
24
25
|
|
|
@@ -58,9 +59,10 @@ class MultiSink:
|
|
|
58
59
|
self.failed += 1
|
|
59
60
|
if first_error is None:
|
|
60
61
|
first_error = err
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
# The class name goes in the *detail*, not in ``where``: only the detail is
|
|
63
|
+
# bounded, and a child's ``__name__`` is a runtime value (SPEC-029 FR-002).
|
|
64
|
+
_diag.absorbed(
|
|
65
|
+
"emitting to a MultiSink child", err, f"{type(sink).__name__} skipped"
|
|
64
66
|
)
|
|
65
67
|
else:
|
|
66
68
|
delivered += 1
|
|
@@ -78,7 +80,4 @@ class MultiSink:
|
|
|
78
80
|
sink.close()
|
|
79
81
|
except Exception as err: # close-all: an earlier failure must not skip the rest
|
|
80
82
|
self.failed += 1
|
|
81
|
-
|
|
82
|
-
f"log-foundry: MultiSink child {type(sink).__name__}.close "
|
|
83
|
-
f"failed and was skipped: {err!r}\n"
|
|
84
|
-
)
|
|
83
|
+
_diag.absorbed("closing a MultiSink child", err, f"{type(sink).__name__} skipped")
|
|
@@ -11,9 +11,10 @@ from __future__ import annotations
|
|
|
11
11
|
|
|
12
12
|
import asyncio
|
|
13
13
|
import json
|
|
14
|
-
import sys
|
|
15
14
|
from typing import Any
|
|
16
15
|
|
|
16
|
+
from log_foundry import _diag
|
|
17
|
+
|
|
17
18
|
__all__ = ["NATSSink"]
|
|
18
19
|
|
|
19
20
|
|
|
@@ -64,7 +65,7 @@ class NATSSink:
|
|
|
64
65
|
await target.publish(self._subject, json.dumps(event).encode("utf-8"))
|
|
65
66
|
except Exception as err: # isolation boundary: never crash the worker (FR-011)
|
|
66
67
|
self.failed += 1
|
|
67
|
-
|
|
68
|
+
_diag.lost("event", 1, f"NATSSink publish, {type(err).__name__}")
|
|
68
69
|
|
|
69
70
|
async def _drain(self) -> None:
|
|
70
71
|
drain = getattr(self._client, "drain", None)
|
|
@@ -10,10 +10,10 @@ convenience is off by default — the user owns their schema and indexes.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import json
|
|
13
|
-
import sys
|
|
14
13
|
import time
|
|
15
14
|
from typing import Any
|
|
16
15
|
|
|
16
|
+
from log_foundry import _diag
|
|
17
17
|
from log_foundry.sinks._chunk import chunk_list, valid_identifier
|
|
18
18
|
|
|
19
19
|
__all__ = ["PostgresSink"]
|
|
@@ -71,9 +71,14 @@ class PostgresSink:
|
|
|
71
71
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
72
72
|
continue
|
|
73
73
|
self.failed += len(batch)
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
74
|
+
# The type, never the repr. ``_row`` binds the whole ``json.dumps(event)`` as a
|
|
75
|
+
# statement parameter, and a psycopg error repr routinely reprints the failing
|
|
76
|
+
# statement *and* its parameters — so the old line reprinted the event, PII
|
|
77
|
+
# included, into a stream nobody was asked to secure (SPEC-029 FR-002, arch §6).
|
|
78
|
+
_diag.lost(
|
|
79
|
+
"event",
|
|
80
|
+
len(batch),
|
|
81
|
+
f"PostgresSink, {self.max_retries + 1} attempts, {type(err).__name__}",
|
|
77
82
|
)
|
|
78
83
|
return
|
|
79
84
|
|
|
@@ -9,9 +9,10 @@ are counted and logged.
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
-
import sys
|
|
13
12
|
from typing import Any
|
|
14
13
|
|
|
14
|
+
from log_foundry import _diag
|
|
15
|
+
|
|
15
16
|
__all__ = ["GooglePubSubSink"]
|
|
16
17
|
|
|
17
18
|
|
|
@@ -41,5 +42,5 @@ class GooglePubSubSink:
|
|
|
41
42
|
future.result()
|
|
42
43
|
except Exception as err: # isolation boundary: never crash the worker (FR-011)
|
|
43
44
|
self.failed += 1
|
|
44
|
-
|
|
45
|
+
_diag.lost("event", 1, f"GooglePubSubSink publish, {type(err).__name__}")
|
|
45
46
|
self._futures.clear()
|
|
@@ -9,10 +9,11 @@ and counted; ``close()`` closes the channel and connection.
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import json
|
|
12
|
-
import sys
|
|
13
12
|
import time
|
|
14
13
|
from typing import Any
|
|
15
14
|
|
|
15
|
+
from log_foundry import _diag
|
|
16
|
+
|
|
16
17
|
__all__ = ["RabbitMQSink"]
|
|
17
18
|
|
|
18
19
|
_BACKOFF_BASE = 0.1
|
|
@@ -83,9 +84,10 @@ class RabbitMQSink:
|
|
|
83
84
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
84
85
|
continue
|
|
85
86
|
self.failed += 1
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
87
|
+
_diag.lost(
|
|
88
|
+
"message",
|
|
89
|
+
1,
|
|
90
|
+
f"RabbitMQSink, {self._max_retries + 1} attempts, {type(err).__name__}",
|
|
89
91
|
)
|
|
90
92
|
return
|
|
91
93
|
|
|
@@ -11,10 +11,11 @@ shadows or requires the real ``redis`` package at import time.
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
13
|
import json
|
|
14
|
-
import sys
|
|
15
14
|
import time
|
|
16
15
|
from typing import Any
|
|
17
16
|
|
|
17
|
+
from log_foundry import _diag
|
|
18
|
+
|
|
18
19
|
__all__ = ["RedisListSink", "RedisStreamsSink"]
|
|
19
20
|
|
|
20
21
|
_BACKOFF_BASE = 0.1
|
|
@@ -49,9 +50,10 @@ class _RedisSink:
|
|
|
49
50
|
time.sleep(_BACKOFF_BASE * (2**attempt))
|
|
50
51
|
continue
|
|
51
52
|
self.failed += len(batch)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
_diag.lost(
|
|
54
|
+
"event",
|
|
55
|
+
len(batch),
|
|
56
|
+
f"{type(self).__name__}, {self.max_retries + 1} attempts, {type(err).__name__}",
|
|
55
57
|
)
|
|
56
58
|
return
|
|
57
59
|
|
|
@@ -8,9 +8,9 @@ request, ≤ 256 KB total). ``boto3`` is the optional ``aws`` extra, imported la
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import json
|
|
11
|
-
import sys
|
|
12
11
|
from typing import Any
|
|
13
12
|
|
|
13
|
+
from log_foundry import _diag
|
|
14
14
|
from log_foundry.sinks._chunk import chunk_items
|
|
15
15
|
|
|
16
16
|
__all__ = ["SNSSink"]
|
|
@@ -53,10 +53,7 @@ class SNSSink:
|
|
|
53
53
|
body = json.dumps(event)
|
|
54
54
|
if len(body.encode("utf-8")) > self.MAX_BYTES:
|
|
55
55
|
self.dropped_oversized += 1
|
|
56
|
-
|
|
57
|
-
f"log-foundry: SNSSink dropped an event exceeding the "
|
|
58
|
-
f"{self.MAX_BYTES}-byte message limit\n"
|
|
59
|
-
)
|
|
56
|
+
_diag.lost("event", 1, f"SNSSink, exceeds the {self.MAX_BYTES}-byte message limit")
|
|
60
57
|
continue
|
|
61
58
|
bodies.append(body)
|
|
62
59
|
return bodies
|
|
@@ -75,8 +72,9 @@ class SNSSink:
|
|
|
75
72
|
entries = [entry for entry in entries if entry["Id"] in failed_ids]
|
|
76
73
|
if attempt >= self.max_retries:
|
|
77
74
|
self.failed += len(entries)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
75
|
+
_diag.lost(
|
|
76
|
+
"message",
|
|
77
|
+
len(entries),
|
|
78
|
+
f"SNSSink, still failing after {self.max_retries + 1} attempts; abandoned",
|
|
81
79
|
)
|
|
82
80
|
return
|
|
@@ -31,10 +31,11 @@ is not done.
|
|
|
31
31
|
from __future__ import annotations
|
|
32
32
|
|
|
33
33
|
import json
|
|
34
|
-
import sys
|
|
35
34
|
from collections.abc import Callable
|
|
36
35
|
from typing import Any, NamedTuple
|
|
37
36
|
|
|
37
|
+
from log_foundry import _diag
|
|
38
|
+
|
|
38
39
|
__all__ = ["SQSSink"]
|
|
39
40
|
|
|
40
41
|
DEFAULT_GROUP_ID = "log-foundry"
|
|
@@ -173,9 +174,10 @@ class SQSSink:
|
|
|
173
174
|
size += len(group_id.encode("utf-8")) + len(dedup_id.encode("utf-8"))
|
|
174
175
|
if size > self.MAX_BYTES:
|
|
175
176
|
self.dropped_oversized += 1
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
177
|
+
_diag.lost(
|
|
178
|
+
"event",
|
|
179
|
+
1,
|
|
180
|
+
f"SQSSink, {size} bytes exceeds the {self.MAX_BYTES}-byte message limit",
|
|
179
181
|
)
|
|
180
182
|
continue
|
|
181
183
|
if current and (
|
|
@@ -223,9 +225,14 @@ class SQSSink:
|
|
|
223
225
|
sender_faults = [item for item in failed if item.get("SenderFault")]
|
|
224
226
|
if sender_faults:
|
|
225
227
|
self.failed += len(sender_faults)
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
228
|
+
# The AWS error code is library-controlled in the sense that matters — it is an
|
|
229
|
+
# enumerated API constant, not the event — and ``_diag`` bounds and escapes it
|
|
230
|
+
# regardless, so a surprising response shape cannot forge a line (SPEC-029).
|
|
231
|
+
_diag.lost(
|
|
232
|
+
"message",
|
|
233
|
+
len(sender_faults),
|
|
234
|
+
f"SQSSink, rejected as invalid (first code: "
|
|
235
|
+
f"{sender_faults[0].get('Code', 'unknown')}); not retried",
|
|
229
236
|
)
|
|
230
237
|
|
|
231
238
|
retryable_ids = {item["Id"] for item in failed if not item.get("SenderFault")}
|
|
@@ -234,8 +241,9 @@ class SQSSink:
|
|
|
234
241
|
entries = [entry for entry in entries if entry["Id"] in retryable_ids]
|
|
235
242
|
if attempt >= self.max_retries:
|
|
236
243
|
self.failed += len(entries)
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
244
|
+
_diag.lost(
|
|
245
|
+
"message",
|
|
246
|
+
len(entries),
|
|
247
|
+
f"SQSSink, still failing after {self.max_retries + 1} attempts; abandoned",
|
|
240
248
|
)
|
|
241
249
|
return
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|