log-foundry 0.10.2.dev13__tar.gz → 0.10.2.dev15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/PKG-INFO +1 -1
  2. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/pyproject.toml +1 -1
  3. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/_socket.py +11 -4
  4. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/clickhouse.py +5 -4
  5. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/elasticsearch.py +2 -2
  6. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/eventhubs.py +8 -8
  7. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/firehose.py +15 -10
  8. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/http.py +14 -6
  9. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/kafka.py +31 -3
  10. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/kinesis.py +15 -11
  11. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/mongodb.py +9 -10
  12. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/multi.py +7 -8
  13. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/nats.py +3 -2
  14. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/postgres.py +9 -4
  15. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/pubsub.py +3 -2
  16. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/rabbitmq.py +6 -4
  17. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/redis.py +6 -4
  18. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/sns.py +6 -8
  19. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/sqs.py +18 -10
  20. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/LICENSE +0 -0
  21. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/README.md +0 -0
  22. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/__init__.py +0 -0
  23. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/_diag.py +0 -0
  24. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/api.py +0 -0
  25. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/config.py +0 -0
  26. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/console.py +0 -0
  27. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/context.py +0 -0
  28. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/decorator.py +0 -0
  29. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/ids.py +0 -0
  30. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/model.py +0 -0
  31. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/py.typed +0 -0
  32. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sanitize.py +0 -0
  33. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/__init__.py +0 -0
  34. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/_batch.py +0 -0
  35. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/_chunk.py +0 -0
  36. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/_time.py +0 -0
  37. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/base.py +0 -0
  38. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/callback.py +0 -0
  39. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/datadog.py +0 -0
  40. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/file.py +0 -0
  41. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/filtering.py +0 -0
  42. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/honeycomb.py +0 -0
  43. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/logging_sink.py +0 -0
  44. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/logstash.py +0 -0
  45. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/loki.py +0 -0
  46. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/newrelic.py +0 -0
  47. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/sentry.py +0 -0
  48. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/splunk.py +0 -0
  49. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/sqlite.py +0 -0
  50. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/stdout.py +0 -0
  51. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/syslog.py +0 -0
  52. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/transform.py +0 -0
  53. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/sinks/util.py +0 -0
  54. {log_foundry-0.10.2.dev13 → log_foundry-0.10.2.dev15}/src/log_foundry/worker.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: log-foundry
3
- Version: 0.10.2.dev13
3
+ Version: 0.10.2.dev15
4
4
  Summary: Generate logs for your console and JSON events for downstream consumption.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -20,7 +20,7 @@ dependencies = [
20
20
  ]
21
21
 
22
22
  # Optional features. Install with: pip install log-foundry[aws]
23
- version = "0.10.2.dev13"
23
+ version = "0.10.2.dev15"
24
24
 
25
25
  [project.optional-dependencies]
26
26
  aws = ["boto3>=1.43.61"] # SQSSink, SNSSink, KinesisSink, FirehoseSink
@@ -12,9 +12,10 @@ substitute a fake socket without any network access.
12
12
  from __future__ import annotations
13
13
 
14
14
  import socket
15
- import sys
16
15
  import time
17
16
 
17
+ from log_foundry import _diag
18
+
18
19
  __all__ = ["SocketTransport"]
19
20
 
20
21
  _BACKOFF_BASE = 0.1 # seconds; delay for retry attempt n is _BACKOFF_BASE * 2**n
@@ -81,9 +82,15 @@ class SocketTransport:
81
82
  time.sleep(_BACKOFF_BASE * (2**attempt))
82
83
  continue
83
84
  self.failed += 1
84
- sys.stderr.write(
85
- f"log-foundry: SocketTransport abandoned a message after "
86
- f"{self._max_retries + 1} attempt(s) ({err!r})\n"
85
+ # Guarded now (SPEC-029 FR-003): this runs on the worker thread, and the bare
86
+ # ``sys.stderr.write`` this replaced could end delivery for good. ``errno``
87
+ # because an ``OSError`` type name alone does not tell "connection refused" from
88
+ # "host unknown", and the code is an integer from the OS — not caller data.
89
+ _diag.lost(
90
+ "message",
91
+ 1,
92
+ f"SocketTransport, {self._max_retries + 1} attempt(s), "
93
+ f"{type(err).__name__} {_diag.errno_of(err)}".rstrip(),
87
94
  )
88
95
  return
89
96
 
@@ -9,10 +9,10 @@ idempotent ``MergeTree`` ``create_table`` convenience is off by default. Write-o
9
9
  from __future__ import annotations
10
10
 
11
11
  import json
12
- import sys
13
12
  import time
14
13
  from typing import Any
15
14
 
15
+ from log_foundry import _diag
16
16
  from log_foundry.sinks._chunk import chunk_list, valid_identifier
17
17
 
18
18
  __all__ = ["ClickHouseSink"]
@@ -97,9 +97,10 @@ class ClickHouseSink:
97
97
  time.sleep(_BACKOFF_BASE * (2**attempt))
98
98
  continue
99
99
  self.failed += len(rows)
100
- sys.stderr.write(
101
- f"log-foundry: ClickHouseSink abandoned {len(rows)} row(s) after "
102
- f"{self.max_retries + 1} attempts ({err!r})\n"
100
+ _diag.lost(
101
+ "row",
102
+ len(rows),
103
+ f"ClickHouseSink, {self.max_retries + 1} attempts, {type(err).__name__}",
103
104
  )
104
105
  return
105
106
 
@@ -10,8 +10,8 @@ without discarding the successfully-indexed items. OpenSearch speaks the same bu
10
10
  from __future__ import annotations
11
11
 
12
12
  import json
13
- import sys
14
13
 
14
+ from log_foundry import _diag
15
15
  from log_foundry.sinks.http import HTTPSink
16
16
 
17
17
  __all__ = ["ElasticsearchSink", "OpenSearchSink"]
@@ -61,7 +61,7 @@ class ElasticsearchSink(HTTPSink):
61
61
  errors += 1
62
62
  if errors:
63
63
  self.item_errors += errors
64
- sys.stderr.write(f"log-foundry: ElasticsearchSink saw {errors} failed bulk item(s)\n")
64
+ _diag.lost("bulk item", errors, f"{type(self).__name__}, rejected by the server")
65
65
 
66
66
 
67
67
  class OpenSearchSink(ElasticsearchSink):
@@ -10,10 +10,11 @@ send errors are retried within a bound. ``close()`` closes the producer.
10
10
  from __future__ import annotations
11
11
 
12
12
  import json
13
- import sys
14
13
  import time
15
14
  from typing import Any
16
15
 
16
+ from log_foundry import _diag
17
+
17
18
  __all__ = ["AzureEventHubsSink"]
18
19
 
19
20
  _BACKOFF_BASE = 0.1
@@ -62,10 +63,7 @@ class AzureEventHubsSink:
62
63
  current = self.producer.create_batch()
63
64
  if not _try_add(current, data):
64
65
  self.dropped_oversized += 1
65
- sys.stderr.write(
66
- "log-foundry: AzureEventHubsSink dropped an event too large for an "
67
- "empty 1 MB batch\n"
68
- )
66
+ _diag.lost("event", 1, "AzureEventHubsSink, too large for an empty 1 MB batch")
69
67
  self._send(current)
70
68
 
71
69
  def close(self) -> None:
@@ -87,9 +85,11 @@ class AzureEventHubsSink:
87
85
  time.sleep(_BACKOFF_BASE * (2**attempt))
88
86
  continue
89
87
  self.failed += len(event_batch)
90
- sys.stderr.write(
91
- f"log-foundry: AzureEventHubsSink abandoned a batch of {len(event_batch)} "
92
- f"event(s) after {self.max_retries + 1} attempts ({err!r})\n"
88
+ _diag.lost(
89
+ "event",
90
+ len(event_batch),
91
+ f"AzureEventHubsSink, one batch, {self.max_retries + 1} attempts, "
92
+ f"{type(err).__name__}",
93
93
  )
94
94
  return
95
95
 
@@ -9,9 +9,9 @@ retried within a bounded count.
9
9
  from __future__ import annotations
10
10
 
11
11
  import json
12
- import sys
13
12
  from typing import Any
14
13
 
14
+ from log_foundry import _diag
15
15
  from log_foundry.sinks._batch import adjudicate_positional, usable_results
16
16
  from log_foundry.sinks._chunk import chunk_items
17
17
 
@@ -69,9 +69,11 @@ class FirehoseSink:
69
69
  data = json.dumps(event).encode("utf-8")
70
70
  if len(data) > self.MAX_RECORD_BYTES:
71
71
  self.dropped_oversized += 1
72
- sys.stderr.write(
73
- f"log-foundry: FirehoseSink dropped an event of {len(data)} bytes exceeding "
74
- f"the {self.MAX_RECORD_BYTES}-byte per-record limit\n"
72
+ _diag.lost(
73
+ "event",
74
+ 1,
75
+ f"FirehoseSink, {len(data)} bytes exceeds the "
76
+ f"{self.MAX_RECORD_BYTES}-byte per-record limit",
75
77
  )
76
78
  continue
77
79
  records.append({"Data": data})
@@ -89,10 +91,12 @@ class FirehoseSink:
89
91
  verdict = adjudicate_positional(records, results)
90
92
  if verdict.unadjudicated:
91
93
  self.dropped_unadjudicated += verdict.unadjudicated
92
- sys.stderr.write(
93
- f"log-foundry: FirehoseSink could not adjudicate a put_record_batch response "
94
+ _diag.lost(
95
+ "record",
96
+ verdict.unadjudicated,
97
+ f"FirehoseSink could not adjudicate a put_record_batch response "
94
98
  f"({len(records)} record(s) sent, {len(results)} result(s) returned); "
95
- f"{verdict.unadjudicated} record(s) abandoned\n"
99
+ f"abandoned, not retried",
96
100
  )
97
101
  return
98
102
  records = verdict.retry
@@ -100,8 +104,9 @@ class FirehoseSink:
100
104
  return
101
105
  if attempt >= self.max_retries:
102
106
  self.failed += len(records)
103
- sys.stderr.write(
104
- f"log-foundry: {len(records)} Firehose record(s) still failing after "
105
- f"{self.max_retries + 1} attempts; abandoned\n"
107
+ _diag.lost(
108
+ "record",
109
+ len(records),
110
+ f"FirehoseSink, still failing after {self.max_retries + 1} attempts; abandoned",
106
111
  )
107
112
  return
@@ -15,13 +15,14 @@ from __future__ import annotations
15
15
 
16
16
  import gzip as _gzip
17
17
  import json
18
- import sys
19
18
  import time
20
19
  import urllib.error
21
20
  import urllib.request
22
21
  from base64 import b64encode
23
22
  from typing import TYPE_CHECKING, Any
24
23
 
24
+ from log_foundry import _diag
25
+
25
26
  if TYPE_CHECKING:
26
27
  from collections.abc import Callable
27
28
 
@@ -128,7 +129,11 @@ class HTTPSink:
128
129
  if attempt < self.max_retries:
129
130
  self._sleep_backoff(attempt, None)
130
131
  continue
131
- self._abandon(f"connection error: {err!r}")
132
+ # The type plus the OS code, never the message: a ``URLError``'s text embeds the
133
+ # URL and whatever the resolver said (SPEC-029 FR-002).
134
+ self._abandon(
135
+ f"connection error, {type(err).__name__} {_diag.errno_of(err)}".rstrip()
136
+ )
132
137
  return None
133
138
  if 200 <= status < 300:
134
139
  return payload
@@ -198,11 +203,14 @@ class HTTPSink:
198
203
  time.sleep(delay)
199
204
 
200
205
  def _abandon(self, reason: str) -> None:
201
- """Count and log a request abandoned past the retry bound (FR-012)."""
206
+ """Count and log a request abandoned past the retry bound (FR-012).
207
+
208
+ ``reason`` carries only library-controlled values — an HTTP status, an exception type, an
209
+ ``errno`` — never a server-supplied body or an exception's text (SPEC-029 FR-002).
210
+ """
202
211
  self.failed += 1
203
- sys.stderr.write(
204
- f"log-foundry: {type(self).__name__} abandoned a request after "
205
- f"{self.max_retries + 1} attempt(s) ({reason})\n"
212
+ _diag.lost(
213
+ "request", 1, f"{type(self).__name__}, {self.max_retries + 1} attempt(s), {reason}"
206
214
  )
207
215
 
208
216
 
@@ -10,9 +10,10 @@ raised out of ``emit``.
10
10
  from __future__ import annotations
11
11
 
12
12
  import json
13
- import sys
14
13
  from typing import Any
15
14
 
15
+ from log_foundry import _diag
16
+
16
17
  __all__ = ["KafkaSink"]
17
18
 
18
19
 
@@ -65,7 +66,34 @@ class KafkaSink:
65
66
  return str(value).encode("utf-8") if value is not None else None
66
67
 
67
68
  def _on_delivery(self, err: object, msg: object) -> None:
68
- """confluent-kafka delivery callback: count and log failures (FR-002)."""
69
+ """confluent-kafka delivery callback: count and log failures (FR-002).
70
+
71
+ ``err`` is a ``KafkaError``, not an exception, so it goes to :func:`~_diag.lost` rather
72
+ than :func:`~_diag.absorbed`. Its ``str`` is a human message that can quote the record, so
73
+ only the type and the numeric ``code()`` are written — the code is librdkafka's own
74
+ enumeration and is what makes a delivery failure diagnosable (SPEC-029 FR-002).
75
+ """
69
76
  if err is not None:
70
77
  self.failed += 1
71
- sys.stderr.write(f"log-foundry: KafkaSink delivery failed: {err!r}\n")
78
+ _diag.lost("message", 1, f"KafkaSink delivery, {type(err).__name__}{_code(err)}")
79
+
80
+
81
+ def _code(err: object) -> str:
82
+ """Render a ``KafkaError``'s numeric code as ``" code=N"``, or ``""`` if it has none.
83
+
84
+ ``code()`` is a method on confluent-kafka's ``KafkaError``, returns one of librdkafka's
85
+ integer constants, and carries no caller data. Deliberately narrow: only an ``int`` is written,
86
+ so a stub or a future version returning something else contributes nothing rather than
87
+ smuggling text past the type-name rule.
88
+
89
+ Total, and the ``int()`` is inside the guard for the same reason ``_diag.errno_of`` puts it
90
+ there: ``isinstance(code, int)`` admits a *subclass*, whose ``__int__`` is Python that can
91
+ raise — and this runs in a delivery callback, so an escaping exception would surface from
92
+ ``emit`` and cost the whole batch. A diagnostic can never be the failure (SPEC-029 FR-003).
93
+ """
94
+ try:
95
+ code = getattr(err, "code", None)
96
+ value = code() if callable(code) else None
97
+ return f" code={int(value)}" if isinstance(value, int) else ""
98
+ except Exception:
99
+ return ""
@@ -10,9 +10,9 @@ unless a sink is built without an injected client. Each incoming batch is re-chu
10
10
  from __future__ import annotations
11
11
 
12
12
  import json
13
- import sys
14
13
  from typing import Any
15
14
 
15
+ from log_foundry import _diag
16
16
  from log_foundry.sinks._batch import adjudicate_positional, usable_results
17
17
  from log_foundry.sinks._chunk import chunk_items
18
18
 
@@ -77,9 +77,11 @@ class KinesisSink:
77
77
  data = json.dumps(event).encode("utf-8")
78
78
  if len(data) > self.MAX_RECORD_BYTES:
79
79
  self.dropped_oversized += 1
80
- sys.stderr.write(
81
- f"log-foundry: KinesisSink dropped an event of {len(data)} bytes exceeding "
82
- f"the {self.MAX_RECORD_BYTES}-byte per-record limit\n"
80
+ _diag.lost(
81
+ "event",
82
+ 1,
83
+ f"KinesisSink, {len(data)} bytes exceeds the "
84
+ f"{self.MAX_RECORD_BYTES}-byte per-record limit",
83
85
  )
84
86
  continue
85
87
  key = str(event.get(self.partition_key_field) or "log-foundry")[:256]
@@ -96,10 +98,11 @@ class KinesisSink:
96
98
  verdict = adjudicate_positional(records, results)
97
99
  if verdict.unadjudicated:
98
100
  self.dropped_unadjudicated += verdict.unadjudicated
99
- sys.stderr.write(
100
- f"log-foundry: KinesisSink could not adjudicate a put_records response "
101
- f"({len(records)} record(s) sent, {len(results)} result(s) returned); "
102
- f"{verdict.unadjudicated} record(s) abandoned\n"
101
+ _diag.lost(
102
+ "record",
103
+ verdict.unadjudicated,
104
+ f"KinesisSink could not adjudicate a put_records response ({len(records)} "
105
+ f"record(s) sent, {len(results)} result(s) returned); abandoned, not retried",
103
106
  )
104
107
  return
105
108
  records = verdict.retry
@@ -107,8 +110,9 @@ class KinesisSink:
107
110
  return
108
111
  if attempt >= self.max_retries:
109
112
  self.failed += len(records)
110
- sys.stderr.write(
111
- f"log-foundry: {len(records)} Kinesis record(s) still failing after "
112
- f"{self.max_retries + 1} attempts; abandoned\n"
113
+ _diag.lost(
114
+ "record",
115
+ len(records),
116
+ f"KinesisSink, still failing after {self.max_retries + 1} attempts; abandoned",
113
117
  )
114
118
  return
@@ -10,10 +10,11 @@ documents retained. Write-only — querying is the downstream tool's job.
10
10
  from __future__ import annotations
11
11
 
12
12
  import json
13
- import sys
14
13
  import time
15
14
  from typing import Any
16
15
 
16
+ from log_foundry import _diag
17
+
17
18
  __all__ = ["MongoDBSink"]
18
19
 
19
20
  _BACKOFF_BASE = 0.1
@@ -66,18 +67,18 @@ class MongoDBSink:
66
67
  # do not retry (successes are already in, a retry would duplicate/re-error).
67
68
  rejects = len(details["writeErrors"])
68
69
  self.failed += rejects
69
- sys.stderr.write(
70
- f"log-foundry: MongoDBSink had {rejects} document(s) rejected by a bulk "
71
- f"write; the rest were inserted\n"
70
+ _diag.lost(
71
+ "document", rejects, "MongoDBSink bulk write; the rest were inserted"
72
72
  )
73
73
  return
74
74
  if attempt < self.max_retries:
75
75
  time.sleep(_BACKOFF_BASE * (2**attempt))
76
76
  continue
77
77
  self.failed += len(documents)
78
- sys.stderr.write(
79
- f"log-foundry: MongoDBSink abandoned {len(documents)} document(s) after "
80
- f"{self.max_retries + 1} attempts ({err!r})\n"
78
+ _diag.lost(
79
+ "document",
80
+ len(documents),
81
+ f"MongoDBSink, {self.max_retries + 1} attempts, {type(err).__name__}",
81
82
  )
82
83
  return
83
84
 
@@ -97,9 +98,7 @@ class MongoDBSink:
97
98
  for event in batch:
98
99
  if len(json.dumps(event).encode("utf-8")) > _MAX_DOC_BYTES:
99
100
  self.dropped_oversized += 1
100
- sys.stderr.write(
101
- "log-foundry: MongoDBSink dropped a document exceeding the 16 MB limit\n"
102
- )
101
+ _diag.lost("document", 1, "MongoDBSink, exceeds the 16 MB limit")
103
102
  continue
104
103
  documents.append(dict(event))
105
104
  return documents
@@ -16,9 +16,10 @@ has nothing to retry.
16
16
 
17
17
  from __future__ import annotations
18
18
 
19
- import sys
20
19
  from typing import TYPE_CHECKING
21
20
 
21
+ from log_foundry import _diag
22
+
22
23
  if TYPE_CHECKING:
23
24
  from log_foundry.sinks.base import Sink
24
25
 
@@ -58,9 +59,10 @@ class MultiSink:
58
59
  self.failed += 1
59
60
  if first_error is None:
60
61
  first_error = err
61
- sys.stderr.write(
62
- f"log-foundry: MultiSink child {type(sink).__name__}.emit "
63
- f"failed and was skipped: {err!r}\n"
62
+ # The class name goes in the *detail*, not in ``where``: only the detail is
63
+ # bounded, and a child's ``__name__`` is a runtime value (SPEC-029 FR-002).
64
+ _diag.absorbed(
65
+ "emitting to a MultiSink child", err, f"{type(sink).__name__} skipped"
64
66
  )
65
67
  else:
66
68
  delivered += 1
@@ -78,7 +80,4 @@ class MultiSink:
78
80
  sink.close()
79
81
  except Exception as err: # close-all: an earlier failure must not skip the rest
80
82
  self.failed += 1
81
- sys.stderr.write(
82
- f"log-foundry: MultiSink child {type(sink).__name__}.close "
83
- f"failed and was skipped: {err!r}\n"
84
- )
83
+ _diag.absorbed("closing a MultiSink child", err, f"{type(sink).__name__} skipped")
@@ -11,9 +11,10 @@ from __future__ import annotations
11
11
 
12
12
  import asyncio
13
13
  import json
14
- import sys
15
14
  from typing import Any
16
15
 
16
+ from log_foundry import _diag
17
+
17
18
  __all__ = ["NATSSink"]
18
19
 
19
20
 
@@ -64,7 +65,7 @@ class NATSSink:
64
65
  await target.publish(self._subject, json.dumps(event).encode("utf-8"))
65
66
  except Exception as err: # isolation boundary: never crash the worker (FR-011)
66
67
  self.failed += 1
67
- sys.stderr.write(f"log-foundry: NATSSink publish failed: {err!r}\n")
68
+ _diag.lost("event", 1, f"NATSSink publish, {type(err).__name__}")
68
69
 
69
70
  async def _drain(self) -> None:
70
71
  drain = getattr(self._client, "drain", None)
@@ -10,10 +10,10 @@ convenience is off by default — the user owns their schema and indexes.
10
10
  from __future__ import annotations
11
11
 
12
12
  import json
13
- import sys
14
13
  import time
15
14
  from typing import Any
16
15
 
16
+ from log_foundry import _diag
17
17
  from log_foundry.sinks._chunk import chunk_list, valid_identifier
18
18
 
19
19
  __all__ = ["PostgresSink"]
@@ -71,9 +71,14 @@ class PostgresSink:
71
71
  time.sleep(_BACKOFF_BASE * (2**attempt))
72
72
  continue
73
73
  self.failed += len(batch)
74
- sys.stderr.write(
75
- f"log-foundry: PostgresSink abandoned {len(batch)} event(s) after "
76
- f"{self.max_retries + 1} attempts ({err!r})\n"
74
+ # The type, never the repr. ``_row`` binds the whole ``json.dumps(event)`` as a
75
+ # statement parameter, and a psycopg error repr routinely reprints the failing
76
+ # statement *and* its parameters — so the old line reprinted the event, PII
77
+ # included, into a stream nobody was asked to secure (SPEC-029 FR-002, arch §6).
78
+ _diag.lost(
79
+ "event",
80
+ len(batch),
81
+ f"PostgresSink, {self.max_retries + 1} attempts, {type(err).__name__}",
77
82
  )
78
83
  return
79
84
 
@@ -9,9 +9,10 @@ are counted and logged.
9
9
  from __future__ import annotations
10
10
 
11
11
  import json
12
- import sys
13
12
  from typing import Any
14
13
 
14
+ from log_foundry import _diag
15
+
15
16
  __all__ = ["GooglePubSubSink"]
16
17
 
17
18
 
@@ -41,5 +42,5 @@ class GooglePubSubSink:
41
42
  future.result()
42
43
  except Exception as err: # isolation boundary: never crash the worker (FR-011)
43
44
  self.failed += 1
44
- sys.stderr.write(f"log-foundry: GooglePubSubSink publish failed: {err!r}\n")
45
+ _diag.lost("event", 1, f"GooglePubSubSink publish, {type(err).__name__}")
45
46
  self._futures.clear()
@@ -9,10 +9,11 @@ and counted; ``close()`` closes the channel and connection.
9
9
  from __future__ import annotations
10
10
 
11
11
  import json
12
- import sys
13
12
  import time
14
13
  from typing import Any
15
14
 
15
+ from log_foundry import _diag
16
+
16
17
  __all__ = ["RabbitMQSink"]
17
18
 
18
19
  _BACKOFF_BASE = 0.1
@@ -83,9 +84,10 @@ class RabbitMQSink:
83
84
  time.sleep(_BACKOFF_BASE * (2**attempt))
84
85
  continue
85
86
  self.failed += 1
86
- sys.stderr.write(
87
- f"log-foundry: RabbitMQSink abandoned a message after "
88
- f"{self._max_retries + 1} attempts ({err!r})\n"
87
+ _diag.lost(
88
+ "message",
89
+ 1,
90
+ f"RabbitMQSink, {self._max_retries + 1} attempts, {type(err).__name__}",
89
91
  )
90
92
  return
91
93
 
@@ -11,10 +11,11 @@ shadows or requires the real ``redis`` package at import time.
11
11
  from __future__ import annotations
12
12
 
13
13
  import json
14
- import sys
15
14
  import time
16
15
  from typing import Any
17
16
 
17
+ from log_foundry import _diag
18
+
18
19
  __all__ = ["RedisListSink", "RedisStreamsSink"]
19
20
 
20
21
  _BACKOFF_BASE = 0.1
@@ -49,9 +50,10 @@ class _RedisSink:
49
50
  time.sleep(_BACKOFF_BASE * (2**attempt))
50
51
  continue
51
52
  self.failed += len(batch)
52
- sys.stderr.write(
53
- f"log-foundry: {type(self).__name__} abandoned {len(batch)} event(s) after "
54
- f"{self.max_retries + 1} attempts ({err!r})\n"
53
+ _diag.lost(
54
+ "event",
55
+ len(batch),
56
+ f"{type(self).__name__}, {self.max_retries + 1} attempts, {type(err).__name__}",
55
57
  )
56
58
  return
57
59
 
@@ -8,9 +8,9 @@ request, ≤ 256 KB total). ``boto3`` is the optional ``aws`` extra, imported la
8
8
  from __future__ import annotations
9
9
 
10
10
  import json
11
- import sys
12
11
  from typing import Any
13
12
 
13
+ from log_foundry import _diag
14
14
  from log_foundry.sinks._chunk import chunk_items
15
15
 
16
16
  __all__ = ["SNSSink"]
@@ -53,10 +53,7 @@ class SNSSink:
53
53
  body = json.dumps(event)
54
54
  if len(body.encode("utf-8")) > self.MAX_BYTES:
55
55
  self.dropped_oversized += 1
56
- sys.stderr.write(
57
- f"log-foundry: SNSSink dropped an event exceeding the "
58
- f"{self.MAX_BYTES}-byte message limit\n"
59
- )
56
+ _diag.lost("event", 1, f"SNSSink, exceeds the {self.MAX_BYTES}-byte message limit")
60
57
  continue
61
58
  bodies.append(body)
62
59
  return bodies
@@ -75,8 +72,9 @@ class SNSSink:
75
72
  entries = [entry for entry in entries if entry["Id"] in failed_ids]
76
73
  if attempt >= self.max_retries:
77
74
  self.failed += len(entries)
78
- sys.stderr.write(
79
- f"log-foundry: {len(entries)} SNS message(s) still failing after "
80
- f"{self.max_retries + 1} attempts; abandoned\n"
75
+ _diag.lost(
76
+ "message",
77
+ len(entries),
78
+ f"SNSSink, still failing after {self.max_retries + 1} attempts; abandoned",
81
79
  )
82
80
  return
@@ -31,10 +31,11 @@ is not done.
31
31
  from __future__ import annotations
32
32
 
33
33
  import json
34
- import sys
35
34
  from collections.abc import Callable
36
35
  from typing import Any, NamedTuple
37
36
 
37
+ from log_foundry import _diag
38
+
38
39
  __all__ = ["SQSSink"]
39
40
 
40
41
  DEFAULT_GROUP_ID = "log-foundry"
@@ -173,9 +174,10 @@ class SQSSink:
173
174
  size += len(group_id.encode("utf-8")) + len(dedup_id.encode("utf-8"))
174
175
  if size > self.MAX_BYTES:
175
176
  self.dropped_oversized += 1
176
- sys.stderr.write(
177
- f"log-foundry: dropped an event of {size} bytes exceeding the "
178
- f"{self.MAX_BYTES}-byte SQS message limit\n"
177
+ _diag.lost(
178
+ "event",
179
+ 1,
180
+ f"SQSSink, {size} bytes exceeds the {self.MAX_BYTES}-byte message limit",
179
181
  )
180
182
  continue
181
183
  if current and (
@@ -223,9 +225,14 @@ class SQSSink:
223
225
  sender_faults = [item for item in failed if item.get("SenderFault")]
224
226
  if sender_faults:
225
227
  self.failed += len(sender_faults)
226
- sys.stderr.write(
227
- f"log-foundry: {len(sender_faults)} SQS message(s) rejected as invalid "
228
- f"(first code: {sender_faults[0].get('Code', 'unknown')}); not retried\n"
228
+ # The AWS error code is library-controlled in the sense that matters — it is an
229
+ # enumerated API constant, not the event — and ``_diag`` bounds and escapes it
230
+ # regardless, so a surprising response shape cannot forge a line (SPEC-029).
231
+ _diag.lost(
232
+ "message",
233
+ len(sender_faults),
234
+ f"SQSSink, rejected as invalid (first code: "
235
+ f"{sender_faults[0].get('Code', 'unknown')}); not retried",
229
236
  )
230
237
 
231
238
  retryable_ids = {item["Id"] for item in failed if not item.get("SenderFault")}
@@ -234,8 +241,9 @@ class SQSSink:
234
241
  entries = [entry for entry in entries if entry["Id"] in retryable_ids]
235
242
  if attempt >= self.max_retries:
236
243
  self.failed += len(entries)
237
- sys.stderr.write(
238
- f"log-foundry: {len(entries)} SQS message(s) still failing after "
239
- f"{self.max_retries + 1} attempts; abandoned\n"
244
+ _diag.lost(
245
+ "message",
246
+ len(entries),
247
+ f"SQSSink, still failing after {self.max_retries + 1} attempts; abandoned",
240
248
  )
241
249
  return