log-foundry 0.6.1.dev2__tar.gz → 0.6.1.dev3__tar.gz

{log_foundry-0.6.1.dev2 → log_foundry-0.6.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.6.1.dev2
+Version: 0.6.1.dev3
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE
@@ -394,7 +394,8 @@ A few conventions hold across every sink below:
 - **Ownership.** A resource the sink opens itself is closed on `shutdown()`; an injected one is left
   open for you to manage.
 - **Never crashes the app.** A failing sink is retried with backoff and then counted (`.failed`,
-  `.dropped_oversized`, …) rather than raised — a broken destination degrades logging, nothing more.
+  `.dropped_oversized`, `.dropped_unadjudicated`, …) rather than raised — a broken destination
+  degrades logging, nothing more.
   The one deliberate exception is a `MultiSink` whose children *all* failed: it re-raises so the
   worker's retry engages, since nothing was delivered and there are no duplicates to risk. That
   still doesn't reach your code — the worker is what catches it.
@@ -513,6 +514,14 @@ chain — log-foundry adds none of its own. Each re-chunks every batch to the se
 limits, retries partial failures, and drops any single event too large to ever fit (counted on
 `.dropped_oversized`).
 
+`KinesisSink` and `FirehoseSink` learn which records failed **positionally** — the response carries a
+parallel array with no ids — so they check that it describes as many records as were sent before
+acting on it. A response that doesn't is not used to adjudicate any record in the chunk: the chunk is
+abandoned rather than re-sent (some of it almost certainly landed), counted on
+`.dropped_unadjudicated`, and named on stderr. A non-zero value there is real loss, and normally
+means the client isn't AWS-shaped. `SQSSink` and `SNSSink` correlate by explicit `Id` instead, so
+they can't mis-pair and have no such counter.
+
 | Sink | Import from | Configure |
 |---|---|---|
 | `SQSSink` | `log_foundry.sinks.sqs` | `SQSSink(queue_url, *, max_retries=3, fifo=None, message_group_id=None, message_deduplication_id=None)` — the headline production path: a durable buffer in front of ELK, absorbing downstream spikes/outages. Standard **and** FIFO queues |

{log_foundry-0.6.1.dev2 → log_foundry-0.6.1.dev3}/README.md

@@ -358,7 +358,8 @@ A few conventions hold across every sink below:
 - **Ownership.** A resource the sink opens itself is closed on `shutdown()`; an injected one is left
   open for you to manage.
 - **Never crashes the app.** A failing sink is retried with backoff and then counted (`.failed`,
-  `.dropped_oversized`, …) rather than raised — a broken destination degrades logging, nothing more.
+  `.dropped_oversized`, `.dropped_unadjudicated`, …) rather than raised — a broken destination
+  degrades logging, nothing more.
   The one deliberate exception is a `MultiSink` whose children *all* failed: it re-raises so the
   worker's retry engages, since nothing was delivered and there are no duplicates to risk. That
   still doesn't reach your code — the worker is what catches it.
@@ -477,6 +478,14 @@ chain — log-foundry adds none of its own. Each re-chunks every batch to the se
 limits, retries partial failures, and drops any single event too large to ever fit (counted on
 `.dropped_oversized`).
 
+`KinesisSink` and `FirehoseSink` learn which records failed **positionally** — the response carries a
+parallel array with no ids — so they check that it describes as many records as were sent before
+acting on it. A response that doesn't is not used to adjudicate any record in the chunk: the chunk is
+abandoned rather than re-sent (some of it almost certainly landed), counted on
+`.dropped_unadjudicated`, and named on stderr. A non-zero value there is real loss, and normally
+means the client isn't AWS-shaped. `SQSSink` and `SNSSink` correlate by explicit `Id` instead, so
+they can't mis-pair and have no such counter.
+
 | Sink | Import from | Configure |
 |---|---|---|
 | `SQSSink` | `log_foundry.sinks.sqs` | `SQSSink(queue_url, *, max_retries=3, fifo=None, message_group_id=None, message_deduplication_id=None)` — the headline production path: a durable buffer in front of ELK, absorbing downstream spikes/outages. Standard **and** FIFO queues |

{log_foundry-0.6.1.dev2 → log_foundry-0.6.1.dev3}/pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
 ]
 
 # Optional features. Install with: pip install log-foundry[aws]
-version = "0.6.1.dev2"
+version = "0.6.1.dev3"
 
 [project.optional-dependencies]
 aws = ["boto3>=1.34"]              # SQSSink, SNSSink, KinesisSink, FirehoseSink

log_foundry-0.6.1.dev3/src/log_foundry/sinks/_batch.py

@@ -0,0 +1,70 @@
+"""Shared adjudication of a positional batch response (SPEC-018).
+
+Some batch APIs report per-record outcomes **positionally**: the response carries a parallel array
+with no identifiers, so entry *i* describes record *i*. That correlation holds only while the two
+arrays are the same length, and a disagreement is evidence they are not aligned — not an
+invitation to use the overlapping prefix. Pairing them anyway truncates silently, which reads
+downstream as "everything landed" for records the destination never confirmed.
+
+This module makes the precondition explicit: either the response describes the whole chunk and the
+caller acts on it, or it does not and the caller abandons the chunk audibly. Id-keyed responses
+(``SQSSink``, ``SNSSink``) select by id and cannot truncate, so they do not come through here.
+"""
+
+from __future__ import annotations
+
+from typing import Any, NamedTuple
+
+__all__ = ["Adjudication", "adjudicate_positional", "usable_results"]
+
+
+class Adjudication[T](NamedTuple):
+    """The outcome of pairing a positional batch response against the records it should describe.
+
+    ``retry`` is non-empty only when ``unadjudicated`` is ``0``: the response either describes the
+    chunk or it does not. The two are never both non-zero.
+
+    Attributes:
+        retry: Records the response explicitly flagged as failed.
+        unadjudicated: Records whose outcome the response did not describe — ``0`` on a
+            well-formed response.
+    """
+
+    retry: list[T]
+    unadjudicated: int
+
+
+def usable_results(results: Any) -> list[dict[str, Any]]:
+    """Return ``results`` if it is a list of mappings, else an empty list.
+
+    The response comes off a client the sink does not control, so the field may be any shape at
+    all — ``None``, a scalar, a list of non-mappings. None of those carry per-record outcomes that
+    can be read, so each is treated as describing nothing, which routes it to the same counted,
+    audible abandonment as a length mismatch. Raising instead would put the malformed-client case
+    back on the path this module exists to take it off.
+    """
+    if isinstance(results, list) and all(isinstance(result, dict) for result in results):
+        return results
+    return []
+
+
+def adjudicate_positional[T](
+    records: list[T],
+    results: list[dict[str, Any]],
+    *,
+    error_key: str = "ErrorCode",
+) -> Adjudication[T]:
+    """Pair a positional batch response against the records it should describe.
+
+    Returns the subset of ``records`` whose paired ``results`` entry carries a truthy ``error_key``.
+    If the arrays disagree in length — including a ``results`` the caller defaulted to empty
+    because the response omitted it — nothing is selected and the chunk is reported unadjudicated.
+    ``error_key`` is a parameter so a future positional response naming its error differently needs
+    no fork of this rule.
+    """
+    if len(results) != len(records):
+        return Adjudication([], len(records))
+    return Adjudication(
+        [record for record, result in zip(records, results, strict=True) if result.get(error_key)],
+        0,
+    )

{log_foundry-0.6.1.dev2 → log_foundry-0.6.1.dev3}/src/log_foundry/sinks/firehose.py

@@ -12,13 +12,23 @@ import json
 import sys
 from typing import Any
 
+from log_foundry.sinks._batch import adjudicate_positional, usable_results
 from log_foundry.sinks._chunk import chunk_items
 
 __all__ = ["FirehoseSink"]
 
 
 class FirehoseSink:
-    """A :class:`~log_foundry.sinks.base.Sink` that writes events to a Firehose delivery stream."""
+    """A :class:`~log_foundry.sinks.base.Sink` that writes events to a Firehose delivery stream.
+
+    Three counters report what was not delivered: ``failed`` (the delivery stream told us these
+    failed, and they still failed after ``max_retries``), ``dropped_oversized`` (too large for the
+    per-record limit to ever accept), and ``dropped_unadjudicated`` (a ``put_record_batch`` response
+    whose ``RequestResponses`` did not describe the chunk that was sent, so no record in it could be
+    paired to an outcome). A non-zero ``dropped_unadjudicated`` means those records were abandoned
+    without the stream ever confirming them — treat it as loss, and as a sign the client is not
+    AWS-shaped.
+    """
 
     MAX_RECORDS = 500  # put_record_batch hard limit: records per request
     MAX_REQUEST_BYTES = 4 * 1024 * 1024  # 4 MB per put_record_batch request
@@ -34,6 +44,7 @@ class FirehoseSink:
         self.max_retries = max_retries
         self.failed = 0
         self.dropped_oversized = 0
+        self.dropped_unadjudicated = 0
 
     def emit(self, batch: list[dict[str, object]]) -> None:
         """Re-chunk to put_record_batch limits and send each chunk, retrying failures (FR-004)."""
@@ -74,14 +85,17 @@ class FirehoseSink:
             )
             if not response.get("FailedPutCount"):
                 return
-            results = response.get("RequestResponses", [])
-            records = [
-                record
-                # strict=False states today's behaviour. A short `results` would silently
-                # truncate this retry list — see the note in kinesis.py.
-                for record, result in zip(records, results, strict=False)
-                if result.get("ErrorCode")
-            ]
+            results = usable_results(response.get("RequestResponses"))
+            verdict = adjudicate_positional(records, results)
+            if verdict.unadjudicated:
+                self.dropped_unadjudicated += verdict.unadjudicated
+                sys.stderr.write(
+                    f"log-foundry: FirehoseSink could not adjudicate a put_record_batch response "
+                    f"({len(records)} record(s) sent, {len(results)} result(s) returned); "
+                    f"{verdict.unadjudicated} record(s) abandoned\n"
+                )
+                return
+            records = verdict.retry
             if not records:
                 return
             if attempt >= self.max_retries:

{log_foundry-0.6.1.dev2 → log_foundry-0.6.1.dev3}/src/log_foundry/sinks/kinesis.py

@@ -13,13 +13,22 @@ import json
 import sys
 from typing import Any
 
+from log_foundry.sinks._batch import adjudicate_positional, usable_results
 from log_foundry.sinks._chunk import chunk_items
 
 __all__ = ["KinesisSink"]
 
 
 class KinesisSink:
-    """A :class:`~log_foundry.sinks.base.Sink` that writes events to a Kinesis Data Stream."""
+    """A :class:`~log_foundry.sinks.base.Sink` that writes events to a Kinesis Data Stream.
+
+    Three counters report what was not delivered: ``failed`` (the stream told us these failed, and
+    they still failed after ``max_retries``), ``dropped_oversized`` (too large for the per-record
+    limit to ever accept), and ``dropped_unadjudicated`` (a ``put_records`` response whose results
+    array did not describe the chunk that was sent, so no record in it could be paired to an
+    outcome). A non-zero ``dropped_unadjudicated`` means those records were abandoned without the
+    stream ever confirming them — treat it as loss, and as a sign the client is not AWS-shaped.
+    """
 
     MAX_RECORDS = 500  # put_records hard limit: records per request
     MAX_REQUEST_BYTES = 5 * 1024 * 1024  # 5 MB per put_records request
@@ -43,6 +52,7 @@ class KinesisSink:
         self.max_retries = max_retries
         self.failed = 0
         self.dropped_oversized = 0
+        self.dropped_unadjudicated = 0
 
     def emit(self, batch: list[dict[str, object]]) -> None:
         """Re-chunk to put_records limits and send each chunk, retrying failures (FR-003)."""
@@ -82,17 +92,17 @@ class KinesisSink:
             response = self.client.put_records(StreamName=self.stream_name, Records=records)
             if not response.get("FailedRecordCount"):
                 return
-            results = response.get("Records", [])
-            records = [
-                record
-                # strict=False states today's behaviour rather than changing it. Note the
-                # latent case it preserves: if `results` came back shorter than `records`
-                # (or empty, via the `.get` default above), zip truncates, `records` empties
-                # and the batch reports success — the silent-loss shape SPEC-017 went after.
-                # Making that raise is a behaviour change, so it belongs in its own spec.
-                for record, result in zip(records, results, strict=False)
-                if result.get("ErrorCode")
-            ]
+            results = usable_results(response.get("Records"))
+            verdict = adjudicate_positional(records, results)
+            if verdict.unadjudicated:
+                self.dropped_unadjudicated += verdict.unadjudicated
+                sys.stderr.write(
+                    f"log-foundry: KinesisSink could not adjudicate a put_records response "
+                    f"({len(records)} record(s) sent, {len(results)} result(s) returned); "
+                    f"{verdict.unadjudicated} record(s) abandoned\n"
+                )
+                return
+            records = verdict.retry
             if not records:
                 return
             if attempt >= self.max_retries: