log-foundry 0.4.1.dev2__py3-none-any.whl → 0.4.1.dev4__py3-none-any.whl

log_foundry/sinks/sqs.py

@@ -14,16 +14,59 @@ The worker (SPEC-004) batches by count and time, but SQS has hard per-request li
 sink re-chunks every incoming batch on both dimensions: ≤ 10 messages **and** ≤ 256 KB per
 ``send_message_batch``. Partial failures (the response ``Failed`` list) are retried; a single
 event too large to ever fit is dropped with a warning rather than crashing the batch.
+
+Both queue types are supported (SPEC-016). A ``.fifo`` queue URL selects FIFO behaviour, where
+every entry additionally carries a ``MessageGroupId`` — SQS orders messages *within* a group, and
+the default group is the event's own ``trace_id``, which is exactly the unit whose events should
+stay ordered. Per-trace groups also keep traces independent, so the queue delivers them in
+parallel instead of serializing the whole process behind one group. Standard queues are
+untouched: their entries carry ``Id`` and ``MessageBody`` and nothing else.
+
+Ordering is best-effort across a retry boundary: if one entry fails while a same-group entry
+ahead of it succeeded, the retried entry lands *after* it. Holding a whole group back on one
+failure would trade log delivery for ordering the consumer can rebuild from ``timestamp``, so it
+is not done.
 """
 
 from __future__ import annotations
 
 import json
 import sys
-from typing import Any
+from collections.abc import Callable
+from typing import Any, NamedTuple
 
 __all__ = ["SQSSink"]
 
+DEFAULT_GROUP_ID = "log-foundry"
+"""Fallback group for an event carrying no usable ``trace_id`` — never send an empty group id."""
+
+MAX_ID_LEN = 128
+"""SQS maximum length for ``MessageGroupId`` and ``MessageDeduplicationId``."""
+
+GroupIdSource = str | Callable[[dict[str, object]], str] | None
+DedupIdSource = Callable[[dict[str, object]], str] | None
+
+
+class _Prepared(NamedTuple):
+    """One event serialized and costed, with its FIFO ids resolved (``None`` on a standard queue).
+
+    The ids are derived **once**, here, rather than again at send time: the deduplication
+    fallback mints a fresh UUID, so deriving twice would bill the byte budget for one value and
+    put a different one on the wire.
+    """
+
+    body: str
+    group_id: str | None
+    dedup_id: str | None
+
+
+def _bounded(raw: str, fallback: str) -> str:
+    """Normalize a derived id: blank falls back, over-long truncates to the SQS maximum."""
+    cleaned = raw.strip()
+    if not cleaned:
+        return fallback
+    return cleaned[:MAX_ID_LEN]
+
 
 class SQSSink:
     """A :class:`~log_foundry.sinks.base.Sink` that sends events to an SQS queue."""
@@ -31,7 +74,20 @@ class SQSSink:
     MAX_BATCH = 10  # SQS SendMessageBatch hard limit: entries per request
     MAX_BYTES = 256 * 1024  # SQS limit: 256 KB per request
 
-    def __init__(self, queue_url: str, client: Any = None, *, max_retries: int = 3) -> None:
+    def __init__(
+        self,
+        queue_url: str,
+        client: Any = None,
+        *,
+        max_retries: int = 3,
+        fifo: bool | None = None,
+        message_group_id: GroupIdSource = None,
+        message_deduplication_id: DedupIdSource = None,
+    ) -> None:
+        if isinstance(message_group_id, str) and not message_group_id.strip():
+            raise ValueError(
+                "message_group_id must be a non-empty string; pass None to group by trace_id"
+            )
         if client is None:
             import boto3  # type: ignore[import-not-found]  # optional 'aws' extra
 
@@ -39,6 +95,11 @@ class SQSSink:
         self.queue_url = queue_url
         self.client = client
         self.max_retries = max_retries
+        # AWS requires every FIFO queue name to end in '.fifo', so the suffix is a contract
+        # rather than a guess — but an explicit flag still wins. Decided once, not per emit.
+        self.fifo = queue_url.endswith(".fifo") if fifo is None else fifo
+        self.message_group_id = message_group_id
+        self.message_deduplication_id = message_deduplication_id
         self.dropped_oversized = 0  # events too large to ever fit one message
         self.failed = 0  # entries still failing after the retry bound
 
@@ -52,15 +113,51 @@ class SQSSink:
 
     # -- internals ----------------------------------------------------------------------
 
-    def _chunks(self, batch: list[dict[str, object]]) -> list[list[str]]:
-        """Split ``batch`` into sends of ≤ ``MAX_BATCH`` bodies and ≤ ``MAX_BYTES`` each.
+    def _group_id(self, event: dict[str, object]) -> str:
+        """Resolve one event's ``MessageGroupId`` (SPEC-016 FR-002).
+
+        A constant is used as given; a callable is asked; otherwise the event's own
+        ``trace_id`` groups it. Every route is bounded by :func:`_bounded`, so a caller's
+        callable returning ``""`` cannot put an empty parameter on the wire.
+        """
+        source = self.message_group_id
+        if isinstance(source, str):
+            raw = source
+        elif source is not None:
+            raw = source(event)
+        else:
+            raw = str(event.get("trace_id", ""))
+        return _bounded(raw, DEFAULT_GROUP_ID)
+
+    def _dedup_id(self, event: dict[str, object]) -> str:
+        """Resolve one event's ``MessageDeduplicationId`` (SPEC-016 FR-003).
+
+        Defaults to the event's ``log_id`` — already a per-event UUID, so SQS's five-minute
+        deduplication window can never collapse two genuinely distinct records. The fallback
+        mints a fresh id for the same reason: a shared constant would make unrelated events
+        deduplicate each other.
+        """
+        source = self.message_deduplication_id
+        raw = source(event) if source is not None else str(event.get("log_id", ""))
+        if raw.strip():
+            return _bounded(raw, "")
+        from log_foundry.ids import new_log_id
+
+        return new_log_id()
+
+    def _chunks(self, batch: list[dict[str, object]]) -> list[list[_Prepared]]:
+        """Split ``batch`` into sends of ≤ ``MAX_BATCH`` entries and ≤ ``MAX_BYTES`` each.
 
         Each event is serialized once with ``json.dumps``. An event whose serialized size
         exceeds ``MAX_BYTES`` on its own can never fit a message, so it is dropped with a
-        counted warning (FR-004) instead of stalling the batch.
+        counted warning (FR-004) instead of stalling the batch — judged on the body alone, so
+        an event's droppability does not depend on which queue type it is bound for.
+
+        On a FIFO queue the resolved ids are costed alongside the body (SPEC-016 FR-005): they
+        travel in the same request, so leaving them out of the budget could overshoot 256 KB.
         """
-        chunks: list[list[str]] = []
-        current: list[str] = []
+        chunks: list[list[_Prepared]] = []
+        current: list[_Prepared] = []
         current_bytes = 0
         for event in batch:
             body = json.dumps(event)
@@ -72,32 +169,66 @@ class SQSSink:
                     f"{self.MAX_BYTES}-byte SQS message limit\n"
                 )
                 continue
+            group_id: str | None = None
+            dedup_id: str | None = None
+            if self.fifo:
+                group_id = self._group_id(event)
+                dedup_id = self._dedup_id(event)
+                size += len(group_id.encode("utf-8")) + len(dedup_id.encode("utf-8"))
             if current and (
                 len(current) >= self.MAX_BATCH or current_bytes + size > self.MAX_BYTES
             ):
                 chunks.append(current)
                 current = []
                 current_bytes = 0
-            current.append(body)
+            current.append(_Prepared(body, group_id, dedup_id))
             current_bytes += size
         if current:
             chunks.append(current)
         return chunks
 
-    def _send(self, bodies: list[str]) -> None:
+    def _send(self, prepared: list[_Prepared]) -> None:
         """Send one valid chunk, retrying only the ``Failed`` entries with a bounded count.
 
         Successfully-sent entries are never re-sent; entries still failing past ``max_retries``
-        are counted (``failed``) and logged, not silently dropped (FR-003).
+        are counted (``failed``) and logged, not silently dropped (FR-003). The FIFO parameters
+        are attached only when the queue is FIFO, so a standard queue's entries are exactly
+        ``Id`` + ``MessageBody`` (SPEC-016 FR-004).
+
+        Entries SQS marks ``SenderFault`` are abandoned rather than retried (SPEC-016 FR-006):
+        a retry re-sends them byte-identical, so a fault in the request itself can only fail
+        the same way. Throttles and internal errors carry ``SenderFault: false`` and are still
+        retried under the bound.
         """
-        entries = [{"Id": str(i), "MessageBody": body} for i, body in enumerate(bodies)]
+        entries: list[dict[str, str]] = []
+        for i, item in enumerate(prepared):
+            entry = {"Id": str(i), "MessageBody": item.body}
+            if item.group_id is not None:
+                entry["MessageGroupId"] = item.group_id
+            if item.dedup_id is not None:
+                entry["MessageDeduplicationId"] = item.dedup_id
+            entries.append(entry)
         for attempt in range(self.max_retries + 1):
             response = self.client.send_message_batch(QueueUrl=self.queue_url, Entries=entries)
             failed = response.get("Failed", [])
             if not failed:
                 return
-            failed_ids = {entry["Id"] for entry in failed}
-            entries = [entry for entry in entries if entry["Id"] in failed_ids]
+            # Abandon sender faults immediately and name the code: the retry would re-send the
+            # entry byte-identical, and the code is the only thing that makes a rejection
+            # diagnosable from the log line alone. A missing flag is treated as retryable, so
+            # an unfamiliar response shape degrades to the old behaviour rather than dropping.
+            sender_faults = [item for item in failed if item.get("SenderFault")]
+            if sender_faults:
+                self.failed += len(sender_faults)
+                sys.stderr.write(
+                    f"log-foundry: {len(sender_faults)} SQS message(s) rejected as invalid "
+                    f"(first code: {sender_faults[0].get('Code', 'unknown')}); not retried\n"
+                )
+
+            retryable_ids = {item["Id"] for item in failed if not item.get("SenderFault")}
+            if not retryable_ids:
+                return
+            entries = [entry for entry in entries if entry["Id"] in retryable_ids]
             if attempt >= self.max_retries:
                 self.failed += len(entries)
                 sys.stderr.write(

{log_foundry-0.4.1.dev2.dist-info → log_foundry-0.4.1.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: log-foundry
-Version: 0.4.1.dev2
+Version: 0.4.1.dev4
 Summary: Generate logs for your console and JSON events for downstream consumption.
 License-Expression: MIT
 License-File: LICENSE
@@ -512,7 +512,7 @@ limits, retries partial failures, and drops any single event too large to ever f
 
 | Sink | Import from | Configure |
 |---|---|---|
-| `SQSSink` | `log_foundry.sinks.sqs` | `SQSSink(queue_url, *, max_retries=3)` — the headline production path: a durable buffer in front of ELK, absorbing downstream spikes/outages |
+| `SQSSink` | `log_foundry.sinks.sqs` | `SQSSink(queue_url, *, max_retries=3, fifo=None, message_group_id=None, message_deduplication_id=None)` — the headline production path: a durable buffer in front of ELK, absorbing downstream spikes/outages. Standard **and** FIFO queues |
 | `SNSSink` | `log_foundry.sinks.sns` | `SNSSink(topic_arn, *, max_retries=3)` |
 | `KinesisSink` | `log_foundry.sinks.kinesis` | `KinesisSink(stream_name, *, partition_key_field="trace_id", max_retries=3)` |
 | `FirehoseSink` | `log_foundry.sinks.firehose` | `FirehoseSink(delivery_stream, *, max_retries=3)` |
@@ -525,6 +525,51 @@ lf.configure(service="payments",
 
 Consuming from the buffer and indexing into ELK is a separate component, outside this library.
 
+`SQSSink` does not retry a message SQS rejects as a **sender fault** — the retry would re-send it
+byte-identical, so it can only fail the same way. Those are counted on `.failed` immediately and
+the SQS error code is named on stderr. Throttles and internal errors are still retried up to
+`max_retries`.
+
+##### FIFO queues
+
+A queue URL ending in `.fifo` switches `SQSSink` into FIFO mode automatically — AWS requires the
+suffix on every FIFO queue, so nothing needs configuring:
+
+```python
+SQSSink(queue_url="https://sqs.us-east-1.amazonaws.com/123456789012/logs.fifo")
+```
+
+Each message then carries a **`MessageGroupId`**, which defaults to the event's own `trace_id`.
+SQS guarantees ordering *within* a group, and a trace is exactly the unit whose events should stay
+ordered — while separate traces land in separate groups, so the queue delivers them in parallel
+instead of serializing your whole process behind one group. (`KinesisSink` partitions on `trace_id`
+by default for the same reason.) The **`MessageDeduplicationId`** defaults to the event's `log_id`,
+already a per-event UUID, so SQS's five-minute deduplication window never collapses two distinct
+records.
+
+Override the group with a constant or a callable:
+
+```python
+# One group for the whole process — strict global ordering, capped at ~300 msg/s.
+SQSSink(queue_url=FIFO_URL, message_group_id="payments")
+
+# Group by anything on the event. Baggage lands in `fields`, so this groups by tenant
+# and falls back to per-trace when unset:
+SQSSink(queue_url=FIFO_URL,
+        message_group_id=lambda e: str(e["fields"].get("tenant_id") or e["trace_id"]))
+```
+
+Pass `fifo=True` or `fifo=False` to override the URL-based detection. Standard queues are entirely
+unaffected — their messages carry neither parameter.
+
+Two things worth knowing:
+
+- **Ordering is best-effort across a retry.** If one message fails and a same-group message ahead
+  of it succeeded, the retry lands after it. Holding a whole group back on a single failure would
+  trade log delivery for ordering you can rebuild from `timestamp`, so the sink doesn't.
+- **FIFO queues cap throughput** at 300 messages/second (3,000 with batching), or higher in
+  high-throughput mode. That's queue-side configuration, not something the library sets.
+
 #### Queue & stream
 
 Each needs its own extra (lazy-imported). All publish + retry within a bound and close cleanly.

{log_foundry-0.4.1.dev2.dist-info → log_foundry-0.4.1.dev4.dist-info}/RECORD

@@ -39,13 +39,13 @@ log_foundry/sinks/sentry.py,sha256=EfOpEjcJZvYZCaSIVMbpLOxzaxwo7e5zHWfFhzNuCJU,5
 log_foundry/sinks/sns.py,sha256=J3ILBbyzl6jwf_A2wuIsrWjKUItcPRMXEm7bj4g2zSs,3304
 log_foundry/sinks/splunk.py,sha256=JlPOC1MRMrRbT7RoV87bMc_azdoPYvghlcfsAJacRuI,1874
 log_foundry/sinks/sqlite.py,sha256=lD4289rsArlqnFNmMGewLeHexdK-Hp-3Z11GsEqKLgM,4621
-log_foundry/sinks/sqs.py,sha256=uQdN9kNpVZeEIKpm2eN8oiWQjvKeAYeD2Ra9FbJxfh0,4799
+log_foundry/sinks/sqs.py,sha256=_Cz7dO8gUFwfhOgmroSH_tcRd3PJM4kNhW5ThLcWJD8,11189
 log_foundry/sinks/stdout.py,sha256=X_hVTUAAsppx1k8znuBveOZZwj3FqcTwLH185_JWJUU,1052
 log_foundry/sinks/syslog.py,sha256=CAQ957jBIlAzrLyrOmtzspqcnAFBfYHGrIN8I35DGMw,3381
 log_foundry/sinks/transform.py,sha256=eeZpaMxc7urrZ07Vc4GQ-sIq2mTYhoA_dI4Ur4Mmb4s,1613
 log_foundry/sinks/util.py,sha256=SCYFxW9Jm-bIVsyOKir5JWWbwP7u5VFnhNmncMIkwBM,2759
 log_foundry/worker.py,sha256=bbBwMwecVOBu1k6rbSLPwM0zzxN-ebGdy5X2Mo5Q14s,11014
-log_foundry-0.4.1.dev2.dist-info/METADATA,sha256=IkkNXeEVzy1S4hQv_Hs8Hyo9LXbgSneoujobMsRrEHk,38154
-log_foundry-0.4.1.dev2.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
-log_foundry-0.4.1.dev2.dist-info/licenses/LICENSE,sha256=HaCckyCX8P6TR8gY5xyEsFi050Hs2kl-JqaO9Vi0Sw0,1072
-log_foundry-0.4.1.dev2.dist-info/RECORD,,
+log_foundry-0.4.1.dev4.dist-info/METADATA,sha256=Ymt2o3X4EiFnPc64Yz6aH2QS_HwL7S6msPr-YSsnpAs,40527
+log_foundry-0.4.1.dev4.dist-info/WHEEL,sha256=eY7nduwzv-ldUxpzbRlxwvC693Hg6PX8bWDjEHjZ_dk,88
+log_foundry-0.4.1.dev4.dist-info/licenses/LICENSE,sha256=HaCckyCX8P6TR8gY5xyEsFi050Hs2kl-JqaO9Vi0Sw0,1072
+log_foundry-0.4.1.dev4.dist-info/RECORD,,