velocity-python 0.1.74__tar.gz → 0.1.75__tar.gz

{velocity_python-0.1.74 → velocity_python-0.1.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.74
+Version: 0.1.75
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.74 → velocity_python-0.1.75}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "velocity-python"
-version = "0.1.74"
+version = "0.1.75"
 authors = [
   { name="Velocity Team", email="info@codeclubs.org" },
 ]

{velocity_python-0.1.74 → velocity_python-0.1.75}/src/velocity/__init__.py

@@ -1,4 +1,4 @@
-__version__ = version = "0.1.74"
+__version__ = version = "0.1.75"
 
 import importlib as _importlib
 

{velocity_python-0.1.74 → velocity_python-0.1.75}/src/velocity/aws/handlers/context.py

@@ -36,6 +36,38 @@ def _get_work_queue_name() -> str:
     return queue_name
 
 
+def _emit_enqueue_failure_metric(action, count):
+    """Emit a CloudWatch ``CaringCent/Enqueue SendFailures`` metric.
+
+    SQS send failures are a pre-queue loss the dead-letter queue can never see
+    (a message that was never sent never reaches the DLQ), so this metric makes
+    send-side losses observable/alarmable the same way DLQ arrivals are.
+    Best-effort: never raises, so a metrics outage cannot break enqueueing.
+    """
+    if not count:
+        return
+    try:
+        stage = config_get_stage("") or "unknown"
+        boto3.client("cloudwatch").put_metric_data(
+            Namespace="CaringCent/Enqueue",
+            MetricData=[
+                {
+                    "MetricName": "SendFailures",
+                    "Value": float(count),
+                    "Unit": "Count",
+                    "Dimensions": [{"Name": "Stage", "Value": str(stage)}],
+                }
+            ],
+        )
+    except Exception:
+        logger.warning(
+            "Failed to emit enqueue SendFailures metric (action=%s, count=%s)",
+            action,
+            count,
+            exc_info=True,
+        )
+
+
 def _get_controls_vars() -> dict:
     app_stage = config_get_stage()
     return {
@@ -266,6 +298,32 @@ class Context:
             self._job_record_cache.pop(job_id, None)
         tx.commit()
 
+    def fail_enqueue(self, tx, job_id=None, reason=None):
+        """Mark a job-activity row Failed when its SQS message could not be sent.
+
+        ``enqueue`` writes the ``aws_job_activity`` row (status ``Initialized``)
+        *before* sending to SQS. If the send fails and is never reconciled, the
+        row is stuck in ``Initialized`` forever — never consumed, and never
+        counted as a failure (the consumer's ``onError`` is the only thing that
+        writes ``Failed``). This flips such an orphaned row to ``Failed`` so it
+        surfaces on the operational dashboard instead of inflating the stuck
+        count. Runs in its own transaction like ``create_job``/``update_job``.
+        """
+        if not job_id:
+            return
+        tx.table(self.JOB_ACTIVITY_TABLE).update(
+            self._sanitize_job_data(
+                {
+                    "status": "Failed",
+                    "message": "Job enqueue failed (SQS send failure)",
+                    "error": reason or "SQS send failed",
+                }
+            ),
+            {"job_id": job_id},
+        )
+        self._job_record_cache.pop(job_id, None)
+        tx.commit()
+
     def _sanitize_job_data(self, data):
         """Sanitize sensitive data before storing in aws_job_activity table."""
         if not isinstance(data, dict):
@@ -444,6 +502,50 @@ class Context:
         messages = []
         if user is None:
             user = self.session().get("email_address") or "EnqueueTasks"
+
+        def _flush(batch):
+            """Send one batch and reconcile SQS send failures.
+
+            A failed send would otherwise leave the job-activity row stuck in
+            ``Initialized`` forever, so transient (non-``SenderFault``) failures
+            are retried once and any message that still never reaches the queue
+            has its job row marked ``Failed`` via ``fail_enqueue``.
+            """
+            nonlocal results
+            if not batch:
+                return
+            result = queue.send_messages(Entries=batch) or {}
+            results = deep_merge(results, result)
+            failed = result.get("Failed") or []
+            if not failed:
+                return
+            by_id = {entry["Id"]: entry for entry in batch}
+            still_failed = {f["Id"]: f for f in failed if f.get("SenderFault")}
+            transient = [
+                by_id[f["Id"]]
+                for f in failed
+                if not f.get("SenderFault") and f.get("Id") in by_id
+            ]
+            if transient:
+                # The retry result is consulted only for entries that still
+                # failed; it is not merged into ``results`` because ``deep_merge``
+                # cannot dedup SQS's list-of-dict ``Successful``/``Failed`` payloads.
+                retry = queue.send_messages(Entries=transient) or {}
+                for f in retry.get("Failed") or []:
+                    still_failed[f["Id"]] = f
+            if not still_failed:
+                return
+            if not suppress_job_activity:
+                for job_id, f in still_failed.items():
+                    self.fail_enqueue(
+                        job_id=job_id,
+                        reason="SQS send failed: {} {}".format(
+                            f.get("Code") or "", f.get("Message") or ""
+                        ).strip(),
+                    )
+            _emit_enqueue_failure_metric(action, len(still_failed))
+            results.setdefault("FailedToEnqueue", []).extend(still_failed.keys())
+
         for item in payload:
             message = {"action": action, "payload": item}
             id = str(uuid.uuid4()).split("-")[0]
@@ -474,13 +576,10 @@ class Context:
                 messages.append({"Id": id, "MessageBody": to_json(message)})
 
             if len(messages) == 10:
-                result = queue.send_messages(Entries=messages)
-                results = deep_merge(results, result)
-                messages.clear()
+                _flush(messages)
+                messages = []
 
-        if messages:
-            result = queue.send_messages(Entries=messages)
-            results = deep_merge(results, result)
+        _flush(messages)
 
         return results
 

{velocity_python-0.1.74 → velocity_python-0.1.75}/src/velocity_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.74
+Version: 0.1.75
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.74 → velocity_python-0.1.75}/src/velocity_python.egg-info/SOURCES.txt

@@ -165,6 +165,7 @@ tests/test_db_credentials_ssm_cascade.py
 tests/test_decorators.py
 tests/test_dirty_pipeline_fast_path.py
 tests/test_email_processing.py
+tests/test_enqueue_send_failures.py
 tests/test_get_cognito_user_provider.py
 tests/test_http_handler_rollback.py
 tests/test_iconv_money_to_cents.py

velocity_python-0.1.75/tests/test_enqueue_send_failures.py

@@ -0,0 +1,146 @@
+import unittest
+from unittest.mock import patch
+
+from velocity.aws.handlers import context as handler_context
+
+
+class _FakeQueue:
+    """Fake SQS queue whose send_messages fails for chosen entry Ids.
+
+    ``fail_mode`` controls the Failed payload:
+      - "transient": fail with SenderFault=False (retryable)
+      - "permanent": fail with SenderFault=True (no retry)
+    ``fail_calls`` lets a transient failure succeed on retry (e.g. [True, False]).
+    """
+
+    def __init__(self, fail_mode=None, fail_calls=None):
+        self.fail_mode = fail_mode
+        self.calls = []
+        self._fail_calls = list(fail_calls) if fail_calls is not None else None
+
+    def send_messages(self, Entries):
+        self.calls.append([e["Id"] for e in Entries])
+        fail_this_call = True
+        if self._fail_calls is not None:
+            fail_this_call = self._fail_calls.pop(0) if self._fail_calls else False
+        if not self.fail_mode or not fail_this_call:
+            return {"Successful": [{"Id": e["Id"]} for e in Entries], "Failed": []}
+        sender_fault = self.fail_mode == "permanent"
+        return {
+            "Successful": [],
+            "Failed": [
+                {
+                    "Id": e["Id"],
+                    "Code": "ServiceUnavailable",
+                    "Message": "throttled",
+                    "SenderFault": sender_fault,
+                }
+                for e in Entries
+            ],
+        }
+
+
+class _FakeSqsResource:
+    def __init__(self, queue):
+        self.queue = queue
+
+    def get_queue_by_name(self, QueueName):
+        return self.queue
+
+
+class _FakeContext:
+    """Captures create_job/fail_enqueue so we can assert reconciliation."""
+
+    def __init__(self):
+        self.jobs = []
+        self.failed = []
+
+    def session(self):
+        return {"email_address": "owner@example.com"}
+
+    def create_job(self, job_data=None):
+        self.jobs.append(job_data)
+
+    def fail_enqueue(self, job_id=None, reason=None):
+        self.failed.append({"job_id": job_id, "reason": reason})
+
+
+def _run_enqueue(fake_context, fake_queue, payload, suppress=False):
+    fake_context.metric_calls = []
+
+    def _record(action, count):
+        fake_context.metric_calls.append((action, count))
+
+    with patch.dict(
+        handler_context.os.environ, {"SqsWorkQueue": "example-queue"}, clear=False
+    ), patch.object(
+        handler_context.boto3, "resource", return_value=_FakeSqsResource(fake_queue)
+    ), patch.object(
+        handler_context, "_emit_enqueue_failure_metric", side_effect=_record
+    ):
+        return handler_context.Context.enqueue(
+            fake_context,
+            "resolve-donor-billing-states",
+            payload=payload,
+            suppress_job_activity=suppress,
+        )
+
+
+class TestEnqueueSendFailures(unittest.TestCase):
+    def test_transient_send_failure_is_retried_then_marked_failed(self):
+        ctx = _FakeContext()
+        queue = _FakeQueue(fail_mode="transient")  # fails every time
+        result = _run_enqueue(ctx, queue, [{"id": 1}, {"id": 2}])
+
+        # Each job row was created (Initialized) then reconciled to Failed.
+        self.assertEqual(2, len(ctx.jobs))
+        failed_ids = {f["job_id"] for f in ctx.failed}
+        created_ids = {j["job_id"] for j in ctx.jobs}
+        self.assertEqual(created_ids, failed_ids)
+        self.assertEqual(sorted(failed_ids), sorted(result["FailedToEnqueue"]))
+        # Transient failure means the batch was retried once (2 send calls).
+        self.assertEqual(2, len(queue.calls))
+        # A SendFailures metric is emitted with the action and failed count.
+        self.assertEqual([("resolve-donor-billing-states", 2)], ctx.metric_calls)
+
+    def test_transient_failure_that_succeeds_on_retry_is_not_marked(self):
+        ctx = _FakeContext()
+        queue = _FakeQueue(fail_mode="transient", fail_calls=[True, False])
+        result = _run_enqueue(ctx, queue, [{"id": 1}])
+
+        self.assertEqual(1, len(ctx.jobs))
+        self.assertEqual([], ctx.failed)
+        self.assertNotIn("FailedToEnqueue", result)
+        self.assertEqual(2, len(queue.calls))  # original + successful retry
+
+    def test_permanent_send_failure_is_not_retried(self):
+        ctx = _FakeContext()
+        queue = _FakeQueue(fail_mode="permanent")
+        _run_enqueue(ctx, queue, [{"id": 1}])
+
+        self.assertEqual(1, len(ctx.failed))
+        self.assertEqual(1, len(queue.calls))  # no retry on SenderFault
+
+    def test_suppressed_jobs_do_not_create_or_fail_rows(self):
+        ctx = _FakeContext()
+        queue = _FakeQueue(fail_mode="transient")
+        _run_enqueue(ctx, queue, [{"id": 1}], suppress=True)
+
+        self.assertEqual([], ctx.jobs)
+        self.assertEqual([], ctx.failed)
+        # Send loss is still metered even when job-activity rows are suppressed.
+        self.assertEqual([("resolve-donor-billing-states", 1)], ctx.metric_calls)
+
+    def test_successful_send_marks_nothing(self):
+        ctx = _FakeContext()
+        queue = _FakeQueue()  # always succeeds
+        result = _run_enqueue(ctx, queue, [{"id": 1}, {"id": 2}])
+
+        self.assertEqual(2, len(ctx.jobs))
+        self.assertEqual([], ctx.failed)
+        self.assertNotIn("FailedToEnqueue", result)
+        self.assertEqual([], ctx.metric_calls)  # no failures -> no metric
+
+
+if __name__ == "__main__":
+    unittest.main()