aws-python-helper 1.8.1__tar.gz → 1.9.1__tar.gz

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-python-helper
-Version: 1.8.1
+Version: 1.9.1
 Summary: AWS Python Helper Framework
 Author-email: Fabian Claros <neufabiae@gmail.com>
 License-Expression: MIT

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/aws_python_helper/__init__.py

@@ -57,11 +57,12 @@ from .invoker import (
 )
 
 
-from .monitoring import process_run, RunReporter, ProcessRunPublisher
+from .monitoring import process_run, resolve_code, RunReporter, ProcessRunPublisher
 
 __all__ = [
     'API',
     'process_run',
+    'resolve_code',
     'RunReporter',
     'ProcessRunPublisher',
     'SQSConsumer',

aws_python_helper-1.9.1/aws_python_helper/monitoring/__init__.py

@@ -0,0 +1,3 @@
+from .run import ProcessRunPublisher, RunReporter, process_run, resolve_code
+
+__all__ = ['process_run', 'resolve_code', 'RunReporter', 'ProcessRunPublisher']

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/aws_python_helper/monitoring/run.py

@@ -4,10 +4,21 @@ Two hand-written publishes would work, and the second one would get forgotten on
 the path that matters: the failure. So the boundary is a context manager — enter
 and it says it started, leave and it says how it went, exception included.
 
-Nothing here can raise into the caller. Monitoring must never be able to break
-the thing it monitors: a topic that does not exist, credentials that are wrong,
-SNS being slow are all reasons to lose a record, never reasons to fail a run that
-otherwise worked.
+The process code is the name of the AWS resource doing the work, taken from the
+environment. Nobody types it twice: a code invented here and typed again in the
+panel is a code that will not match one day, and a mismatch is silent — the runs
+arrive, core finds no process, and nothing is recorded. Taking it from the
+environment means the panel can offer the real resource list and the two sides
+cannot drift.
+
+Nothing here decides whether a run is kept. It publishes; core has the database
+and filters. Wrapping something in `process_run` is not turning monitoring on —
+registering the process in the panel is.
+
+Nothing here can raise into the caller either. Monitoring must never be able to
+break the thing it monitors: a topic that does not exist, credentials that are
+wrong, SNS being slow are all reasons to lose a record, never reasons to fail a
+run that otherwise worked.
 """
 
 import logging
@@ -17,6 +28,7 @@ from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from typing import Any, Dict, Optional
 
+from ..context.session import get_session
 from ..sns.publisher import SNSPublisher
 
 logger = logging.getLogger(__name__)
@@ -28,6 +40,11 @@ FAILED = "failed"
 
 TOPIC_ENV = "PROCESS_RUNS_TOPIC_ARN"
 
+# De dónde sale el código del proceso, en orden. `PROCESS_CODE` es para lo que no
+# es una lambda —una tarea de Fargate lo recibe en su definición— y el nombre de
+# la función es lo que AWS deja puesto solo.
+CODE_ENVS = ("PROCESS_CODE", "AWS_LAMBDA_FUNCTION_NAME")
+
 
 class ProcessRunPublisher(SNSPublisher):
     pass
@@ -81,12 +98,25 @@ def _now() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+def resolve_code(explicit: Optional[str] = None) -> Optional[str]:
+    """El código del proceso: el nombre del recurso que está corriendo."""
+    if explicit:
+        return explicit
+    for name in CODE_ENVS:
+        value = (os.getenv(name) or "").strip()
+        if value:
+            return value
+    return None
+
+
 @asynccontextmanager
 async def process_run(
-    process_code: str,
+    process_code: Optional[str] = None,
     *,
     run_id: Optional[str] = None,
+    parent_run_id: Optional[str] = None,
     scope: Optional[str] = None,
+    state: Optional[str] = None,
     resource_arn: Optional[str] = None,
     request_id: Optional[str] = None,
 ):
@@ -101,47 +131,65 @@ async def process_run(
     se está vigilando, y registrarlo solo llenaría la pantalla de ejecuciones que
     nadie puede configurar.
     """
-    reporter = RunReporter(run_id or str(uuid.uuid4()), process_code)
+    code = resolve_code(process_code)
+    if not code:
+        # Sin código no hay a qué proceso pertenecer. Se sigue adelante sin
+        # reportar: el trabajo importa más que su registro.
+        logger.warning(
+            "No process code: pass one, or set %s. The run was not reported",
+            " or ".join(CODE_ENVS),
+        )
+        yield RunReporter("", "")
+        return
+
+    # El estado sale de la sesión: core identifica un proceso por código *y*
+    # estado, porque una misma lambda corre para varios con frecuencias
+    # distintas.
+    if state is None:
+        session = get_session()
+        state = session.state if session else None
+
+    reporter = RunReporter(run_id or str(uuid.uuid4()), code)
     started_at = _now()
+    common = {
+        "run_id": reporter.run_id,
+        "process_code": code,
+        "state": state,
+        "parent_run_id": parent_run_id,
+        "request_id": request_id,
+    }
 
     await _publish({
-        "run_id": reporter.run_id,
-        "process_code": process_code,
+        **common,
         "status": LAUNCHED,
         "started_at": started_at,
         "scope": scope,
-        # Lo único que AWS deja en el ambiente. El ARN completo y el id de la
-        # invocación viven en el `context` del handler, que acá no se tiene: quien
-        # lo tenga los pasa por `resource_arn` y `request_id`.
-        "resource_arn": resource_arn or os.getenv("AWS_LAMBDA_FUNCTION_NAME") or None,
-        "request_id": request_id,
+        # El ARN completo y el id de la invocación viven en el `context` del
+        # handler, que acá no se tiene: quien lo tenga los pasa por parámetro.
+        "resource_arn": resource_arn,
     })
 
     try:
         yield reporter
     except Exception as error:
         await _publish({
-            "run_id": reporter.run_id,
-            "process_code": process_code,
+            **common,
             "status": FAILED,
             "started_at": started_at,
             "finished_at": _now(),
             "error": f"{type(error).__name__}: {error}"[:2000],
             "counters": reporter.counters,
             "scope": reporter.scope or scope,
-            "request_id": request_id,
         })
         # La excepción sigue su camino: el monitoreo anota lo que pasó, no lo
         # decide. Tragarla acá convertiría una falla en un éxito silencioso.
         raise
     else:
         await _publish({
-            "run_id": reporter.run_id,
-            "process_code": process_code,
+            **common,
             "status": SUCCESS,
             "started_at": started_at,
             "finished_at": _now(),
             "counters": reporter.counters,
             "scope": reporter.scope or scope,
-            "request_id": request_id,
         })

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/aws_python_helper/repository/audit.py

@@ -124,6 +124,23 @@ def _merge_into_set(update: Dict[str, Any], values: Dict[str, Any], operator: st
         section.setdefault(key, value)
 
 
+def _written_elsewhere(update: Dict[str, Any], key: str, operator: str) -> bool:
+    """Si otro operador de esta misma actualización ya escribe ese campo.
+
+    Mongo rechaza el update entero si dos operadores tocan la misma ruta —
+    `$set: {status}` junto a `$setOnInsert: {status}` da "would create a
+    conflict at 'status'" — y el sello no tiene por qué ser el que lo provoque.
+    Si el llamador ya escribe el campo, el insert va a quedar con su valor, así
+    que el sello ahí sobra.
+    """
+    for other, section in update.items():
+        if other == operator or not other.startswith('$') or not isinstance(section, dict):
+            continue
+        if key in section:
+            return True
+    return False
+
+
 def stamp_update(
     update: Union[Dict[str, Any], List[Dict[str, Any]]],
     upsert: bool = False,
@@ -164,6 +181,11 @@ def stamp_update(
         born = {CREATED_AT: now, CREATED_BY: who}
         if statuses:
             born[STATUS] = default_status or statuses[0]
+        born = {
+            key: value
+            for key, value in born.items()
+            if not _written_elsewhere(update, key, '$setOnInsert')
+        }
         _merge_into_set(update, born, '$setOnInsert')
     return update
 

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/aws_python_helper/sns/publisher.py

@@ -15,6 +15,11 @@ from abc import ABC
 from ..context.session import get_session
 
 
+# Lo que SNS acepta en una sola llamada a PublishBatch. No es configurable: es
+# un límite del servicio, y pasarse falla el lote entero sin publicar nada.
+MAX_BATCH_ENTRIES = 10
+
+
 class SNSPublisher(ABC):
     """
     Base class to publish messages to SNS
@@ -172,35 +177,52 @@ class SNSPublisher(ABC):
         message_ids_success = []
         message_ids_failed = []
 
-        params = {
-            'TopicArn': self.topic_arn
-        }
-        
         for i, message in enumerate(messages):
             try:
                 message_to_publish.append(self._format_message(message, i))
             except Exception as e:
                 self.logger.error(f"Error publishing message {i} in batch: {e}")
+                message_ids_failed.append(None)
                 # Continue with the other messages
                 continue
 
-        params['PublishBatchRequestEntries'] = message_to_publish
+        # De a diez, que es lo que SNS acepta por llamada. Mandar más falla el
+        # lote entero sin publicar nada, así que una lista larga no se pierde a
+        # medias: se pierde completa.
+        for start in range(0, len(message_to_publish), MAX_BATCH_ENTRIES):
+            chunk = message_to_publish[start:start + MAX_BATCH_ENTRIES]
 
-        try:
-            result = self.sns_client.publish_batch(**params)
-        except Exception as e:
-            self.logger.error(f"Error calling publish_batch to {self.topic_arn}: {e}")
-            raise
+            # El Id sólo tiene que ser único dentro de su propia llamada.
+            entries = [
+                {**entry, 'Id': str(position)}
+                for position, entry in enumerate(chunk)
+            ]
+
+            try:
+                result = self.sns_client.publish_batch(
+                    TopicArn=self.topic_arn,
+                    PublishBatchRequestEntries=entries,
+                )
+            except Exception as e:
+                # Se sigue con los demás lotes: cortar acá dejaría sin publicar
+                # todo lo que venía detrás por un problema que puede ser de este
+                # lote solo.
+                self.logger.error(
+                    f"Error calling publish_batch to {self.topic_arn} "
+                    f"({start}-{start + len(chunk) - 1} of {len(message_to_publish)}): {e}"
+                )
+                message_ids_failed.extend([None] * len(chunk))
+                continue
 
-        for message in result.get('Successful', []):
-            message_ids_success.append(message['MessageId'])
+            for message in result.get('Successful', []):
+                message_ids_success.append(message['MessageId'])
 
-        for message in result.get('Failed', []):
-            self.logger.error(
-                f"Failed to publish message {message.get('Id')} to {self.topic_arn}: "
-                f"Code={message.get('Code')}, Message={message.get('Message')}"
-            )
-            message_ids_failed.append(None)
+            for message in result.get('Failed', []):
+                self.logger.error(
+                    f"Failed to publish message {message.get('Id')} to {self.topic_arn}: "
+                    f"Code={message.get('Code')}, Message={message.get('Message')}"
+                )
+                message_ids_failed.append(None)
 
         self.logger.info(
             f"Batch publish to {self.topic_arn}: "

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/aws_python_helper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aws-python-helper
-Version: 1.8.1
+Version: 1.9.1
 Summary: AWS Python Helper Framework
 Author-email: Fabian Claros <neufabiae@gmail.com>
 License-Expression: MIT

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/aws_python_helper.egg-info/SOURCES.txt

@@ -55,4 +55,5 @@ aws_python_helper/sqs/handler.py
 aws_python_helper/utils/__init__.py
 aws_python_helper/utils/json_encoder.py
 aws_python_helper/utils/response.py
-aws_python_helper/utils/serializer.py
+aws_python_helper/utils/serializer.py
+tests/test_audit_stamp.py

{aws_python_helper-1.8.1 → aws_python_helper-1.9.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "aws-python-helper"
-version = "1.8.1"
+version = "1.9.1"
 description = "AWS Python Helper Framework"
 readme = "README.md"
 requires-python = ">=3.9"

aws_python_helper-1.9.1/tests/test_audit_stamp.py

@@ -0,0 +1,88 @@
+"""What the audit stamp must never do to a write.
+
+Runs on its own, no dependencies:
+
+    python tests/test_audit_stamp.py
+
+These are here because the stamp broke a write in production once: an upsert
+that set `status` itself collided with the `$setOnInsert` the stamp added, and
+Mongo refused the whole update with "would create a conflict at 'status'". It
+failed at runtime, on one endpoint, in a log nobody was reading.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from aws_python_helper.repository.audit import stamp_update  # noqa: E402
+
+STATUSES = ["active", "inactive"]
+
+failures = []
+
+
+def check(label, condition, detail=""):
+    mark = "ok  " if condition else "FAIL"
+    print(f"  {mark} {label}{'  ' + detail if detail else ''}")
+    if not condition:
+        failures.append(label)
+
+
+def conflicts(update):
+    """Los campos que dos operadores de un mismo update escriben a la vez.
+
+    Es lo que Mongo rechaza, y rechaza el update entero: no escribe nada.
+    """
+    sections = [
+        (name, section) for name, section in update.items()
+        if name.startswith("$") and isinstance(section, dict)
+    ]
+    clash = set()
+    for index, (_, first) in enumerate(sections):
+        for _, second in sections[index + 1:]:
+            clash |= set(first) & set(second)
+    return clash
+
+
+print("no two operators may write the same path")
+for label, update in [
+    ("upsert that sets status itself", {"$set": {"name": "ct", "status": "active"}}),
+    ("upsert that sets created_at itself", {"$set": {"created_at": "fixed"}}),
+    ("upsert that sets created_by itself", {"$set": {"created_by": {"name": "x"}}}),
+    ("plain upsert", {"$set": {"name": "ct"}}),
+    ("upsert with $inc alongside", {"$set": {"name": "ct"}, "$inc": {"hits": 1}}),
+]:
+    result = stamp_update(dict(update), upsert=True, statuses=STATUSES, default_status="active")
+    found = conflicts(result)
+    check(label, not found, f"conflicts={sorted(found)}" if found else "")
+
+print("\nthe stamp is still complete when nothing conflicts")
+result = stamp_update({"$set": {"name": "ct"}}, upsert=True, statuses=STATUSES, default_status="active")
+check("status on insert", result["$setOnInsert"].get("status") == "active")
+check("created_at on insert", "created_at" in result["$setOnInsert"])
+check("created_by on insert", "created_by" in result["$setOnInsert"])
+check("updated_at always", "updated_at" in result["$set"])
+
+print("\nthe caller's own value wins")
+result = stamp_update({"$set": {"status": "inactive"}}, upsert=True, statuses=STATUSES, default_status="active")
+check("status stays what the caller set", result["$set"]["status"] == "inactive")
+check("no default forced on top", "status" not in result.get("$setOnInsert", {}))
+
+print("\nwithout upsert there is no insert stamp")
+result = stamp_update({"$set": {"status": "inactive"}}, upsert=False, statuses=STATUSES)
+check("no $setOnInsert", "$setOnInsert" not in result)
+
+print("\nwithout a vocabulary no status is invented")
+result = stamp_update({"$set": {"name": "ct"}}, upsert=True, statuses=None)
+check("no status", "status" not in result.get("$setOnInsert", {}))
+
+print("\na pipeline update takes stages, not operators")
+result = stamp_update([{"$set": {"name": "ct"}}], upsert=True, statuses=STATUSES)
+check("still a list", isinstance(result, list))
+check("stamp appended last", result[-1]["$set"].get("updated_at") is not None)
+
+if failures:
+    print(f"\n{len(failures)} failed: {failures}")
+    raise SystemExit(1)
+print("\nall good")

aws_python_helper-1.8.1/aws_python_helper/monitoring/__init__.py

@@ -1,3 +0,0 @@
-from .run import ProcessRunPublisher, RunReporter, process_run
-
-__all__ = ['process_run', 'RunReporter', 'ProcessRunPublisher']