velocity-python 0.1.74__tar.gz → 0.1.76__tar.gz

{velocity_python-0.1.74 → velocity_python-0.1.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.74
+Version: 0.1.76
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.74 → velocity_python-0.1.76}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "velocity-python"
-version = "0.1.74"
+version = "0.1.76"
 authors = [
   { name="Velocity Team", email="info@codeclubs.org" },
 ]

{velocity_python-0.1.74 → velocity_python-0.1.76}/src/velocity/__init__.py

@@ -1,4 +1,4 @@
-__version__ = version = "0.1.74"
+__version__ = version = "0.1.76"
 
 import importlib as _importlib
 

{velocity_python-0.1.74 → velocity_python-0.1.76}/src/velocity/aws/handlers/context.py

@@ -36,6 +36,38 @@ def _get_work_queue_name() -> str:
     return queue_name
 
 
+def _emit_enqueue_failure_metric(action, count):
+    """Emit a CloudWatch ``CaringCent/Enqueue SendFailures`` metric.
+
+    SQS send failures are a pre-queue loss the dead-letter queue can never see
+    (a message that was never sent never reaches the DLQ), so this metric makes
+    send-side losses observable/alarmable the same way DLQ arrivals are.
+    Best-effort: never raises, so a metrics outage cannot break enqueueing.
+    """
+    if not count:
+        return
+    try:
+        stage = config_get_stage("") or "unknown"
+        boto3.client("cloudwatch").put_metric_data(
+            Namespace="CaringCent/Enqueue",
+            MetricData=[
+                {
+                    "MetricName": "SendFailures",
+                    "Value": float(count),
+                    "Unit": "Count",
+                    "Dimensions": [{"Name": "Stage", "Value": str(stage)}],
+                }
+            ],
+        )
+    except Exception:
+        logger.warning(
+            "Failed to emit enqueue SendFailures metric (action=%s, count=%s)",
+            action,
+            count,
+            exc_info=True,
+        )
+
+
 def _get_controls_vars() -> dict:
     app_stage = config_get_stage()
     return {
@@ -266,6 +298,32 @@ class Context:
             self._job_record_cache.pop(job_id, None)
         tx.commit()
 
+    def fail_enqueue(self, tx, job_id=None, reason=None):
+        """Mark a job-activity row Failed when its SQS message could not be sent.
+
+        ``enqueue`` writes the ``aws_job_activity`` row (status ``Initialized``)
+        *before* sending to SQS. If the send fails and is never reconciled, the
+        row is stuck in ``Initialized`` forever — never consumed, and never
+        counted as a failure (the consumer's ``onError`` is the only thing that
+        writes ``Failed``). This flips such an orphaned row to ``Failed`` so it
+        surfaces on the operational dashboard instead of inflating the stuck
+        count. Runs in its own transaction like ``create_job``/``update_job``.
+        """
+        if not job_id:
+            return
+        tx.table(self.JOB_ACTIVITY_TABLE).update(
+            self._sanitize_job_data(
+                {
+                    "status": "Failed",
+                    "message": "Job enqueue failed (SQS send failure)",
+                    "error": reason or "SQS send failed",
+                }
+            ),
+            {"job_id": job_id},
+        )
+        self._job_record_cache.pop(job_id, None)
+        tx.commit()
+
     def _sanitize_job_data(self, data):
         """Sanitize sensitive data before storing in aws_job_activity table."""
         if not isinstance(data, dict):
@@ -444,6 +502,50 @@ class Context:
         messages = []
         if user is None:
             user = self.session().get("email_address") or "EnqueueTasks"
+
+        def _flush(batch):
+            """Send one batch and reconcile SQS send failures.
+
+            A failed send would otherwise leave the job-activity row stuck in
+            ``Initialized`` forever, so transient (non-``SenderFault``) failures
+            are retried once and any message that still never reaches the queue
+            has its job row marked ``Failed`` via ``fail_enqueue``.
+            """
+            nonlocal results
+            if not batch:
+                return
+            result = queue.send_messages(Entries=batch) or {}
+            results = deep_merge(results, result)
+            failed = result.get("Failed") or []
+            if not failed:
+                return
+            by_id = {entry["Id"]: entry for entry in batch}
+            still_failed = {f["Id"]: f for f in failed if f.get("SenderFault")}
+            transient = [
+                by_id[f["Id"]]
+                for f in failed
+                if not f.get("SenderFault") and f.get("Id") in by_id
+            ]
+            if transient:
+                # The retry result is consulted only for entries that still
+                # failed; it is not merged into ``results`` because ``deep_merge``
+                # cannot dedup SQS's list-of-dict ``Successful``/``Failed`` payloads.
+                retry = queue.send_messages(Entries=transient) or {}
+                for f in retry.get("Failed") or []:
+                    still_failed[f["Id"]] = f
+            if not still_failed:
+                return
+            if not suppress_job_activity:
+                for job_id, f in still_failed.items():
+                    self.fail_enqueue(
+                        job_id=job_id,
+                        reason="SQS send failed: {} {}".format(
+                            f.get("Code") or "", f.get("Message") or ""
+                        ).strip(),
+                    )
+            _emit_enqueue_failure_metric(action, len(still_failed))
+            results.setdefault("FailedToEnqueue", []).extend(still_failed.keys())
+
         for item in payload:
             message = {"action": action, "payload": item}
             id = str(uuid.uuid4()).split("-")[0]
@@ -474,13 +576,10 @@ class Context:
                 messages.append({"Id": id, "MessageBody": to_json(message)})
 
             if len(messages) == 10:
-                result = queue.send_messages(Entries=messages)
-                results = deep_merge(results, result)
-                messages.clear()
+                _flush(messages)
+                messages = []
 
-        if messages:
-            result = queue.send_messages(Entries=messages)
-            results = deep_merge(results, result)
+        _flush(messages)
 
         return results
 

velocity_python-0.1.76/src/velocity/db/maintenance.py

@@ -0,0 +1,319 @@
+"""PostgreSQL database maintenance toolkit.
+
+Safe, efficient database-admin operations that velocity's normal transactional
+model can't express. ``CREATE``/``DROP``/``RENAME DATABASE`` and template clones
+must run in **autocommit** and must **not** run while connected to the source or
+destination database. ``DatabaseMaintenance`` connects to a neutral *maintenance*
+database (default ``postgres``) so the source/dest are never the active session,
+runs DDL in autocommit, and terminates other sessions as needed.
+
+    from velocity.db import maintenance
+
+    m = maintenance.DatabaseMaintenance()      # env (DBHost/DBUser/...), maint db 'postgres'
+
+    # Server-side template clone -- instant, no dump/restore round-trip:
+    m.refresh("caringcent-production", "caringcent-develop")  # snapshot dest, then clone source -> dest
+    m.snapshot("caringcent-develop")                          # -> caringcent-develop-snapshot-<ts>
+    m.prune_snapshots("caringcent-develop", keep=3)           # drop all but the 3 newest snapshots
+
+Every database name is identifier-quoted; string-literal comparisons are escaped.
+A configurable ``protected`` set blocks destructive ops (drop / rename / overwrite)
+on critical databases (default: ``caringcent-production``).
+
+Snapshot databases are named ``<db>-snapshot-<YYYYMMDD-HHMMSS>`` -- a **sortable**
+timestamp, so listing/pruning by name is chronological.
+"""
+
+from __future__ import annotations
+
+import datetime
+import os
+from typing import Iterable, List, Optional
+
+from velocity.db.servers import postgres
+
+SNAPSHOT_INFIX = "-snapshot-"
+TS_FORMAT = "%Y%m%d-%H%M%S"  # sortable: lexical order == chronological order
+DEFAULT_PROTECTED = frozenset({"caringcent-production"})
+
+_BACKUP_FORMAT_FLAGS = {"custom": "-Fc", "plain": "-Fp", "directory": "-Fd", "tar": "-Ft"}
+
+
+def quote_identifier(name: str) -> str:
+    """Quote a database identifier (doubles embedded double-quotes)."""
+    return '"' + str(name).replace('"', '""') + '"'
+
+
+def quote_literal(value: str) -> str:
+    """Quote a string literal (doubles embedded single-quotes)."""
+    return "'" + str(value).replace("'", "''") + "'"
+
+
+def timestamp() -> str:
+    return datetime.datetime.now().strftime(TS_FORMAT)
+
+
+class DatabaseMaintenance:
+    """Connection-pooled, autocommit-aware PostgreSQL admin operations."""
+
+    def __init__(
+        self,
+        *,
+        maintenance_db: str = "postgres",
+        config: Optional[dict] = None,
+        protected: Iterable[str] = DEFAULT_PROTECTED,
+        engine=None,
+    ):
+        self.maintenance_db = maintenance_db
+        self.protected = frozenset(protected or ())
+        self._config = dict(config or {})
+        # Lazily initialized so construction never opens a connection (keeps the
+        # class unit-testable without a live database).
+        self._engine = engine
+
+    # ------------------------------------------------------------------ engine
+    def _get_engine(self):
+        if self._engine is None:
+            cfg = {"dbname": self.maintenance_db}
+            cfg.update(self._config)
+            self._engine = postgres.initialize(config=cfg)
+        return self._engine
+
+    def _exec(self, *statements: str) -> None:
+        """Run DDL/admin statements in autocommit (required for CREATE/DROP/RENAME DATABASE)."""
+        engine = self._get_engine()
+
+        @engine.transaction
+        def _run(tx):
+            tx.rollback()  # discard the implicit transaction opened on checkout
+            tx.connection.autocommit = True
+            for sql in statements:
+                if sql:
+                    tx.execute(sql)
+
+        _run()
+
+    def _query(self, sql: str) -> List[dict]:
+        engine = self._get_engine()
+
+        @engine.transaction
+        def _run(tx):
+            return list(tx.execute(sql).as_dict())
+
+        return _run()
+
+    def _conn_params(self) -> dict:
+        return {
+            "host": self._config.get("host") or os.environ.get("DBHost"),
+            "port": self._config.get("port") or os.environ.get("DBPort"),
+            "user": self._config.get("user") or os.environ.get("DBUser"),
+            "password": self._config.get("password") or os.environ.get("DBPassword"),
+        }
+
+    # ------------------------------------------------------------------ guards
+    def _guard(self, db: str, action: str) -> None:
+        if db in self.protected:
+            raise PermissionError(
+                f"Refusing to {action} protected database {db!r}. "
+                "Construct with protected=... to override."
+            )
+
+    # ------------------------------------------------------------------ SQL helpers
+    @staticmethod
+    def _terminate_sql(db: str) -> str:
+        return (
+            "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
+            f"WHERE datname = {quote_literal(db)} AND pid <> pg_backend_pid()"
+        )
+
+    # ------------------------------------------------------------------ queries
+    def exists(self, db: str) -> bool:
+        return bool(
+            self._query(f"SELECT 1 AS x FROM pg_database WHERE datname = {quote_literal(db)}")
+        )
+
+    def list_databases(self, *, pattern: Optional[str] = None) -> List[str]:
+        where = "WHERE datistemplate = false"
+        if pattern:
+            where += f" AND datname LIKE {quote_literal(pattern)}"
+        rows = self._query(f"SELECT datname FROM pg_database {where} ORDER BY datname")
+        return [r["datname"] for r in rows]
+
+    def list_snapshots(self, db: str) -> List[str]:
+        """Snapshot databases for ``db``, oldest first (sortable timestamp suffix)."""
+        return sorted(self.list_databases(pattern=f"{db}{SNAPSHOT_INFIX}%"))
+
+    # ------------------------------------------------------------------ connection control
+    def terminate_connections(self, db: str) -> None:
+        self._exec(self._terminate_sql(db))
+
+    # ------------------------------------------------------------------ core ops
+    def clone(self, source: str, dest: str, *, drop_existing: bool = False) -> str:
+        """``CREATE DATABASE dest WITH TEMPLATE source`` (server-side, near-instant)."""
+        self._guard(dest, "overwrite")
+        if self.exists(dest):
+            if not drop_existing:
+                raise ValueError(
+                    f"Destination {dest!r} already exists. Pass drop_existing=True to replace it."
+                )
+            self.drop(dest)
+        # A template clone requires no other sessions on the source.
+        self._exec(
+            self._terminate_sql(source),
+            f"CREATE DATABASE {quote_identifier(dest)} WITH TEMPLATE {quote_identifier(source)}",
+        )
+        return dest
+
+    def snapshot(self, db: str, *, label: Optional[str] = None) -> str:
+        """Clone ``db`` to ``<db>-snapshot-<ts>`` (or a custom ``label``)."""
+        dest = f"{db}{SNAPSHOT_INFIX}{label or timestamp()}"
+        return self.clone(db, dest)
+
+    def rename(self, old: str, new: str) -> str:
+        self._guard(old, "rename")
+        self._exec(
+            self._terminate_sql(old),
+            f"ALTER DATABASE {quote_identifier(old)} RENAME TO {quote_identifier(new)}",
+        )
+        return new
+
+    def drop(self, db: str, *, if_exists: bool = True) -> None:
+        self._guard(db, "drop")
+        clause = "DROP DATABASE IF EXISTS" if if_exists else "DROP DATABASE"
+        self._exec(self._terminate_sql(db), f"{clause} {quote_identifier(db)}")
+
+    def refresh(self, source: str, dest: str, *, keep_snapshot: bool = True) -> str:
+        """Replace ``dest`` with a fresh template-clone of ``source``.
+
+        Safe sequence: terminate ``dest`` sessions -> rename ``dest`` to
+        ``dest-snapshot-<ts>`` (or drop it when ``keep_snapshot=False``) -> clone
+        ``source`` -> ``dest``. Keeping the snapshot makes the refresh reversible.
+        """
+        self._guard(dest, "overwrite")
+        if self.exists(dest):
+            if keep_snapshot:
+                self.rename(dest, f"{dest}{SNAPSHOT_INFIX}{timestamp()}")
+            else:
+                self.drop(dest)
+        return self.clone(source, dest)
+
+    def prune_snapshots(
+        self,
+        db: str,
+        *,
+        keep: int = 3,
+        older_than_days: Optional[int] = None,
+        dry_run: bool = False,
+    ) -> List[str]:
+        """Drop old snapshot databases for ``db``.
+
+        With ``older_than_days`` set, drops snapshots whose timestamp is older than
+        the cutoff. Otherwise keeps the ``keep`` newest and drops the rest. Returns
+        the names dropped (or that would be dropped, when ``dry_run=True``).
+        """
+        snaps = self.list_snapshots(db)  # ascending (oldest first)
+        to_drop: List[str] = []
+        if older_than_days is not None:
+            cutoff = datetime.datetime.now() - datetime.timedelta(days=older_than_days)
+            prefix = f"{db}{SNAPSHOT_INFIX}"
+            for name in snaps:
+                suffix = name[len(prefix):]
+                try:
+                    when = datetime.datetime.strptime(suffix, TS_FORMAT)
+                except ValueError:
+                    continue  # unrecognized suffix -> leave it alone
+                if when < cutoff:
+                    to_drop.append(name)
+        elif keep >= 0 and len(snaps) > keep:
+            to_drop = snaps[: len(snaps) - keep]
+
+        if not dry_run:
+            for name in to_drop:
+                self.drop(name)
+        return to_drop
+
+    # ------------------------------------------------------------------ portable backup/restore
+    def backup_to_file(
+        self, db: str, path: str, *, fmt: str = "custom", extra_args: Iterable[str] = ()
+    ) -> str:
+        """Portable off-instance backup via ``pg_dump`` (use for cross-server moves;
+        prefer ``snapshot``/``clone`` for same-server copies). ``extra_args`` are
+        passed through to ``pg_dump`` (e.g. ``("-O",)`` for no-owner)."""
+        if fmt not in _BACKUP_FORMAT_FLAGS:
+            raise ValueError(f"Unknown fmt {fmt!r}; choose one of {sorted(_BACKUP_FORMAT_FLAGS)}.")
+        p = self._conn_params()
+        self._run_tool(
+            ["pg_dump", "-h", str(p["host"]), "-p", str(p["port"]), "-U", str(p["user"]),
+             _BACKUP_FORMAT_FLAGS[fmt], *extra_args, "-f", path, db]
+        )
+        return path
+
+    def restore_from_file(
+        self,
+        path: str,
+        dest: str,
+        *,
+        fmt: str = "custom",
+        drop_existing: bool = False,
+        extra_args: Iterable[str] = (),
+    ) -> str:
+        """Restore a ``pg_dump`` file into a freshly created ``dest`` database.
+        ``extra_args`` are passed to ``pg_restore``/``psql``."""
+        self._guard(dest, "overwrite")
+        if self.exists(dest):
+            if not drop_existing:
+                raise ValueError(
+                    f"Destination {dest!r} already exists. Pass drop_existing=True to replace it."
+                )
+            self.drop(dest)
+        self._exec(f"CREATE DATABASE {quote_identifier(dest)}")
+        p = self._conn_params()
+        base = ["-h", str(p["host"]), "-p", str(p["port"]), "-U", str(p["user"])]
+        if fmt == "plain":
+            self._run_tool(["psql", *base, *extra_args, "-d", dest, "-f", path])
+        else:
+            self._run_tool(["pg_restore", *base, *extra_args, "-d", dest, path])
+        return dest
+
+    def _run_tool(self, cmd: List[str]) -> None:
+        import subprocess
+
+        env = dict(os.environ)
+        password = self._conn_params().get("password")
+        if password:
+            env["PGPASSWORD"] = str(password)
+        subprocess.run(cmd, check=True, env=env, capture_output=True, text=True)
+
+    # ------------------------------------------------------------------ maintenance
+    def vacuum(self, db: str, *, analyze: bool = True, full: bool = False) -> None:
+        """``VACUUM`` the target database (autocommit; cannot run in a transaction block)."""
+        opts = []
+        if full:
+            opts.append("FULL")
+        if analyze:
+            opts.append("ANALYZE")
+        stmt = "VACUUM" + (f" ({', '.join(opts)})" if opts else "")
+        self._exec_in(db, stmt)
+
+    def analyze(self, db: str) -> None:
+        self._exec_in(db, "ANALYZE")
+
+    def reindex(self, db: str) -> None:
+        """``REINDEX DATABASE`` the target database."""
+        self._exec_in(db, f"REINDEX DATABASE {quote_identifier(db)}")
+
+    def _exec_in(self, db: str, *statements: str) -> None:
+        """Run autocommit statements against ``db`` itself (not the maintenance db)."""
+        cfg = dict(self._config)
+        cfg["dbname"] = db
+        engine = postgres.initialize(config=cfg)
+
+        @engine.transaction
+        def _run(tx):
+            tx.rollback()
+            tx.connection.autocommit = True
+            for sql in statements:
+                if sql:
+                    tx.execute(sql)
+
+        _run()

{velocity_python-0.1.74 → velocity_python-0.1.76}/src/velocity_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.74
+Version: 0.1.76
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.74 → velocity_python-0.1.76}/src/velocity_python.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ src/velocity/aws/tests/test_lambda_handler_json_serialization.py
 src/velocity/aws/tests/test_response.py
 src/velocity/db/__init__.py
 src/velocity/db/exceptions.py
+src/velocity/db/maintenance.py
 src/velocity/db/migrations.py
 src/velocity/db/utils.py
 src/velocity/db/core/__init__.py
@@ -162,9 +163,11 @@ tests/test_connection_pool.py
 tests/test_connection_resilience.py
 tests/test_context_job_descriptions.py
 tests/test_db_credentials_ssm_cascade.py
+tests/test_db_maintenance.py
 tests/test_decorators.py
 tests/test_dirty_pipeline_fast_path.py
 tests/test_email_processing.py
+tests/test_enqueue_send_failures.py
 tests/test_get_cognito_user_provider.py
 tests/test_http_handler_rollback.py
 tests/test_iconv_money_to_cents.py