velocity-python 0.1.75__tar.gz → 0.1.77__tar.gz

{velocity_python-0.1.75 → velocity_python-0.1.77}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.75
+Version: 0.1.77
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.75 → velocity_python-0.1.77}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "velocity-python"
-version = "0.1.75"
+version = "0.1.77"
 authors = [
   { name="Velocity Team", email="info@codeclubs.org" },
 ]

{velocity_python-0.1.75 → velocity_python-0.1.77}/src/velocity/__init__.py

@@ -1,4 +1,4 @@
-__version__ = version = "0.1.75"
+__version__ = version = "0.1.77"
 
 import importlib as _importlib
 

velocity_python-0.1.77/src/velocity/db/maintenance.py

@@ -0,0 +1,329 @@
+"""PostgreSQL database maintenance toolkit.
+
+Safe, efficient database-admin operations that velocity's normal transactional
+model can't express. ``CREATE``/``DROP``/``RENAME DATABASE`` and template clones
+must run in **autocommit** and must **not** run while connected to the source or
+destination database. ``DatabaseMaintenance`` connects to a neutral *maintenance*
+database (default ``postgres``) so the source/dest are never the active session,
+runs DDL in autocommit, and terminates other sessions as needed.
+
+    from velocity.db import maintenance
+
+    m = maintenance.DatabaseMaintenance()      # env (DBHost/DBUser/...), maint db 'postgres'
+
+    # Server-side template clone -- instant, no dump/restore round-trip:
+    m.refresh("caringcent-production", "caringcent-develop")  # snapshot dest, then clone source -> dest
+    m.snapshot("caringcent-develop")                          # -> caringcent-develop-snapshot-<ts>
+    m.prune_snapshots("caringcent-develop", keep=3)           # drop all but the 3 newest snapshots
+
+Every database name is identifier-quoted; string-literal comparisons are escaped.
+A configurable ``protected`` set blocks destructive ops (drop / rename / overwrite)
+on critical databases (default: ``caringcent-production``).
+
+Snapshot databases are named ``<db>-snapshot-<YYYYMMDD-HHMMSS>`` -- a **sortable**
+timestamp, so listing/pruning by name is chronological.
+"""
+
+from __future__ import annotations
+
+import datetime
+import os
+from typing import Iterable, List, Optional
+
+from velocity.db.servers import postgres
+
+SNAPSHOT_INFIX = "-snapshot-"
+TS_FORMAT = "%Y%m%d-%H%M%S"  # sortable: lexical order == chronological order
+DEFAULT_PROTECTED = frozenset({"caringcent-production"})
+
+_BACKUP_FORMAT_FLAGS = {"custom": "-Fc", "plain": "-Fp", "directory": "-Fd", "tar": "-Ft"}
+
+
+def quote_identifier(name: str) -> str:
+    """Quote a database identifier (doubles embedded double-quotes)."""
+    return '"' + str(name).replace('"', '""') + '"'
+
+
+def quote_literal(value: str) -> str:
+    """Quote a string literal (doubles embedded single-quotes)."""
+    return "'" + str(value).replace("'", "''") + "'"
+
+
+def timestamp() -> str:
+    return datetime.datetime.now().strftime(TS_FORMAT)
+
+
+class DatabaseMaintenance:
+    """Connection-pooled, autocommit-aware PostgreSQL admin operations."""
+
+    def __init__(
+        self,
+        *,
+        maintenance_db: str = "postgres",
+        config: Optional[dict] = None,
+        protected: Iterable[str] = DEFAULT_PROTECTED,
+        engine=None,
+    ):
+        self.maintenance_db = maintenance_db
+        self.protected = frozenset(protected or ())
+        self._config = dict(config or {})
+        # Lazily initialized so construction never opens a connection (keeps the
+        # class unit-testable without a live database).
+        self._engine = engine
+
+    # ------------------------------------------------------------------ engine
+    def _engine_config(self, dbname: str) -> dict:
+        """Connection config that connects to ``dbname``.
+
+        The source/destination database is operated *on*, never connected *to*, so
+        any caller-supplied database name is dropped and replaced. (libpq uses
+        ``dbname``; ``postgres.initialize`` remaps a stray ``database`` key, which
+        would otherwise override the forced connection database.)
+        """
+        cfg = dict(self._config)
+        cfg.pop("database", None)
+        cfg.pop("dbname", None)
+        cfg["dbname"] = dbname
+        return cfg
+
+    def _get_engine(self):
+        if self._engine is None:
+            self._engine = postgres.initialize(config=self._engine_config(self.maintenance_db))
+        return self._engine
+
+    def _exec(self, *statements: str) -> None:
+        """Run DDL/admin statements in autocommit (required for CREATE/DROP/RENAME DATABASE)."""
+        engine = self._get_engine()
+
+        @engine.transaction
+        def _run(tx):
+            tx.rollback()  # discard the implicit transaction opened on checkout
+            tx.connection.autocommit = True
+            for sql in statements:
+                if sql:
+                    tx.execute(sql)
+
+        _run()
+
+    def _query(self, sql: str) -> List[dict]:
+        engine = self._get_engine()
+
+        @engine.transaction
+        def _run(tx):
+            return list(tx.execute(sql).as_dict())
+
+        return _run()
+
+    def _conn_params(self) -> dict:
+        return {
+            "host": self._config.get("host") or os.environ.get("DBHost"),
+            "port": self._config.get("port") or os.environ.get("DBPort"),
+            "user": self._config.get("user") or os.environ.get("DBUser"),
+            "password": self._config.get("password") or os.environ.get("DBPassword"),
+        }
+
+    # ------------------------------------------------------------------ guards
+    def _guard(self, db: str, action: str) -> None:
+        if db in self.protected:
+            raise PermissionError(
+                f"Refusing to {action} protected database {db!r}. "
+                "Construct with protected=... to override."
+            )
+
+    # ------------------------------------------------------------------ SQL helpers
+    @staticmethod
+    def _terminate_sql(db: str) -> str:
+        return (
+            "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
+            f"WHERE datname = {quote_literal(db)} AND pid <> pg_backend_pid()"
+        )
+
+    # ------------------------------------------------------------------ queries
+    def exists(self, db: str) -> bool:
+        return bool(
+            self._query(f"SELECT 1 AS x FROM pg_database WHERE datname = {quote_literal(db)}")
+        )
+
+    def list_databases(self, *, pattern: Optional[str] = None) -> List[str]:
+        where = "WHERE datistemplate = false"
+        if pattern:
+            where += f" AND datname LIKE {quote_literal(pattern)}"
+        rows = self._query(f"SELECT datname FROM pg_database {where} ORDER BY datname")
+        return [r["datname"] for r in rows]
+
+    def list_snapshots(self, db: str) -> List[str]:
+        """Snapshot databases for ``db``, oldest first (sortable timestamp suffix)."""
+        return sorted(self.list_databases(pattern=f"{db}{SNAPSHOT_INFIX}%"))
+
+    # ------------------------------------------------------------------ connection control
+    def terminate_connections(self, db: str) -> None:
+        self._exec(self._terminate_sql(db))
+
+    # ------------------------------------------------------------------ core ops
+    def clone(self, source: str, dest: str, *, drop_existing: bool = False) -> str:
+        """``CREATE DATABASE dest WITH TEMPLATE source`` (server-side, near-instant)."""
+        self._guard(dest, "overwrite")
+        if self.exists(dest):
+            if not drop_existing:
+                raise ValueError(
+                    f"Destination {dest!r} already exists. Pass drop_existing=True to replace it."
+                )
+            self.drop(dest)
+        # A template clone requires no other sessions on the source.
+        self._exec(
+            self._terminate_sql(source),
+            f"CREATE DATABASE {quote_identifier(dest)} WITH TEMPLATE {quote_identifier(source)}",
+        )
+        return dest
+
+    def snapshot(self, db: str, *, label: Optional[str] = None) -> str:
+        """Clone ``db`` to ``<db>-snapshot-<ts>`` (or a custom ``label``)."""
+        dest = f"{db}{SNAPSHOT_INFIX}{label or timestamp()}"
+        return self.clone(db, dest)
+
+    def rename(self, old: str, new: str) -> str:
+        self._guard(old, "rename")
+        self._exec(
+            self._terminate_sql(old),
+            f"ALTER DATABASE {quote_identifier(old)} RENAME TO {quote_identifier(new)}",
+        )
+        return new
+
+    def drop(self, db: str, *, if_exists: bool = True) -> None:
+        self._guard(db, "drop")
+        clause = "DROP DATABASE IF EXISTS" if if_exists else "DROP DATABASE"
+        self._exec(self._terminate_sql(db), f"{clause} {quote_identifier(db)}")
+
+    def refresh(self, source: str, dest: str, *, keep_snapshot: bool = True) -> str:
+        """Replace ``dest`` with a fresh template-clone of ``source``.
+
+        Safe sequence: terminate ``dest`` sessions -> rename ``dest`` to
+        ``dest-snapshot-<ts>`` (or drop it when ``keep_snapshot=False``) -> clone
+        ``source`` -> ``dest``. Keeping the snapshot makes the refresh reversible.
+        """
+        self._guard(dest, "overwrite")
+        if self.exists(dest):
+            if keep_snapshot:
+                self.rename(dest, f"{dest}{SNAPSHOT_INFIX}{timestamp()}")
+            else:
+                self.drop(dest)
+        return self.clone(source, dest)
+
+    def prune_snapshots(
+        self,
+        db: str,
+        *,
+        keep: int = 3,
+        older_than_days: Optional[int] = None,
+        dry_run: bool = False,
+    ) -> List[str]:
+        """Drop old snapshot databases for ``db``.
+
+        With ``older_than_days`` set, drops snapshots whose timestamp is older than
+        the cutoff. Otherwise keeps the ``keep`` newest and drops the rest. Returns
+        the names dropped (or that would be dropped, when ``dry_run=True``).
+        """
+        snaps = self.list_snapshots(db)  # ascending (oldest first)
+        to_drop: List[str] = []
+        if older_than_days is not None:
+            cutoff = datetime.datetime.now() - datetime.timedelta(days=older_than_days)
+            prefix = f"{db}{SNAPSHOT_INFIX}"
+            for name in snaps:
+                suffix = name[len(prefix):]
+                try:
+                    when = datetime.datetime.strptime(suffix, TS_FORMAT)
+                except ValueError:
+                    continue  # unrecognized suffix -> leave it alone
+                if when < cutoff:
+                    to_drop.append(name)
+        elif keep >= 0 and len(snaps) > keep:
+            to_drop = snaps[: len(snaps) - keep]
+
+        if not dry_run:
+            for name in to_drop:
+                self.drop(name)
+        return to_drop
+
+    # ------------------------------------------------------------------ portable backup/restore
+    def backup_to_file(
+        self, db: str, path: str, *, fmt: str = "custom", extra_args: Iterable[str] = ()
+    ) -> str:
+        """Portable off-instance backup via ``pg_dump`` (use for cross-server moves;
+        prefer ``snapshot``/``clone`` for same-server copies). ``extra_args`` are
+        passed through to ``pg_dump`` (e.g. ``("-O",)`` for no-owner)."""
+        if fmt not in _BACKUP_FORMAT_FLAGS:
+            raise ValueError(f"Unknown fmt {fmt!r}; choose one of {sorted(_BACKUP_FORMAT_FLAGS)}.")
+        p = self._conn_params()
+        self._run_tool(
+            ["pg_dump", "-h", str(p["host"]), "-p", str(p["port"]), "-U", str(p["user"]),
+             _BACKUP_FORMAT_FLAGS[fmt], *extra_args, "-f", path, db]
+        )
+        return path
+
+    def restore_from_file(
+        self,
+        path: str,
+        dest: str,
+        *,
+        fmt: str = "custom",
+        drop_existing: bool = False,
+        extra_args: Iterable[str] = (),
+    ) -> str:
+        """Restore a ``pg_dump`` file into a freshly created ``dest`` database.
+        ``extra_args`` are passed to ``pg_restore``/``psql``."""
+        self._guard(dest, "overwrite")
+        if self.exists(dest):
+            if not drop_existing:
+                raise ValueError(
+                    f"Destination {dest!r} already exists. Pass drop_existing=True to replace it."
+                )
+            self.drop(dest)
+        self._exec(f"CREATE DATABASE {quote_identifier(dest)}")
+        p = self._conn_params()
+        base = ["-h", str(p["host"]), "-p", str(p["port"]), "-U", str(p["user"])]
+        if fmt == "plain":
+            self._run_tool(["psql", *base, *extra_args, "-d", dest, "-f", path])
+        else:
+            self._run_tool(["pg_restore", *base, *extra_args, "-d", dest, path])
+        return dest
+
+    def _run_tool(self, cmd: List[str]) -> None:
+        import subprocess
+
+        env = dict(os.environ)
+        password = self._conn_params().get("password")
+        if password:
+            env["PGPASSWORD"] = str(password)
+        subprocess.run(cmd, check=True, env=env, capture_output=True, text=True)
+
+    # ------------------------------------------------------------------ maintenance
+    def vacuum(self, db: str, *, analyze: bool = True, full: bool = False) -> None:
+        """``VACUUM`` the target database (autocommit; cannot run in a transaction block)."""
+        opts = []
+        if full:
+            opts.append("FULL")
+        if analyze:
+            opts.append("ANALYZE")
+        stmt = "VACUUM" + (f" ({', '.join(opts)})" if opts else "")
+        self._exec_in(db, stmt)
+
+    def analyze(self, db: str) -> None:
+        self._exec_in(db, "ANALYZE")
+
+    def reindex(self, db: str) -> None:
+        """``REINDEX DATABASE`` the target database."""
+        self._exec_in(db, f"REINDEX DATABASE {quote_identifier(db)}")
+
+    def _exec_in(self, db: str, *statements: str) -> None:
+        """Run autocommit statements against ``db`` itself (not the maintenance db)."""
+        engine = postgres.initialize(config=self._engine_config(db))
+
+        @engine.transaction
+        def _run(tx):
+            tx.rollback()
+            tx.connection.autocommit = True
+            for sql in statements:
+                if sql:
+                    tx.execute(sql)
+
+        _run()

{velocity_python-0.1.75 → velocity_python-0.1.77}/src/velocity_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: velocity-python
-Version: 0.1.75
+Version: 0.1.77
 Summary: A rapid application development library for interfacing with data storage
 Author-email: Velocity Team <info@codeclubs.org>
 License-Expression: MIT

{velocity_python-0.1.75 → velocity_python-0.1.77}/src/velocity_python.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ src/velocity/aws/tests/test_lambda_handler_json_serialization.py
 src/velocity/aws/tests/test_response.py
 src/velocity/db/__init__.py
 src/velocity/db/exceptions.py
+src/velocity/db/maintenance.py
 src/velocity/db/migrations.py
 src/velocity/db/utils.py
 src/velocity/db/core/__init__.py
@@ -162,6 +163,7 @@ tests/test_connection_pool.py
 tests/test_connection_resilience.py
 tests/test_context_job_descriptions.py
 tests/test_db_credentials_ssm_cascade.py
+tests/test_db_maintenance.py
 tests/test_decorators.py
 tests/test_dirty_pipeline_fast_path.py
 tests/test_email_processing.py

velocity_python-0.1.77/tests/test_db_maintenance.py

@@ -0,0 +1,198 @@
+import unittest
+
+from velocity.db import maintenance
+
+
+class _RecordingMaintenance(maintenance.DatabaseMaintenance):
+    """In-memory model: records emitted SQL and tracks database existence so the
+    internal exists()/drop()/rename() checks stay consistent -- no live DB."""
+
+    def __init__(self, *, existing=(), snapshots=(), **kwargs):
+        super().__init__(**kwargs)
+        self._existing = set(existing)
+        self._snapshots = list(snapshots)
+        self.executed = []
+
+    def _exec(self, *statements):
+        self.executed.extend(s for s in statements if s)
+
+    def exists(self, db):
+        return db in self._existing
+
+    def list_snapshots(self, db):
+        return sorted(self._snapshots)
+
+    # state-tracking wrappers (call through to the real logic, then update state)
+    def clone(self, source, dest, **kwargs):
+        result = super().clone(source, dest, **kwargs)
+        self._existing.add(dest)
+        return result
+
+    def drop(self, db, **kwargs):
+        super().drop(db, **kwargs)
+        self._existing.discard(db)
+
+    def rename(self, old, new):
+        result = super().rename(old, new)
+        if old in self._existing:
+            self._existing.discard(old)
+            self._existing.add(new)
+        return result
+
+    @property
+    def sql(self):
+        return " | ".join(self.executed)
+
+    def _index(self, needle):
+        return next(i for i, s in enumerate(self.executed) if needle in s)
+
+
+class TestClone(unittest.TestCase):
+    def test_emits_terminate_then_template(self):
+        m = _RecordingMaintenance()
+        m.clone("src", "dst")
+        self.assertIn("pg_terminate_backend", m.sql)
+        self.assertIn('CREATE DATABASE "dst" WITH TEMPLATE "src"', m.sql)
+        # source connections are terminated BEFORE the template clone
+        self.assertLess(self.m_terminate_src(m), m._index("CREATE DATABASE"))
+
+    @staticmethod
+    def m_terminate_src(m):
+        return next(i for i, s in enumerate(m.executed) if "pg_terminate" in s and "'src'" in s)
+
+    def test_existing_without_drop_raises(self):
+        m = _RecordingMaintenance(existing={"dst"})
+        with self.assertRaises(ValueError):
+            m.clone("src", "dst")
+
+    def test_existing_with_drop_replaces(self):
+        m = _RecordingMaintenance(existing={"dst"})
+        m.clone("src", "dst", drop_existing=True)
+        self.assertIn('DROP DATABASE IF EXISTS "dst"', m.sql)
+        self.assertIn('CREATE DATABASE "dst" WITH TEMPLATE "src"', m.sql)
+        self.assertLess(m._index("DROP DATABASE"), m._index("CREATE DATABASE"))
+
+
+class TestRefresh(unittest.TestCase):
+    def test_keeps_snapshot_then_clones(self):
+        m = _RecordingMaintenance(existing={"dev"})
+        m.refresh("prod", "dev")
+        self.assertRegex(
+            m.sql, r'ALTER DATABASE "dev" RENAME TO "dev-snapshot-\d{8}-\d{6}"'
+        )
+        self.assertIn('CREATE DATABASE "dev" WITH TEMPLATE "prod"', m.sql)
+        self.assertLess(m._index("RENAME TO"), m._index("CREATE DATABASE"))
+
+    def test_no_snapshot_drops(self):
+        m = _RecordingMaintenance(existing={"dev"})
+        m.refresh("prod", "dev", keep_snapshot=False)
+        self.assertIn('DROP DATABASE IF EXISTS "dev"', m.sql)
+        self.assertNotIn("RENAME TO", m.sql)
+        self.assertIn('CREATE DATABASE "dev" WITH TEMPLATE "prod"', m.sql)
+
+    def test_dest_absent_just_clones(self):
+        m = _RecordingMaintenance(existing=set())
+        m.refresh("prod", "dev")
+        self.assertNotIn("RENAME TO", m.sql)
+        self.assertNotIn("DROP DATABASE", m.sql)
+        self.assertIn('CREATE DATABASE "dev" WITH TEMPLATE "prod"', m.sql)
+
+
+class TestSnapshotAndPrune(unittest.TestCase):
+    def test_snapshot_custom_label(self):
+        m = _RecordingMaintenance()
+        name = m.snapshot("dev", label="manual")
+        self.assertEqual(name, "dev-snapshot-manual")
+        self.assertIn('CREATE DATABASE "dev-snapshot-manual" WITH TEMPLATE "dev"', m.sql)
+
+    def test_prune_keeps_newest(self):
+        snaps = [
+            "dev-snapshot-20260101-000000",
+            "dev-snapshot-20260201-000000",
+            "dev-snapshot-20260301-000000",
+        ]
+        m = _RecordingMaintenance(snapshots=snaps)
+        dropped = m.prune_snapshots("dev", keep=1)
+        self.assertEqual(dropped, snaps[:2])
+        self.assertIn('DROP DATABASE IF EXISTS "dev-snapshot-20260101-000000"', m.sql)
+        self.assertNotIn("20260301", m.sql)  # newest is kept
+
+    def test_prune_dry_run_executes_nothing(self):
+        snaps = ["dev-snapshot-20260101-000000", "dev-snapshot-20260201-000000"]
+        m = _RecordingMaintenance(snapshots=snaps)
+        dropped = m.prune_snapshots("dev", keep=1, dry_run=True)
+        self.assertEqual(dropped, snaps[:1])
+        self.assertEqual(m.executed, [])
+
+    def test_prune_older_than_days(self):
+        snaps = ["dev-snapshot-20200101-000000", "dev-snapshot-29990101-000000"]
+        m = _RecordingMaintenance(snapshots=snaps)
+        dropped = m.prune_snapshots("dev", older_than_days=30)
+        self.assertEqual(dropped, ["dev-snapshot-20200101-000000"])
+
+    def test_prune_keep_all_when_under_limit(self):
+        snaps = ["dev-snapshot-20260101-000000"]
+        m = _RecordingMaintenance(snapshots=snaps)
+        self.assertEqual(m.prune_snapshots("dev", keep=3), [])
+        self.assertEqual(m.executed, [])
+
+
+class TestGuards(unittest.TestCase):
+    def test_protected_blocks_drop(self):
+        m = _RecordingMaintenance(existing={"caringcent-production"})
+        with self.assertRaises(PermissionError):
+            m.drop("caringcent-production")
+
+    def test_protected_blocks_overwrite_via_clone(self):
+        m = _RecordingMaintenance()
+        with self.assertRaises(PermissionError):
+            m.clone("x", "caringcent-production")
+
+    def test_protected_blocks_refresh_dest(self):
+        m = _RecordingMaintenance()
+        with self.assertRaises(PermissionError):
+            m.refresh("x", "caringcent-production")
+
+    def test_protected_blocks_rename(self):
+        m = _RecordingMaintenance(existing={"caringcent-production"})
+        with self.assertRaises(PermissionError):
+            m.rename("caringcent-production", "x")
+
+    def test_override_protected_allows(self):
+        m = _RecordingMaintenance(existing={"caringcent-production"}, protected=())
+        m.drop("caringcent-production")
+        self.assertIn('DROP DATABASE IF EXISTS "caringcent-production"', m.sql)
+
+
+class TestQuoting(unittest.TestCase):
+    def test_identifier_quoting_doubles_double_quotes(self):
+        self.assertEqual(maintenance.quote_identifier('a"b'), '"a""b"')
+
+    def test_literal_quoting_doubles_single_quotes(self):
+        self.assertEqual(maintenance.quote_literal("a'b"), "'a''b'")
+
+    def test_terminate_sql_escapes_name(self):
+        sql = maintenance.DatabaseMaintenance._terminate_sql("o'brien")
+        self.assertIn("datname = 'o''brien'", sql)
+        self.assertIn("pid <> pg_backend_pid()", sql)
+
+
+class TestEngineConfig(unittest.TestCase):
+    def test_forces_maintenance_db_and_drops_caller_db_name(self):
+        m = maintenance.DatabaseMaintenance(
+            config={"database": "caringcent-develop", "host": "h", "user": "u"}
+        )
+        cfg = m._engine_config(m.maintenance_db)
+        self.assertEqual(cfg["dbname"], "postgres")
+        self.assertNotIn("database", cfg)  # the stray key that would override dbname is gone
+        self.assertEqual(cfg["host"], "h")
+        self.assertEqual(cfg["user"], "u")
+
+    def test_target_db_overrides_caller_dbname(self):
+        m = maintenance.DatabaseMaintenance(config={"dbname": "x", "host": "h"})
+        cfg = m._engine_config("mydb")
+        self.assertEqual(cfg["dbname"], "mydb")
+
+
+if __name__ == "__main__":
+    unittest.main()