keepup-db 0.4.0__tar.gz

keepup_db-0.4.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: keepup-db
+Version: 0.4.0
+Summary: The abstraction that holds SQLAlchemy: the table language, the pool, the dialect registry and the creation of declared tables.
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: keepup-admin>=0.4.0
+Requires-Dist: sqlalchemy<3,>=2.0
+
+# keepup-db
+
+The abstraction that holds SQLAlchemy: the table language, the pool, the dialect registry and the creation of declared tables.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-db`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_db-0.4.0/README.md

@@ -0,0 +1,10 @@
+# keepup-db
+
+The abstraction that holds SQLAlchemy: the table language, the pool, the dialect registry and the creation of declared tables.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-db`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_db-0.4.0/keepup_db/__init__.py

@@ -0,0 +1,22 @@
+"""The abstraction that holds SQLAlchemy: the table language, the pool, the dialect registry and the creation of declared tables.
+
+This distribution provides `datasource` and declares one plugin,
+`db`, in the `keepup.plugins` group. It depends on `keepup-admin`
+for the constructor -- the descriptor, the catalogue, the service registry and
+the lifecycle -- and on nothing of another capability.
+
+`keepup.db` moved here in keepup-124, and the old name keeps working for one
+release: `keepup.compat` answers for it and names this distribution when it is not
+installed. The three names below are the declaration the moved module carried.
+"""
+
+__version__ = "0.4.0"
+
+from keepup_db import tables  # noqa: E402 - the table language, with the manager
+from keepup_db.manager import (  # noqa: E402 - after the version, as a public face
+    DatabaseConfig,
+    DatabaseManagerV2,
+    db_config,
+)
+
+__all__ = ["DatabaseConfig", "DatabaseManagerV2", "db_config", "tables"]

keepup_db-0.4.0/keepup_db/manager.py

@@ -0,0 +1,603 @@
+"""Access to the database: configuration and the pooled manager.
+
+`DatabaseManagerV2` is the one way in -- a SQLAlchemy engine with a pool and a
+session context manager, named parameters only. The legacy `DatabaseManager`,
+a layer over raw drivers with positional parameters that opened a connection
+per call, is gone (0.2.0); code written against a cursor takes one from the
+pool with `DatabaseManagerV2.raw_connection()`.
+
+Where the database is comes from the environment when `DB_TYPE` is set and from
+`config/postgres.properties` otherwise. Both paths are read from the working
+directory of the started application, which is how two applications on one
+framework keep separate databases. SQLite exists for development only.
+"""
+
+import asyncio
+import contextvars
+import os
+import threading
+import time
+import warnings
+from contextlib import contextmanager
+from typing import Optional, List, Dict, Any
+
+from sqlalchemy import create_engine, event, exc, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import sessionmaker, Session
+
+_RETURNED_AT = "keepup_returned_at"
+
+
+def _ping_when_idle(engine, idle_seconds: int) -> None:
+    """Ping a connection at checkout only when it has lain in the pool a while.
+
+    The pool used to ping on every checkout (pool_pre_ping), and a request that
+    takes three connections pinged three times -- on the load stand (keepup-53)
+    a fifth of what a replica spent. A connection handed back a moment ago has
+    had no time to be closed by the database or the network; one that has lain
+    longer than ``idle_seconds`` is pinged, and one that fails the ping is
+    replaced, exactly as the built-in ping does (keepup-86). The ping is the
+    dialect's own. A connection the database dropped while it was busy a
+    moment ago still fails its next statement once, as it would have after a
+    ping made just before the drop.
+    """
+    if not isinstance(engine, Engine):
+        # A stand-in engine (a test's mock) has no pool events to listen to.
+        return
+
+    @event.listens_for(engine, "connect")
+    def _fresh(dbapi_connection, record):
+        record.info[_RETURNED_AT] = time.monotonic()
+
+    @event.listens_for(engine, "checkin")
+    def _returned(dbapi_connection, record):
+        if record is not None:
+            record.info[_RETURNED_AT] = time.monotonic()
+
+    @event.listens_for(engine, "checkout")
+    def _ping(dbapi_connection, record, proxy):
+        returned = record.info.get(_RETURNED_AT)
+        if idle_seconds and returned is not None and time.monotonic() - returned < idle_seconds:
+            return
+        try:
+            engine.dialect.do_ping(dbapi_connection)
+        except Exception as error:
+            # The pool discards the connection and hands out a fresh one.
+            raise exc.DisconnectionError(f"connection failed its ping: {error}") from error
+
+
+def _reads_only(query: str) -> bool:
+    """Whether the statement is a plain read: a SELECT that takes no lock."""
+    body = query.lstrip().lstrip("(").lstrip()
+    return body[:6].upper() == "SELECT" and "FOR UPDATE" not in query.upper() \
+        and "FOR SHARE" not in query.upper()
+
+
+#: The session of the shared_session() block running on this thread, if any,
+#: with the thread it belongs to.
+_SHARED_SESSION: contextvars.ContextVar = contextvars.ContextVar(
+    "keepup_shared_session", default=None)
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = [
+    "DatabaseConfig",
+    "DatabaseManagerV2",
+    "db_config",
+]
+
+
+class DatabaseConfig:
+    """Database configuration."""
+
+    def __init__(self):
+        self.db_type = 'sqlite'
+        self.db_host = 'localhost'
+        self.db_port = '5432'
+        self.db_name = 'keepup_app'
+        self.db_user = 'postgres'
+        self.db_password = ''
+        self.db_path = 'data/keepup_app.db'
+        # The defaults come first and the configuration over them: the other way
+        # round, DB_POOL_SIZE and db.pool.size were read and then written over
+        # with these very numbers, and never applied (keepup-38).
+        self.pool_size = 5
+        self.pool_max_overflow = 10
+        self.pool_timeout = 30
+        self.pool_recycle = 3600
+        #: Seconds a connection may lie in the pool and still be handed out
+        #: without a ping; 0 pings on every checkout (keepup-86).
+        self.pool_ping_after_idle = 10
+        self._load_config()
+
+    def apply_pool(self, size=None, max_overflow=None, timeout=None, recycle=None,
+                   ping_after_idle=None) -> bool:
+        """Take the application's pool values over the deployment's; True if any changed."""
+        changed = False
+        for field_name, value in (("pool_size", size), ("pool_max_overflow", max_overflow),
+                                  ("pool_timeout", timeout), ("pool_recycle", recycle),
+                                  ("pool_ping_after_idle", ping_after_idle)):
+            if value is not None and getattr(self, field_name) != int(value):
+                setattr(self, field_name, int(value))
+                changed = True
+        return changed
+
+    def _load_config(self):
+        self.db_type = os.getenv('DB_TYPE', '').lower()
+        if not self.db_type:
+            self._load_from_file()
+        else:
+            self._load_from_env()
+
+    def _load_from_env(self):
+        self.db_host = os.getenv('DB_HOST', 'localhost')
+        self.db_port = os.getenv('DB_PORT', '5432')
+        self.db_name = os.getenv('DB_NAME', 'keepup')
+        self.db_user = os.getenv('DB_USER', 'postgres')
+        self.db_password = os.getenv('DB_PASSWORD', '')
+        self.db_path = os.getenv('DB_PATH', 'data/keepup.db')
+
+        self.pool_size = int(os.getenv('DB_POOL_SIZE', '5'))
+        self.pool_max_overflow = int(os.getenv('DB_POOL_MAX_OVERFLOW', '10'))
+        self.pool_timeout = int(os.getenv('DB_POOL_TIMEOUT', '30'))
+        self.pool_recycle = int(os.getenv('DB_POOL_RECYCLE', '3600'))
+        self.pool_ping_after_idle = int(os.getenv('DB_POOL_PING_AFTER_IDLE', '10'))
+
+        if self.db_type == 'sqlite':
+            os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
+
+    def _load_from_file(self):
+        self.config_file = "config/postgres.properties"
+        config = {}
+        if os.path.exists(self.config_file):
+            with open(self.config_file, 'r') as f:
+                for line in f:
+                    line = line.strip()
+                    if line and not line.startswith('#'):
+                        if '=' in line:
+                            key, value = line.split('=', 1)
+                            config[key.strip()] = value.strip()
+
+        self.db_type = config.get('db.type', 'sqlite').lower()
+        self.db_host = config.get('db.host', 'localhost')
+        self.db_port = config.get('db.port', '5432')
+        self.db_name = config.get('db.name', 'keepup')
+        self.db_user = config.get('db.user', 'postgres')
+        # The file names where the database is; the password never lives in it.
+        self.db_password = os.getenv('DB_PASSWORD') or config.get('db.password', '')
+        self.db_path = config.get('db.path', 'data/keepup.db')
+
+        self.pool_size = int(config.get('db.pool.size', '5'))
+        self.pool_max_overflow = int(config.get('db.pool.max_overflow', '10'))
+        self.pool_timeout = int(config.get('db.pool.timeout', '30'))
+        self.pool_recycle = int(config.get('db.pool.recycle', '3600'))
+
+        if self.db_type == 'sqlite':
+            db_dir = os.path.dirname(self.db_path)
+            if db_dir:
+                os.makedirs(db_dir, exist_ok=True)
+
+    def get_connection_string(self) -> str:
+        if self.db_type == 'postgres':
+            # The driver is named, not left to SQLAlchemy: from 2.1 a bare
+            # postgresql:// means psycopg 3, which this package does not
+            # install, and every fresh installation failed to reach the
+            # database at all (keepup-84). psycopg2 is the declared dependency.
+            return (f"postgresql+psycopg2://{self.db_user}:{self.db_password}"
+                    f"@{self.db_host}:{self.db_port}/{self.db_name}")
+        else:
+            return f"sqlite:///{self.db_path}"
+
+    def get_sqlalchemy_engine_params(self) -> dict:
+        params = {}
+        if self.is_postgres():
+            params.update({
+                'pool_size': self.pool_size,
+                'max_overflow': self.pool_max_overflow,
+                'pool_timeout': self.pool_timeout,
+                'pool_recycle': self.pool_recycle,
+                # Pinged by _ping_when_idle() instead: on every checkout the
+                # ping was a fifth of what a replica spent (keepup-86).
+                'pool_pre_ping': False,
+                'echo': os.getenv('SQL_ECHO', 'false').lower() == 'true'
+            })
+        else:
+            params.update({
+                'pool_size': self.pool_size,
+                'pool_timeout': self.pool_timeout,
+            })
+            params['connect_args'] = {'check_same_thread': False}
+        return params
+
+    def is_postgres(self) -> bool:
+        return self.db_type == 'postgres'
+
+    def is_sqlite(self) -> bool:
+        return self.db_type == 'sqlite'
+
+    def __str__(self):
+        if self.is_postgres():
+            return (f"DatabaseConfig(type={self.db_type}, host={self.db_host}, "
+                    f"port={self.db_port}, db={self.db_name}, user={self.db_user}, "
+                    f"pool_size={self.pool_size})")
+        else:
+            return f"DatabaseConfig(type={self.db_type}, path={self.db_path})"
+
+
+
+class DatabaseManagerV2:
+    """Database access through SQLAlchemy with a connection pool."""
+
+    _engine = None
+    _session_factory = None
+
+    @classmethod
+    def initialize(cls):
+        """Initialise the SQLAlchemy engine."""
+        if cls._engine is None:
+            connection_string = db_config.get_connection_string()
+            engine_params = db_config.get_sqlalchemy_engine_params()
+            cls._engine = create_engine(connection_string, **engine_params)
+            _ping_when_idle(cls._engine, db_config.pool_ping_after_idle)
+            cls._session_factory = sessionmaker(bind=cls._engine)
+            import logging
+            logging.getLogger(__name__).info(f"DatabaseManagerV2 initialized with pool_size={db_config.pool_size}")
+
+    @classmethod
+    @contextmanager
+    def get_session(cls) -> Session:
+        """Context manager yielding a session, committing or rolling back on exit.
+
+        Inside ``shared_session()`` on the same thread it yields that block's
+        session instead, and leaves committing to the block.
+        """
+        shared = _SHARED_SESSION.get()
+        if shared is not None and shared[0] == threading.get_ident():
+            yield shared[1]
+            return
+        if cls._engine is None:
+            cls.initialize()
+        session = cls._session_factory()
+        try:
+            yield session
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            raise e
+        finally:
+            session.close()
+
+    @classmethod
+    @contextmanager
+    def shared_session(cls):
+        """Run every query of this block on one session: one connection, one commit.
+
+        Each call of the query helpers takes a connection from the pool on its
+        own -- a checkout, a ping, the query and a commit -- and a path that
+        asks three questions paid that three times. The session check on every
+        request did: it was half of what a replica spent (keepup-85). Nested
+        blocks join the outer one. Bound to the thread that opened it, so work
+        handed to another thread from inside the block takes a session of its
+        own rather than sharing one across threads.
+        """
+        shared = _SHARED_SESSION.get()
+        if shared is not None and shared[0] == threading.get_ident():
+            yield shared[1]
+            return
+        with cls.get_session() as session:
+            token = _SHARED_SESSION.set((threading.get_ident(), session))
+            try:
+                yield session
+            finally:
+                _SHARED_SESSION.reset(token)
+
+    # --- the same calls, awaitable ------------------------------------------
+    #
+    # Every query here blocks its thread until the database answers, and in a
+    # coroutine that thread is the event loop's: while it waits, the process
+    # serves nobody. These run the synchronous call in a worker thread
+    # (asyncio.to_thread) and await it, so async code never calls the database
+    # on the loop. They call the synchronous methods by name, so whatever
+    # replaces one of those (a test's stand-in, a recorder) is used here too.
+
+    @classmethod
+    async def execute_async(cls, query: str, params: Optional[dict] = None) -> List[Dict]:
+        return await asyncio.to_thread(cls.execute, query, params)
+
+    @classmethod
+    async def execute_one_async(cls, query: str, params: Optional[dict] = None) -> Optional[Dict]:
+        return await asyncio.to_thread(cls.execute_one, query, params)
+
+    @classmethod
+    async def execute_commit_async(cls, query: str, params: Optional[dict] = None) -> int:
+        return await asyncio.to_thread(cls.execute_commit, query, params)
+
+    @classmethod
+    async def execute_many_async(cls, query: str, params_list: List[dict]) -> int:
+        return await asyncio.to_thread(cls.execute_many, query, params_list)
+
+    @classmethod
+    async def execute_commit_returning_async(cls, query: str, params: Optional[dict] = None,
+                                             returning: str = "id") -> Optional[Dict]:
+        return await asyncio.to_thread(cls.execute_commit_returning, query, params, returning)
+
+    @classmethod
+    def test_connection(cls) -> Dict[str, Any]:
+        """Whether the database answers, and which one it is; never raises."""
+        try:
+            if db_config.is_postgres():
+                version = cls.execute_one("SELECT version() AS version")
+                db_type = "PostgreSQL"
+            else:
+                version = cls.execute_one("SELECT sqlite_version() AS version")
+                db_type = "SQLite"
+            return {
+                "success": True,
+                "database_type": db_type,
+                "version": (version or {}).get("version", "Unknown"),
+                "config": str(db_config),
+            }
+        except Exception as e:
+            return {"success": False, "error": str(e), "config": str(db_config)}
+
+    @classmethod
+    def raw_connection(cls):
+        """A driver connection out of the pool, for code written against a cursor.
+
+        The application's table hook and the first-start accounts take a
+        cursor; the statements run on it are the driver's own (``?`` on
+        SQLite, ``%s`` on PostgreSQL) and rows come back as the driver makes
+        them. ``close()`` hands the connection back to the pool.
+        """
+        if cls._engine is None:
+            cls.initialize()
+        return cls._engine.raw_connection()
+
+    @classmethod
+    @contextmanager
+    def _statement_scope(cls, query: str):
+        """Where one statement of execute()/execute_one() runs.
+
+        A plain read runs on a connection in autocommit: it wrote nothing, and
+        the transaction around it cost a BEGIN and a COMMIT -- a round trip to
+        the database for every read (keepup-87). Anything else, and anything
+        inside shared_session(), keeps its transaction as before: these helpers
+        are also called with INSERT ... RETURNING, and a SELECT ... FOR UPDATE
+        means its lock.
+        """
+        shared = _SHARED_SESSION.get()
+        if _reads_only(query) and not (shared is not None and shared[0] == threading.get_ident()):
+            if cls._engine is None:
+                cls.initialize()
+            with cls._engine.connect() as connection:
+                yield connection.execution_options(isolation_level="AUTOCOMMIT")
+            return
+        with cls.get_session() as session:
+            yield session
+
+    @classmethod
+    def execute(cls, query: str, params: Optional[dict] = None) -> List[Dict]:
+        """Run a SQL query.
+
+        NOTE: takes NAMED parameters (:param_name), not positional ones.
+        """
+        with cls._statement_scope(query) as session:
+            result = session.execute(text(query), params or {})
+            if result.returns_rows:
+                result = [dict(row._mapping) for row in result.fetchall()]
+                return result
+            return []
+
+    @classmethod
+    def execute_one(cls, query: str, params: Optional[dict] = None) -> Optional[Dict]:
+        """Run a SQL query and return a single row.
+
+        NOTE: takes NAMED parameters (:param_name), not positional ones.
+        """
+        with cls._statement_scope(query) as session:
+            result = session.execute(text(query), params or {})
+            if result.returns_rows:
+                row = result.fetchone()
+                return dict(row._mapping) if row else None
+            return None
+
+    @classmethod
+    def execute_commit(cls, query: str, params: Optional[dict] = None) -> int:
+        """Run a SQL query and commit.
+
+        NOTE: takes NAMED parameters (:param_name), not positional ones.
+        """
+        with cls.get_session() as session:
+            result = session.execute(text(query), params or {})
+            return result.rowcount
+
+    @classmethod
+    def column_exists(cls, table: str, column: str) -> bool:
+        """Whether the column is already on the table, asked of the database.
+
+        Each dialect is asked the way it answers: PostgreSQL keeps a catalogue,
+        SQLite answers a pragma. Neither reads an error message, which is the
+        whole point -- see add_column_if_missing.
+        """
+        if db_config.is_postgres():
+            found = cls.execute_one(
+                "SELECT 1 AS present FROM information_schema.columns "
+                "WHERE table_name = :table AND column_name = :column",
+                {"table": table, "column": column})
+            return found is not None
+
+        # PRAGMA takes no parameters, so the name is inlined -- and therefore
+        # checked: see keepup.tables.identifier for why this is a refusal and
+        # not a quoting.
+        from keepup.tables import identifier
+
+        rows = cls.execute(f"PRAGMA table_info({identifier(table)})")
+        return any(str(row.get("name")) == column for row in rows or [])
+
+    @classmethod
+    def add_column_if_missing(cls, table: str, column: str,
+                              definition: str) -> bool:
+        """Add a column unless it is already there. True if this call added it.
+
+        There are no migrations in this project: every start re-runs its DDL, so
+        an ALTER that has already been applied has to be a no-op rather than an
+        error. That used to be decided by matching the text of the failure
+        against "duplicate column" -- which is what SQLite says and PostgreSQL
+        does not. PostgreSQL answers `(psycopg2.errors.DuplicateColumn) column
+        "x" of relation "y" already exists`: no such substring, so the exception
+        was re-raised, the plugin failed to initialise, and every one of its
+        routes answered 404 while the application itself reported healthy. On
+        the stand that happened on the second start after any new column -- the
+        first one added it, the next one fell over.
+
+        A method rather than a function beside the class on purpose: the test
+        harnesses swap this module and rebind the manager by name, so a helper
+        of its own would quietly talk to a different database than its caller.
+        """
+        from keepup.tables import identifier
+
+        if db_config.is_postgres():
+            # PostgreSQL can say it in the statement, and then a repeat start
+            # raises nothing at all -- no aborted transaction to recover from.
+            cls.execute_commit(
+                f"ALTER TABLE {identifier(table)} "
+                f"ADD COLUMN IF NOT EXISTS {identifier(column)} {definition}")
+            return True
+
+        # SQLite has no such form: try, and ask the catalogue if it failed.
+        try:
+            cls.execute_commit(
+                f"ALTER TABLE {identifier(table)} ADD COLUMN {identifier(column)} {definition}")
+            return True
+        except Exception:
+            if cls.column_exists(table, column):
+                return False
+            raise
+
+    @classmethod
+    def execute_many(cls, query: str, params_list: List[dict]) -> int:
+        """Run a SQL query for many parameter sets.
+
+        NOTE: takes NAMED parameters (:param_name), not positional ones.
+        """
+        total = 0
+        with cls.get_session() as session:
+            for params in params_list:
+                result = session.execute(text(query), params)
+                total += result.rowcount
+        return total
+
+    @classmethod
+    def get_pool_status(cls) -> Dict[str, Any]:
+        """Return the connection pool status."""
+        if cls._engine is None:
+            cls.initialize()
+        pool = cls._engine.pool
+        return {
+            'size': pool.size(),
+            'checked_in': pool.checkedin(),
+            'overflow': pool.overflow(),
+            'total': pool.size() + pool.overflow(),  # total = size + overflow
+            'checked_out': pool.checkedout(),
+        }
+
+    @classmethod
+    def dispose(cls):
+        """Close every connection."""
+        if cls._engine:
+            cls._engine.dispose()
+            cls._engine = None
+            cls._session_factory = None
+
+    @classmethod
+    def get_last_insert_rowid(cls, session=None) -> int:
+        """Return the id of the last inserted row.
+
+        Works on both PostgreSQL and SQLite -- but only with the session that
+        made the insert. Without one it opens a new session, which may be
+        another pooled connection, and answers another insert's id or 0
+        (keepup-82); that form is deprecated. Use execute_commit_returning().
+        """
+        if session is None:
+            warnings.warn(
+                "get_last_insert_rowid() without the inserting session may answer "
+                "another insert's id; use execute_commit_returning()",
+                DeprecationWarning, stacklevel=2)
+        if cls._engine is None:
+            cls.initialize()
+
+        if db_config.is_postgres():
+            query = "SELECT lastval() as id"
+            result = cls.execute_one(query)
+            return result["id"] if result else 0
+        else:
+            if session:
+                result = session.execute(text("SELECT last_insert_rowid() as id"))
+                row = result.fetchone()
+                return row[0] if row else 0
+            else:
+                with cls.get_session() as sess:
+                    result = sess.execute(text("SELECT last_insert_rowid() as id"))
+                    row = result.fetchone()
+                    return row[0] if row else 0
+
+    @classmethod
+    def execute_commit_returning(cls, query: str, params: Optional[dict] = None, returning: str = "id") -> Optional[
+        Dict]:
+        """Run a SQL query and return the inserted record.
+
+        Uses RETURNING on PostgreSQL and last_insert_rowid on SQLite.
+        """
+        if db_config.is_postgres():
+            modified_query = f"{query} RETURNING {returning}"
+            with cls.get_session() as session:
+                result = session.execute(text(modified_query), params or {})
+                row = result.fetchone()
+                if row:
+                    return dict(row._mapping)
+                return None
+        else:
+            with cls.get_session() as session:
+                session.execute(text(query), params or {})
+                result = session.execute(text(f"SELECT last_insert_rowid() as {returning}"))
+                row = result.fetchone()
+                if row:
+                    return {returning: row[0]}
+                return None
+
+    @classmethod
+    def execute_commit_with_positional(cls, query: str, params: tuple) -> int:
+        """Run a SQL query with POSITIONAL parameters (?) and commit.
+
+        Deprecated: a leftover of the transition from the manager that took
+        positional parameters, removed in 0.2.0 (keepup-55). Named parameters
+        are what this manager takes; statements still written with `?` go
+        through the application's own `positional()`. Removed in the next minor
+        release, once the applications that still call it have moved.
+        """
+        warnings.warn(
+            "DatabaseManagerV2.execute_commit_with_positional is deprecated; write "
+            "the statement with named parameters", DeprecationWarning, stacklevel=2)
+        with cls.get_session() as session:
+            # SQLAlchemy only binds named parameters, so ? placeholders are rewritten to :paramN.
+            if '?' in query:
+                param_count = query.count('?')
+                for i in range(param_count):
+                    query = query.replace('?', f':param{i}', 1)
+
+                named_params = {f'param{i}': params[i] for i in range(len(params))}
+                result = session.execute(text(query), named_params)
+            else:
+                result = session.execute(text(query), params or {})
+            return result.rowcount
+
+
+db_config = DatabaseConfig()
+
+# Log the resolved configuration on import, so a mis-pointed run is visible early.
+
+if __name__ != "__main__":
+    import logging
+
+    logger = logging.getLogger(__name__)
+    logger.info(f"Database configuration loaded: {db_config}")

keepup_db-0.4.0/keepup_db/positional_sql.py

@@ -0,0 +1,125 @@
+"""Positional ``?`` parameters for DatabaseManagerV2, which takes named ones.
+
+The manager removed in 0.2.0 took SQL with ``?`` placeholders and a tuple; the
+pooled DatabaseManagerV2 takes ``:name`` parameters. Statements written for the
+old one -- several assembled at run time, with lists of ``?`` -- are converted
+here, at the call, rather than rewritten by hand one by one:
+
+    DatabaseManagerV2.execute(*positional("SELECT ... WHERE id = ?", (5,)))
+
+``?`` (or ``%s``) outside quotes becomes ``:p0``, ``:p1``...; a colon that
+SQLAlchemy would take for a parameter of its own -- a time in a literal,
+``'12:30'`` -- is escaped, and ``::`` casts are left alone.
+
+Three applications kept identical copies of this module, held equal by a test,
+because the framework did not have it (keepup-56). Moving to this release, an
+application imports it from here and deletes its copy.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = [
+    "execute_many",
+    "insert_returning_id",
+    "positional",
+    "positional_many",
+    "raw_connection",
+]
+
+
+def positional(query: str, params: Optional[Sequence[Any]] = None) -> Tuple[str, Dict[str, Any]]:
+    """The query with named placeholders, and its parameters as a dictionary."""
+    out, count, i, quote = [], 0, 0, None
+    while i < len(query):
+        ch = query[i]
+        if quote:
+            if ch == quote:
+                # A doubled quote inside a literal is the quote itself.
+                if i + 1 < len(query) and query[i + 1] == quote:
+                    out.append(ch * 2)
+                    i += 2
+                    continue
+                quote = None
+            if ch == ":" and _takes_colon(query, i):
+                out.append("\\:")
+            else:
+                out.append(ch)
+        elif ch in ("'", '"'):
+            quote = ch
+            out.append(ch)
+        elif ch == "?" or (ch == "%" and query[i + 1:i + 2] == "s"):
+            # %s is the driver's own style, which some PostgreSQL branches used.
+            out.append(f":p{count}")
+            count += 1
+            i += 2 if ch == "%" else 1
+            continue
+        elif ch == ":" and _takes_colon(query, i):
+            out.append("\\:")
+        else:
+            out.append(ch)
+        i += 1
+    values = list(params or ())
+    if len(values) != count:
+        raise ValueError(f"{count} placeholders and {len(values)} parameters in: {query[:120]}")
+    return "".join(out), {f"p{n}": value for n, value in enumerate(values)}
+
+
+def _takes_colon(query: str, i: int) -> bool:
+    """Whether SQLAlchemy would read the colon at i as the start of a parameter."""
+    before = query[i - 1] if i else ""
+    after = query[i + 1] if i + 1 < len(query) else ""
+    return before != ":" and after != ":" and (after.isalnum() or after == "_")
+
+
+def positional_many(query: str, rows: Sequence[Sequence[Any]]) -> Tuple[str, List[Dict[str, Any]]]:
+    """The query with named placeholders, and one parameter dictionary per row."""
+    named, _ = positional(query, rows[0] if rows else [None] * query.count("?"))
+    return named, [positional(query, row)[1] for row in rows]
+
+
+def insert_returning_id(query: str, params: Optional[Sequence[Any]] = None) -> Optional[int]:
+    """Run one INSERT and return the id of the row it made, on both dialects.
+
+    What the removed manager's ``execute_commit`` answered; the query carries no RETURNING
+    of its own -- DatabaseManagerV2 adds it on PostgreSQL.
+    """
+    from keepup_db import DatabaseManagerV2
+
+    row = DatabaseManagerV2.execute_commit_returning(*positional(query, params), "id")
+    return row["id"] if row else None
+
+
+def execute_many(query: str, rows: Sequence[Sequence[Any]]) -> bool:
+    """Run one statement for many positional rows in one transaction.
+
+    True on success and False on a failure, which rolls the whole batch back --
+    what the removed manager's ``executemany_commit`` answered.
+    """
+    if not rows:
+        return True
+    try:
+        from keepup_db import DatabaseManagerV2
+
+        DatabaseManagerV2.execute_many(*positional_many(query, rows))
+        return True
+    except Exception as e:
+        logging.error(f"Batch statement failed and was rolled back: {e}")
+        return False
+
+
+def raw_connection():
+    """A driver connection out of DatabaseManagerV2's pool, for code written
+    against a cursor; ``close()`` hands it back to the pool.
+
+    The statements run on it are the driver's own: ``?`` on SQLite, ``%s`` on
+    PostgreSQL, with no conversion.
+    """
+    from keepup_db import DatabaseManagerV2
+
+    with DatabaseManagerV2.get_session() as session:
+        engine = session.get_bind()
+    return engine.raw_connection()

keepup_db-0.4.0/keepup_db/tables.py

@@ -0,0 +1,304 @@
+"""Tables declared once and created on either dialect.
+
+Until this module every table was written out twice -- once for PostgreSQL and
+once for SQLite -- and every column added later was a third and fourth copy in
+an ``ALTER``. The copies drifted: a foreign key on one side only, ``VARCHAR``
+on one and ``TEXT`` on the other, a column added to the PostgreSQL branch and
+forgotten in the SQLite one.
+
+A table is now a SQLAlchemy ``Table`` in the owner's data-access module, and
+``ensure_tables()`` makes the database match it, idempotently and on every
+start -- there are still no migrations:
+
+- the table is created with ``CREATE TABLE IF NOT EXISTS``, which, unlike
+  inspecting first and creating second, is safe when two replicas start at
+  once;
+- a declared column the table lacks is added -- one catalogue read per table,
+  not an ``ALTER`` per column per start, which on PostgreSQL would take a lock
+  each time;
+- declared indexes are created ``IF NOT EXISTS``.
+
+What a column cannot say for both dialects at once is said with SQLAlchemy's
+own means: ``with_variant()`` for a type, ``ddl_if(dialect=...)`` for a
+constraint that only one dialect had. The DDL is compiled for the dialect in
+force and run through ``DatabaseManagerV2.execute_commit`` -- the same path as
+any other statement, so a test harness that rebinds the manager sees it too.
+
+See doc/database_tables.md.
+"""
+
+import logging
+import re
+from typing import Dict, Iterable, List, Set
+
+from sqlalchemy import BigInteger, Column, ForeignKeyConstraint, Index, Integer, MetaData, Sequence, Table
+from sqlalchemy import text as sql_text
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.sql.expression import ColumnElement
+from sqlalchemy.dialects import postgresql, sqlite
+from sqlalchemy.schema import CreateColumn, CreateIndex, CreateSequence, CreateTable
+
+from keepup import db as _db
+
+#: What an application may import from this module. Everything else is
+#: internal and may change without notice -- see doc/keepup.md.
+__all__ = [
+    "NOW",
+    "auto_id",
+    "ensure_columns",
+    "ensure_tables",
+    "foreign_key",
+    "identifier",
+    "per_dialect",
+    "table",
+]
+
+logger = logging.getLogger(__name__)
+
+#: One catalogue for every declared table of the process: a foreign key names
+#: its target table, and SQLAlchemy resolves it here.
+metadata = MetaData()
+
+#: The default "now" of the hand-written DDL, kept as the database's own.
+NOW = sql_text("CURRENT_TIMESTAMP")
+
+_DIALECTS = {"postgres": postgresql.dialect(), "sqlite": sqlite.dialect()}
+
+
+def dialect():
+    """The SQLAlchemy dialect of the database in force."""
+    return _DIALECTS["postgres"] if _db.db_config.is_postgres() else _DIALECTS["sqlite"]
+
+
+def table(name: str, *columns, **kwargs) -> Table:
+    """Declare a table in the shared catalogue.
+
+    ``sqlite_autoincrement`` is on, as it was in every hand-written
+    ``INTEGER PRIMARY KEY AUTOINCREMENT``: without it SQLite reuses the ids of
+    deleted rows, and an id that once meant another order is a bug waiting.
+    """
+    kwargs.setdefault("sqlite_autoincrement", True)
+    # Test harnesses re-import data-access modules; a second import replaces
+    # the declaration instead of failing. Two different modules declaring one
+    # name is caught by tests/schema_declaration_tests.py, not here.
+    kwargs.setdefault("extend_existing", True)
+    declared = Table(name, metadata, *columns, **kwargs)
+    # SQLAlchemy writes NOT NULL on every key column; the hand-written SQLite
+    # DDL never did on a single-column key (``id INTEGER PRIMARY KEY
+    # AUTOINCREMENT``). On PostgreSQL the primary key makes the column NOT NULL
+    # by itself, so dropping the explicit word changes nothing there and keeps
+    # SQLite as it was. Composite keys were written with NOT NULL on each part
+    # and keep whatever the declaration says.
+    key = list(declared.primary_key.columns)
+    if len(key) == 1:
+        key[0].nullable = True
+    return declared
+
+
+def sequence(name: str, start: int = 1, increment: int = 1) -> Sequence:
+    """A PostgreSQL sequence read by hand (``nextval``); SQLite has none."""
+    return Sequence(name, start=start, increment=increment, metadata=metadata)
+
+
+def ensure_sequences(*sequences: Sequence) -> None:
+    """Create these sequences on PostgreSQL; nothing to do on SQLite."""
+    if not _db.db_config.is_postgres():
+        return
+    for declared in sequences:
+        _db.DatabaseManagerV2.execute_commit(
+            str(CreateSequence(declared, if_not_exists=True).compile(dialect=dialect())).strip())
+
+
+class per_dialect(ColumnElement):
+    """A server default (or any SQL fragment) that the two dialects spell differently.
+
+    ``server_default=per_dialect(postgres="NOW()", sqlite="CURRENT_TIMESTAMP")``.
+    SQLAlchemy has ``with_variant`` for types but nothing for defaults, and a
+    default that differed between the hand-written branches must keep differing.
+    """
+
+    inherit_cache = False
+
+    def __init__(self, postgres: str, sqlite: str):
+        self.postgres = postgres
+        self.sqlite = sqlite
+
+
+@compiles(per_dialect)
+def _compile_per_dialect(element, compiler, **kw):
+    return element.postgres if compiler.dialect.name == "postgresql" else element.sqlite
+
+
+def _wanted_here(item, current) -> bool:
+    """Whether an index or constraint marked ``ddl_if(dialect=...)`` belongs to this dialect."""
+    condition = getattr(item, "_ddl_if", None)
+    if condition is None or not condition.dialect:
+        return True
+    wanted = condition.dialect if isinstance(condition.dialect, (list, tuple, set)) else [condition.dialect]
+    return current.name in wanted
+
+
+def foreign_key(columns, target: str, target_columns=("id",), **kwargs) -> ForeignKeyConstraint:
+    """A foreign key to a table that may be declared by another module, later.
+
+    SQLAlchemy resolves the target in the catalogue when compiling. Modules are
+    imported in whatever order start-up and tests happen to use, so the target
+    may not be declared yet; a placeholder holding just the referenced columns
+    stands in (``keep_existing``: it never overrides a real declaration), and
+    the real declaration replaces it when it comes (``extend_existing``).
+    The placeholder is never created: ``ensure_tables`` makes only the tables
+    it is given.
+    """
+    if target not in metadata.tables:
+        Table(target, metadata, *[Column(name, Integer, primary_key=True) for name in target_columns],
+              keep_existing=True)
+    columns = [columns] if isinstance(columns, str) else list(columns)
+    return ForeignKeyConstraint(columns, [f"{target}.{c}" for c in target_columns], **kwargs)
+
+
+def auto_id(name: str = "id") -> Column:
+    """``SERIAL PRIMARY KEY`` / ``INTEGER PRIMARY KEY AUTOINCREMENT``."""
+    return Column(name, Integer, primary_key=True, autoincrement=True)
+
+
+def big_auto_id(name: str = "id") -> Column:
+    """``BIGSERIAL PRIMARY KEY`` on PostgreSQL, the same integer key on SQLite."""
+    return Column(name, BigInteger().with_variant(Integer(), "sqlite"),
+                  primary_key=True, autoincrement=True)
+
+
+def create_statements(tables: Iterable[Table]) -> List[str]:
+    """The DDL ``ensure_tables`` runs for these tables, compiled for the dialect in force."""
+    current = dialect()
+    statements = []
+    for declared in tables:
+        statements.append(str(CreateTable(declared, if_not_exists=True).compile(dialect=current)).strip())
+        for index in sorted(declared.indexes, key=lambda i: i.name or ""):
+            if not _wanted_here(index, current):
+                continue
+            statements.append(str(CreateIndex(index, if_not_exists=True).compile(dialect=current)).strip())
+    return statements
+
+
+def existing_columns(name: str) -> Set[str]:
+    """The columns the table has now, asked of the database in one read."""
+    manager = _db.DatabaseManagerV2
+    if _db.db_config.is_postgres():
+        # current_schema(): a table of the same name in another schema must not
+        # make a column look present when it is not.
+        rows = manager.execute(
+            "SELECT column_name FROM information_schema.columns "
+            "WHERE table_name = :table AND table_schema = current_schema()",
+            {"table": name})
+        return {row["column_name"] for row in rows or []}
+    # PRAGMA takes no parameters; the name comes from a declaration, never a request.
+    rows = manager.execute(f'PRAGMA table_info("{identifier(name)}")')
+    return {row["name"] for row in rows or []}
+
+
+#: What a table or column may be called. Checked rather than escaped: an
+#: identifier of a schema is written by the code that owns it, so anything
+#: outside this shape is a mistake to stop, not a string to quote. The package
+#: offers a template where a table name is data (a table per application), and
+#: an author who copies it without a check of their own would otherwise reach
+#: the DDL with whatever arrived (task keepup-15).
+IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def identifier(name: str) -> str:
+    """The name, if it is one.
+
+    Args:
+        name: a table or column name about to be written into a statement.
+
+    Returns:
+        The same name.
+
+    Raises:
+        ValueError: when it is not a plain identifier.
+    """
+    if not isinstance(name, str) or not IDENTIFIER.match(name):
+        raise ValueError(f"{name!r} is not a usable table or column name")
+    return name
+
+
+def column_definition(column: Column) -> str:
+    """What follows the column name in ``ADD COLUMN``, for the dialect in force."""
+    spec = str(CreateColumn(column).compile(dialect=dialect())).strip()
+    quoted = dialect().identifier_preparer.quote(column.name)
+    for prefix in (quoted + " ", column.name + " "):
+        if spec.startswith(prefix):
+            return spec[len(prefix):]
+    return spec
+
+
+def ensure_tables(*tables: Table) -> Dict[str, List[str]]:
+    """Make the database hold these tables, their columns and their indexes.
+
+    Returns, per table, the columns this call added -- empty when the table was
+    already complete. Order matters only for foreign keys, and the caller passes
+    tables in the order their references need, as the hand-written DDL did.
+    """
+    manager = _db.DatabaseManagerV2
+    current = dialect()
+    added: Dict[str, List[str]] = {}
+    for declared in tables:
+        manager.execute_commit(
+            str(CreateTable(declared, if_not_exists=True).compile(dialect=current)).strip())
+        added[declared.name] = _add_missing(declared.name, declared.columns)
+        for index in sorted(declared.indexes, key=lambda i: i.name or ""):
+            if not _wanted_here(index, current):
+                continue
+            manager.execute_commit(
+                str(CreateIndex(index, if_not_exists=True).compile(dialect=current)).strip())
+    return added
+
+
+def _add_missing(name: str, columns: Iterable[Column]) -> List[str]:
+    manager = _db.DatabaseManagerV2
+    present = existing_columns(name)
+    missing = [c for c in columns if c.name not in present]
+    for column in missing:
+        manager.add_column_if_missing(name, column.name, column_definition(column))
+    return [c.name for c in missing]
+
+
+def ensure_columns(name: str, *columns: Column) -> List[str]:
+    """Add columns to a table another module declares.
+
+    The application extends tables of the framework (``users`` gets a tenant's
+    SSH key, a Telegram persona); the framework cannot know those columns, and
+    declaring ``users`` a second time would make two owners of one table. The
+    columns are declared here, by the module that needs them, and added to the
+    existing table -- never creating it: that is its owner's job.
+    """
+    for column in columns:
+        if getattr(column, "table", None) is None:
+            # A free column cannot be compiled; a throw-away table outside the
+            # catalogue gives it a parent without declaring anything.
+            Table(f"_columns_of_{name}", MetaData(), column)
+    return _add_missing(name, columns)
+
+
+def ensure_indexes(name: str, *indexes: Index) -> None:
+    """Create indexes on a table another module declares.
+
+    The counterpart of ``ensure_columns``: a plugin that joins against a table
+    it does not own may need an index there, and declaring the table a second
+    time to carry it would make two owners of one table. Each index is given
+    by name and column names; the table is never created here.
+    """
+    current = dialect()
+    for index in indexes:
+        if index.table is None:
+            # Unbound columns cannot be compiled; a throw-away table outside the
+            # catalogue gives the index its columns without declaring anything.
+            Table(f"_indexes_of_{name}", MetaData(),
+                  *[Column(expression, Integer) for expression in index.expressions], index)
+        if not _wanted_here(index, current):
+            continue
+        text = str(CreateIndex(index, if_not_exists=True).compile(dialect=current)).strip()
+        # The throw-away table's name is not the real one.
+        text = text.replace(f"_indexes_of_{name}", name)
+        _db.DatabaseManagerV2.execute_commit(text)
+

keepup_db-0.4.0/keepup_db.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: keepup-db
+Version: 0.4.0
+Summary: The abstraction that holds SQLAlchemy: the table language, the pool, the dialect registry and the creation of declared tables.
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: keepup-admin>=0.4.0
+Requires-Dist: sqlalchemy<3,>=2.0
+
+# keepup-db
+
+The abstraction that holds SQLAlchemy: the table language, the pool, the dialect registry and the creation of declared tables.
+
+One capability of [keepup](https://github.com/FedorSergeev/keepup-admin), the framework over FastAPI for admin
+panels: the base is installed as `keepup-admin` and imported as `keepup`, and
+this distribution carries one part of it. It is installed on its own
+(`pip install keepup-db`) or together with the base (`pip install "keepup-admin[panel]"`),
+and it declares itself to the kernel through its
+`[project.entry-points."keepup.plugins"]` entry point.

keepup_db-0.4.0/keepup_db.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+keepup_db/__init__.py
+keepup_db/manager.py
+keepup_db/positional_sql.py
+keepup_db/tables.py
+keepup_db.egg-info/PKG-INFO
+keepup_db.egg-info/SOURCES.txt
+keepup_db.egg-info/dependency_links.txt
+keepup_db.egg-info/entry_points.txt
+keepup_db.egg-info/requires.txt
+keepup_db.egg-info/top_level.txt

keepup_db-0.4.0/keepup_db.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

keepup_db-0.4.0/keepup_db.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[keepup.plugins]
+db = keepup_db:DbPlugin

keepup_db-0.4.0/keepup_db.egg-info/requires.txt

@@ -0,0 +1,2 @@
+keepup-admin>=0.4.0
+sqlalchemy<3,>=2.0

keepup_db-0.4.0/keepup_db.egg-info/top_level.txt

@@ -0,0 +1 @@
+keepup_db

keepup_db-0.4.0/pyproject.toml

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "keepup-db"
+version = "0.4.0"
+description = "The abstraction that holds SQLAlchemy: the table language, the pool, the dialect registry and the creation of declared tables."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "keepup-admin>=0.4.0",
+    "sqlalchemy>=2.0,<3",
+]
+
+[project.entry-points."keepup.plugins"]
+db = "keepup_db:DbPlugin"
+
+[tool.setuptools]
+packages = ["keepup_db"]

keepup_db-0.4.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+