toolframe-sql-postgres 0.2.0__py3-none-any.whl

toolframe_sql_postgres/__init__.py

@@ -0,0 +1 @@
+"""PostgreSQL persistence for plugin-owned stores, independent of Core state."""

toolframe_sql_postgres/backup.py

@@ -0,0 +1,163 @@
+"""Private database dumps, with bounded subprocesses and no credentials in argv."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import re
+import time
+import uuid
+from pathlib import Path
+
+import psycopg
+from psycopg.conninfo import conninfo_to_dict, make_conninfo
+from toolframe_sdk import PluginError
+
+
+def command_environment(dsn):
+    values = conninfo_to_dict(dsn)
+    environment = {
+        key: value
+        for key, value in os.environ.items()
+        if not key.startswith("PG") and not key.startswith("TOOLFRAME_POSTGRES")
+    }
+    password = values.pop("password", None)
+    if password is not None:
+        environment["PGPASSWORD"] = password
+    return make_conninfo(**values), environment
+
+
+async def run_tool(program, arguments, dsn, *, timeout):
+    public_dsn, environment = command_environment(dsn)
+    process = None
+    try:
+        process = await asyncio.create_subprocess_exec(
+            program,
+            *arguments,
+            "--no-password",
+            "--dbname",
+            public_dsn,
+            env=environment,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        async with asyncio.timeout(max(0.01, timeout)):
+            output, _ = await process.communicate()
+        if process.returncode:
+            raise PluginError(
+                "postgres_tool_failed",
+                "PostgreSQL client failed; check client version and database access",
+            )
+        return output
+    except OSError, TimeoutError:
+        raise PluginError(
+            "postgres_tool_failed",
+            "PostgreSQL client is unavailable or exceeded its deadline",
+        ) from None
+    finally:
+        if process and process.returncode is None:
+            process.kill()
+            await process.wait()
+
+
+async def client_major(program):
+    process = None
+    try:
+        process = await asyncio.create_subprocess_exec(
+            program,
+            "--version",
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        async with asyncio.timeout(5):
+            output, _ = await process.communicate()
+        match = re.search(rb"\(PostgreSQL\) (\d+)", output)
+        if process.returncode or not match:
+            raise ValueError
+        return int(match[1])
+    except OSError, TimeoutError, ValueError:
+        raise PluginError(
+            "postgres_tool_failed", "PostgreSQL client version is unavailable"
+        ) from None
+    finally:
+        if process and process.returncode is None:
+            process.kill()
+            await process.wait()
+
+
+async def require_matching_client(program, dsn, expected=None):
+    async with await psycopg.AsyncConnection.connect(
+        dsn, connect_timeout=5
+    ) as connection:
+        major = connection.info.server_version // 10000
+    if await client_major(program) != major or (
+        expected is not None and expected != major
+    ):
+        raise PluginError(
+            "postgres_version_mismatch",
+            "Use PostgreSQL client tools and restore target with the same major version as the source database",
+        )
+    return major
+
+
+async def create_backup(settings, directory, plan, deadline):
+    major = await require_matching_client(settings.pg_dump, settings.dsn)
+    directory = Path(directory) / "backups"
+    directory.mkdir(parents=True, exist_ok=True, mode=0o700)
+    identifier = f"{int(time.time())}-{uuid.uuid4().hex}"
+    path = directory / f"{identifier}.dump"
+    temporary = directory / f".{identifier}.partial"
+    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    os.close(descriptor)
+    try:
+        await run_tool(
+            settings.pg_dump,
+            ["--format=custom", "--file", str(temporary)],
+            settings.dsn,
+            timeout=deadline - time.time(),
+        )
+        with temporary.open("rb") as source:
+            checksum = hashlib.file_digest(source, "sha256").hexdigest()
+        metadata = {
+            "format": 1,
+            "application_id": settings.application_id,
+            "owners": sorted(settings.stores),
+            "sha256": checksum,
+            "plan": plan,
+            "created": time.time(),
+            "core_state_included": False,
+            "postgres_major": major,
+        }
+        temporary.replace(path)
+        manifest = path.with_suffix(".json")
+        descriptor = os.open(manifest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(descriptor, "w") as stream:
+            json.dump(metadata, stream, sort_keys=True)
+        return path
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def verify_backup(path):
+    path = Path(path)
+    metadata = json.loads(path.with_suffix(".json").read_text())
+    with path.open("rb") as source:
+        checksum = hashlib.file_digest(source, "sha256").hexdigest()
+    if (
+        metadata.get("format") != 1
+        or checksum != metadata.get("sha256")
+        or not isinstance(metadata.get("application_id"), str)
+        or not isinstance(metadata.get("postgres_major"), int)
+        or not isinstance(metadata.get("owners"), list)
+        or not metadata["owners"]
+        or any(
+            not isinstance(owner, str)
+            or not re.fullmatch(r"[a-z][a-z0-9-]{0,63}", owner)
+            for owner in metadata["owners"]
+        )
+    ):
+        raise PluginError("backup_invalid", "Backup checksum or format does not match")
+    return metadata

toolframe_sql_postgres/cli.py

@@ -0,0 +1,221 @@
+"""Provisioning and restore utilities, separate from the runtime entry point."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import stat
+import sys
+from pathlib import Path
+
+import psycopg
+from psycopg import sql
+from psycopg.conninfo import make_conninfo
+from toolframe_sdk import PluginError
+
+from .backup import require_matching_client, run_tool, verify_backup
+from .config import names
+from .provision import provision
+
+
+def read_environment(path):
+    path = Path(path)
+    mode = path.lstat().st_mode
+    if not stat.S_ISREG(mode) or mode & 0o077:
+        raise PluginError(
+            "credentials_permissions",
+            "Environment file must be a private regular file (0600)",
+        )
+    values = json.loads(path.read_text())
+    if not isinstance(values, dict) or any(
+        not isinstance(k, str)
+        or not isinstance(v, str)
+        or not k.startswith("TOOLFRAME_POSTGRES_")
+        for k, v in values.items()
+    ):
+        raise PluginError(
+            "credentials_invalid", "Invalid PostgreSQL environment document"
+        )
+    if (
+        "TOOLFRAME_POSTGRES_SETUP_DSN" in values
+        or "TOOLFRAME_POSTGRES_TEST_ADMIN_DSN" in values
+    ):
+        raise PluginError(
+            "credentials_invalid",
+            "Provisioning credentials must not be stored in runtime output",
+        )
+    return values
+
+
+def save_environment(path, values):
+    path = Path(path)
+    if path.is_symlink():
+        raise PluginError(
+            "credentials_invalid", "Environment output cannot be a symbolic link"
+        )
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    temporary = path.with_name(path.name + ".tmp")
+    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    try:
+        with os.fdopen(descriptor, "w") as stream:
+            json.dump(values, stream, indent=2)
+            stream.write("\n")
+            stream.flush()
+            os.fsync(stream.fileno())
+        temporary.replace(path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+async def restore(admin_dsn, database, backup, *, pg_restore="pg_restore"):
+    """Restore only into a newly provisioned database. Failure leaves it for diagnosis."""
+    metadata = verify_backup(backup)
+    await require_matching_client(pg_restore, admin_dsn, metadata["postgres_major"])
+    application = metadata["application_id"]
+    environment = provision(admin_dsn, database, application, metadata["owners"])
+    target = make_conninfo(admin_dsn, dbname=database)
+    await run_tool(
+        pg_restore,
+        [
+            "--clean",
+            "--if-exists",
+            "--no-owner",
+            "--no-privileges",
+            "--exit-on-error",
+            str(Path(backup).resolve()),
+        ],
+        target,
+        timeout=300,
+    )
+    manager = names(application, database=database)["manager"]
+    with psycopg.connect(target) as connection:
+        schemas = {"_toolframe_platform": manager}
+        for owner in metadata["owners"]:
+            ns = names(application, owner, database=database)
+            schemas.update(
+                {ns["schema"]: ns["controller"], ns["metadata"]: ns["controller"]}
+            )
+        actual = {
+            row[0]
+            for row in connection.execute(
+                "SELECT nspname FROM pg_namespace WHERE nspname NOT LIKE 'pg_%' AND nspname NOT IN ('information_schema','public')"
+            ).fetchall()
+        }
+        if actual != set(schemas):
+            raise PluginError(
+                "backup_inventory_mismatch",
+                "Restored schema inventory differs from the backup owners; target remains isolated for inspection",
+            )
+        for schema, owner_role in schemas.items():
+            connection.execute(
+                sql.SQL("ALTER SCHEMA {} OWNER TO {}").format(
+                    sql.Identifier(schema), sql.Identifier(owner_role)
+                )
+            )
+            tables = connection.execute(
+                "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=%s AND c.relkind IN ('r','p')",
+                (schema,),
+            ).fetchall()
+            for (table,) in tables:
+                connection.execute(
+                    sql.SQL("ALTER TABLE {}.{} OWNER TO {}").format(
+                        sql.Identifier(schema),
+                        sql.Identifier(table),
+                        sql.Identifier(owner_role),
+                    )
+                )
+    # Reapply grants and defaults to both restored and subsequently created objects.
+    provision(
+        admin_dsn, database, application, metadata["owners"], previous=environment
+    )
+    return environment
+
+
+def parser():
+    result = argparse.ArgumentParser(
+        description="Provision and restore Toolframe PostgreSQL stores"
+    )
+    commands = result.add_subparsers(dest="command", required=True)
+    setup = commands.add_parser("setup")
+    setup.add_argument("--application-id", required=True)
+    setup.add_argument("--database", required=True)
+    setup.add_argument("--owner", action="append", required=True)
+    setup.add_argument("--output", required=True, type=Path)
+    recovery = commands.add_parser("restore")
+    recovery.add_argument("--database", required=True)
+    recovery.add_argument("--backup", required=True, type=Path)
+    recovery.add_argument("--output", required=True, type=Path)
+    run = commands.add_parser("run")
+    run.add_argument("--env-file", required=True, type=Path)
+    run.add_argument("argv", nargs=argparse.REMAINDER)
+    return result
+
+
+def main(argv=None):
+    args = parser().parse_args(argv)
+    try:
+        if args.command == "run":
+            values = read_environment(args.env_file)
+            command = args.argv[1:] if args.argv[:1] == ["--"] else args.argv
+            if not command:
+                raise PluginError("command_missing", "Supply a command after --")
+            environment = dict(os.environ)
+            for key in (
+                "TOOLFRAME_POSTGRES_SETUP_DSN",
+                "TOOLFRAME_POSTGRES_TEST_ADMIN_DSN",
+            ):
+                environment.pop(key, None)
+            environment.update(values)
+            os.execvpe(command[0], command, environment)
+        admin = os.environ.get("TOOLFRAME_POSTGRES_SETUP_DSN")
+        if not admin:
+            raise PluginError(
+                "setup_credentials_missing",
+                "Set TOOLFRAME_POSTGRES_SETUP_DSN for provisioning",
+            )
+        if args.command == "setup":
+            previous = read_environment(args.output) if args.output.exists() else None
+            values = provision(
+                admin, args.database, args.application_id, args.owner, previous=previous
+            )
+        else:
+            if args.output.exists():
+                raise PluginError(
+                    "output_exists",
+                    "Restore output must be a new private environment file",
+                )
+            values = asyncio.run(
+                restore(
+                    admin,
+                    args.database,
+                    args.backup,
+                    pg_restore=os.environ.get(
+                        "TOOLFRAME_POSTGRES_PG_RESTORE", "pg_restore"
+                    ),
+                )
+            )
+        save_environment(args.output, values)
+        print(
+            json.dumps(
+                {
+                    "database": args.database,
+                    "environment_file": str(args.output),
+                    "credentials_printed": False,
+                }
+            )
+        )
+    except PluginError as error:
+        print(f"{error.code}: {error}", file=sys.stderr)
+        raise SystemExit(1) from None
+    except psycopg.Error, ValueError, OSError:
+        print(
+            "PostgreSQL setup/restore failed; inspect connectivity, client tools and resource permissions. No credentials were printed.",
+            file=sys.stderr,
+        )
+        raise SystemExit(1) from None
+
+
+if __name__ == "__main__":
+    main()

toolframe_sql_postgres/config.py

@@ -0,0 +1,104 @@
+"""Runtime credentials come exclusively from the process environment."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from dataclasses import dataclass
+
+from psycopg.conninfo import conninfo_to_dict, make_conninfo
+from toolframe_sdk import PluginError
+
+
+def names(
+    application_id: str, owner: str | None = None, *, database: str
+) -> dict[str, str]:
+    application = hashlib.sha256(
+        (application_id + "\0" + database).encode()
+    ).hexdigest()[:20]
+    if owner is None:
+        return {"manager": f"tf_{application}_manager"}
+    suffix = hashlib.sha256(owner.encode()).hexdigest()[:20]
+    return {
+        "schema": f"tf_{suffix}",
+        "metadata": f"tf_{suffix}_meta",
+        "controller": f"tf_{application}_{suffix}_ctl",
+        "runtime": f"tf_{application}_{suffix}_run",
+    }
+
+
+@dataclass(frozen=True, repr=False)
+class Settings:
+    application_id: str
+    dsn: str
+    stores: dict[str, dict[str, str]]
+    pg_dump: str
+    pg_restore: str
+    connection_limit: int
+
+    @classmethod
+    def from_env(cls):
+        try:
+            application = os.environ["TOOLFRAME_POSTGRES_APPLICATION_ID"]
+            dsn = os.environ["TOOLFRAME_POSTGRES_DSN"]
+            stores = json.loads(os.environ["TOOLFRAME_POSTGRES_STORES"])
+            connection = conninfo_to_dict(dsn)
+            database = connection["dbname"]
+            if (
+                connection.get("user")
+                != names(application, database=database)["manager"]
+            ):
+                raise ValueError("Use the application-scoped manager from setup")
+            if (
+                not application
+                or len(application) > 256
+                or not isinstance(stores, dict)
+            ):
+                raise ValueError
+            for owner, credentials in stores.items():
+                if not re.fullmatch(r"[a-z][a-z0-9-]{0,63}", owner):
+                    raise ValueError
+                if set(credentials) != {"user", "password"}:
+                    raise ValueError
+                if (
+                    credentials["user"]
+                    != names(application, owner, database=database)["controller"]
+                ):
+                    raise ValueError
+                if (
+                    not isinstance(credentials["password"], str)
+                    or not credentials["password"]
+                ):
+                    raise ValueError
+            limit = int(os.environ.get("TOOLFRAME_POSTGRES_CONNECTION_LIMIT", "16"))
+            if not 1 <= limit <= 128:
+                raise ValueError
+            return cls(
+                application,
+                dsn,
+                stores,
+                os.environ.get("TOOLFRAME_POSTGRES_PG_DUMP", "pg_dump"),
+                os.environ.get("TOOLFRAME_POSTGRES_PG_RESTORE", "pg_restore"),
+                limit,
+            )
+        except Exception:  # noqa: BLE001 - malformed secrets must never appear in diagnostics
+            raise PluginError(
+                "postgres_config_invalid",
+                "Set TOOLFRAME_POSTGRES_APPLICATION_ID, TOOLFRAME_POSTGRES_DSN and "
+                "TOOLFRAME_POSTGRES_STORES in the runtime environment using setup output",
+            ) from None
+
+    def store_dsn(self, owner):
+        if owner not in self.stores:
+            raise PluginError(
+                "store_not_provisioned",
+                "Provision the plugin store with toolframe-postgres setup",
+            )
+        return make_conninfo(self.dsn, **self.stores[owner])
+
+    def store_names(self, owner):
+        return names(
+            self.application_id, owner, database=conninfo_to_dict(self.dsn)["dbname"]
+        )