dbreduce 0.1.0__py3-none-any.whl

dbreduce/__init__.py

@@ -0,0 +1 @@
+"""PostgreSQL reproducer reduction."""

dbreduce/__main__.py

@@ -0,0 +1,3 @@
+from dbreduce.cli import app
+
+app()

dbreduce/cache/store.py

@@ -0,0 +1,32 @@
+import hashlib
+from pathlib import Path
+
+
+def fingerprint(snapshot: Path) -> str:
+    """Hash schema/sequence statements and COPY rows as a multiset per table."""
+    digest = hashlib.sha256()
+    with snapshot.open("rb") as stream:
+        for line in stream:
+            digest.update(line)
+            if line.startswith(b"COPY ") and line.rstrip(b"\r\n").endswith(b" FROM stdin;"):
+                rows = []
+                for line in stream:
+                    if line.rstrip(b"\r\n") == b"\\.":
+                        break
+                    rows.append(line)
+                for row in sorted(rows):
+                    digest.update(row)
+                digest.update(line)
+    return digest.hexdigest()
+
+
+class Cache:
+    def __init__(self) -> None:
+        self.results: dict[str, bool] = {}
+        self.hits = 0
+
+    def get(self, key: str) -> bool | None:
+        result = self.results.get(key)
+        if result is not None:
+            self.hits += 1
+        return result

dbreduce/cli/__init__.py

@@ -0,0 +1,193 @@
+import json
+import os
+import shutil
+import tempfile
+import time
+from pathlib import Path
+from typing import Annotated
+
+import psycopg
+import typer
+
+from dbreduce.cache.store import Cache
+from dbreduce.graph.dependencies import components, dependencies
+from dbreduce.oracle.runner import Oracle
+from dbreduce.postgres.backend import PostgresBackend
+from dbreduce.postgres.database import Workspace, read_archive_settings, read_settings
+from dbreduce.postgres.dump import dump
+from dbreduce.postgres.introspection import check_extension_tables, inspect_database
+from dbreduce.reducer.engine import reduce as reduce_state
+
+app = typer.Typer(
+    no_args_is_help=True, help="Minimize PostgreSQL bug reproducers in a disposable copy."
+)
+
+
+def publish_result(exported: Path, output: Path, report: Path, result: dict[str, object]) -> None:
+    """Prepare both files, then publish without overwriting existing paths."""
+    pending: list[Path] = []
+    created: list[Path] = []
+    try:
+        with tempfile.NamedTemporaryFile(
+            dir=output.parent, prefix=".dbreduce-", delete=False
+        ) as file:
+            pending.append(Path(file.name))
+            with exported.open("rb") as source:
+                shutil.copyfileobj(source, file)
+        with tempfile.NamedTemporaryFile(
+            mode="w",
+            encoding="utf-8",
+            dir=report.parent,
+            prefix=".dbreduce-",
+            delete=False,
+        ) as file:
+            pending.append(Path(file.name))
+            json.dump(result, file, indent=2)
+            file.write("\n")
+        for prepared_path, destination in zip(pending, (output, report), strict=True):
+            os.link(prepared_path, destination)
+            created.append(destination)
+    except BaseException:
+        for path in created:
+            path.unlink(missing_ok=True)
+        raise
+    finally:
+        for path in pending:
+            path.unlink(missing_ok=True)
+
+
+def require_clients() -> None:
+    for name in ("pg_dump", "pg_restore"):
+        if not shutil.which(name):
+            raise ValueError(f"Required PostgreSQL client tool is missing: {name}")
+
+
+@app.command("inspect")
+def inspect_command(database: Annotated[str, typer.Option(help="PostgreSQL source DSN")]) -> None:
+    """Show tables, exact row counts, keys and dependency graph (read-only)."""
+    try:
+        with psycopg.connect(database, connect_timeout=10) as conn:
+            conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
+            schema = inspect_database(conn)
+        for table in schema.tables:
+            typer.echo(
+                f"{table.label}\n  rows: {table.rows}\n  pk: {', '.join(table.primary_key) or '-'}"
+            )
+            for fk in schema.foreign_keys:
+                if fk.child == table.key:
+                    typer.echo(
+                        f"  FK {', '.join(fk.columns)} -> {'.'.join(fk.parent)}"
+                        f"({', '.join(fk.target_columns)})"
+                    )
+        typer.echo("Dependencies (child -> parent):")
+        for child, parents in dependencies(schema).items():
+            typer.echo(
+                f"  {'.'.join(child)} -> {', '.join('.'.join(p) for p in sorted(parents)) or '-'}"
+            )
+        typer.echo(f"Strongly connected components: {components(schema)}")
+    except (ValueError, psycopg.Error) as error:
+        fail(error)
+
+
+def fail(error: Exception) -> None:
+    message = (
+        "PostgreSQL operation failed; check connection and permissions"
+        if isinstance(error, psycopg.Error)
+        else str(error)
+    )
+    typer.echo(f"Error: {message}", err=True)
+    raise typer.Exit(1)
+
+
+@app.command("reduce")
+def reduce_command(
+    oracle: Annotated[str, typer.Option(help="Shell command; nonzero exit reproduces the bug")],
+    database: Annotated[str | None, typer.Option(help="Source PostgreSQL DSN")] = None,
+    input_dump: Annotated[
+        Path | None, typer.Option("--dump", help="Trusted pg_dump custom-format archive")
+    ] = None,
+    admin_database: Annotated[str | None, typer.Option(help="DSN with CREATEDB privilege")] = None,
+    confirm: Annotated[int, typer.Option(min=1)] = 1,
+    timeout: Annotated[float, typer.Option(min=0.01)] = 60,
+    output: Annotated[Path, typer.Option()] = Path("dbreduce.min.sql"),
+    report: Annotated[Path, typer.Option()] = Path("dbreduce-report.json"),
+) -> None:
+    """Reduce a source database or dump; export SQL and a JSON report."""
+    started = time.monotonic()
+    try:
+        if (database is None) == (input_dump is None) or database == "":
+            raise ValueError("Provide exactly one of --database or --dump")
+        admin = admin_database or database
+        if not admin:
+            raise ValueError("--admin-database is required with --dump")
+        if output.resolve() == report.resolve() or output.exists() or report.exists():
+            raise ValueError("Output and report must be distinct paths that do not already exist")
+        require_clients()
+        with tempfile.TemporaryDirectory(prefix="dbreduce-") as temporary:
+            snapshot = Path(temporary) / "accepted.dump"
+            if database:
+                with psycopg.connect(database, connect_timeout=10) as conn:
+                    conn.execute("SET TRANSACTION READ ONLY")
+                    check_extension_tables(conn)
+                dump(database, snapshot)
+            else:
+                assert input_dump is not None
+                shutil.copyfile(input_dump, snapshot)
+            settings = read_settings(database) if database else read_archive_settings(snapshot)
+            with Workspace(admin, settings=settings) as workspace:
+                workspace.reset(snapshot)
+                with psycopg.connect(workspace.dsn, connect_timeout=10) as conn:
+                    check_extension_tables(conn)
+                    schema = inspect_database(conn)
+                # Keep one normalized snapshot for all probes.
+                dump(workspace.dsn, snapshot)
+                initial_rows = sum(table.rows for table in schema.tables)
+                typer.echo(f"Initial database: {len(schema.tables)} tables, {initial_rows} rows")
+                runner = Oracle(oracle, confirm=confirm, timeout=timeout)
+                cache = Cache()
+                if not runner.fails(workspace.url, lambda: workspace.reset(snapshot)):
+                    raise ValueError(
+                        "Oracle passed on the initial copy; failure does not reproduce"
+                    )
+                typer.echo("Oracle: FAIL\nReducing tables, row groups and individual rows...")
+                workspace.reset(snapshot)
+                backend = PostgresBackend(workspace, schema, snapshot, runner, cache)
+                final = reduce_state(backend, typer.echo)
+                # A fresh uncached final confirmation catches some flaky-oracle failures.
+                if not runner.fails(workspace.url, lambda: workspace.reset(snapshot)):
+                    raise ValueError(
+                        "Final oracle confirmation passed; no verified result exported"
+                    )
+                workspace.reset(snapshot)
+                exported = Path(temporary) / "result.sql"
+                dump(workspace.dsn, exported, archive=False, create_database=True)
+                final_rows = sum(map(len, final.values()))
+                result = {
+                    "initial_tables": len(schema.tables),
+                    "initial_rows": initial_rows,
+                    "final_tables": len(final),
+                    "final_rows": final_rows,
+                    "oracle_executions": runner.executions,
+                    "cache_hits": cache.hits,
+                    "duration_seconds": time.monotonic() - started,
+                    "rows_by_table": {
+                        schema_name: {
+                            name: len(final[(schema_name, name)])
+                            for schema, name in final
+                            if schema == schema_name
+                        }
+                        for schema_name, _ in final
+                    },
+                    "constraint_rejections": backend.constraint_rejections,
+                    "raise_exception_rejections": backend.raise_rejections,
+                    "confirm": confirm,
+                    "oracle": "FAIL",
+                    "restore_database": workspace.name,
+                }
+            publish_result(exported, output, report, result)
+            typer.echo(
+                f"Final: {final_rows} rows, oracle: FAIL, "
+                f"executions: {runner.executions}, cache hits: {cache.hits}"
+            )
+    except (OSError, ValueError, RuntimeError, psycopg.Error) as error:
+        fail(error)

dbreduce/graph/dependencies.py

@@ -0,0 +1,33 @@
+from dbreduce.models.schema import Schema, TableKey
+
+
+def dependencies(schema: Schema) -> dict[TableKey, set[TableKey]]:
+    graph: dict[TableKey, set[TableKey]] = {table.key: set() for table in schema.tables}
+    for fk in schema.foreign_keys:
+        graph[fk.child].add(fk.parent)
+    return graph
+
+
+def components(schema: Schema) -> list[tuple[TableKey, ...]]:
+    """Iterative reachability SCCs; small table graphs favor a simple implementation."""
+    graph = dependencies(schema)
+    reach = {}
+    for node in graph:
+        seen: set[TableKey] = set()
+        pending = [node]
+        while pending:
+            current = pending.pop()
+            if current not in seen:
+                seen.add(current)
+                pending.extend(graph[current] - seen)
+        reach[node] = seen
+    remaining = set(graph)
+    result = []
+    while remaining:
+        node = min(remaining)
+        group = tuple(
+            sorted(other for other in remaining if other in reach[node] and node in reach[other])
+        )
+        result.append(group)
+        remaining.difference_update(group)
+    return result

dbreduce/models/schema.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass
+
+type TableKey = tuple[str, str]
+type RowKey = tuple[str, int]
+type State = dict[TableKey, list[RowKey]]
+
+
+@dataclass(frozen=True)
+class Table:
+    key: TableKey
+    primary_key: tuple[str, ...]
+    rows: int
+
+    @property
+    def label(self) -> str:
+        return ".".join(self.key)
+
+
+@dataclass(frozen=True)
+class ForeignKey:
+    name: str
+    child: TableKey
+    parent: TableKey
+    columns: tuple[str, ...]
+    target_columns: tuple[str, ...]
+    operators: tuple[tuple[str, str], ...]
+    collations: tuple[tuple[str, str] | None, ...] = ()
+
+
+@dataclass(frozen=True)
+class Schema:
+    tables: tuple[Table, ...]
+    foreign_keys: tuple[ForeignKey, ...]

dbreduce/oracle/runner.py

@@ -0,0 +1,52 @@
+import os
+import signal
+import subprocess
+from collections.abc import Callable
+
+
+class OracleError(RuntimeError):
+    pass
+
+
+class Oracle:
+    def __init__(self, command: str, *, confirm: int = 1, timeout: float = 60) -> None:
+        if confirm < 1 or timeout <= 0:
+            raise ValueError("confirm and timeout must be positive")
+        self.command = command
+        self.confirm = confirm
+        self.timeout = timeout
+        self.executions = 0
+
+    def fails(self, url: str, prepare: Callable[[], None]) -> bool:
+        env = os.environ.copy()
+        env.update(DATABASE_URL=url, DBREDUCE_DATABASE_URL=url)
+        for _ in range(self.confirm):
+            prepare()
+            self.executions += 1
+            with subprocess.Popen(
+                self.command,
+                shell=True,
+                env=env,
+                start_new_session=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            ) as process:
+                try:
+                    code = process.wait(timeout=self.timeout)
+                except BaseException as error:
+                    os.killpg(process.pid, signal.SIGKILL)
+                    process.wait()
+                    if isinstance(error, subprocess.TimeoutExpired):
+                        raise OracleError(
+                            "Oracle timed out; this is not a reproduced failure"
+                        ) from error
+                    raise
+                finally:
+                    # Do not leave background children connected to the disposable database.
+                    try:
+                        os.killpg(process.pid, signal.SIGKILL)
+                    except ProcessLookupError:
+                        pass
+            if code == 0:
+                return False
+        return True

dbreduce/postgres/backend.py

@@ -0,0 +1,64 @@
+import uuid
+from pathlib import Path
+
+import psycopg
+
+from dbreduce.cache.store import Cache, fingerprint
+from dbreduce.models.schema import RowKey, Schema, State, TableKey
+from dbreduce.oracle.runner import Oracle
+from dbreduce.postgres.database import Workspace
+from dbreduce.postgres.dump import dump
+from dbreduce.postgres.rows import CandidateRejected, delete_rows, read_rows
+
+
+class PostgresBackend:
+    def __init__(
+        self, workspace: Workspace, schema: Schema, snapshot: Path, oracle: Oracle, cache: Cache
+    ) -> None:
+        self.workspace = workspace
+        self.schema = schema
+        self.snapshot = snapshot
+        self.oracle = oracle
+        self.cache = cache
+        self.constraint_rejections = 0
+        self.raise_rejections = 0
+        self.restrict_key = uuid.uuid4().hex
+        with psycopg.connect(workspace.dsn, connect_timeout=10) as conn:
+            self.current, _ = read_rows(conn, schema)
+
+    def state(self) -> State:
+        return self.current
+
+    def attempt(self, table: TableKey, rows: list[RowKey]) -> bool:
+        self.workspace.reset(self.snapshot)
+        try:
+            with psycopg.connect(self.workspace.dsn, connect_timeout=10) as conn:
+                conn.execute("SET statement_timeout = '600s'")
+                delete_rows(conn, self.schema, table, rows)
+        except psycopg.errors.IntegrityConstraintViolation:
+            self.constraint_rejections += 1
+            return False
+        except CandidateRejected:
+            self.raise_rejections += 1
+            return False
+        candidate_dump = self.snapshot.with_name("candidate.dump")
+        dump(self.workspace.dsn, candidate_dump)
+        # Compare and fingerprint the state the oracle will actually see after restore.
+        self.workspace.reset(candidate_dump)
+        with psycopg.connect(self.workspace.dsn, connect_timeout=10) as conn:
+            candidate, _ = read_rows(conn, self.schema)
+        if sum(map(len, candidate.values())) >= sum(map(len, self.current.values())):
+            return False
+        fingerprint_dump = self.snapshot.with_name("fingerprint.sql")
+        dump(self.workspace.dsn, fingerprint_dump, archive=False, restrict_key=self.restrict_key)
+        key = fingerprint(fingerprint_dump)
+        accepted = self.cache.get(key)
+        if accepted is None:
+            accepted = self.oracle.fails(
+                self.workspace.url, lambda: self.workspace.reset(candidate_dump)
+            )
+            self.cache.results[key] = accepted
+        if accepted:
+            candidate_dump.replace(self.snapshot)
+            self.current = candidate
+        return accepted

dbreduce/postgres/database.py

@@ -0,0 +1,238 @@
+import hashlib
+import os
+import re
+import subprocess
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from types import TracebackType
+from urllib.parse import quote, urlencode
+
+import psycopg
+from psycopg import sql
+from psycopg.conninfo import conninfo_to_dict, make_conninfo
+
+from dbreduce.postgres.dump import restore
+
+
+@dataclass(frozen=True)
+class DatabaseSettings:
+    encoding: str
+    lc_collate: str
+    lc_ctype: str
+    provider: str
+    locale: str | None
+    icu_rules: str | None
+
+
+def read_settings(dsn: str) -> DatabaseSettings:
+    with psycopg.connect(dsn, connect_timeout=10) as conn:
+        row = conn.execute("""
+            SELECT pg_encoding_to_char(d.encoding), d.datcollate, d.datctype,
+                   coalesce(to_jsonb(d)->>'datlocprovider', 'c'),
+                   coalesce(to_jsonb(d)->>'datlocale',
+                            to_jsonb(d)->>'daticulocale'),
+                   to_jsonb(d)->>'daticurules'
+            FROM pg_database d WHERE d.datname = current_database()
+        """).fetchone()
+    if row is None:
+        raise ValueError("Cannot read source database settings")
+    return DatabaseSettings(*row)
+
+
+def read_archive_settings(path: Path) -> DatabaseSettings:
+    """Use pg_restore's own CREATE DATABASE statement as archive metadata."""
+    with path.open("rb") as archive:
+        try:
+            result = subprocess.run(
+                [
+                    "pg_restore",
+                    "--create",
+                    "--schema-only",
+                    "--use-list",
+                    os.devnull,
+                    "--file",
+                    "-",
+                ],
+                stdin=archive,
+                capture_output=True,
+                timeout=600,
+                env={**os.environ, "LC_ALL": "C"},
+                check=False,
+            )
+        except subprocess.TimeoutExpired as error:
+            raise ValueError("Timed out reading archive database settings") from error
+    if result.returncode:
+        if b"database name contains a newline or carriage return" in result.stderr:
+            raise ValueError(
+                "Custom archive has a database name containing a newline or carriage return; "
+                "pg_restore cannot safely read its settings"
+            )
+        raise ValueError("Cannot read database settings from custom archive")
+    statement = next(
+        (
+            line
+            for line in result.stdout.splitlines()
+            if line.startswith(b"CREATE DATABASE ") and b" WITH TEMPLATE = " in line
+        ),
+        None,
+    )
+    if statement is None:
+        raise ValueError("Archive does not expose source database settings")
+
+    encoding_match = re.search(rb"\bENCODING\s*=\s*'([^']+)'", statement)
+    if encoding_match is None:
+        raise ValueError("Archive database encoding is missing")
+    encoding = encoding_match.group(1).decode("ascii")
+    codec = (
+        "cp" + encoding[3:]
+        if encoding.startswith("WIN")
+        else "ascii"
+        if encoding == "SQL_ASCII"
+        else encoding
+    )
+
+    def setting(name: str) -> str | None:
+        match = re.search(rb"\b" + name.encode() + rb"\s*=\s*'((?:''|[^'])*)'", statement)
+        if not match:
+            return None
+        try:
+            return match.group(1).replace(b"''", b"'").decode(codec)
+        except (UnicodeError, LookupError) as error:
+            raise ValueError("Archive locale encoding cannot be decoded") from error
+
+    collate = setting("LC_COLLATE") or setting("LOCALE")
+    ctype = setting("LC_CTYPE") or setting("LOCALE")
+    provider_match = re.search(rb"\bLOCALE_PROVIDER\s*=\s*(\w+)", statement)
+    provider = {"libc": "c", "icu": "i", "builtin": "b"}.get(
+        provider_match.group(1).decode("ascii") if provider_match else "libc"
+    )
+    if not encoding or not collate or not ctype or provider is None:
+        raise ValueError("Archive database encoding/locale is incomplete")
+    locale = setting("ICU_LOCALE") or setting("BUILTIN_LOCALE") or setting("LOCALE")
+    return DatabaseSettings(encoding, collate, ctype, provider, locale, setting("ICU_RULES"))
+
+
+def database_url(dsn: str, name: str) -> str:
+    params = conninfo_to_dict(dsn)
+    params.pop("dbname", None)
+    user = str(params.pop("user", ""))
+    password = params.pop("password", None)
+    authority = quote(user, safe="")
+    if password is not None:
+        authority += ":" + quote(str(password), safe="")
+    if authority:
+        authority += "@"
+    host = str(params.get("host", ""))
+    # Keep socket paths and multi-host libpq settings in the query string.
+    if host and "/" not in host and "," not in host:
+        params.pop("host")
+        authority += f"[{host}]" if ":" in host else host
+        port = params.pop("port", None)
+        if port is not None:
+            authority += ":" + str(port)
+    query = "?" + urlencode(params) if params else ""
+    return f"postgresql://{authority}/{quote(name, safe='')}{query}"
+
+
+class Workspace:
+    """Own exactly one random database; no destructive method accepts a user database name."""
+
+    def __init__(self, admin_dsn: str, settings: DatabaseSettings | None = None) -> None:
+        self.admin_dsn = admin_dsn
+        self.name = f"dbreduce_{uuid.uuid4().hex}"
+        self.lock_key = int.from_bytes(
+            hashlib.sha256(self.name.encode()).digest()[:8], "big", signed=True
+        )
+        self.dsn = make_conninfo(admin_dsn, dbname=self.name)
+        self.url = database_url(admin_dsn, self.name)
+        self.settings = settings
+        self.created = False
+
+    def __enter__(self) -> "Workspace":
+        return self
+
+    def create(self) -> None:
+        self.close()
+        with psycopg.connect(self.admin_dsn, autocommit=True, connect_timeout=10) as conn:
+            conn.execute("SET statement_timeout = '600s'")
+            # Keep cleanup behind a CREATE that continues after its client loses the reply.
+            conn.execute("SELECT pg_advisory_lock(%s)", (self.lock_key,))
+            statement = sql.SQL("CREATE DATABASE {} TEMPLATE template0").format(
+                sql.Identifier(self.name)
+            )
+            if self.settings is not None:
+                settings = self.settings
+                statement += sql.SQL(" ENCODING {} LC_COLLATE {} LC_CTYPE {}").format(
+                    sql.Literal(settings.encoding),
+                    sql.Literal(settings.lc_collate),
+                    sql.Literal(settings.lc_ctype),
+                )
+                if settings.provider == "i":
+                    if settings.locale is None:
+                        raise ValueError("Source ICU locale is missing")
+                    statement += sql.SQL(" LOCALE_PROVIDER icu ICU_LOCALE {}").format(
+                        sql.Literal(settings.locale)
+                    )
+                    if settings.icu_rules:
+                        statement += sql.SQL(" ICU_RULES {}").format(
+                            sql.Literal(settings.icu_rules)
+                        )
+                elif settings.provider == "b":
+                    if settings.locale is None:
+                        raise ValueError("Source builtin locale is missing")
+                    statement += sql.SQL(" LOCALE_PROVIDER builtin BUILTIN_LOCALE {}").format(
+                        sql.Literal(settings.locale)
+                    )
+                elif settings.provider != "c":
+                    raise ValueError(f"Unsupported locale provider: {settings.provider}")
+            # CREATE may commit even if its reply is lost; cleanup must still try the name.
+            self.created = True
+            try:
+                conn.execute(statement)
+            except psycopg.errors.DuplicateDatabase:
+                # This name existed before our CREATE; it is not ours to drop.
+                self.created = False
+                raise
+
+    def reset(self, snapshot: Path) -> None:
+        self.create()
+        restore(self.dsn, snapshot)
+
+    def close(self) -> None:
+        if self.created:
+            try:
+                with psycopg.connect(self.admin_dsn, autocommit=True, connect_timeout=10) as conn:
+                    conn.execute("SET statement_timeout = '600s'")
+                    conn.execute("SELECT pg_advisory_lock(%s)", (self.lock_key,))
+                    conn.execute(
+                        sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(
+                            sql.Identifier(self.name)
+                        )
+                    )
+            except psycopg.Error as error:
+                raise RuntimeError(
+                    f"Could not confirm cleanup of workspace database {self.name}; "
+                    "inspect and remove it if present"
+                ) from error
+            self.created = False
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        try:
+            self.close()
+        except RuntimeError as cleanup_error:
+            if isinstance(exc, Exception):
+                original = (
+                    "PostgreSQL operation failed; check connection and permissions"
+                    if isinstance(exc, psycopg.Error)
+                    else str(exc) or type(exc).__name__
+                )
+                raise RuntimeError(
+                    f"Original failure: {original}; {cleanup_error}"
+                ) from cleanup_error
+            raise