django-data-shape 0.1.0__py3-none-any.whl

django_data_shape/__init__.py

@@ -0,0 +1,33 @@
+"""A realistically shaped test database from Django models."""
+
+from django_data_shape.build import build
+from django_data_shape.build_result import BuildResult
+from django_data_shape.distributions.constant import Constant
+from django_data_shape.distributions.distribution import Distribution
+from django_data_shape.distributions.sequential import Sequential
+from django_data_shape.distributions.skew import Skew
+from django_data_shape.distributions.uniform import Uniform
+from django_data_shape.invalid_shape import InvalidShape
+from django_data_shape.shape import Shape
+from django_data_shape.shape_not_empty import ShapeNotEmpty
+from django_data_shape.table import Table
+from django_data_shape.table_result import TableResult
+from django_data_shape.unsupported_backend import UnsupportedBackend
+from django_data_shape.version import __version__
+
+__all__ = [
+    "BuildResult",
+    "Constant",
+    "Distribution",
+    "InvalidShape",
+    "Sequential",
+    "Shape",
+    "ShapeNotEmpty",
+    "Skew",
+    "Table",
+    "TableResult",
+    "Uniform",
+    "UnsupportedBackend",
+    "__version__",
+    "build",
+]

django_data_shape/build.py

@@ -0,0 +1,144 @@
+"""Turning a declaration into a database the planner can reason about."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django.core.management.color import no_style
+from django.db import DEFAULT_DB_ALIAS, connections, transaction
+
+from django_data_shape.build_result import BuildResult
+from django_data_shape.generate_rows import generate_rows
+from django_data_shape.require_postgres import require_postgres
+from django_data_shape.shape import Shape
+from django_data_shape.shape_not_empty import ShapeNotEmpty
+from django_data_shape.table import Table
+from django_data_shape.table_result import TableResult
+
+
+def build(shape: Shape, using: str = DEFAULT_DB_ALIAS) -> BuildResult:
+    """Generate, load, reset sequences and analyze every table in ``shape``.
+
+    The order of those steps is the whole function, and it is not
+    interchangeable. Loading rows into a table that was analyzed while empty
+    leaves the planner holding statistics from the old contents and applying
+    them to the new row count -- a worse lie than having no statistics at all,
+    and the one that produced a thirty-thousand-fold misestimate in the
+    measurements this package was designed from. So the ``ANALYZE`` is here, at
+    the end, owned by the library rather than left to the caller to remember.
+
+    That is also why a bare ``ANALYZE`` is in this release at all, when the
+    statistics work proper -- per-column targets, and caching a built database
+    as a template -- comes later. A loader that leaves its table unanalyzed
+    ships the exact state this package exists to condemn.
+    """
+    connection = connections[using]
+    require_postgres(connection, "Building a shape")
+
+    results: list[TableResult] = []
+    # One transaction around every table. Without it a shape whose second table
+    # fails leaves the first committed and analyzed, and the natural next action
+    # -- fix the shape, run it again -- fails on a duplicate key rather than on
+    # the original problem.
+    with transaction.atomic(using=using):
+        for table in shape.tables:
+            _require_empty(connection, table)
+            loaded = _load(connection, table, shape.seed)
+            _reset_sequence(connection, table)
+            _analyze(connection, table)
+            results.append(TableResult(table=table.db_table, rows=loaded))
+    return BuildResult(tables=tuple(results))
+
+
+def _require_empty(connection: Any, table: Table) -> None:
+    """Refuse to build on top of rows that are already there.
+
+    The keys this package assigns start at 1 every time, so a second build over
+    the same table collides on the primary key. That surfaced as a bare
+    UniqueViolation naming an index, which tells the reader nothing about what
+    they did or what to do instead.
+    """
+    with connection.cursor() as cursor:
+        cursor.execute(f"SELECT EXISTS (SELECT 1 FROM {connection.ops.quote_name(table.db_table)})")
+        row = cursor.fetchone()
+    if row[0]:
+        raise ShapeNotEmpty(
+            f"{table.db_table} already holds rows, and this package assigns primary keys from 1, "
+            "so building over them would collide. Empty the table first."
+        )
+
+
+# The connection is typed loosely on purpose. ``cursor.copy()`` is psycopg 3's
+# API reached through Django's cursor wrapper, and it appears on no Django base
+# class, so annotating the real wrapper type would mean asserting the checker
+# out of the way on every line that uses it.
+def _load(connection: Any, table: Table, seed: int) -> int:
+    """Stream generated rows into the table with ``COPY FROM STDIN``.
+
+    ``bulk_create`` is the obvious alternative and is roughly an order of
+    magnitude too slow at the row counts that make a plan meaningful. ``COPY``
+    is also why the generator yields tuples rather than model instances: there
+    is no instance to build, and no ``save`` to run.
+
+    Each declared value is passed through its field's ``get_db_prep_save``
+    first, and that is not a formality. Skipping it is how a naive datetime got
+    written five hours away from where ``save()`` would have put it under a
+    non-UTC ``TIME_ZONE`` -- silently, on the exact column ``Sequential`` exists
+    to make realistic -- and how a ``JSONField`` failed to load at all. The
+    generator stays backend-neutral because the preparation happens here rather
+    than inside it.
+
+    Returns the number of rows the database actually took, not the number
+    declared. They are the same today and stop being so once deduplicated
+    many-to-many edges arrive.
+    """
+    quote = connection.ops.quote_name
+    pk_column = table.model._meta.pk.column
+    columns = [quote(pk_column)] + [quote(field.column) for _, field in table.columns()]
+    statement = f"COPY {quote(table.db_table)} ({', '.join(columns)}) FROM STDIN"
+    prepare = [field.get_db_prep_save for _, field in table.columns()]
+
+    with connection.cursor() as cursor:
+        # ``copy`` is not in Django's WRAP_ERROR_ATTRS, so without this a
+        # Postgres error escapes as a raw psycopg exception: the caller cannot
+        # catch django.db.IntegrityError, and -- worse -- an enclosing atomic
+        # block never learns it needs a rollback, so the next query inside it
+        # fails with "current transaction is aborted" instead of a Django error.
+        with connection.wrap_database_errors, cursor.copy(statement) as copy:
+            for row in generate_rows(table, seed):
+                copy.write_row(
+                    (
+                        row[0],
+                        *(
+                            prep(value, connection)
+                            for prep, value in zip(prepare, row[1:], strict=True)
+                        ),
+                    )
+                )
+        return int(cursor.rowcount)
+
+
+def _reset_sequence(connection: Any, table: Table) -> None:
+    """Move the identity sequence past the keys this package just assigned.
+
+    Skipping this is the first bug the design invites: rows exist at ids 1..N
+    while the sequence still starts at 1, so the very first ``objects.create()``
+    inside a test raises ``IntegrityError`` on a primary key that is already
+    taken. Django's own backend operation is used rather than a hand-written
+    ``setval`` because it already knows how the column's sequence is named.
+    """
+    with connection.cursor() as cursor:
+        for statement in connection.ops.sequence_reset_sql(no_style(), [table.model]):
+            cursor.execute(statement)
+
+
+def _analyze(connection: Any, table: Table) -> None:
+    """Populate the statistics the planner reads.
+
+    Rows alone change nothing: without this the planner falls back to a default
+    selectivity and commits to it, which is how a two-million-row table gets
+    bitmap-scanned through an index for a value matching 98% of it. Measured at
+    81 ms on that table, because ``ANALYZE`` samples rather than scans.
+    """
+    with connection.cursor() as cursor:
+        cursor.execute(f"ANALYZE {connection.ops.quote_name(table.db_table)}")

django_data_shape/build_result.py

@@ -0,0 +1,27 @@
+"""What a completed build reports back."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from django_data_shape.table_result import TableResult
+
+
+@dataclass(frozen=True)
+class BuildResult:
+    """The outcome of building a shape, table by table.
+
+    Returned rather than logged because the counts are worth asserting on: the
+    row count a shape declares and the row count a table holds are the same
+    number today, but they stop being the same as soon as deduplication enters
+    the picture with many-to-many edges. Reporting achieved counts from the
+    start means that release changes what this says, not what callers have to
+    start checking.
+    """
+
+    tables: tuple[TableResult, ...]
+
+    @property
+    def rows(self) -> int:
+        """Rows loaded across every table."""
+        return sum(result.rows for result in self.tables)

django_data_shape/distributions/__init__.py

@@ -0,0 +1,9 @@
+"""The declared value distributions."""
+
+from django_data_shape.distributions.constant import Constant
+from django_data_shape.distributions.distribution import Distribution
+from django_data_shape.distributions.sequential import Sequential
+from django_data_shape.distributions.skew import Skew
+from django_data_shape.distributions.uniform import Uniform
+
+__all__ = ["Constant", "Distribution", "Sequential", "Skew", "Uniform"]

django_data_shape/distributions/constant.py

@@ -0,0 +1,26 @@
+"""The same value in every row."""
+
+from __future__ import annotations
+
+
+class Constant:
+    """Every row gets ``value``.
+
+    Present because a column that never varies is a real shape, not a missing
+    declaration: a tenant id on a single-tenant fixture, or a flag that is false
+    for every row in the dataset under test. Declaring it says so, where leaving
+    it out would mean the field simply had no distribution.
+
+    It also has a planner consequence worth knowing: a single-valued column has
+    exactly one most-common value at frequency 1.0, so a filter on it is either
+    everything or nothing, and an index on it is never usable.
+    """
+
+    def __init__(self, value: object) -> None:
+        self._value = value
+
+    def value(self, row: int, draw: float) -> object:
+        return self._value
+
+    def __repr__(self) -> str:
+        return f"Constant({self._value!r})"

django_data_shape/distributions/distribution.py

@@ -0,0 +1,24 @@
+"""The one thing every declared value has in common."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class Distribution(Protocol):
+    """Produces the value of one field for one row.
+
+    Both arguments are supplied to every implementation because the two kinds of
+    distribution need different halves: a categorical or numeric one consumes
+    ``draw`` and ignores the row, while a monotonic one consumes ``row`` and
+    ignores the draw. Passing both keeps the protocol single-method, and a
+    single-method protocol is what allows a distribution to be a plain object
+    rather than a class hierarchy.
+
+    ``draw`` is uniform in [0, 1) and depends only on the field and the row, so
+    an implementation must not carry state between calls. One that did would
+    make the same shape produce different data depending on generation order,
+    which is the property the placement work in a later release depends on.
+    """
+
+    def value(self, row: int, draw: float) -> object: ...

django_data_shape/distributions/sequential.py

@@ -0,0 +1,31 @@
+"""A column that advances with the row, instead of scattering."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class Sequential:
+    """``start`` plus ``row`` steps: monotonic, and correlated with the key.
+
+    The point is the correlation, not the convenience. Postgres records a
+    correlation statistic per column and costs an index scan differently
+    depending on it, so a timestamp column filled with shuffled dates plans
+    differently from one that advances the way real rows arrive. Shuffling is
+    the easy thing to do by accident and it is wrong in a way that only shows up
+    in plan choice.
+
+    Works for anything supporting ``start + row * step``, which covers numbers
+    and ``datetime`` with a ``timedelta``. It ignores ``draw`` entirely: this is
+    the one distribution whose value is a function of position alone.
+    """
+
+    def __init__(self, start: Any, step: Any) -> None:
+        self._start = start
+        self._step = step
+
+    def value(self, row: int, draw: float) -> object:
+        return self._start + row * self._step
+
+    def __repr__(self) -> str:
+        return f"Sequential({self._start!r}, {self._step!r})"

django_data_shape/distributions/skew.py

@@ -0,0 +1,71 @@
+"""A categorical column with a declared imbalance."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from typing import Any
+
+from django_data_shape.invalid_shape import InvalidShape
+
+
+class Skew:
+    """Values drawn from a weighted set, in a fixed order.
+
+    This is the distribution the package exists for. A status column that is 98%
+    one value is what decides whether an index on it is usable at all, and it is
+    the thing a fixtures loop never expresses -- ten rows with one of each says
+    the opposite of what production says.
+
+    ``Mapping[Any, float]`` rather than ``dict[object, float]``: ``dict`` is
+    invariant in both parameters, so a caller's prepared ``dict[str, float]`` --
+    the obvious way to build one of these outside a call -- would be rejected by
+    a type checker for no reason a reader could act on. ``Mapping`` is covariant
+    in its value type, so integer counts are accepted too.
+
+    Weights are relative and need not sum to 1: the readable form is often
+    counts, and normalising here is cheaper than making every caller do it. They
+    must be positive, because a zero-weight value is a value that never appears,
+    which is better said by leaving it out than by declaring it and meaning not.
+    """
+
+    def __init__(self, weights: Mapping[Any, float]) -> None:
+        if not weights:
+            raise InvalidShape("Skew needs at least one value; an empty distribution has none.")
+        # ``not (w > 0)`` rather than ``w <= 0`` because NaN compares False to
+        # both, so the obvious spelling let it through -- and a NaN weight makes
+        # every cumulative bound NaN, so no draw ever matches and the fallthrough
+        # returns the last value for every row. The declared distribution comes
+        # out inverted, silently.
+        bad = sorted(repr(v) for v, w in weights.items() if not (w > 0) or not math.isfinite(w))
+        if bad:
+            raise InvalidShape(
+                "Skew weights must be positive and finite, and these are not: "
+                + ", ".join(bad)
+                + ". A value that never occurs is said by omitting it."
+            )
+        total = sum(weights.values())
+        # The cumulative bounds are precomputed once rather than per row: this
+        # runs a million times or more, and the alternative is re-summing the
+        # weights inside the hot loop.
+        self._values: list[object] = []
+        self._bounds: list[float] = []
+        running = 0.0
+        for value, weight in weights.items():
+            running += weight
+            self._values.append(value)
+            self._bounds.append(running / total)
+        self._weights = dict(weights)
+
+    def value(self, row: int, draw: float) -> object:
+        for value, bound in zip(self._values, self._bounds, strict=True):
+            if draw < bound:
+                return value
+        # Reachable only when floating-point accumulation leaves the final bound
+        # a hair below 1.0, since a draw is guaranteed below 1.0 by construction.
+        # The last value is the correct answer there, and falling through to it
+        # is cheaper than renormalising on every draw.
+        return self._values[-1]
+
+    def __repr__(self) -> str:
+        return f"Skew({self._weights!r})"

django_data_shape/distributions/uniform.py

@@ -0,0 +1,50 @@
+"""A numeric column spread evenly across a range."""
+
+from __future__ import annotations
+
+import math
+from decimal import Decimal
+
+from django_data_shape.invalid_shape import InvalidShape
+
+
+class Uniform:
+    """Values spread evenly between ``low`` and ``high``.
+
+    Deliberately the least interesting distribution in the package, and named
+    plainly so it reads as a choice. Most real columns are not uniform, and a
+    uniform declaration on a column that matters is usually a placeholder
+    somebody meant to come back to.
+
+    ``places`` rounds the result. Not because the column would reject the
+    unrounded value -- Postgres rounds a float to a ``numeric(10, 2)`` happily,
+    and only overflowing the declared precision is an error -- but because a
+    money column whose values carry full binary float noise is not what the
+    application would ever have written, and this package's whole claim is that
+    the loaded rows are ones it could have. Rounding to ``Decimal`` rather than
+    ``float`` keeps the value exact on the way into ``COPY``; ``places=0`` is
+    how a plain integer column is filled.
+    """
+
+    def __init__(self, low: float, high: float, places: int | None = None) -> None:
+        # isfinite first: NaN compares False to every ordering, so ``high <= low``
+        # accepts a NaN bound and then returns NaN for every row.
+        if not math.isfinite(low) or not math.isfinite(high):
+            raise InvalidShape(f"Uniform needs finite bounds, got low={low}, high={high}.")
+        if high <= low:
+            raise InvalidShape(f"Uniform needs high greater than low, got low={low}, high={high}.")
+        if places is not None and places < 0:
+            raise InvalidShape(f"Uniform places cannot be negative, got {places}.")
+        self._low = low
+        self._high = high
+        self._places = places
+
+    def value(self, row: int, draw: float) -> object:
+        raw = self._low + draw * (self._high - self._low)
+        if self._places is None:
+            return raw
+        return round(Decimal(repr(raw)), self._places)
+
+    def __repr__(self) -> str:
+        places = "" if self._places is None else f", places={self._places}"
+        return f"Uniform({self._low!r}, {self._high!r}{places})"

django_data_shape/generate_rows.py

@@ -0,0 +1,43 @@
+"""Turning a declared table into the tuples COPY will consume."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+from django_data_shape.table import Table
+from django_data_shape.utils import draw, field_stream
+
+
+def generate_rows(table: Table, seed: int) -> Iterator[tuple[Any, ...]]:
+    """Yield one tuple per row: the primary key, then each declared column.
+
+    Rows, not model instances. The ORM is the wrong tool at these counts -- it
+    is the difference between a load measured in seconds and one measured in
+    minutes -- and nothing here needs a model instance, because no ``save`` will
+    run and no signal should fire.
+
+    Primary keys are a dense ``1..N`` because this package assigns them. That is
+    what will let a child row's foreign key be satisfied by construction with no
+    lookup, and it is why a self-referential tree is acyclic for free. It also
+    obliges the caller to reset the sequence afterwards; see ``build``.
+
+    A generator rather than a list: a million rows of tuples is real memory, and
+    psycopg writes them one at a time anyway, so materialising the whole set
+    would buy nothing but peak RSS.
+    """
+    columns = table.columns()
+    # Stream ids are derived once per field rather than per row. At a million
+    # rows this loop runs a million times per column, so anything hoistable out
+    # of it is worth hoisting.
+    streams = [field_stream(seed, table.db_table, name) for name, _ in columns]
+    distributions = [table.fields[name] for name, _ in columns]
+
+    for row in range(table.rows):
+        yield (
+            row + 1,
+            *(
+                distribution.value(row, draw(stream, row))
+                for distribution, stream in zip(distributions, streams, strict=True)
+            ),
+        )

django_data_shape/invalid_shape.py

@@ -0,0 +1,18 @@
+"""Raised when a declaration cannot describe a database."""
+
+from __future__ import annotations
+
+
+class InvalidShape(Exception):
+    """A shape declaration is contradictory, incomplete or unsatisfiable.
+
+    Its own type, and raised as early as the contradiction can be seen -- at
+    declaration time wherever possible, rather than at load time. The reason is
+    the package's own bar: a generated database that is wrong is worse than one
+    that refuses to exist, because the test suite it feeds will assert on data
+    that could never occur and pass or fail for reasons unrelated to the code.
+
+    Every message names the model, the field or the constraint at fault. An
+    error that says only that something is inconsistent leaves the reader to
+    re-derive what this code already knew.
+    """

django_data_shape/require_postgres.py

@@ -0,0 +1,46 @@
+"""The backend gate, in one place and testable without a database."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django_data_shape.unsupported_backend import UnsupportedBackend
+
+
+def require_postgres(connection: Any, operation: str) -> None:
+    """Refuse anything but PostgreSQL, naming what was refused and why.
+
+    Takes the connection and reads ``vendor`` off it rather than importing a
+    backend or opening a cursor, which is what lets every refusal path in this
+    package be covered by passing an object with a vendor. A degradation path
+    reachable only by running the whole suite on the backend it refuses is a
+    path the coverage gate cannot see, and this package gates coverage on
+    Postgres precisely because that is where its real work happens.
+    """
+    # The driver is read off the connection, exactly like ``vendor`` above, and
+    # for the same reason: a refusal that could only be covered by installing
+    # the driver it refuses is a refusal the coverage gate cannot see. Django
+    # sets ``Database`` to the driver module -- ``psycopg`` on 3, ``psycopg2``
+    # on 2 -- so a stub can supply it and this branch is testable everywhere.
+    driver = getattr(connection.Database, "__name__", "")
+    if connection.vendor == "postgresql" and driver != "psycopg":
+        # Django 6.1 still ships the psycopg 2 fallback, so this is a live
+        # configuration rather than a legacy one. Without this check the vendor
+        # gate passes and the load fails deep inside the package with
+        # "'psycopg2.extensions.cursor' object has no attribute 'copy'" -- a
+        # traceback pointing here, which a user would reasonably file as a bug
+        # in this package rather than as a missing driver.
+        raise UnsupportedBackend(
+            f"{operation} needs psycopg 3; connection '{connection.alias}' is using "
+            f"{driver or 'an unknown driver'}. "
+            "Rows are streamed straight into COPY FROM STDIN, which psycopg 2 cannot do without "
+            "materialising them first -- the cost this package exists to avoid. Install the "
+            "'postgres' extra: pip install django-data-shape[postgres]."
+        )
+    if connection.vendor != "postgresql":
+        raise UnsupportedBackend(
+            f"{operation} needs PostgreSQL; connection '{connection.alias}' is "
+            f"{connection.vendor}. Generation and cardinality are backend-neutral, but "
+            "COPY loading and planner statistics are not, and a shaped database whose plans "
+            "mean nothing is worse than no shaped database at all."
+        )

django_data_shape/shape.py

@@ -0,0 +1,42 @@
+"""A whole declared database."""
+
+from __future__ import annotations
+
+from django_data_shape.invalid_shape import InvalidShape
+from django_data_shape.table import Table
+
+
+class Shape:
+    """The tables to build, and the seed that makes them reproducible.
+
+    A declaration and nothing else: it holds no connection, opens nothing, and
+    has no ``build`` method. Building lives in a separate function on purpose,
+    because a shape has to stay inert data for two things that come later --
+    hashing it into a template-database cache key, and emitting one from a real
+    database's statistics. An object that could act would be an object with
+    state worth not hashing.
+
+    The seed is part of the declaration rather than an argument to the build,
+    for the same reason: two builds of the same shape must produce byte-identical
+    databases, and a seed passed at build time would let them differ while the
+    declaration claimed they could not.
+    """
+
+    def __init__(self, *tables: Table, seed: int = 0) -> None:
+        if not tables:
+            raise InvalidShape("A shape needs at least one table.")
+        seen: dict[str, Table] = {}
+        for table in tables:
+            key = table.db_table
+            if key in seen:
+                raise InvalidShape(
+                    f"{key} is declared twice. One table gets one row count and one set of "
+                    "distributions; two declarations would silently mean whichever came last."
+                )
+            seen[key] = table
+        self.tables = tables
+        self.seed = seed
+
+    def __repr__(self) -> str:
+        names = ", ".join(table.model.__name__ for table in self.tables)
+        return f"Shape({names}, seed={self.seed})"

django_data_shape/shape_not_empty.py

@@ -0,0 +1,17 @@
+"""Raised when a table already holds rows a build would collide with."""
+
+from __future__ import annotations
+
+
+class ShapeNotEmpty(Exception):
+    """The destination table is not empty, so the assigned keys would collide.
+
+    Its own type rather than a reused one because the caller's remedy is
+    specific and nothing else in this package shares it: empty the table, then
+    build. Raised before any row is written, so a build that fails this way has
+    changed nothing.
+
+    The alternative was letting the database report it, which it did -- as a
+    unique-violation naming an index. That says what went wrong at the storage
+    layer and nothing about what the caller did or what to do instead.
+    """

django_data_shape/table.py

@@ -0,0 +1,207 @@
+"""One model's declared shape."""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+from django.db.models import Field, IntegerField, Model
+from django.db.models.fields import NOT_PROVIDED
+
+from django_data_shape.distributions.constant import Constant
+from django_data_shape.distributions.distribution import Distribution
+from django_data_shape.invalid_shape import InvalidShape
+
+
+class Table:
+    """How many rows of one model, and how each column is distributed.
+
+    Field distributions are given as keyword arguments because that is the form
+    a reader scans fastest. ``fields=`` is the escape hatch, and it is not
+    optional politeness: a model may legitimately have a column called ``rows``
+    or ``model``, and Python's own argument binding would silently hand that
+    keyword to this signature instead. Without the mapping form those models
+    would simply be undeclarable.
+
+    Every refusal below happens here, at declaration time, rather than during
+    the load. A shape that cannot describe a database should say so before it
+    has spent a minute generating rows, and the message should name the field --
+    a reader who has to re-derive which column was meant has been given an error
+    that knows more than it says.
+    """
+
+    def __init__(
+        self,
+        model: type[Model],
+        rows: int,
+        fields: dict[str, Distribution] | None = None,
+        **field_distributions: Distribution,
+    ) -> None:
+        if rows < 0:
+            raise InvalidShape(f"{model.__name__} cannot have {rows} rows.")
+
+        declared: dict[str, Distribution] = dict(fields or {})
+        overlap = sorted(set(declared) & set(field_distributions))
+        if overlap:
+            raise InvalidShape(
+                f"{model.__name__} declares {', '.join(overlap)} twice, once in fields= and "
+                "once as a keyword. Use one or the other."
+            )
+        declared.update(field_distributions)
+
+        self.model = model
+        self.rows = rows
+        self.fields = declared
+        self._validate()
+
+    @property
+    def db_table(self) -> str:
+        return str(self.model._meta.db_table)
+
+    def columns(self) -> tuple[tuple[str, Field[Any, Any]], ...]:
+        """The declared fields, in a stable order, with their model fields.
+
+        Sorted by name rather than left in declaration order: the order decides
+        the column list of the ``COPY`` statement, and a shape whose generated
+        SQL changes when two keyword arguments are swapped would hash to a
+        different cache key for no reason.
+        """
+        meta = self.model._meta
+        # cast because get_field's return type covers reverse relations too,
+        # which _validate has already ruled out for every name reaching here.
+        return tuple(
+            (name, cast("Field[Any, Any]", meta.get_field(name))) for name in sorted(self.fields)
+        )
+
+    def _validate(self) -> None:
+        meta = self.model._meta
+        known = {field.name: field for field in meta.concrete_fields}
+
+        unknown = sorted(name for name in self.fields if name not in known)
+        if unknown:
+            raise InvalidShape(
+                f"{self.model.__name__} has no field named {', '.join(unknown)}. "
+                f"Its concrete fields are: {', '.join(sorted(known))}."
+            )
+
+        pk_fields = [field for field in known.values() if field.primary_key]
+        for field in pk_fields:
+            # The dense 1..N range this package assigns is integers, and nothing
+            # downstream converts it. Given a CharField primary key the load
+            # used to succeed and write "1", "2", "3" -- values the application
+            # could never produce, with a whole statistics picture built on top
+            # of them. Refusing is the only honest answer until a key strategy
+            # can be declared per table.
+            if not isinstance(field, IntegerField):
+                raise InvalidShape(
+                    f"{self.model.__name__}.{field.name} is a "
+                    f"{type(field).__name__} primary key, and this package assigns primary keys "
+                    "itself as a dense 1..N integer range. Only integer primary keys are "
+                    "supported."
+                )
+
+        pk_names = {name for name, field in known.items() if field.primary_key}
+        declared_pk = sorted(pk_names & set(self.fields))
+        if declared_pk:
+            raise InvalidShape(
+                f"{self.model.__name__}.{', '.join(declared_pk)} is the primary key, which this "
+                "package assigns itself as a dense 1..N range. That is what lets a foreign key "
+                "be satisfied without a lookup, so it is not available to declare."
+            )
+
+        # Relations are the next release's work, and generating a foreign key
+        # column from a value distribution would produce ids pointing at rows
+        # that may not exist. Refusing is the only honest answer until fan-out
+        # can be declared as a distribution over the parents.
+        relations = sorted(
+            name for name, field in known.items() if field.is_relation and name in self.fields
+        )
+        if relations:
+            raise InvalidShape(
+                f"{self.model.__name__}.{', '.join(relations)} is a relation, and relations are "
+                "not supported yet. Declaring fan-out as a distribution is the next release."
+            )
+
+        self._resolve_defaults(known)
+
+    def _resolve_defaults(self, known: dict[str, Field[Any, Any]]) -> None:
+        """Decide what happens to every field the caller did not declare.
+
+        The subtlety that makes this its own method: **a Django ``default=`` is
+        not a database default.** It is applied by ``save()``, and this package
+        never calls ``save()`` -- it streams tuples into ``COPY``. So a column
+        that is ``NOT NULL`` with a Python-level default has nothing behind it
+        at the database level, and omitting it from the load fails on a
+        not-null violation rather than quietly taking the default.
+
+        Filling it in with the same value ``save()`` would have written keeps
+        the caller's mental model true instead of making them restate a default
+        they already declared on the model.
+        """
+        undeclared = [
+            (name, field)
+            for name, field in known.items()
+            if name not in self.fields and not field.primary_key
+        ]
+
+        # Declaring a relation is refused in _validate; omitting a required one
+        # used to be accepted and then fail inside COPY with a not-null
+        # violation. Both directions have to refuse, or the contract only holds
+        # for the callers who tried the unsupported thing explicitly.
+        required_relations = sorted(
+            name
+            for name, field in undeclared
+            if field.is_relation and not field.null and not field.has_default()
+        )
+        if required_relations:
+            raise InvalidShape(
+                f"{self.model.__name__}.{', '.join(required_relations)} is a relation that "
+                "cannot be null, and relations are not supported yet, so this shape cannot be "
+                "built. Declaring fan-out as a distribution is the next release."
+            )
+        undeclared = [(name, field) for name, field in undeclared if not field.is_relation]
+
+        # A callable default is refused rather than guessed. This package cannot
+        # know whether it varies per row -- ``uuid4`` does, ``dict`` does not --
+        # and both readings produce data the application would never have
+        # written: one duplicates a value meant to be unique, the other invents
+        # variation where the model promised none.
+        callables = sorted(
+            name for name, field in undeclared if field.has_default() and callable(field.default)
+        )
+        if callables:
+            raise InvalidShape(
+                f"{self.model.__name__}.{', '.join(callables)} has a callable default, which "
+                "this package will not call on your behalf: it cannot tell a per-row default "
+                "from a shared one, and guessing either way writes rows the application never "
+                "would. Declare a distribution for it."
+            )
+
+        missing: list[str] = []
+        for name, field in undeclared:
+            if field.has_default():
+                self.fields[name] = Constant(field.get_default())
+            elif field.null or self._has_db_default(field):
+                continue
+            else:
+                missing.append(name)
+
+        if missing:
+            raise InvalidShape(
+                f"{self.model.__name__}.{', '.join(sorted(missing))} cannot be null and has no "
+                "default, so it has to be declared. A column left to chance is the column whose "
+                "selectivity the plan then depends on."
+            )
+
+    @staticmethod
+    def _has_db_default(field: Field[Any, Any]) -> bool:
+        """Whether the database itself will supply a value.
+
+        ``db_default`` arrived in Django 5.0 and this package supports 4.2, so
+        the attribute cannot be assumed to exist. Unlike ``default``, this one
+        is real DDL, which is why a column carrying it can be left out of the
+        ``COPY`` entirely.
+        """
+        return getattr(field, "db_default", NOT_PROVIDED) is not NOT_PROVIDED
+
+    def __repr__(self) -> str:
+        return f"Table({self.model.__name__}, rows={self.rows})"

django_data_shape/table_result.py

@@ -0,0 +1,18 @@
+"""What one table's load actually produced."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class TableResult:
+    """Rows loaded into one table.
+
+    Carries the database table name rather than the model, because this is what
+    a caller prints or asserts on, and because the two stop being one-to-one as
+    soon as through tables are generated alongside the models that declare them.
+    """
+
+    table: str
+    rows: int

django_data_shape/unsupported_backend.py

@@ -0,0 +1,18 @@
+"""Raised when the connection cannot do what was asked of it."""
+
+from __future__ import annotations
+
+
+class UnsupportedBackend(Exception):
+    """The database backend cannot support the operation requested.
+
+    Separate from :class:`~django_data_shape.invalid_shape.InvalidShape`
+    because nothing is wrong with the declaration: the same shape is valid, and
+    would build, against Postgres. Only the destination is unsuitable.
+
+    Raised rather than warned, and never quietly degraded to a slower path. The
+    whole claim of this package is that the loaded database is one the planner
+    can reason about; a backend without ``COPY`` or column statistics cannot
+    produce that, and silently producing something else would be the failure
+    mode the package was written to expose.
+    """

django_data_shape/utils.py

@@ -0,0 +1,45 @@
+"""Helpers used across more than one module."""
+
+from __future__ import annotations
+
+import hashlib
+
+_MASK64 = (1 << 64) - 1
+_GOLDEN = 0x9E3779B97F4A7C15
+
+
+def field_stream(seed: int, table: str, field: str) -> int:
+    """A stable 64-bit stream id for one table's one field.
+
+    Derived once per field rather than per row, so the cost of a real hash is
+    paid a handful of times instead of millions. ``hash()`` is deliberately not
+    used: it is salted per interpreter run, which would make a seeded shape
+    reproduce only within a single process.
+    """
+    digest = hashlib.blake2b(f"{table}.{field}".encode(), digest_size=8).digest()
+    return (int.from_bytes(digest, "big") ^ seed) & _MASK64
+
+
+def draw(stream: int, row: int) -> float:
+    """A uniform float in [0, 1) for one field of one row.
+
+    Deterministic in ``(stream, row)`` alone, which is the property the whole
+    design rests on: a value does not depend on how many rows were generated
+    before it, so rows can later be emitted in an order different from the one
+    they were assigned in. That separation is what lets physical placement be
+    declared without buffering a group in memory.
+
+    SplitMix64's finalizer rather than ``random.Random``: seeding a Mersenne
+    Twister per value costs far more than the value is worth at these row
+    counts, and this needs no sequential state to be reproducible.
+    """
+    z = (stream + (row + 1) * _GOLDEN) & _MASK64
+    z = ((z ^ (z >> 30)) * 0xBF58476D1CE4E5B9) & _MASK64
+    z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & _MASK64
+    z = z ^ (z >> 31)
+    # 53 bits over 2**53 rather than 64 bits over 2**64: the latter can round up
+    # to exactly 1.0 for the top 1024 finalizer outputs, and the finalizer is a
+    # bijection so all of them are reachable. That would break the [0, 1)
+    # contract every distribution is written against. This is the same
+    # construction the standard library uses for ``random.random()``.
+    return (z >> 11) / 9007199254740992.0

django_data_shape/version.py

@@ -0,0 +1,7 @@
+"""The single source of truth for this package's version."""
+
+from __future__ import annotations
+
+__version__: str = "0.1.0"
+
+__all__ = ["__version__"]

django_data_shape-0.1.0.dist-info/METADATA

@@ -0,0 +1,110 @@
+Metadata-Version: 2.5
+Name: django-data-shape
+Version: 0.1.0
+Summary: A realistically shaped test database from Django models: declare cardinality, skew and fan-out, load by COPY, and make the query planner believe it.
+Project-URL: Homepage, https://github.com/Artui/django-data-shape
+Project-URL: Repository, https://github.com/Artui/django-data-shape
+Project-URL: Issues, https://github.com/Artui/django-data-shape/issues
+Author-email: Artur Veres <artur8118@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: django,fixtures,performance,postgres,query-planner,test-data,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Web Environment
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 4.2
+Classifier: Framework :: Django :: 5.0
+Classifier: Framework :: Django :: 5.1
+Classifier: Framework :: Django :: 5.2
+Classifier: Framework :: Django :: 6.0
+Classifier: Framework :: Django :: 6.1
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: django>=4.2
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgres'
+Description-Content-Type: text/markdown
+
+# django-data-shape
+
+[![CI](https://github.com/Artui/django-data-shape/workflows/tests/badge.svg)](https://github.com/Artui/django-data-shape/actions/workflows/tests.yml)
+[![PyPI](https://img.shields.io/pypi/v/django-data-shape.svg)](https://pypi.org/project/django-data-shape/)
+[![Python versions](https://img.shields.io/pypi/pyversions/django-data-shape.svg)](https://pypi.org/project/django-data-shape/)
+[![Django versions](https://img.shields.io/pypi/djversions/django-data-shape.svg)](https://pypi.org/project/django-data-shape/)
+[![Docs](https://img.shields.io/badge/docs-artui.github.io-blue.svg)](https://artui.github.io/django-data-shape/)
+[![Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Artui/django-data-shape/gh-pages/coverage.json)](https://github.com/Artui/django-data-shape/actions/workflows/tests.yml)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![License](https://img.shields.io/pypi/l/django-data-shape.svg)](LICENSE)
+
+A realistically shaped test database from Django models.
+
+Declare the shape of your data -- cardinality, value skew, foreign-key fan-out as
+a distribution with a long tail, and where related rows physically sit -- then
+load it by `COPY` and `ANALYZE` it, so the query planner makes the same choices
+it will make in production.
+
+It exists because a plan over ten rows is a lie, and because the loop it replaces
+is not merely smaller: uniform fan-out makes the planner always right, and
+generating children parent-by-parent clusters them perfectly, which flatters
+every index scan. A test database can be wrong in the flattering direction, and
+usually is.
+
+## Install
+
+```bash
+pip install django-data-shape[postgres]
+```
+
+## Use
+
+```python
+import datetime
+
+from django_data_shape import Sequential, Shape, Skew, Table, Uniform, build
+
+shape = Shape(
+    Table(
+        Order,
+        rows=1_000_000,
+        status=Skew({"complete": 0.98, "pending": 0.015, "cancelled": 0.005}),
+        total=Uniform(0, 500, places=2),
+        created_at=Sequential(
+            datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc),
+            datetime.timedelta(seconds=3),
+        ),
+    ),
+    seed=1234,
+)
+
+build(shape)
+```
+
+`build()` generates the rows, loads them with `COPY`, moves the identity sequence
+past the keys it assigned, and runs `ANALYZE` so the planner can see the shape.
+It raises on any backend that is not PostgreSQL rather than degrading quietly.
+
+## Status
+
+Early. This release covers single tables. Foreign-key fan-out as a distribution,
+physical placement, per-group invariants and template-database reuse are the
+releases after it; declaring a relation raises today rather than generating ids
+that point at nothing.
+
+Full documentation: <https://artui.github.io/django-data-shape/>
+
+## License
+
+MIT

django_data_shape-0.1.0.dist-info/RECORD

@@ -0,0 +1,24 @@
+django_data_shape/__init__.py,sha256=_IVSrdleAm0zSEg0CZRlsPzMLWp7XYt5w56xx5SC8cQ,1107
+django_data_shape/build.py,sha256=rCzCPBmQafifRS6wB_9oOpquoa74Y0BV3asr1ZK95Nw,7042
+django_data_shape/build_result.py,sha256=z_I8DeVO9YC-pOztXAz6dk-1iFho2gf1J_3JeGU4-Kc,871
+django_data_shape/generate_rows.py,sha256=weXqNSA-ANKfbZe9ub66RoMPvKVRBxlOshTIFXaD_gI,1821
+django_data_shape/invalid_shape.py,sha256=xC9sUqiLo5EkDMvFoBpqeqIiZTOlDQyDl5AXPIetqOU,804
+django_data_shape/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_data_shape/require_postgres.py,sha256=S5y3PPwfa0DpRskXamFzt-L0F0P7yQm0W2hlM6UCEwY,2618
+django_data_shape/shape.py,sha256=2hcASfhoRHNz6JsE_m4T2tH4p7NpHZl741gc8Dmp85E,1727
+django_data_shape/shape_not_empty.py,sha256=0fNK9IjiBiZ14gWPEnXVwNfTdTBE-CVTtyjX06oSZAw,712
+django_data_shape/table.py,sha256=uzViWZogjpA1qRR-thomG2kuIh4zR2aC8E6ohDyPrpg,9553
+django_data_shape/table_result.py,sha256=f5M3irpbCy4m0sUSwc0iVDhCL5zXXmLAnQ7tiC-bxrc,478
+django_data_shape/unsupported_backend.py,sha256=38lea85ttw8yLs83i8yU0AtdI3la4wkH3oAidrNK3j4,799
+django_data_shape/utils.py,sha256=3rdIL0KTBRATL6fyoIEjO0YVhlv2ezJF4fTxygr_AhM,2001
+django_data_shape/version.py,sha256=IGt2-6r71nTeUfPEK_x1lNeMEBrz0uYGAfUvfX0lTVs,152
+django_data_shape/distributions/__init__.py,sha256=_qvSk36zHtq33S0ewBJCZjlQR9eaWWeXJOThnj2MhIQ,426
+django_data_shape/distributions/constant.py,sha256=lwEcfqrtaAdRdIrClnw-nT5UswUKSFQT2ZiLp773Tlk,889
+django_data_shape/distributions/distribution.py,sha256=gjcpRo2eFY2UaUDJwHzbgN8nHK4W3ijxZW2m1Lb6H4M,1009
+django_data_shape/distributions/sequential.py,sha256=Ag7sbIURwG3_47sKfJVnGSNs6rEX-Mw-_x4j3UH9FCQ,1170
+django_data_shape/distributions/skew.py,sha256=KbjYPxUUMPr_r0bMlpTVyho5Hh5WDPLLReRDpxcbzYA,3297
+django_data_shape/distributions/uniform.py,sha256=p-RC0AENioKBCJjEEYscH1Cmgg7cPlYAjwl7Q20asIo,2245
+django_data_shape-0.1.0.dist-info/METADATA,sha256=qgtGCYr9JuILW_4MirC7KzIwliLZ41jU0FkFJsqfnEw,4693
+django_data_shape-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+django_data_shape-0.1.0.dist-info/licenses/LICENSE,sha256=0rCDnUPa87Fm67SxYMrdqEdLV6UiMENH6NCPLZ_KT5M,1068
+django_data_shape-0.1.0.dist-info/RECORD,,

django_data_shape-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

django_data_shape-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Artur Veres
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.