django-data-shape 0.23.0__tar.gz → 0.24.0__tar.gz

{django_data_shape-0.23.0 → django_data_shape-0.24.0}/.github/workflows/release.yml

@@ -56,7 +56,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
-      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
           enable-cache: true
       - name: Install
@@ -68,7 +68,7 @@ jobs:
 
       - name: Publish to PyPI
         if: steps.prepare.outputs.released == 'true'
-        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
         with:
           # Idempotent on purpose. What gates a release is the git tag, not the
           # index (see `prepare`), so re-running this job after a failure in a

{django_data_shape-0.23.0 → django_data_shape-0.24.0}/.github/workflows/tests.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
           enable-cache: true
       - name: Install
@@ -37,7 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
           enable-cache: true
       - name: Install with docs group
@@ -79,7 +79,7 @@ jobs:
       PGDATABASE: django_data_shape
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
           enable-cache: true
       # A floor is a claim about the *oldest* version that works, and a lockfile
@@ -199,7 +199,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
           enable-cache: true
       - name: Install Python ${{ matrix.python-version }}
@@ -257,7 +257,7 @@ jobs:
       PGDATABASE: django_data_shape
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
           enable-cache: true
       - name: Install Python 3.13

{django_data_shape-0.23.0 → django_data_shape-0.24.0}/.github/workflows/upstream-drift.yml

@@ -72,14 +72,14 @@ jobs:
       PGDATABASE: django_data_shape
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
+      - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
         with:
           enable-cache: false
 
       # One cell, not the matrix. The matrix answers "does this combination
       # work"; this answers "has an upstream moved under us", and the newest
       # supported Python is where a new release lands first.
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
         with:
           python-version: "3.14"
 

{django_data_shape-0.23.0 → django_data_shape-0.24.0}/CHANGELOG.md

@@ -7,6 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.24.0] — 2026-11-07
+
+### Added
+
+- **`derive_template(template, step, *, key)` caches a world the application
+  finishes.** Some columns are an application's own arithmetic, such as running
+  totals a replay through its services computes, and no declaration can describe
+  them. It copies a template `template_database` (or `derive_template`) returned,
+  runs `step(using)` once over the copy with the alias pointed at it, runs
+  `ANALYZE` over the whole database -- not only the declared tables, since the
+  step may write to others -- and keeps the result as a template of its own, so a
+  session pays a clone rather than the replay. `key` is required: a callable
+  cannot be hashed, so the caller names the step and changes the key when the
+  step changes. The derived name is a digest of the parent's name and the key,
+  so it also moves with everything the parent's name moves with. Steps chain, a
+  step that raises leaves nothing behind and is raised again as `StepFailed`
+  naming the key, and a parent this package did not finish is refused.
+  `strategy=` is the copy's, as for `clone_database`.
+- **`template_fixture(template)`, a session-scoped pytest fixture that makes the
+  test database a clone of a template**, which is how a test reaches a derived
+  one. `template` is a callable taking the connection alias and returning a
+  template's name. It requests `django_db_setup` and then replaces the test
+  database pytest-django made, under the same name, so xdist suffixes and
+  pytest-django's teardown work as before. Only a test database is replaced: in a
+  run whose tests touch no database the alias still names your configured one
+  (pytest-django 4.11 and later), and the fixture leaves it alone. The pytest page lists what replacing the
+  database means for tests that ran earlier and for transactional tests.
+- **`StepFailed`**, raised from the step's own exception.
+- **Documentation:** the statistics page has a section on a world the
+  application finishes, and the pytest page one on `template_fixture`.
+
 ## [0.23.0] — 2026-10-23
 
 ### Added
@@ -1751,7 +1782,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   into `COPY FROM STDIN`, which psycopg 2 cannot do without materialising them
   first.
 
-[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.23.0...HEAD
+[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.24.0...HEAD
+[0.24.0]: https://github.com/Artui/django-data-shape/compare/v0.23.0...v0.24.0
 [0.23.0]: https://github.com/Artui/django-data-shape/compare/v0.22.0...v0.23.0
 [0.22.0]: https://github.com/Artui/django-data-shape/compare/v0.21.0...v0.22.0
 [0.21.0]: https://github.com/Artui/django-data-shape/compare/v0.20.0...v0.21.0

{django_data_shape-0.23.0 → django_data_shape-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: django-data-shape
-Version: 0.23.0
+Version: 0.24.0
 Summary: A realistically shaped test database from Django models: declare cardinality, skew and fan-out, load by COPY, and make the query planner believe it.
 Project-URL: Homepage, https://github.com/Artui/django-data-shape
 Project-URL: Repository, https://github.com/Artui/django-data-shape
@@ -279,7 +279,12 @@ history no forward migration can fix, such as a base ahead of the migrations on
 disk, is refused.
 A shape holding a `Derived` or a `KeyFunction` is refused rather than hashed --
 there is no honest digest of a callable, and every way of guessing one agrees
-while the data has changed. See
+while the data has changed. A world your own code finishes after the build -- a
+replay that fills running totals -- is cached too:
+`derive_template(template, step, key="replay-3")` runs the step once over a copy
+of the template, analyzes the whole database afterwards and keeps the result as
+a template of its own, under a name keyed on the parent and the key you change
+when the step does. See
 [Statistics and reuse](https://artui.github.io/django-data-shape/statistics/).
 
 ## From pytest
@@ -294,7 +299,9 @@ world = scale_fixture(Shape(Table(Order, rows=100, status=Constant("complete")))
 ```
 
 `orders` is one world built once for the whole session, composed with
-pytest-django rather than replacing it. `world` is the **scale protocol**: make
+pytest-django rather than replacing it. `template_fixture` is the other way in:
+it makes the test database a clone of a template, a derived one included, for
+the whole session. `world` is the **scale protocol**: make
 the world be at factor F, then let the caller run its block, which is what a
 query count asserted to be `O(1)` rather than `O(N)` needs.
 

{django_data_shape-0.23.0 → django_data_shape-0.24.0}/README.md

@@ -236,7 +236,12 @@ history no forward migration can fix, such as a base ahead of the migrations on
 disk, is refused.
 A shape holding a `Derived` or a `KeyFunction` is refused rather than hashed --
 there is no honest digest of a callable, and every way of guessing one agrees
-while the data has changed. See
+while the data has changed. A world your own code finishes after the build -- a
+replay that fills running totals -- is cached too:
+`derive_template(template, step, key="replay-3")` runs the step once over a copy
+of the template, analyzes the whole database afterwards and keeps the result as
+a template of its own, under a name keyed on the parent and the key you change
+when the step does. See
 [Statistics and reuse](https://artui.github.io/django-data-shape/statistics/).
 
 ## From pytest
@@ -251,7 +256,9 @@ world = scale_fixture(Shape(Table(Order, rows=100, status=Constant("complete")))
 ```
 
 `orders` is one world built once for the whole session, composed with
-pytest-django rather than replacing it. `world` is the **scale protocol**: make
+pytest-django rather than replacing it. `template_fixture` is the other way in:
+it makes the test database a clone of a template, a derived one included, for
+the whole session. `world` is the **scale protocol**: make
 the world be at factor F, then let the caller run its block, which is what a
 query count asserted to be `O(1)` rather than `O(N)` needs.
 

{django_data_shape-0.23.0 → django_data_shape-0.24.0}/django_data_shape/__init__.py

@@ -3,8 +3,10 @@
 from django_data_shape.backends.require_clone_strategy import require_clone_strategy
 from django_data_shape.backends.unsupported_backend import UnsupportedBackend
 from django_data_shape.databases.clone_database import clone_database
+from django_data_shape.databases.derive_template import derive_template
 from django_data_shape.databases.drop_database import drop_database
 from django_data_shape.databases.shape_digest import shape_digest
+from django_data_shape.databases.step_failed import StepFailed
 from django_data_shape.databases.template_database import template_database
 from django_data_shape.databases.unhashable_shape import UnhashableShape
 from django_data_shape.databases.unusable_base import UnusableBase
@@ -103,6 +105,7 @@ __all__ = [
     "Skew",
     "SqlKeys",
     "SqlValue",
+    "StepFailed",
     "Table",
     "TableResult",
     "UnhashableShape",
@@ -117,6 +120,7 @@ __all__ = [
     "check_constraints",
     "check_invariants",
     "clone_database",
+    "derive_template",
     "drop_database",
     "fan_out_sizes",
     "require_clone_strategy",

django_data_shape-0.24.0/django_data_shape/databases/derive_template.py

@@ -0,0 +1,231 @@
+"""A template made from another by running the application's own code over it."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable
+from typing import Any
+
+from django.db import DEFAULT_DB_ALIAS, connections
+from django.db.transaction import TransactionManagementError
+
+from django_data_shape.backends.require_clone_strategy import require_clone_strategy
+from django_data_shape.backends.require_postgres import require_postgres
+from django_data_shape.databases.clone_database import clone_database
+from django_data_shape.databases.step_failed import StepFailed
+from django_data_shape.utils import (
+    FINISHED_TEMPLATE,
+    PARTIAL_SUFFIX,
+    TEMPLATE_KEY_BYTES,
+    TEMPLATE_PREFIX,
+    absorb,
+    pointed_at,
+    template_lock_id,
+)
+
+# Its own format tag, so a derived name can never be the name template_database
+# would give some shape: the two digests absorb different things, and the tag is
+# what keeps them in different spaces.
+_FORMAT = "django-data-shape derived template 1"
+
+
+def derive_template(
+    template: str,
+    step: Callable[[str], object],
+    *,
+    key: str,
+    using: str = DEFAULT_DB_ALIAS,
+    strategy: str | None = "file_copy",
+) -> str:
+    """Make sure a template holding ``template`` with ``step`` run over it exists, and return its name.
+
+    For the world an application finishes itself. A shape describes what can be
+    declared; some columns are the application's own arithmetic -- running
+    totals a replay through its services fills in -- and no declaration can
+    describe them. Run that code over a fresh test database every session and
+    the cache pays for the cheap half only: measured on a consumer's world, the
+    template took 2.5 s to build and the replay over it 93 s. This puts the
+    replay inside the cache. The step runs once per machine, over a copy of
+    ``template``, and the result is a template like any other, cloned with
+    :func:`~django_data_shape.databases.clone_database.clone_database` in a
+    fraction of a second::
+
+        template = template_database(shape)
+        finished = derive_template(template, replay_draws, key="replay-draws-3")
+        clone_database(finished, "test_myapp", replace=True)
+
+    **What the step is handed.** ``step(using)`` is called with the connection
+    alias pointed at the copy, under its working name, the way the build itself
+    reaches a template: the ORM, ``connections[using]`` and anything else going
+    through that alias write into the copy. Its return value is ignored. It runs
+    outside any transaction this function opens, so what it commits is what the
+    template holds; a step that leaves another connection attached to the copy
+    stops the copy being renamed, as anything attached to a database does.
+
+    **``key`` is required, and it is the step's identity.** A callable cannot be
+    hashed, for the reason a shape holding a
+    :class:`~django_data_shape.derivations.derived.Derived` is refused: two
+    closures agree while the data differs, which is the one direction a cache key
+    must never be wrong. So the caller names the step, and **changes the key when
+    the step changes**. The template's name is a digest of ``template``'s name
+    and ``key``, and ``template``'s name is itself a digest of the shape, the
+    schema, the settings, the base and this package's version, so a change to any
+    of those makes a new template here as well, without the key having to say so.
+
+    **The cache trusts the step to be a function of the world it is given.** A
+    step reading the clock, drawing unseeded randomness or reading anything
+    outside the database produces whatever its first run produced, under a key
+    that claims more. That is stated rather than detected, as the rows a base
+    holds are for ``template_database``.
+
+    **The whole database is analyzed after the step**, not the tables the shape
+    declared: the step may have written to tables the shape never named, and a
+    table with no statistics is worse than a small one. Anything else that
+    writes to a world after it is built leaves its statistics stale, whether or
+    not it goes through this function.
+
+    **Steps chain**, because a derived template is a template: derive from one
+    to run a second step over the first, and changing the second step's key
+    re-runs only the second step. Changing ``key`` never rebuilds what is under
+    it, which is why this derives from a template rather than taking the step as
+    an argument to ``template_database``.
+
+    **A step that raises leaves nothing behind.** The copy is dropped, so the
+    next call starts again from ``template``, and the exception is raised again
+    as :class:`~django_data_shape.databases.step_failed.StepFailed`, naming the
+    key, with the step's own exception as its cause. A ``KeyboardInterrupt`` or
+    any other ``BaseException`` drops the copy and goes on unwrapped.
+
+    **What it refuses**, each before anything is created:
+
+    - **A ``template`` this package did not finish.** Its name has to be one
+      ``template_database`` or this function returned, a prefix and a digest,
+      because the derived name is keyed on it: a database of the caller's own
+      keeps its name while its rows change, and a template derived from it would
+      be served stale. A partial is refused for the same reason, and so is a
+      name holding a double quote, which no generated name does. ``ValueError``,
+      because what is wrong is an argument.
+    - **A ``template`` that does not exist**, when the derived one is not there
+      either. ``ValueError`` again, naming it. On a hit the parent is not
+      needed and not looked for, so a parent dropped after deriving from it
+      costs nothing until the derived template goes too.
+    - **Anything but PostgreSQL**, and **being called inside a transaction**,
+      for the reasons ``template_database`` gives: there is no ``CREATE
+      DATABASE ... TEMPLATE`` elsewhere, and pointing the connection at the copy
+      closes it, which leaves an atomic block unusable.
+    - **A ``strategy`` the server cannot take.** It means what it means for
+      :func:`~django_data_shape.databases.clone_database.clone_database`, which
+      makes the copy: ``file_copy`` by default, because a world worth deriving
+      is a world worth copying cheaply, and ``None`` for a PostgreSQL older than
+      15. It is not part of the key, because it decides what the copy costs and
+      not what it holds.
+
+    Parallel workers asking for the same derived template wait on one advisory
+    lock, taken on its name, while one of them runs the step. The finished
+    template refuses connections, as every template this package makes does;
+    nothing removes it, and :func:`~django_data_shape.databases.drop_database.drop_database`
+    is how a stale one goes.
+    """
+    # Loosely typed for the reason every other backend-specific reach in this
+    # package is: ``_nodb_cursor`` is the wrapper's, not the base class's.
+    connection: Any = connections[using]
+    require_postgres(connection, "Deriving a template database")
+    if connection.in_atomic_block:
+        raise TransactionManagementError(
+            "Deriving a template database points the connection at a copy and closes it, which "
+            "cannot be done inside an atomic block -- the connection would be unusable for the "
+            "rest of it. Call this from test session setup, before anything has opened a "
+            "transaction."
+        )
+    # Before the name is looked at, so a strategy this server cannot take is
+    # refused on a hit as well as on a miss
+    # (test_a_strategy_it_cannot_take_is_refused_before_anything_is_looked_up).
+    require_clone_strategy(connection, strategy)
+    if not FINISHED_TEMPLATE.fullmatch(template):
+        raise ValueError(
+            f"{template!r} is not a template this package finished, so nothing can be derived "
+            "from it. A derived template is named after its parent's name and the key, which "
+            "keeps up with the data only when the parent's name is itself a digest of "
+            "everything under it, as the names template_database and derive_template return "
+            "are: a database of your own keeps its name while its rows change, and a template "
+            "derived from it would be served stale. Pass a name one of those two returned."
+        )
+    name = f"{TEMPLATE_PREFIX}{_key(template, key)}"
+    quote = connection.ops.quote_name
+    with connection._nodb_cursor() as cursor:
+        # Held for the whole check-then-create, for the reason template_database
+        # holds its own: two workers that both read "not there" would both run
+        # the step, and the second would fail on a name the first had taken.
+        cursor.execute("SELECT pg_advisory_lock(%s)", [template_lock_id(name)])
+        try:
+            cursor.execute("SELECT 1 FROM pg_database WHERE datname = %s", [name])
+            if cursor.fetchone() is None:
+                _create(connection, cursor, template, step, key, name, using, quote, strategy)
+        finally:
+            cursor.execute("SELECT pg_advisory_unlock(%s)", [template_lock_id(name)])
+    return name
+
+
+def _create(
+    connection: Any,
+    cursor: Any,
+    template: str,
+    step: Callable[[str], object],
+    key: str,
+    name: str,
+    using: str,
+    quote: Any,
+    strategy: str | None,
+) -> None:
+    """Copy the parent under a working name, run the step over it, analyze, then adopt it.
+
+    The rename is the commit, as it is for a built template: the derived name
+    exists only once the step and the ``ANALYZE`` after it have finished, so a
+    copy interrupted halfway is never mistaken for a finished one, and one a
+    killed process left behind is dropped first.
+    """
+    partial = f"{name}{PARTIAL_SUFFIX}"
+    cursor.execute(f"DROP DATABASE IF EXISTS {quote(partial)}")
+    # Looked up here rather than left to CREATE DATABASE, whose own refusal
+    # names neither this function nor what to do about it
+    # (test_a_parent_that_does_not_exist_is_refused_by_name).
+    cursor.execute("SELECT 1 FROM pg_database WHERE datname = %s", [template])
+    if cursor.fetchone() is None:
+        raise ValueError(
+            f"The template {template!r} does not exist, so nothing can be derived from it. Make "
+            "it again with template_database, or with derive_template if it was derived, and "
+            "derive from the name that returns."
+        )
+    clone_database(template, partial, using=using, strategy=strategy)
+    try:
+        with pointed_at(connection, partial):
+            step(using)
+            # The whole database rather than the tables the shape declared,
+            # because the step may have written to tables the shape never named
+            # (test_the_statistics_describe_the_world_after_the_step).
+            with connection.cursor() as analyzing:
+                analyzing.execute("ANALYZE")
+    except Exception as failed:
+        cursor.execute(f"DROP DATABASE IF EXISTS {quote(partial)}")
+        raise StepFailed(
+            f"The step deriving a template from {template!r} under key {key!r} raised "
+            f"{type(failed).__name__}: {failed}. Nothing was kept: the copy it ran in was "
+            "dropped, so the next call runs the step again from the same template."
+        ) from failed
+    except BaseException:
+        # Not a failure of the step, so not wrapped; the copy goes all the same,
+        # because a partial is never a finished template
+        # (test_an_interrupt_is_not_wrapped_and_leaves_nothing).
+        cursor.execute(f"DROP DATABASE IF EXISTS {quote(partial)}")
+        raise
+    cursor.execute(f"ALTER DATABASE {quote(partial)} RENAME TO {quote(name)}")
+    cursor.execute(f"ALTER DATABASE {quote(name)} WITH ALLOW_CONNECTIONS false")
+
+
+def _key(template: str, key: str) -> str:
+    """The parent's name and the caller's key, digested to the length a template name takes."""
+    hasher = hashlib.blake2b(digest_size=TEMPLATE_KEY_BYTES)
+    absorb(hasher, _FORMAT)
+    absorb(hasher, template)
+    absorb(hasher, key)
+    return hasher.hexdigest()

django_data_shape-0.24.0/django_data_shape/databases/step_failed.py

@@ -0,0 +1,24 @@
+"""Raised when the step a derived template is made with fails."""
+
+from __future__ import annotations
+
+
+class StepFailed(Exception):
+    """The step given to ``derive_template`` raised, and nothing was kept.
+
+    Raised from the step's own exception, which is its ``__cause__``, so the
+    traceback shows both: what went wrong inside the step, and which template and
+    key it was deriving. The key is the part worth naming, because it is the only
+    thing that tells one step from another -- a callable has no name a cache can
+    trust, which is why ``derive_template`` asks for a key at all.
+
+    Its own type rather than the step's exception passed through, because the
+    step's exception says what failed and not where: a ``KeyError`` from a
+    replay deep inside a session fixture reads the same as one from the test.
+    Only an ``Exception`` is wrapped. A ``KeyboardInterrupt`` or ``SystemExit``
+    is not a failure of the step, and goes on as it came.
+
+    Nothing is left behind either way: the partial copy the step ran in is
+    dropped, so the next call starts again from the parent template and runs the
+    step again.
+    """