django-data-shape 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.github/workflows/tests.yml +7 -1
- django_data_shape-0.4.0/CHANGELOG.md +222 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/PKG-INFO +83 -13
- django_data_shape-0.4.0/README.md +153 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/__init__.py +14 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/build.py +107 -24
- django_data_shape-0.4.0/django_data_shape/fixtures/__init__.py +19 -0
- django_data_shape-0.4.0/django_data_shape/fixtures/scale_fixture.py +76 -0
- django_data_shape-0.4.0/django_data_shape/fixtures/shape_fixture.py +94 -0
- django_data_shape-0.4.0/django_data_shape/fixtures/skip_unless_postgres.py +36 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/generate_rows.py +9 -5
- django_data_shape-0.4.0/django_data_shape/infer_key_strategy.py +27 -0
- django_data_shape-0.4.0/django_data_shape/keys/__init__.py +8 -0
- django_data_shape-0.4.0/django_data_shape/keys/key_function.py +44 -0
- django_data_shape-0.4.0/django_data_shape/keys/key_strategy.py +29 -0
- django_data_shape-0.4.0/django_data_shape/keys/sequential_keys.py +22 -0
- django_data_shape-0.4.0/django_data_shape/keys/uuid_keys.py +37 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/require_postgres.py +9 -2
- django_data_shape-0.4.0/django_data_shape/scale_protocol.py +48 -0
- django_data_shape-0.4.0/django_data_shape/scaled_shape.py +102 -0
- django_data_shape-0.4.0/django_data_shape/scaled_world.py +77 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/table.py +35 -16
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/version.py +1 -1
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/docs/index.md +26 -9
- django_data_shape-0.4.0/docs/keys.md +85 -0
- django_data_shape-0.4.0/docs/pytest.md +332 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/docs/reference.md +24 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/docs/relations.md +7 -1
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/mkdocs.yml +2 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/pyproject.toml +15 -1
- django_data_shape-0.4.0/tests/keys/test_key_function.py +37 -0
- django_data_shape-0.4.0/tests/keys/test_sequential_keys.py +19 -0
- django_data_shape-0.4.0/tests/keys/test_uuid_keys.py +34 -0
- django_data_shape-0.4.0/tests/scale_protocol_consumers.py +54 -0
- django_data_shape-0.4.0/tests/scale_protocol_impostors.py +32 -0
- django_data_shape-0.4.0/tests/test_build_keys.py +132 -0
- django_data_shape-0.4.0/tests/test_build_portable.py +125 -0
- django_data_shape-0.4.0/tests/test_documentation.py +53 -0
- django_data_shape-0.4.0/tests/test_fixtures.py +139 -0
- django_data_shape-0.4.0/tests/test_infer_key_strategy.py +24 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_require_postgres.py +21 -0
- django_data_shape-0.4.0/tests/test_scale_protocol.py +46 -0
- django_data_shape-0.4.0/tests/test_scaled_shape.py +139 -0
- django_data_shape-0.4.0/tests/test_scaled_world.py +132 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_table.py +55 -2
- django_data_shape-0.4.0/tests/testapp/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/testapp/models.py +45 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/uv.lock +7 -1
- django_data_shape-0.2.0/CHANGELOG.md +0 -114
- django_data_shape-0.2.0/README.md +0 -86
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.github/CODEOWNERS +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.github/SECURITY.md +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.github/dependabot.yml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.github/workflows/release.yml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.github/workflows/upstream-drift.yml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.gitignore +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/.pre-commit-config.yaml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/CLAUDE.md +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/LICENSE +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/Makefile +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/build_result.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/bounded.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/constant.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/distribution.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/sequential.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/skew.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/uniform.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/distributions/zipf.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/fan_out_plan.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/invalid_shape.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/order_tables.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/py.typed +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/resolve_fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/shape.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/shape_not_empty.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/table_result.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/unsupported_backend.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/django_data_shape/utils.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/scripts/release-publish.sh +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/conftest.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/conftest_settings.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/distributions/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/distributions/test_constant.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/distributions/test_sequential.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/distributions/test_skew.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/distributions/test_uniform.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/distributions/test_zipf.py +0 -0
- {django_data_shape-0.2.0/tests/testapp → django_data_shape-0.4.0/tests/keys}/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_build.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_build_graph.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_generate_rows.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_order_tables.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_resolve_fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_shape.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_utils.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.4.0}/tests/test_version.py +0 -0
|
@@ -114,8 +114,14 @@ jobs:
|
|
|
114
114
|
run: uv sync --frozen --all-groups --all-extras --python 3.10
|
|
115
115
|
- name: Name the versions that produced
|
|
116
116
|
run: uv pip list
|
|
117
|
+
# No coverage gate here either. This job's claim is that the declared
|
|
118
|
+
# floors resolve and the suite passes against them -- not that every line
|
|
119
|
+
# runs. It inherited the gate from the default addopts, which went
|
|
120
|
+
# unnoticed until a test skipped below Django 5.2 left one branch
|
|
121
|
+
# uncovered and failed a job that was never meant to measure coverage.
|
|
122
|
+
# One gate, on the Postgres job, is the design.
|
|
117
123
|
- name: Test
|
|
118
|
-
run: uv run --no-sync pytest
|
|
124
|
+
run: uv run --no-sync pytest --cov-fail-under=0
|
|
119
125
|
|
|
120
126
|
# The step above installs every extra, and an extra can hold a shared
|
|
121
127
|
# dependency *above* the floor being claimed -- masking the lie rather
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.4.0] — 2026-09-02
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **The pytest surface**, in `django_data_shape.fixtures`. `shape_fixture(shape)`
|
|
14
|
+
returns a session-scoped fixture that builds a shape once for a whole run;
|
|
15
|
+
bind it to a name in `conftest.py` and request that name from a test. It
|
|
16
|
+
composes with pytest-django rather than replacing it, asking for
|
|
17
|
+
`django_db_setup` and `django_db_blocker` by name so the coupling is to two
|
|
18
|
+
fixture names and not to pytest-django's internals. Session scope is
|
|
19
|
+
load-bearing: pytest creates higher-scoped fixtures first, so the rows are
|
|
20
|
+
committed before the transaction that wraps a test is opened, and every test
|
|
21
|
+
sees them while everything a test writes is rolled back with it.
|
|
22
|
+
- **The scale protocol**, which is what a growth assertion asks a world for:
|
|
23
|
+
make the world be at factor F, then let the caller run its block.
|
|
24
|
+
`scaled_world(shape, factor)` is a context manager that builds it and undoes
|
|
25
|
+
it; `scale_fixture(shape)` is the same thing as a fixture; and `ScaleProtocol`
|
|
26
|
+
is the structural type both satisfy, so a consumer asserting that a query
|
|
27
|
+
count is `O(1)` rather than `O(N)` depends on the shape of the call rather
|
|
28
|
+
than on this package. What the context manager yields is a plain row count,
|
|
29
|
+
not a `BuildResult`, for the same reason: a seam a stranger cannot implement
|
|
30
|
+
is not a seam.
|
|
31
|
+
- `scaled_shape(shape, factor)`, the declaration transform underneath it. **A
|
|
32
|
+
factor varies the declaration rather than subsetting one larger build**, because
|
|
33
|
+
a subset is not a smaller database but the same database with a filter -- the
|
|
34
|
+
statistics still describe every row, and the block under test would have to
|
|
35
|
+
cooperate by restricting itself, which puts the harness inside the thing being
|
|
36
|
+
measured. Every table scales, parents included, so the average fan-out is the
|
|
37
|
+
same at every factor and two worlds differ in size alone. The scaled tables go
|
|
38
|
+
through `Table`'s own constructor, so a declaration that only holds at its
|
|
39
|
+
original size is refused at the factor that breaks it, naming the factor.
|
|
40
|
+
- `skip_unless_postgres(connection, operation)`, the pytest twin of the backend
|
|
41
|
+
refusal. Both fixtures skip with the refusal's own message as the reason where
|
|
42
|
+
a shaped database cannot exist, so a suite that also runs on SQLite reports
|
|
43
|
+
what it did not check rather than passing over a database nobody shaped.
|
|
44
|
+
- A `pytest` extra. It is an extra rather than a dependency because the rest of
|
|
45
|
+
the package has nothing to do with pytest, which is also why these fixtures are
|
|
46
|
+
not re-exported from the top-level `__init__`: importing them is what requires
|
|
47
|
+
pytest, not importing the package.
|
|
48
|
+
- **`build(shape, require_statistics=False)` loads rows on any backend.** It asks
|
|
49
|
+
for rows and cardinality rather than for a database the planner can reason
|
|
50
|
+
about, and it is written as a requirement being dropped rather than as work
|
|
51
|
+
being skipped: on PostgreSQL it changes nothing at all, since `COPY` and
|
|
52
|
+
`ANALYZE` are both free and leaving them out would manufacture the unanalyzed
|
|
53
|
+
table this package exists to condemn. Elsewhere the rows are inserted in chunks
|
|
54
|
+
and no statistics are gathered. SQLite's own `ANALYZE` is deliberately not run,
|
|
55
|
+
because running it would claim the plan realism this package says it will not
|
|
56
|
+
claim. The driver check is unaffected: psycopg 2 is still refused on a
|
|
57
|
+
PostgreSQL connection, because the vendor picks the route and not the caller.
|
|
58
|
+
- **The growth harness works on every backend Django supports** and no longer
|
|
59
|
+
skips. A query count is an ORM property and means the same anywhere, so a
|
|
60
|
+
growth assertion is honest off PostgreSQL where a plan assertion is not, and
|
|
61
|
+
the scale harness was the only thing standing between a consumer on SQLite and
|
|
62
|
+
the milestone's headline seam. `shape_fixture` still skips: it exists to build
|
|
63
|
+
a world a planner will believe.
|
|
64
|
+
|
|
65
|
+
### Fixed
|
|
66
|
+
- `ScaleProtocol` rejected the implementations its own docstring offered. A
|
|
67
|
+
structural type matches parameter names too, so a hand-rolled callable taking
|
|
68
|
+
`n` rather than `factor` did not satisfy it -- exactly the five-line callable
|
|
69
|
+
the documentation tells a consumer to write. The factor is positional-only now,
|
|
70
|
+
and two files of consumers and impostors are type-checked by the suite, because
|
|
71
|
+
a type-level claim with no type-level test is what let this ship.
|
|
72
|
+
- `ShapeNotEmpty` names the likely cause and not only the remedy. The first
|
|
73
|
+
consumer met it by composing both fixtures over one model, where the rows are
|
|
74
|
+
real, correct and written by a fixture the failing test never mentions -- so
|
|
75
|
+
"empty the table first" read as advice about somebody else's data.
|
|
76
|
+
|
|
77
|
+
## [0.3.0] — 2026-09-01
|
|
78
|
+
|
|
79
|
+
### Fixed
|
|
80
|
+
- **A UUID primary key no longer refuses to load.** It was refused outright,
|
|
81
|
+
which made this package unusable for a whole class of Django project. The
|
|
82
|
+
reframing is the fix: the design never needed integers, it needed a
|
|
83
|
+
deterministic injection from row index to key -- which is what lets a foreign
|
|
84
|
+
key be satisfied without a lookup, what makes a self-referential tree acyclic
|
|
85
|
+
on the index rather than the value, and what makes two builds of one shape
|
|
86
|
+
agree. Integers were only the most obvious such function.
|
|
87
|
+
- The primary key is prepared by its own field, like every other value. It did
|
|
88
|
+
not need to be while keys were always integers, and a UUID works either way
|
|
89
|
+
because psycopg adapts it -- but a key type needing conversion was stored
|
|
90
|
+
verbatim, which is the bug already found once on an ordinary column.
|
|
91
|
+
- Two documentation examples were syntax errors -- `...` after keyword arguments,
|
|
92
|
+
in the README and on the relations page -- and would have failed the moment a
|
|
93
|
+
reader pasted them.
|
|
94
|
+
- The install line is quoted. `pip install django-data-shape[postgres]` fails in
|
|
95
|
+
zsh with `no matches found`, which is the default shell on macOS.
|
|
96
|
+
- The README's usage example imports the model it uses, and documents the
|
|
97
|
+
refusals a first attempt actually meets: PostgreSQL and psycopg 3, integer
|
|
98
|
+
primary keys, empty tables, and callable model defaults.
|
|
99
|
+
|
|
100
|
+
### Added
|
|
101
|
+
- **Key strategies.** A table's primary keys come from a deterministic function
|
|
102
|
+
of the row index rather than from a hard-coded dense `1..N` range. Integer keys
|
|
103
|
+
count from one, `UUIDField` keys are derived from the seed, and `KeyFunction`
|
|
104
|
+
declares one for any other type.
|
|
105
|
+
- `SequentialKeys`, `UuidKeys`, `KeyFunction` and the `KeyStrategy` protocol,
|
|
106
|
+
plus a `keys=` argument on `Table`.
|
|
107
|
+
- A composite primary key is refused by name. It is not among a model's concrete
|
|
108
|
+
fields, because it has no column of its own, so the package used to raise a
|
|
109
|
+
bare `StopIteration` from inside itself. The message says `keys=` cannot help
|
|
110
|
+
either: a strategy maps a row index to one value, and this is arity rather than
|
|
111
|
+
type.
|
|
112
|
+
- The documentation's Python examples are parsed by the test suite. A docs
|
|
113
|
+
example is the first code anybody runs, so it gets a guard rather than a
|
|
114
|
+
convention.
|
|
115
|
+
|
|
116
|
+
## [0.2.0] — 2026-09-01
|
|
117
|
+
|
|
118
|
+
### Added
|
|
119
|
+
- `FanOut`, which declares how a foreign key's children spread across their
|
|
120
|
+
parents: a size distribution, a `childless` share for parents with no children
|
|
121
|
+
at all, a `null` share for nullable columns, and `placement`.
|
|
122
|
+
- `Zipf`, the heavy-tailed weight distribution fan-out is realistically drawn
|
|
123
|
+
from. A table where every parent has ten children is not merely tidy -- it is
|
|
124
|
+
the one shape in which the planner is never wrong, because its `n_distinct`
|
|
125
|
+
average is the truth.
|
|
126
|
+
- Tables load in dependency order, and a cycle of fan-outs is refused by name.
|
|
127
|
+
|
|
128
|
+
### The two representation decisions
|
|
129
|
+
- **Fan-out reads the parent's real keys rather than assuming the dense `1..N`
|
|
130
|
+
range this package assigns.** The case that matters is the hybrid: a project
|
|
131
|
+
builds its fifty companies with the ORM, where the row count is small and the
|
|
132
|
+
ORM is the right tool, and asks this package only for the two million orders.
|
|
133
|
+
Referential integrity then holds by construction, because every key emitted
|
|
134
|
+
came out of the parent table.
|
|
135
|
+
- **A fan-out is a partition of the child key range, not a per-child draw.**
|
|
136
|
+
Parent `j` owns rows `[start, end)`. A per-child draw cannot be inverted, and
|
|
137
|
+
"which children belong to parent T" is what a mirrored collection needs. The
|
|
138
|
+
childless tail and `placement` both fall out of the partition for free.
|
|
139
|
+
|
|
140
|
+
### Notes
|
|
141
|
+
- `placement` defaults to `arrival`. Emitting children parent by parent gives a
|
|
142
|
+
perfectly clustered table that no production system has and that flatters
|
|
143
|
+
every index scan over the foreign key.
|
|
144
|
+
- A self-referential fan-out is refused: it would read keys from a table still
|
|
145
|
+
empty at load time. Self-referential trees are their own feature.
|
|
146
|
+
- A relation needs a `FanOut` and a plain column refuses one, in both
|
|
147
|
+
directions.
|
|
148
|
+
|
|
149
|
+
## [0.1.1] — 2026-09-01
|
|
150
|
+
|
|
151
|
+
### Added
|
|
152
|
+
- `Bounded`, an optional second protocol for distributions that can say how many
|
|
153
|
+
distinct values they produce. `Constant` and `Skew` implement it. It is
|
|
154
|
+
separate from `Distribution` on purpose: adding the method there would make it
|
|
155
|
+
required, so a custom distribution written against the single-method protocol
|
|
156
|
+
would stop satisfying it.
|
|
157
|
+
- A declaration that provably cannot be loaded is now refused at declaration
|
|
158
|
+
time. A `Constant` on a unique column with more than one row, or a `Skew` with
|
|
159
|
+
fewer values than rows, is arithmetic rather than a subtle problem, and it used
|
|
160
|
+
to be discovered by the database partway through a load that had already
|
|
161
|
+
written most of a table. Only single-column uniqueness is checked; multi-column
|
|
162
|
+
constraints are satisfiable through combinations across independently declared
|
|
163
|
+
columns, which is an analysis rather than a comparison.
|
|
164
|
+
|
|
165
|
+
### Fixed
|
|
166
|
+
- `Uniform` with `places` raised `decimal.InvalidOperation` past 28 significant
|
|
167
|
+
digits -- Python's default context precision -- from inside the `COPY` loop, on
|
|
168
|
+
a column such as `numeric(30, 2)` that would have accepted the value. The
|
|
169
|
+
precision needed is now derived from the declared bounds.
|
|
170
|
+
- `Table` and `Shape` attributes are read-only. Every rule they enforce runs once
|
|
171
|
+
in `__init__`, so while the attributes were writable a declaration could be
|
|
172
|
+
edited afterwards into one that would have been refused, with nothing
|
|
173
|
+
re-checking it.
|
|
174
|
+
|
|
175
|
+
## [0.1.0] — 2026-09-01
|
|
176
|
+
|
|
177
|
+
### Added
|
|
178
|
+
- The shape vocabulary: `Shape`, `Table`, and the `Skew`, `Uniform`, `Sequential`
|
|
179
|
+
and `Constant` distributions, behind a single-method `Distribution` protocol.
|
|
180
|
+
- `build()`, which generates rows, loads them with `COPY FROM STDIN`, moves the
|
|
181
|
+
identity sequence past the keys it assigned, and runs `ANALYZE`. The order is
|
|
182
|
+
owned by the library: loading into a table analyzed while empty leaves the
|
|
183
|
+
planner applying old statistics to a new row count, which is a worse lie than
|
|
184
|
+
having no statistics at all.
|
|
185
|
+
- `InvalidShape`, raised at declaration time, and `UnsupportedBackend`, raised
|
|
186
|
+
for any connection that is not PostgreSQL. Generation is backend-neutral;
|
|
187
|
+
`COPY` and planner statistics are not, and degrading quietly would produce the
|
|
188
|
+
false confidence this package exists to remove.
|
|
189
|
+
- Primary keys are assigned as a dense `1..N` range rather than declared. That is
|
|
190
|
+
what will let a foreign key be satisfied without a lookup once relations land,
|
|
191
|
+
and what makes a self-referential tree acyclic by construction.
|
|
192
|
+
- A field carrying a Django `default=` is filled with the value `save()` would
|
|
193
|
+
have written, because defaults are applied by `save()` and nothing here calls
|
|
194
|
+
it. A callable default is refused instead of guessed: `uuid4` varies per row
|
|
195
|
+
and `dict` does not, and nothing on the field distinguishes them.
|
|
196
|
+
|
|
197
|
+
- `ShapeNotEmpty`, raised before anything is written when a target table already
|
|
198
|
+
holds rows. Keys start at 1 on every build, so the collision was previously a
|
|
199
|
+
unique-violation naming an index, which says nothing about what to do instead.
|
|
200
|
+
- The whole build runs in one transaction, so a shape whose second table fails
|
|
201
|
+
leaves nothing behind and can be re-run after a fix.
|
|
202
|
+
- Every declared value passes through its field's `get_db_prep_save`. Without it
|
|
203
|
+
a naive datetime was stored verbatim rather than localised -- hours from where
|
|
204
|
+
`save()` puts it under a non-UTC `TIME_ZONE` -- and a `JSONField` could not be
|
|
205
|
+
written at all.
|
|
206
|
+
|
|
207
|
+
### Notes
|
|
208
|
+
- Relations are refused in both directions: declaring one raises, and so does
|
|
209
|
+
omitting one that cannot be null. An optional foreign key may be omitted and
|
|
210
|
+
loads entirely `NULL`, which is documented rather than left to be discovered.
|
|
211
|
+
- Only integer primary keys are supported. Any other kind is refused, rather than
|
|
212
|
+
a dense `1..N` integer range being written into a character column.
|
|
213
|
+
- psycopg 3 is required and psycopg 2 is refused by name. Rows stream straight
|
|
214
|
+
into `COPY FROM STDIN`, which psycopg 2 cannot do without materialising them
|
|
215
|
+
first.
|
|
216
|
+
|
|
217
|
+
[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.4.0...HEAD
|
|
218
|
+
[0.4.0]: https://github.com/Artui/django-data-shape/compare/v0.3.0...v0.4.0
|
|
219
|
+
[0.3.0]: https://github.com/Artui/django-data-shape/compare/v0.2.0...v0.3.0
|
|
220
|
+
[0.2.0]: https://github.com/Artui/django-data-shape/compare/v0.1.1...v0.2.0
|
|
221
|
+
[0.1.1]: https://github.com/Artui/django-data-shape/compare/v0.1.0...v0.1.1
|
|
222
|
+
[0.1.0]: https://github.com/Artui/django-data-shape/compare/v0.0.0...v0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: django-data-shape
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: A realistically shaped test database from Django models: declare cardinality, skew and fan-out, load by COPY, and make the query planner believe it.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Artui/django-data-shape
|
|
6
6
|
Project-URL: Repository, https://github.com/Artui/django-data-shape
|
|
@@ -36,6 +36,9 @@ Requires-Python: >=3.10
|
|
|
36
36
|
Requires-Dist: django>=4.2
|
|
37
37
|
Provides-Extra: postgres
|
|
38
38
|
Requires-Dist: psycopg[binary]>=3.2.0; extra == 'postgres'
|
|
39
|
+
Provides-Extra: pytest
|
|
40
|
+
Requires-Dist: pytest-django>=4.9.0; extra == 'pytest'
|
|
41
|
+
Requires-Dist: pytest>=8.0.0; extra == 'pytest'
|
|
39
42
|
Description-Content-Type: text/markdown
|
|
40
43
|
|
|
41
44
|
# django-data-shape
|
|
@@ -65,9 +68,12 @@ usually is.
|
|
|
65
68
|
## Install
|
|
66
69
|
|
|
67
70
|
```bash
|
|
68
|
-
pip install django-data-shape[postgres]
|
|
71
|
+
pip install 'django-data-shape[postgres]'
|
|
69
72
|
```
|
|
70
73
|
|
|
74
|
+
The quotes are not decoration: zsh globs the brackets and reports
|
|
75
|
+
`no matches found` without them.
|
|
76
|
+
|
|
71
77
|
## Use
|
|
72
78
|
|
|
73
79
|
```python
|
|
@@ -75,6 +81,8 @@ import datetime
|
|
|
75
81
|
|
|
76
82
|
from django_data_shape import Sequential, Shape, Skew, Table, Uniform, build
|
|
77
83
|
|
|
84
|
+
from myapp.models import Order
|
|
85
|
+
|
|
78
86
|
shape = Shape(
|
|
79
87
|
Table(
|
|
80
88
|
Order,
|
|
@@ -94,19 +102,29 @@ build(shape)
|
|
|
94
102
|
|
|
95
103
|
`build()` generates the rows, loads them with `COPY`, moves the identity sequence
|
|
96
104
|
past the keys it assigned, and runs `ANALYZE` so the planner can see the shape.
|
|
97
|
-
It raises on any backend that is not PostgreSQL rather than degrading quietly
|
|
105
|
+
It raises on any backend that is not PostgreSQL rather than degrading quietly --
|
|
106
|
+
unless you say `require_statistics=False`, which asks for rows and cardinality
|
|
107
|
+
instead of a database the planner can reason about, and is what the growth
|
|
108
|
+
harness below is built on.
|
|
98
109
|
|
|
99
110
|
## Relations
|
|
100
111
|
|
|
101
112
|
```python
|
|
102
|
-
Table
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
113
|
+
from django_data_shape import Constant, FanOut, Shape, Table, Zipf, build
|
|
114
|
+
|
|
115
|
+
build(
|
|
116
|
+
Shape(
|
|
117
|
+
Table(Company, rows=50, name=Constant("acme")),
|
|
118
|
+
Table(
|
|
119
|
+
Order,
|
|
120
|
+
rows=2_000_000,
|
|
121
|
+
# A distribution, not a number: giving every parent ten children is
|
|
122
|
+
# the one shape in which the planner is never wrong, because its
|
|
123
|
+
# n_distinct average is then the truth.
|
|
124
|
+
company=FanOut(Zipf(1.2), childless=0.35),
|
|
125
|
+
status=Constant("complete"),
|
|
126
|
+
),
|
|
127
|
+
)
|
|
110
128
|
)
|
|
111
129
|
```
|
|
112
130
|
|
|
@@ -114,10 +132,62 @@ The parents can be rows this package built or rows your own code did -- their
|
|
|
114
132
|
real keys are read, not assumed, so the ORM can own the small tables while this
|
|
115
133
|
owns the large ones.
|
|
116
134
|
|
|
135
|
+
## From pytest
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
# conftest.py
|
|
139
|
+
from django_data_shape import Constant, Shape, Table
|
|
140
|
+
from django_data_shape.fixtures import scale_fixture, shape_fixture
|
|
141
|
+
|
|
142
|
+
orders = shape_fixture(Shape(Table(Order, rows=100_000, status=Constant("complete"))))
|
|
143
|
+
world = scale_fixture(Shape(Table(Order, rows=100, status=Constant("complete"))))
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
`orders` is one world built once for the whole session, composed with
|
|
147
|
+
pytest-django rather than replacing it. `world` is the **scale protocol**: make
|
|
148
|
+
the world be at factor F, then let the caller run its block, which is what a
|
|
149
|
+
query count asserted to be `O(1)` rather than `O(N)` needs.
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
def test_the_dashboard_does_not_grow(world, django_assert_num_queries):
|
|
153
|
+
for factor in (1, 10):
|
|
154
|
+
with world(factor):
|
|
155
|
+
with django_assert_num_queries(3):
|
|
156
|
+
dashboard()
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
A factor varies the declaration rather than subsetting one larger build, and
|
|
160
|
+
`pip install 'django-data-shape[pytest]'` is what these two need. **The growth
|
|
161
|
+
harness works on any backend Django supports**, because a query count is an ORM
|
|
162
|
+
property and means the same everywhere; the session world **skips with a stated
|
|
163
|
+
reason** where a shaped database cannot exist, because a plan over it is the
|
|
164
|
+
thing it exists to make honest.
|
|
165
|
+
|
|
166
|
+
## What it expects, and what it refuses
|
|
167
|
+
|
|
168
|
+
A declaration that cannot describe a database raises before a row is generated,
|
|
169
|
+
naming the field. In particular:
|
|
170
|
+
|
|
171
|
+
- **PostgreSQL and psycopg 3.** Rows stream into `COPY FROM STDIN`, which
|
|
172
|
+
psycopg 2 cannot do without materialising them first. Both are refused by name
|
|
173
|
+
rather than degraded around. PostgreSQL is required for the statistics half
|
|
174
|
+
only: `build(shape, require_statistics=False)` loads rows on any backend and
|
|
175
|
+
claims nothing about a plan. psycopg 2 is refused either way, because the
|
|
176
|
+
vendor picks the route and not the caller.
|
|
177
|
+
- **A key type it can assign.** Integer keys count from one and UUID keys are
|
|
178
|
+
derived from the seed; anything else is refused rather than guessed, and
|
|
179
|
+
`keys=KeyFunction(...)` declares one.
|
|
180
|
+
- **Empty tables.** Keys start at 1 on every build, so `build()` checks first and
|
|
181
|
+
raises rather than colliding partway through.
|
|
182
|
+
- **A callable model default** such as `default=uuid4` must be declared as a
|
|
183
|
+
distribution: `uuid4` varies per row and `dict` does not, and nothing on the
|
|
184
|
+
field distinguishes them.
|
|
185
|
+
|
|
117
186
|
## Status
|
|
118
187
|
|
|
119
|
-
Early. Single tables and the
|
|
120
|
-
along a join, per-group invariants and template-database reuse
|
|
188
|
+
Early. Single tables, the model graph and the pytest surface. Derived fields,
|
|
189
|
+
collections copied along a join, per-group invariants and template-database reuse
|
|
190
|
+
come next.
|
|
121
191
|
|
|
122
192
|
Full documentation: <https://artui.github.io/django-data-shape/>
|
|
123
193
|
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# django-data-shape
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Artui/django-data-shape/actions/workflows/tests.yml)
|
|
4
|
+
[](https://pypi.org/project/django-data-shape/)
|
|
5
|
+
[](https://pypi.org/project/django-data-shape/)
|
|
6
|
+
[](https://pypi.org/project/django-data-shape/)
|
|
7
|
+
[](https://artui.github.io/django-data-shape/)
|
|
8
|
+
[](https://github.com/Artui/django-data-shape/actions/workflows/tests.yml)
|
|
9
|
+
[](https://github.com/astral-sh/ruff)
|
|
10
|
+
[](LICENSE)
|
|
11
|
+
|
|
12
|
+
A realistically shaped test database from Django models.
|
|
13
|
+
|
|
14
|
+
Declare the shape of your data -- cardinality, value skew, foreign-key fan-out as
|
|
15
|
+
a distribution with a long tail, and where related rows physically sit -- then
|
|
16
|
+
load it by `COPY` and `ANALYZE` it, so the query planner makes the same choices
|
|
17
|
+
it will make in production.
|
|
18
|
+
|
|
19
|
+
It exists because a plan over ten rows is a lie, and because the loop it replaces
|
|
20
|
+
is not merely smaller: uniform fan-out makes the planner always right, and
|
|
21
|
+
generating children parent-by-parent clusters them perfectly, which flatters
|
|
22
|
+
every index scan. A test database can be wrong in the flattering direction, and
|
|
23
|
+
usually is.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install 'django-data-shape[postgres]'
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The quotes are not decoration: zsh globs the brackets and reports
|
|
32
|
+
`no matches found` without them.
|
|
33
|
+
|
|
34
|
+
## Use
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
import datetime
|
|
38
|
+
|
|
39
|
+
from django_data_shape import Sequential, Shape, Skew, Table, Uniform, build
|
|
40
|
+
|
|
41
|
+
from myapp.models import Order
|
|
42
|
+
|
|
43
|
+
shape = Shape(
|
|
44
|
+
Table(
|
|
45
|
+
Order,
|
|
46
|
+
rows=1_000_000,
|
|
47
|
+
status=Skew({"complete": 0.98, "pending": 0.015, "cancelled": 0.005}),
|
|
48
|
+
total=Uniform(0, 500, places=2),
|
|
49
|
+
created_at=Sequential(
|
|
50
|
+
datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc),
|
|
51
|
+
datetime.timedelta(seconds=3),
|
|
52
|
+
),
|
|
53
|
+
),
|
|
54
|
+
seed=1234,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
build(shape)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`build()` generates the rows, loads them with `COPY`, moves the identity sequence
|
|
61
|
+
past the keys it assigned, and runs `ANALYZE` so the planner can see the shape.
|
|
62
|
+
It raises on any backend that is not PostgreSQL rather than degrading quietly --
|
|
63
|
+
unless you say `require_statistics=False`, which asks for rows and cardinality
|
|
64
|
+
instead of a database the planner can reason about, and is what the growth
|
|
65
|
+
harness below is built on.
|
|
66
|
+
|
|
67
|
+
## Relations
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from django_data_shape import Constant, FanOut, Shape, Table, Zipf, build
|
|
71
|
+
|
|
72
|
+
build(
|
|
73
|
+
Shape(
|
|
74
|
+
Table(Company, rows=50, name=Constant("acme")),
|
|
75
|
+
Table(
|
|
76
|
+
Order,
|
|
77
|
+
rows=2_000_000,
|
|
78
|
+
# A distribution, not a number: giving every parent ten children is
|
|
79
|
+
# the one shape in which the planner is never wrong, because its
|
|
80
|
+
# n_distinct average is then the truth.
|
|
81
|
+
company=FanOut(Zipf(1.2), childless=0.35),
|
|
82
|
+
status=Constant("complete"),
|
|
83
|
+
),
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
The parents can be rows this package built or rows your own code did -- their
|
|
89
|
+
real keys are read, not assumed, so the ORM can own the small tables while this
|
|
90
|
+
owns the large ones.
|
|
91
|
+
|
|
92
|
+
## From pytest
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
# conftest.py
|
|
96
|
+
from django_data_shape import Constant, Shape, Table
|
|
97
|
+
from django_data_shape.fixtures import scale_fixture, shape_fixture
|
|
98
|
+
|
|
99
|
+
orders = shape_fixture(Shape(Table(Order, rows=100_000, status=Constant("complete"))))
|
|
100
|
+
world = scale_fixture(Shape(Table(Order, rows=100, status=Constant("complete"))))
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`orders` is one world built once for the whole session, composed with
|
|
104
|
+
pytest-django rather than replacing it. `world` is the **scale protocol**: make
|
|
105
|
+
the world be at factor F, then let the caller run its block, which is what a
|
|
106
|
+
query count asserted to be `O(1)` rather than `O(N)` needs.
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
def test_the_dashboard_does_not_grow(world, django_assert_num_queries):
|
|
110
|
+
for factor in (1, 10):
|
|
111
|
+
with world(factor):
|
|
112
|
+
with django_assert_num_queries(3):
|
|
113
|
+
dashboard()
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
A factor varies the declaration rather than subsetting one larger build, and
|
|
117
|
+
`pip install 'django-data-shape[pytest]'` is what these two need. **The growth
|
|
118
|
+
harness works on any backend Django supports**, because a query count is an ORM
|
|
119
|
+
property and means the same everywhere; the session world **skips with a stated
|
|
120
|
+
reason** where a shaped database cannot exist, because a plan over it is the
|
|
121
|
+
thing it exists to make honest.
|
|
122
|
+
|
|
123
|
+
## What it expects, and what it refuses
|
|
124
|
+
|
|
125
|
+
A declaration that cannot describe a database raises before a row is generated,
|
|
126
|
+
naming the field. In particular:
|
|
127
|
+
|
|
128
|
+
- **PostgreSQL and psycopg 3.** Rows stream into `COPY FROM STDIN`, which
|
|
129
|
+
psycopg 2 cannot do without materialising them first. Both are refused by name
|
|
130
|
+
rather than degraded around. PostgreSQL is required for the statistics half
|
|
131
|
+
only: `build(shape, require_statistics=False)` loads rows on any backend and
|
|
132
|
+
claims nothing about a plan. psycopg 2 is refused either way, because the
|
|
133
|
+
vendor picks the route and not the caller.
|
|
134
|
+
- **A key type it can assign.** Integer keys count from one and UUID keys are
|
|
135
|
+
derived from the seed; anything else is refused rather than guessed, and
|
|
136
|
+
`keys=KeyFunction(...)` declares one.
|
|
137
|
+
- **Empty tables.** Keys start at 1 on every build, so `build()` checks first and
|
|
138
|
+
raises rather than colliding partway through.
|
|
139
|
+
- **A callable model default** such as `default=uuid4` must be declared as a
|
|
140
|
+
distribution: `uuid4` varies per row and `dict` does not, and nothing on the
|
|
141
|
+
field distinguishes them.
|
|
142
|
+
|
|
143
|
+
## Status
|
|
144
|
+
|
|
145
|
+
Early. Single tables, the model graph and the pytest surface. Derived fields,
|
|
146
|
+
collections copied along a join, per-group invariants and template-database reuse
|
|
147
|
+
come next.
|
|
148
|
+
|
|
149
|
+
Full documentation: <https://artui.github.io/django-data-shape/>
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
MIT
|
|
@@ -11,6 +11,13 @@ from django_data_shape.distributions.uniform import Uniform
|
|
|
11
11
|
from django_data_shape.distributions.zipf import Zipf
|
|
12
12
|
from django_data_shape.fan_out import FanOut
|
|
13
13
|
from django_data_shape.invalid_shape import InvalidShape
|
|
14
|
+
from django_data_shape.keys.key_function import KeyFunction
|
|
15
|
+
from django_data_shape.keys.key_strategy import KeyStrategy
|
|
16
|
+
from django_data_shape.keys.sequential_keys import SequentialKeys
|
|
17
|
+
from django_data_shape.keys.uuid_keys import UuidKeys
|
|
18
|
+
from django_data_shape.scale_protocol import ScaleProtocol
|
|
19
|
+
from django_data_shape.scaled_shape import scaled_shape
|
|
20
|
+
from django_data_shape.scaled_world import scaled_world
|
|
14
21
|
from django_data_shape.shape import Shape
|
|
15
22
|
from django_data_shape.shape_not_empty import ShapeNotEmpty
|
|
16
23
|
from django_data_shape.table import Table
|
|
@@ -25,6 +32,11 @@ __all__ = [
|
|
|
25
32
|
"Distribution",
|
|
26
33
|
"FanOut",
|
|
27
34
|
"InvalidShape",
|
|
35
|
+
"KeyFunction",
|
|
36
|
+
"KeyStrategy",
|
|
37
|
+
"ScaleProtocol",
|
|
38
|
+
"SequentialKeys",
|
|
39
|
+
"UuidKeys",
|
|
28
40
|
"Sequential",
|
|
29
41
|
"Shape",
|
|
30
42
|
"ShapeNotEmpty",
|
|
@@ -36,4 +48,6 @@ __all__ = [
|
|
|
36
48
|
"UnsupportedBackend",
|
|
37
49
|
"__version__",
|
|
38
50
|
"build",
|
|
51
|
+
"scaled_shape",
|
|
52
|
+
"scaled_world",
|
|
39
53
|
]
|