django-data-shape 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.github/workflows/tests.yml +7 -1
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/CHANGELOG.md +41 -1
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/PKG-INFO +39 -10
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/README.md +38 -9
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/__init__.py +8 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/build.py +7 -10
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/generate_rows.py +9 -5
- django_data_shape-0.3.0/django_data_shape/infer_key_strategy.py +27 -0
- django_data_shape-0.3.0/django_data_shape/keys/__init__.py +8 -0
- django_data_shape-0.3.0/django_data_shape/keys/key_function.py +44 -0
- django_data_shape-0.3.0/django_data_shape/keys/key_strategy.py +29 -0
- django_data_shape-0.3.0/django_data_shape/keys/sequential_keys.py +22 -0
- django_data_shape-0.3.0/django_data_shape/keys/uuid_keys.py +37 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/table.py +35 -16
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/version.py +1 -1
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/docs/index.md +8 -7
- django_data_shape-0.3.0/docs/keys.md +85 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/docs/reference.md +10 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/docs/relations.md +1 -1
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/mkdocs.yml +1 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/pyproject.toml +1 -1
- django_data_shape-0.3.0/tests/keys/test_key_function.py +37 -0
- django_data_shape-0.3.0/tests/keys/test_sequential_keys.py +19 -0
- django_data_shape-0.3.0/tests/keys/test_uuid_keys.py +34 -0
- django_data_shape-0.3.0/tests/test_build_keys.py +132 -0
- django_data_shape-0.3.0/tests/test_documentation.py +53 -0
- django_data_shape-0.3.0/tests/test_infer_key_strategy.py +24 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_table.py +55 -2
- django_data_shape-0.3.0/tests/testapp/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/testapp/models.py +32 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.github/CODEOWNERS +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.github/SECURITY.md +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.github/dependabot.yml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.github/workflows/release.yml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.github/workflows/upstream-drift.yml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.gitignore +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/.pre-commit-config.yaml +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/CLAUDE.md +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/LICENSE +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/Makefile +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/build_result.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/bounded.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/constant.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/distribution.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/sequential.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/skew.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/uniform.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/distributions/zipf.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/fan_out_plan.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/invalid_shape.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/order_tables.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/py.typed +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/require_postgres.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/resolve_fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/shape.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/shape_not_empty.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/table_result.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/unsupported_backend.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/django_data_shape/utils.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/scripts/release-publish.sh +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/conftest.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/conftest_settings.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/distributions/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/distributions/test_constant.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/distributions/test_sequential.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/distributions/test_skew.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/distributions/test_uniform.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/distributions/test_zipf.py +0 -0
- {django_data_shape-0.2.0/tests/testapp → django_data_shape-0.3.0/tests/keys}/__init__.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_build.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_build_graph.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_generate_rows.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_order_tables.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_require_postgres.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_resolve_fan_out.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_shape.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_utils.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/tests/test_version.py +0 -0
- {django_data_shape-0.2.0 → django_data_shape-0.3.0}/uv.lock +0 -0
|
@@ -114,8 +114,14 @@ jobs:
|
|
|
114
114
|
run: uv sync --frozen --all-groups --all-extras --python 3.10
|
|
115
115
|
- name: Name the versions that produced
|
|
116
116
|
run: uv pip list
|
|
117
|
+
# No coverage gate here either. This job's claim is that the declared
|
|
118
|
+
# floors resolve and the suite passes against them -- not that every line
|
|
119
|
+
# runs. It inherited the gate from the default addopts, which went
|
|
120
|
+
# unnoticed until a test skipped below Django 5.2 left one branch
|
|
121
|
+
# uncovered and failed a job that was never meant to measure coverage.
|
|
122
|
+
# One gate, on the Postgres job, is the design.
|
|
117
123
|
- name: Test
|
|
118
|
-
run: uv run --no-sync pytest
|
|
124
|
+
run: uv run --no-sync pytest --cov-fail-under=0
|
|
119
125
|
|
|
120
126
|
# The step above installs every extra, and an extra can hold a shared
|
|
121
127
|
# dependency *above* the floor being claimed -- masking the lie rather
|
|
@@ -7,6 +7,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.0] — 2026-09-01
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
- **A UUID primary key no longer refuses to load.** It was refused outright,
|
|
14
|
+
which made this package unusable for a whole class of Django project. The
|
|
15
|
+
reframing is the fix: the design never needed integers, it needed a
|
|
16
|
+
deterministic injection from row index to key -- which is what lets a foreign
|
|
17
|
+
key be satisfied without a lookup, what makes a self-referential tree acyclic
|
|
18
|
+
on the index rather than the value, and what makes two builds of one shape
|
|
19
|
+
agree. Integers were only the most obvious such function.
|
|
20
|
+
- The primary key is prepared by its own field, like every other value. It did
|
|
21
|
+
not need to be while keys were always integers, and a UUID works either way
|
|
22
|
+
because psycopg adapts it -- but a key type needing conversion was stored
|
|
23
|
+
verbatim, which is the bug already found once on an ordinary column.
|
|
24
|
+
- Two documentation examples were syntax errors -- `...` after keyword arguments,
|
|
25
|
+
in the README and on the relations page -- and would have failed the moment a
|
|
26
|
+
reader pasted them.
|
|
27
|
+
- The install line is quoted. `pip install django-data-shape[postgres]` fails in
|
|
28
|
+
zsh with `no matches found`, which is the default shell on macOS.
|
|
29
|
+
- The README's usage example imports the model it uses, and documents the
|
|
30
|
+
refusals a first attempt actually meets: PostgreSQL and psycopg 3, integer
|
|
31
|
+
primary keys, empty tables, and callable model defaults.
|
|
32
|
+
|
|
33
|
+
### Added
|
|
34
|
+
- **Key strategies.** A table's primary keys come from a deterministic function
|
|
35
|
+
of the row index rather than from a hard-coded dense `1..N` range. Integer keys
|
|
36
|
+
count from one, `UUIDField` keys are derived from the seed, and `KeyFunction`
|
|
37
|
+
declares one for any other type.
|
|
38
|
+
- `SequentialKeys`, `UuidKeys`, `KeyFunction` and the `KeyStrategy` protocol,
|
|
39
|
+
plus a `keys=` argument on `Table`.
|
|
40
|
+
- A composite primary key is refused by name. It is not among a model's concrete
|
|
41
|
+
fields, because it has no column of its own, so the package used to raise a
|
|
42
|
+
bare `StopIteration` from inside itself. The message says `keys=` cannot help
|
|
43
|
+
either: a strategy maps a row index to one value, and this is arity rather than
|
|
44
|
+
type.
|
|
45
|
+
- The documentation's Python examples are parsed by the test suite. A docs
|
|
46
|
+
example is the first code anybody runs, so it gets a guard rather than a
|
|
47
|
+
convention.
|
|
48
|
+
|
|
10
49
|
## [0.2.0] — 2026-09-01
|
|
11
50
|
|
|
12
51
|
### Added
|
|
@@ -108,7 +147,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
108
147
|
into `COPY FROM STDIN`, which psycopg 2 cannot do without materialising them
|
|
109
148
|
first.
|
|
110
149
|
|
|
111
|
-
[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.
|
|
150
|
+
[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.3.0...HEAD
|
|
151
|
+
[0.3.0]: https://github.com/Artui/django-data-shape/compare/v0.2.0...v0.3.0
|
|
112
152
|
[0.2.0]: https://github.com/Artui/django-data-shape/compare/v0.1.1...v0.2.0
|
|
113
153
|
[0.1.1]: https://github.com/Artui/django-data-shape/compare/v0.1.0...v0.1.1
|
|
114
154
|
[0.1.0]: https://github.com/Artui/django-data-shape/compare/v0.0.0...v0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: django-data-shape
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A realistically shaped test database from Django models: declare cardinality, skew and fan-out, load by COPY, and make the query planner believe it.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Artui/django-data-shape
|
|
6
6
|
Project-URL: Repository, https://github.com/Artui/django-data-shape
|
|
@@ -65,9 +65,12 @@ usually is.
|
|
|
65
65
|
## Install
|
|
66
66
|
|
|
67
67
|
```bash
|
|
68
|
-
pip install django-data-shape[postgres]
|
|
68
|
+
pip install 'django-data-shape[postgres]'
|
|
69
69
|
```
|
|
70
70
|
|
|
71
|
+
The quotes are not decoration: zsh globs the brackets and reports
|
|
72
|
+
`no matches found` without them.
|
|
73
|
+
|
|
71
74
|
## Use
|
|
72
75
|
|
|
73
76
|
```python
|
|
@@ -75,6 +78,8 @@ import datetime
|
|
|
75
78
|
|
|
76
79
|
from django_data_shape import Sequential, Shape, Skew, Table, Uniform, build
|
|
77
80
|
|
|
81
|
+
from myapp.models import Order
|
|
82
|
+
|
|
78
83
|
shape = Shape(
|
|
79
84
|
Table(
|
|
80
85
|
Order,
|
|
@@ -99,14 +104,21 @@ It raises on any backend that is not PostgreSQL rather than degrading quietly.
|
|
|
99
104
|
## Relations
|
|
100
105
|
|
|
101
106
|
```python
|
|
102
|
-
Table
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
107
|
+
from django_data_shape import Constant, FanOut, Shape, Table, Zipf, build
|
|
108
|
+
|
|
109
|
+
build(
|
|
110
|
+
Shape(
|
|
111
|
+
Table(Company, rows=50, name=Constant("acme")),
|
|
112
|
+
Table(
|
|
113
|
+
Order,
|
|
114
|
+
rows=2_000_000,
|
|
115
|
+
# A distribution, not a number: giving every parent ten children is
|
|
116
|
+
# the one shape in which the planner is never wrong, because its
|
|
117
|
+
# n_distinct average is then the truth.
|
|
118
|
+
company=FanOut(Zipf(1.2), childless=0.35),
|
|
119
|
+
status=Constant("complete"),
|
|
120
|
+
),
|
|
121
|
+
)
|
|
110
122
|
)
|
|
111
123
|
```
|
|
112
124
|
|
|
@@ -114,6 +126,23 @@ The parents can be rows this package built or rows your own code did -- their
|
|
|
114
126
|
real keys are read, not assumed, so the ORM can own the small tables while this
|
|
115
127
|
owns the large ones.
|
|
116
128
|
|
|
129
|
+
## What it expects, and what it refuses
|
|
130
|
+
|
|
131
|
+
A declaration that cannot describe a database raises before a row is generated,
|
|
132
|
+
naming the field. In particular:
|
|
133
|
+
|
|
134
|
+
- **PostgreSQL and psycopg 3.** Rows stream into `COPY FROM STDIN`, which
|
|
135
|
+
psycopg 2 cannot do without materialising them first. Both are refused by name
|
|
136
|
+
rather than degraded around.
|
|
137
|
+
- **A key type it can assign.** Integer keys count from one and UUID keys are
|
|
138
|
+
derived from the seed; anything else is refused rather than guessed, and
|
|
139
|
+
`keys=KeyFunction(...)` declares one.
|
|
140
|
+
- **Empty tables.** Keys start at 1 on every build, so `build()` checks first and
|
|
141
|
+
raises rather than colliding partway through.
|
|
142
|
+
- **A callable model default** such as `default=uuid4` must be declared as a
|
|
143
|
+
distribution: `uuid4` varies per row and `dict` does not, and nothing on the
|
|
144
|
+
field distinguishes them.
|
|
145
|
+
|
|
117
146
|
## Status
|
|
118
147
|
|
|
119
148
|
Early. Single tables and the model graph. Derived fields, collections copied
|
|
@@ -25,9 +25,12 @@ usually is.
|
|
|
25
25
|
## Install
|
|
26
26
|
|
|
27
27
|
```bash
|
|
28
|
-
pip install django-data-shape[postgres]
|
|
28
|
+
pip install 'django-data-shape[postgres]'
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
+
The quotes are not decoration: zsh globs the brackets and reports
|
|
32
|
+
`no matches found` without them.
|
|
33
|
+
|
|
31
34
|
## Use
|
|
32
35
|
|
|
33
36
|
```python
|
|
@@ -35,6 +38,8 @@ import datetime
|
|
|
35
38
|
|
|
36
39
|
from django_data_shape import Sequential, Shape, Skew, Table, Uniform, build
|
|
37
40
|
|
|
41
|
+
from myapp.models import Order
|
|
42
|
+
|
|
38
43
|
shape = Shape(
|
|
39
44
|
Table(
|
|
40
45
|
Order,
|
|
@@ -59,14 +64,21 @@ It raises on any backend that is not PostgreSQL rather than degrading quietly.
|
|
|
59
64
|
## Relations
|
|
60
65
|
|
|
61
66
|
```python
|
|
62
|
-
Table
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
from django_data_shape import Constant, FanOut, Shape, Table, Zipf, build
|
|
68
|
+
|
|
69
|
+
build(
|
|
70
|
+
Shape(
|
|
71
|
+
Table(Company, rows=50, name=Constant("acme")),
|
|
72
|
+
Table(
|
|
73
|
+
Order,
|
|
74
|
+
rows=2_000_000,
|
|
75
|
+
# A distribution, not a number: giving every parent ten children is
|
|
76
|
+
# the one shape in which the planner is never wrong, because its
|
|
77
|
+
# n_distinct average is then the truth.
|
|
78
|
+
company=FanOut(Zipf(1.2), childless=0.35),
|
|
79
|
+
status=Constant("complete"),
|
|
80
|
+
),
|
|
81
|
+
)
|
|
70
82
|
)
|
|
71
83
|
```
|
|
72
84
|
|
|
@@ -74,6 +86,23 @@ The parents can be rows this package built or rows your own code did -- their
|
|
|
74
86
|
real keys are read, not assumed, so the ORM can own the small tables while this
|
|
75
87
|
owns the large ones.
|
|
76
88
|
|
|
89
|
+
## What it expects, and what it refuses
|
|
90
|
+
|
|
91
|
+
A declaration that cannot describe a database raises before a row is generated,
|
|
92
|
+
naming the field. In particular:
|
|
93
|
+
|
|
94
|
+
- **PostgreSQL and psycopg 3.** Rows stream into `COPY FROM STDIN`, which
|
|
95
|
+
psycopg 2 cannot do without materialising them first. Both are refused by name
|
|
96
|
+
rather than degraded around.
|
|
97
|
+
- **A key type it can assign.** Integer keys count from one and UUID keys are
|
|
98
|
+
derived from the seed; anything else is refused rather than guessed, and
|
|
99
|
+
`keys=KeyFunction(...)` declares one.
|
|
100
|
+
- **Empty tables.** Keys start at 1 on every build, so `build()` checks first and
|
|
101
|
+
raises rather than colliding partway through.
|
|
102
|
+
- **A callable model default** such as `default=uuid4` must be declared as a
|
|
103
|
+
distribution: `uuid4` varies per row and `dict` does not, and nothing on the
|
|
104
|
+
field distinguishes them.
|
|
105
|
+
|
|
77
106
|
## Status
|
|
78
107
|
|
|
79
108
|
Early. Single tables and the model graph. Derived fields, collections copied
|
|
@@ -11,6 +11,10 @@ from django_data_shape.distributions.uniform import Uniform
|
|
|
11
11
|
from django_data_shape.distributions.zipf import Zipf
|
|
12
12
|
from django_data_shape.fan_out import FanOut
|
|
13
13
|
from django_data_shape.invalid_shape import InvalidShape
|
|
14
|
+
from django_data_shape.keys.key_function import KeyFunction
|
|
15
|
+
from django_data_shape.keys.key_strategy import KeyStrategy
|
|
16
|
+
from django_data_shape.keys.sequential_keys import SequentialKeys
|
|
17
|
+
from django_data_shape.keys.uuid_keys import UuidKeys
|
|
14
18
|
from django_data_shape.shape import Shape
|
|
15
19
|
from django_data_shape.shape_not_empty import ShapeNotEmpty
|
|
16
20
|
from django_data_shape.table import Table
|
|
@@ -25,6 +29,10 @@ __all__ = [
|
|
|
25
29
|
"Distribution",
|
|
26
30
|
"FanOut",
|
|
27
31
|
"InvalidShape",
|
|
32
|
+
"KeyFunction",
|
|
33
|
+
"KeyStrategy",
|
|
34
|
+
"SequentialKeys",
|
|
35
|
+
"UuidKeys",
|
|
28
36
|
"Sequential",
|
|
29
37
|
"Shape",
|
|
30
38
|
"ShapeNotEmpty",
|
|
@@ -118,10 +118,13 @@ def _load(connection: Any, table: Table, seed: int, plans: dict[str, FanOutPlan]
|
|
|
118
118
|
many-to-many edges arrive.
|
|
119
119
|
"""
|
|
120
120
|
quote = connection.ops.quote_name
|
|
121
|
-
|
|
122
|
-
columns = [quote(
|
|
121
|
+
pk_field = table.model._meta.pk
|
|
122
|
+
columns = [quote(pk_field.column)] + [quote(field.column) for _, field in table.columns()]
|
|
123
123
|
statement = f"COPY {quote(table.db_table)} ({', '.join(columns)}) FROM STDIN"
|
|
124
|
-
|
|
124
|
+
# The primary key is prepared like every other value. It did not need to be
|
|
125
|
+
# while keys were always integers; a UUID key does, and a strategy the
|
|
126
|
+
# caller wrote could return anything its column accepts.
|
|
127
|
+
prepare = [pk_field.get_db_prep_save] + [field.get_db_prep_save for _, field in table.columns()]
|
|
125
128
|
|
|
126
129
|
with connection.cursor() as cursor:
|
|
127
130
|
# ``copy`` is not in Django's WRAP_ERROR_ATTRS, so without this a
|
|
@@ -132,13 +135,7 @@ def _load(connection: Any, table: Table, seed: int, plans: dict[str, FanOutPlan]
|
|
|
132
135
|
with connection.wrap_database_errors, cursor.copy(statement) as copy:
|
|
133
136
|
for row in generate_rows(table, seed, plans):
|
|
134
137
|
copy.write_row(
|
|
135
|
-
(
|
|
136
|
-
row[0],
|
|
137
|
-
*(
|
|
138
|
-
prep(value, connection)
|
|
139
|
-
for prep, value in zip(prepare, row[1:], strict=True)
|
|
140
|
-
),
|
|
141
|
-
)
|
|
138
|
+
tuple(prep(value, connection) for prep, value in zip(prepare, row, strict=True))
|
|
142
139
|
)
|
|
143
140
|
return int(cursor.rowcount)
|
|
144
141
|
|
|
@@ -22,10 +22,12 @@ def generate_rows(
|
|
|
22
22
|
and nothing here needs an instance, because no ``save`` will run and no
|
|
23
23
|
signal should fire.
|
|
24
24
|
|
|
25
|
-
Primary keys
|
|
26
|
-
|
|
27
|
-
a
|
|
28
|
-
|
|
25
|
+
Primary keys come from the table's key strategy, which is a deterministic
|
|
26
|
+
function of the row index -- ``row + 1`` for an integer key, a derived UUID
|
|
27
|
+
for a UUID one, a caller's own function for anything else. Determinism is the
|
|
28
|
+
requirement, not the integers: it is what lets a child compute its parent's
|
|
29
|
+
key without a lookup, and what makes a self-referential tree acyclic on the
|
|
30
|
+
index rather than on the value.
|
|
29
31
|
|
|
30
32
|
``plans`` carries the resolved fan-out for each relation column. Resolving
|
|
31
33
|
happens outside this function because it has to read the parent's real keys
|
|
@@ -37,6 +39,8 @@ def generate_rows(
|
|
|
37
39
|
nothing but peak RSS.
|
|
38
40
|
"""
|
|
39
41
|
plans = plans or {}
|
|
42
|
+
keys = table.keys
|
|
43
|
+
key_stream = field_stream(seed, table.db_table, ":key")
|
|
40
44
|
# Each column is reduced to one callable of the row index before the loop
|
|
41
45
|
# starts. At a million rows the loop body runs a million times per column, so
|
|
42
46
|
# the branch between a fan-out and a value distribution is worth deciding
|
|
@@ -53,7 +57,7 @@ def generate_rows(
|
|
|
53
57
|
)
|
|
54
58
|
|
|
55
59
|
for row in range(table.rows):
|
|
56
|
-
yield (row
|
|
60
|
+
yield (keys.key_for(row, key_stream), *(produce(row) for produce in emit))
|
|
57
61
|
|
|
58
62
|
|
|
59
63
|
def _from_distribution(distribution: Distribution, stream: int, row: int) -> object:
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Choosing a key strategy from the model, when the caller did not."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from django.db.models import Field, IntegerField, UUIDField
|
|
8
|
+
|
|
9
|
+
from django_data_shape.keys.key_strategy import KeyStrategy
|
|
10
|
+
from django_data_shape.keys.sequential_keys import SequentialKeys
|
|
11
|
+
from django_data_shape.keys.uuid_keys import UuidKeys
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def infer_key_strategy(field: Field[Any, Any]) -> KeyStrategy | None:
|
|
15
|
+
"""The obvious strategy for a primary key type, or None if there is none.
|
|
16
|
+
|
|
17
|
+
Only the two types where the right answer is unambiguous. Everything else
|
|
18
|
+
returns None and is refused unless the caller declares a strategy, because
|
|
19
|
+
inventing values for a semantic column is how a character primary key once
|
|
20
|
+
got loaded with the strings "1", "2" and "3" -- data the application could
|
|
21
|
+
never have written, with a whole statistics picture built on top of it.
|
|
22
|
+
"""
|
|
23
|
+
if isinstance(field, IntegerField):
|
|
24
|
+
return SequentialKeys()
|
|
25
|
+
if isinstance(field, UUIDField):
|
|
26
|
+
return UuidKeys()
|
|
27
|
+
return None
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""How a table's primary keys are decided."""
|
|
2
|
+
|
|
3
|
+
from django_data_shape.keys.key_function import KeyFunction
|
|
4
|
+
from django_data_shape.keys.key_strategy import KeyStrategy
|
|
5
|
+
from django_data_shape.keys.sequential_keys import SequentialKeys
|
|
6
|
+
from django_data_shape.keys.uuid_keys import UuidKeys
|
|
7
|
+
|
|
8
|
+
__all__ = ["KeyFunction", "KeyStrategy", "SequentialKeys", "UuidKeys"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Keys from a function the caller supplies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
|
|
7
|
+
from django_data_shape.invalid_shape import InvalidShape
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class KeyFunction:
|
|
11
|
+
"""A caller's own deterministic mapping from row index to key.
|
|
12
|
+
|
|
13
|
+
The escape hatch for a key this package cannot infer -- a natural key, a
|
|
14
|
+
prefixed slug, an external identifier. Integer and UUID primary keys are
|
|
15
|
+
inferred and need none of this; anything else is declared rather than
|
|
16
|
+
guessed, because a guessed value in a semantic column is how a character
|
|
17
|
+
primary key once got loaded with the strings "1", "2" and "3".
|
|
18
|
+
|
|
19
|
+
The function must be a pure function of the row index. That is checked on a
|
|
20
|
+
sample at construction rather than trusted: a key that varies between calls
|
|
21
|
+
would break reproducibility, and it would break it silently, in the one
|
|
22
|
+
column every foreign key points at.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, function: Callable[[int], object], *, sample: int = 64) -> None:
|
|
26
|
+
first = [function(row) for row in range(sample)]
|
|
27
|
+
if [function(row) for row in range(sample)] != first:
|
|
28
|
+
raise InvalidShape(
|
|
29
|
+
"KeyFunction must be a pure function of the row index, and this one returned "
|
|
30
|
+
"different keys for the same rows on a second call. A key that varies between "
|
|
31
|
+
"calls breaks reproducibility in the column every foreign key points at."
|
|
32
|
+
)
|
|
33
|
+
if len(set(first)) != len(first):
|
|
34
|
+
raise InvalidShape(
|
|
35
|
+
f"KeyFunction produced duplicate keys within its first {sample} rows, and a "
|
|
36
|
+
"primary key has to be unique. It must be an injection from the row index."
|
|
37
|
+
)
|
|
38
|
+
self._function = function
|
|
39
|
+
|
|
40
|
+
def key_for(self, row: int, stream: int) -> object:
|
|
41
|
+
return self._function(row)
|
|
42
|
+
|
|
43
|
+
def __repr__(self) -> str:
|
|
44
|
+
return f"KeyFunction({getattr(self._function, '__name__', self._function)!r})"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""How a table's primary keys are decided."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Protocol
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class KeyStrategy(Protocol):
|
|
9
|
+
"""Turns a row index into that row's primary key.
|
|
10
|
+
|
|
11
|
+
The generalisation of what used to be a hard-coded dense ``1..N`` range. The
|
|
12
|
+
range was never the requirement: what the design actually rests on is that
|
|
13
|
+
the key is a **deterministic function of the row index**, and integers were
|
|
14
|
+
only the most obvious such function.
|
|
15
|
+
|
|
16
|
+
Everything the dense range bought is bought by determinism instead. A child
|
|
17
|
+
can compute its parent's key from the parent's *index*, so a foreign key is
|
|
18
|
+
satisfied without a lookup whatever the key type. A self-referential tree is
|
|
19
|
+
acyclic because ``parent_index < child_index`` holds on the index, not on the
|
|
20
|
+
value. And two builds of one shape agree because the same seed produces the
|
|
21
|
+
same keys.
|
|
22
|
+
|
|
23
|
+
``stream`` is a per-table value derived from the seed, so a strategy that
|
|
24
|
+
needs entropy has some. One that does not -- a counter, or a caller's own
|
|
25
|
+
function -- ignores it, exactly as a positional distribution ignores its
|
|
26
|
+
draw.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def key_for(self, row: int, stream: int) -> object: ...
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Dense integer keys, counting from one."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class SequentialKeys:
|
|
7
|
+
"""``row + 1``: the default for any integer primary key.
|
|
8
|
+
|
|
9
|
+
Counting from one rather than zero because that is what a database sequence
|
|
10
|
+
does, and a test database whose keys start at zero is subtly unlike every
|
|
11
|
+
other one the reader has seen.
|
|
12
|
+
|
|
13
|
+
This is the strategy that obliges the sequence reset after loading. It is
|
|
14
|
+
also the only one that does: a key type with no sequence behind it has
|
|
15
|
+
nothing to move.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def key_for(self, row: int, stream: int) -> object:
|
|
19
|
+
return row + 1
|
|
20
|
+
|
|
21
|
+
def __repr__(self) -> str:
|
|
22
|
+
return "SequentialKeys()"
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Version-4 shaped UUID keys, derived rather than random."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import uuid
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class UuidKeys:
|
|
10
|
+
"""A UUID per row, deterministic in the seed and the row index.
|
|
11
|
+
|
|
12
|
+
Derived from a hash rather than drawn from ``uuid4`` for the reason the
|
|
13
|
+
whole package is built around: two builds of one shape have to agree, and a
|
|
14
|
+
random key would make the primary key -- and therefore every foreign key
|
|
15
|
+
pointing at it -- differ between runs.
|
|
16
|
+
|
|
17
|
+
A full 128 bits from the digest, not a float draw. A draw carries 53 bits,
|
|
18
|
+
which sounds ample until birthday collisions arrive around ninety million
|
|
19
|
+
rows; a table that large is exactly the kind this package exists to build.
|
|
20
|
+
|
|
21
|
+
The version and variant bits are stamped so the result is a well-formed
|
|
22
|
+
version 4 UUID. Applications store v4 keys, so a test database holding
|
|
23
|
+
something that merely looks UUID-shaped would be unlike the thing it stands
|
|
24
|
+
in for.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def key_for(self, row: int, stream: int) -> object:
|
|
28
|
+
digest = hashlib.blake2b(
|
|
29
|
+
stream.to_bytes(8, "big") + row.to_bytes(8, "big"), digest_size=16
|
|
30
|
+
).digest()
|
|
31
|
+
raw = bytearray(digest)
|
|
32
|
+
raw[6] = (raw[6] & 0x0F) | 0x40
|
|
33
|
+
raw[8] = (raw[8] & 0x3F) | 0x80
|
|
34
|
+
return uuid.UUID(bytes=bytes(raw))
|
|
35
|
+
|
|
36
|
+
def __repr__(self) -> str:
|
|
37
|
+
return "UuidKeys()"
|
|
@@ -6,14 +6,16 @@ from collections.abc import Mapping
|
|
|
6
6
|
from types import MappingProxyType
|
|
7
7
|
from typing import Any, cast
|
|
8
8
|
|
|
9
|
-
from django.db.models import Field,
|
|
9
|
+
from django.db.models import Field, Model
|
|
10
10
|
from django.db.models.fields import NOT_PROVIDED
|
|
11
11
|
|
|
12
12
|
from django_data_shape.distributions.bounded import Bounded
|
|
13
13
|
from django_data_shape.distributions.constant import Constant
|
|
14
14
|
from django_data_shape.distributions.distribution import Distribution
|
|
15
15
|
from django_data_shape.fan_out import FanOut
|
|
16
|
+
from django_data_shape.infer_key_strategy import infer_key_strategy
|
|
16
17
|
from django_data_shape.invalid_shape import InvalidShape
|
|
18
|
+
from django_data_shape.keys.key_strategy import KeyStrategy
|
|
17
19
|
|
|
18
20
|
|
|
19
21
|
class Table:
|
|
@@ -38,6 +40,7 @@ class Table:
|
|
|
38
40
|
model: type[Model],
|
|
39
41
|
rows: int,
|
|
40
42
|
fields: Mapping[str, Distribution | FanOut] | None = None,
|
|
43
|
+
keys: KeyStrategy | None = None,
|
|
41
44
|
**field_distributions: Distribution | FanOut,
|
|
42
45
|
) -> None:
|
|
43
46
|
if rows < 0:
|
|
@@ -55,6 +58,7 @@ class Table:
|
|
|
55
58
|
self._model = model
|
|
56
59
|
self._rows = rows
|
|
57
60
|
self._fields = declared
|
|
61
|
+
self._keys = keys
|
|
58
62
|
self._validate()
|
|
59
63
|
|
|
60
64
|
# Read-only, because every rule in this class is enforced once, in
|
|
@@ -70,6 +74,15 @@ class Table:
|
|
|
70
74
|
def rows(self) -> int:
|
|
71
75
|
return self._rows
|
|
72
76
|
|
|
77
|
+
@property
|
|
78
|
+
def keys(self) -> KeyStrategy:
|
|
79
|
+
"""How this table's primary keys are decided.
|
|
80
|
+
|
|
81
|
+
Never None by the time anyone can read it: _validate either inferred a
|
|
82
|
+
strategy from the primary key's type or refused the declaration.
|
|
83
|
+
"""
|
|
84
|
+
return cast("KeyStrategy", self._keys)
|
|
85
|
+
|
|
73
86
|
@property
|
|
74
87
|
def fields(self) -> Mapping[str, Distribution | FanOut]:
|
|
75
88
|
"""The declared distributions, including defaults filled in from the model."""
|
|
@@ -114,21 +127,27 @@ class Table:
|
|
|
114
127
|
f"Its concrete fields are: {', '.join(sorted(known))}."
|
|
115
128
|
)
|
|
116
129
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
#
|
|
120
|
-
#
|
|
121
|
-
#
|
|
122
|
-
#
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
pk_field = next((field for field in known.values() if field.primary_key), None)
|
|
131
|
+
if pk_field is None:
|
|
132
|
+
# A composite primary key is not among the concrete fields, because
|
|
133
|
+
# it has no column of its own. Without this the package raised a
|
|
134
|
+
# bare StopIteration from inside itself, which says nothing about
|
|
135
|
+
# what the caller did.
|
|
136
|
+
raise InvalidShape(
|
|
137
|
+
f"{self.model.__name__} has a composite primary key, which this package cannot "
|
|
138
|
+
"assign. A key strategy maps a row index to one value and a composite key is "
|
|
139
|
+
"several columns, so keys= cannot help either: this is arity, not type."
|
|
140
|
+
)
|
|
141
|
+
if self._keys is None:
|
|
142
|
+
self._keys = infer_key_strategy(pk_field)
|
|
143
|
+
if self._keys is None:
|
|
144
|
+
raise InvalidShape(
|
|
145
|
+
f"{self.model.__name__}.{pk_field.name} is a {type(pk_field).__name__} primary "
|
|
146
|
+
"key, and only integer and UUID keys are inferred. Pass keys= with a strategy "
|
|
147
|
+
"for it -- KeyFunction takes any deterministic function of the row index. "
|
|
148
|
+
"Inventing values for a key column is how a character primary key once got "
|
|
149
|
+
'loaded with the strings "1", "2" and "3".'
|
|
150
|
+
)
|
|
132
151
|
|
|
133
152
|
pk_names = {name for name, field in known.items() if field.primary_key}
|
|
134
153
|
declared_pk = sorted(pk_names & set(self.fields))
|
|
@@ -58,13 +58,14 @@ print(result.rows)
|
|
|
58
58
|
|
|
59
59
|
## What it decides for you
|
|
60
60
|
|
|
61
|
-
- **Primary keys are assigned here**,
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
61
|
+
- **Primary keys are assigned here**, by a *key strategy*: a deterministic
|
|
62
|
+
function of the row index. Integer keys count from one and the identity
|
|
63
|
+
sequence is moved past them afterwards; UUID keys are derived from the seed, so
|
|
64
|
+
two builds of one shape agree. Determinism is the requirement, not integers --
|
|
65
|
+
it is what lets a foreign key be satisfied without a lookup and what makes a
|
|
66
|
+
self-referential tree acyclic. A key type with no obvious strategy is refused
|
|
67
|
+
rather than guessed, and `keys=KeyFunction(...)` declares one. See
|
|
68
|
+
[Keys](keys.md).
|
|
68
69
|
- **Every value goes through its field's `get_db_prep_save`.** Without it a naive
|
|
69
70
|
datetime lands in the database verbatim rather than localised, which under a
|
|
70
71
|
non-UTC `TIME_ZONE` is hours from where `save()` would have put it, and a
|