django-data-shape 0.13.0__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.pre-commit-config.yaml +12 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/CHANGELOG.md +140 -1
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/PKG-INFO +1 -1
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/__init__.py +4 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/build.py +41 -7
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/check_constraints.py +11 -6
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/fan_out_plan.py +9 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/generate_rows.py +25 -3
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/md5_keys.py +19 -10
- django_data_shape-0.15.0/django_data_shape/paired.py +82 -0
- django_data_shape-0.15.0/django_data_shape/paired_plan.py +145 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/projection.py +165 -27
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/resolve_fan_out.py +23 -2
- django_data_shape-0.15.0/django_data_shape/resolve_paired.py +145 -0
- django_data_shape-0.15.0/django_data_shape/shape_from_factory.py +228 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/table.py +46 -3
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/version.py +1 -1
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/index.md +59 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/projections.md +35 -1
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/reference.md +4 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/relations.md +71 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/pyproject.toml +1 -1
- django_data_shape-0.15.0/scripts/check-changelog +148 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/keys/test_md5_keys.py +46 -0
- django_data_shape-0.15.0/tests/test_build_m2m.py +100 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_build_projection.py +96 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_check_constraints.py +45 -0
- django_data_shape-0.15.0/tests/test_paired.py +34 -0
- django_data_shape-0.15.0/tests/test_paired_plan.py +116 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_projection.py +138 -3
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_resolve_fan_out.py +43 -0
- django_data_shape-0.15.0/tests/test_resolve_paired.py +117 -0
- django_data_shape-0.15.0/tests/test_shape_from_factory.py +194 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_table.py +60 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/testapp/models.py +79 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.github/CODEOWNERS +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.github/SECURITY.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.github/dependabot.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.github/workflows/release.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.github/workflows/tests.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.github/workflows/upstream-drift.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/.gitignore +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/CLAUDE.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/LICENSE +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/Makefile +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/README.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/apply_statistics_targets.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/build_result.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/canonical.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/check_invariants.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/children_per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/clone_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivation_queried_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/after.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/aligned.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/derivation.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/derived.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/given.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/derivations/scope.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/ascending.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/bounded.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/categorical.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/constant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/distinct.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/distribution.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/sequential.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/skew.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/uniform.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/distributions/zipf.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/drop_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/fan_out.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/fan_out_sizes.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/fixtures/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/fixtures/scale_fixture.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/fixtures/shape_fixture.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/fixtures/skip_unless_postgres.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/infer_key_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/invalid_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/invariant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/invariant_violated.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/disjoint.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/key_function.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/key_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/sequential_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/sql_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/keys/uuid_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/order_derivations.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/order_tables.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/py.typed +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/refuse_queries.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/require_clone_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/require_postgres.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/scale_protocol.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/scaled_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/scaled_world.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/shape_digest.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/shape_not_empty.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/table_result.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/template_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/unhashable_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/unsupported_backend.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/utils.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/world_changed.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/derivations.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/invariants.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/keys.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/pytest.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/docs/statistics.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/mkdocs.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/scripts/release-publish.sh +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/conftest.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/conftest_settings.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/derivations/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/derivations/test_after.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/derivations/test_aligned.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/derivations/test_derived.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/derivations/test_given.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/derivations/test_per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/distributions/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/distributions/test_constant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/distributions/test_sequential.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/distributions/test_skew.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/distributions/test_uniform.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/distributions/test_zipf.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/keys/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/keys/test_key_function.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/keys/test_sequential_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/keys/test_uuid_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/scale_protocol_consumers.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/scale_protocol_impostors.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_apply_statistics_targets.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_build.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_build_derivations.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_build_graph.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_build_invariants.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_build_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_build_portable.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_children_per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_documentation.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_fan_out.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_fan_out_sizes.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_fixtures.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_generate_rows.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_infer_key_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_invariant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_order_derivations.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_order_tables.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_reference_documents_the_public_surface.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_refuse_queries.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_require_clone_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_require_postgres.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_scale_protocol.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_scaled_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_scaled_world.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_shape_digest.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_template_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_utils.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/test_version.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/tests/testapp/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.15.0}/uv.lock +0 -0
|
@@ -65,3 +65,15 @@ repos:
|
|
|
65
65
|
language: pygrep
|
|
66
66
|
types: [text]
|
|
67
67
|
exclude: '^\.pre-commit-config\.yaml$'
|
|
68
|
+
|
|
69
|
+
# Structural rather than a pattern, so it is a script where the guards
|
|
70
|
+
# above are pygrep. A merge or a rebase can rearrange this file without
|
|
71
|
+
# conflicting: a released heading and an unreleased entry insert after the
|
|
72
|
+
# same line, git resolves them silently, and shipped work ends up filed
|
|
73
|
+
# under a version already published. It has happened four times across two
|
|
74
|
+
# repositories and every catch was a habit rather than a gate.
|
|
75
|
+
- id: check-changelog
|
|
76
|
+
name: "check CHANGELOG structure (merge and rebase leave it silently wrong)"
|
|
77
|
+
entry: scripts/check-changelog
|
|
78
|
+
language: script
|
|
79
|
+
files: '^CHANGELOG\.md$'
|
|
@@ -7,6 +7,143 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.15.0] — 2026-09-04
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **`shape_from_factory` runs a factory you already have and returns source, not
|
|
14
|
+
a `Shape`.** That is the design rather than a limitation of it: a shape this
|
|
15
|
+
package builds is *declared*, which is what makes it reviewable and
|
|
16
|
+
assertable, and a shape learned from a sample is neither -- one used directly
|
|
17
|
+
would change whenever the factory did, silently. So it returns text, and a
|
|
18
|
+
person decides what of it to keep.
|
|
19
|
+
- **What it finds matters more than what it writes**, because a faithful
|
|
20
|
+
reading of a typical factory produces the flat world this package exists to
|
|
21
|
+
argue against. Factories are written for single-object tests, so they fix
|
|
22
|
+
values and reach foreign keys in the two most unrealistic ways there are.
|
|
23
|
+
Every such column is a **finding**, and the report leads with them.
|
|
24
|
+
- **A sub-factory is the sharpest case and the reason to run it.**
|
|
25
|
+
`company = SubFactory(CompanyFactory)` creates one parent per child, which is
|
|
26
|
+
a fan-out of degree one: every parent has exactly one row, the average is the
|
|
27
|
+
truth, and a join over it cannot be misestimated. It is invisible in the
|
|
28
|
+
factory's own source, so it is detected by watching which other tables grew
|
|
29
|
+
and by how much. A round-robin over four parents is the same defect with a
|
|
30
|
+
different number, and is reported too.
|
|
31
|
+
- A relation is rendered as a `FanOut` and never as a value distribution,
|
|
32
|
+
because `Table` refuses the second -- emitting it would hand back source that
|
|
33
|
+
cannot be built, which is worse than a wrong number. A factory that already
|
|
34
|
+
varies everything gets a declaration and **no findings**: if the quiet case
|
|
35
|
+
were not quiet, the loud one would stop meaning anything.
|
|
36
|
+
- Nothing is left behind. The calls run inside a transaction that is rolled
|
|
37
|
+
back, so it can be pointed at a development database without writing to one.
|
|
38
|
+
The sample size is stated in the output, because the answer moves with it.
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
### Fixed
|
|
42
|
+
- **`Md5Keys` could not fill a projection on about half of all tables.**
|
|
43
|
+
`key_sql` wrote `to_hex(<stream>::bigint)`, asking PostgreSQL to re-derive a
|
|
44
|
+
number Python produced as **unsigned** -- and `bigint` is signed, so any
|
|
45
|
+
stream above 2^63 raised `NumericValueOutOfRange` before a row was written.
|
|
46
|
+
The stream is a hash of the table and field names, so it is a coin flip per
|
|
47
|
+
table rather than anything about a schema or a seed: measured, 49% of table
|
|
48
|
+
names land above the limit. The stream is a constant by the time the statement
|
|
49
|
+
is built, so the sixteen hex digits are embedded and nothing is re-derived.
|
|
50
|
+
- **Reported by a consumer, and it had never executed for them.** In 0.13.0
|
|
51
|
+
the join-ambiguity refusal answered first, so `key_sql` was unreachable;
|
|
52
|
+
fixing that in 0.14.0 with `through=` is what exposed a feature 0.13.0 had
|
|
53
|
+
shipped and nobody could run.
|
|
54
|
+
- **The test that was supposed to prove the two halves agree used
|
|
55
|
+
`stream=12345`** -- a number chosen for a test rather than one the producer
|
|
56
|
+
makes. It now uses a pair `field_stream` actually produced, one either side
|
|
57
|
+
of the limit, and asserts that the pair straddles it so a later change to
|
|
58
|
+
how streams are derived cannot quietly make both tests vacuous.
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
## [0.14.0] — 2026-09-03
|
|
62
|
+
|
|
63
|
+
### Added
|
|
64
|
+
|
|
65
|
+
- **`Paired` fills the second key of a many-to-many, and the edge count stays
|
|
66
|
+
exact.** Two fan-outs on one through table are refused because a fan-out is a
|
|
67
|
+
partition computed from the row index alone, so two of them partition the same
|
|
68
|
+
rows without either seeing the other -- nothing enumerates the pairs and a
|
|
69
|
+
collision is a matter of the seed. `person=Paired("company", Zipf())` is the
|
|
70
|
+
half that looks: `company` partitions the rows as any fan-out does, and within
|
|
71
|
+
each of its groups `person` takes that many **distinct** partners. A duplicate
|
|
72
|
+
pair is impossible by construction rather than removed afterwards, so nothing
|
|
73
|
+
is deduplicated and the build never reports an achieved count against a
|
|
74
|
+
requested one. The design notes for this milestone had given that up; they
|
|
75
|
+
were wrong, and the rule that cardinality is declared survives M2M.
|
|
76
|
+
- **What binds is the busiest group, not the product.** Every row of one
|
|
77
|
+
group needs a different partner, so the constraint is
|
|
78
|
+
`largest group <= partners` rather than `rows <= groups x partners` -- and a
|
|
79
|
+
heavy tail puts a large share of every edge on one group, `Zipf(1.2)` over
|
|
80
|
+
five thousand of them putting 21% on the top one. A declaration that looks
|
|
81
|
+
sparse against the product can still be impossible, and it cannot be known
|
|
82
|
+
until the partition is resolved: this is the one structural refusal that
|
|
83
|
+
waits for the build rather than firing where the shape is written.
|
|
84
|
+
- **The second side is derived and approximate, and the numbers are
|
|
85
|
+
published.** Both marginals plus the edge count is over-determined, and
|
|
86
|
+
fixing all three is a constraint satisfaction problem that cannot stream
|
|
87
|
+
into `COPY`. Measured against exact weighted sampling over 200,000 edges,
|
|
88
|
+
the busiest partner comes out 1.09 times as busy, the 99th percentile about
|
|
89
|
+
30% high, and about 7% fewer partners are touched.
|
|
90
|
+
- **Partners are allocated across weight bands and strided within one**, so
|
|
91
|
+
nothing is ever drawn and asked whether it is taken. Rejection sampling
|
|
92
|
+
degenerates exactly where the shape is most interesting -- 245 probes per
|
|
93
|
+
row on that world -- and the usual fix, sampling the ones to leave out when
|
|
94
|
+
a group wants more than half, is ten times faster and **a different sampling
|
|
95
|
+
rule**, so a group one row over the halfway mark would get a materially
|
|
96
|
+
different membership from one row under. The band count is **derived from
|
|
97
|
+
the declared weights** rather than chosen, which systematic allocation is
|
|
98
|
+
what makes possible: it converges as bands get finer where largest-remainder
|
|
99
|
+
runs away, so "enough" is a limit rather than a number somebody tuned.
|
|
100
|
+
|
|
101
|
+
- **`FanOutPlan.group_of`** answers which parent a row belongs to, which a
|
|
102
|
+
pairing needs to choose partners inside this row's group.
|
|
103
|
+
|
|
104
|
+
- **`Projection(..., through=Model)` says which model a derived join runs on.**
|
|
105
|
+
An abstract base carrying `created_by`/`updated_by` to `User` -- a pattern most
|
|
106
|
+
Django schemas have -- makes **any two models in the schema joinable**, so the
|
|
107
|
+
derivation had candidates everywhere and could settle nothing: not for one
|
|
108
|
+
pair but for every pair, which put the derived form out of reach for a whole
|
|
109
|
+
application and left `sql=` as the only route to this feature's own motivating
|
|
110
|
+
example. The refusal now names `through=` first, and names a model that can
|
|
111
|
+
actually resolve the join rather than the first one alphabetically -- an audit
|
|
112
|
+
model is reached by two edges from each side, so it is a real candidate and
|
|
113
|
+
never a usable answer. Where every candidate is reached more than once,
|
|
114
|
+
`through=` cannot narrow anything and the refusal says so instead of
|
|
115
|
+
suggesting it.
|
|
116
|
+
|
|
117
|
+
### Changed
|
|
118
|
+
|
|
119
|
+
- **The two-fan-out refusal names the declaration that replaces it.** Its remedy
|
|
120
|
+
used to be a statement of your own, because nothing else could keep the
|
|
121
|
+
constraint. It now names `Paired` first and keeps `Projection(columns=, sql=)`
|
|
122
|
+
as the escape hatch for an edge set that has to be a particular one.
|
|
123
|
+
|
|
124
|
+
### Fixed
|
|
125
|
+
|
|
126
|
+
- **A rounded `Uniform` can size a fan-out.**
|
|
127
|
+
`FanOut(Uniform(1, 10, places=0))` was refused with "needs numeric fan-out
|
|
128
|
+
sizes, but produced `Decimal('3')`" -- which reads as a contradiction, and was
|
|
129
|
+
one keyword away from `FanOut(Uniform(1, 10))`, the spelling this package's own
|
|
130
|
+
docstring recommends and two of its refusals suggest. `places=0` is the natural
|
|
131
|
+
way to say that a fan-out size is a count. The guard was
|
|
132
|
+
`isinstance(weight, (int, float))` while the next line already did
|
|
133
|
+
`float(weight)`, so it was stricter than the arithmetic it protected. It is now
|
|
134
|
+
the numeric tower plus `Decimal`, which `numbers.Real` deliberately excludes,
|
|
135
|
+
and `Fraction` and numpy scalars come along with it. `bool` stays refused. The
|
|
136
|
+
message also names a remedy now, which it was alone in not doing.
|
|
137
|
+
|
|
138
|
+
- **`columns=` accepts the column an `INSERT` actually lists.** The refusal that
|
|
139
|
+
sends a reader there says "the select's columns are what the insert lists", and
|
|
140
|
+
what an insert lists for a foreign key is `event_id` -- which was then refused
|
|
141
|
+
as "no field named event_id". That is a contradiction rather than a correction,
|
|
142
|
+
and the surrounding documentation reinforced the wrong reading. Both spellings
|
|
143
|
+
are accepted and resolve to the same column, the messages say so, and the same
|
|
144
|
+
dict is read for `statistics=` so the two cannot disagree about what a column
|
|
145
|
+
is called.
|
|
146
|
+
|
|
10
147
|
## [0.13.0] — 2026-09-03
|
|
11
148
|
|
|
12
149
|
### Added
|
|
@@ -881,7 +1018,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
881
1018
|
into `COPY FROM STDIN`, which psycopg 2 cannot do without materialising them
|
|
882
1019
|
first.
|
|
883
1020
|
|
|
884
|
-
[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.
|
|
1021
|
+
[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.15.0...HEAD
|
|
1022
|
+
[0.15.0]: https://github.com/Artui/django-data-shape/compare/v0.14.0...v0.15.0
|
|
1023
|
+
[0.14.0]: https://github.com/Artui/django-data-shape/compare/v0.13.0...v0.14.0
|
|
885
1024
|
[0.13.0]: https://github.com/Artui/django-data-shape/compare/v0.12.0...v0.13.0
|
|
886
1025
|
[0.12.0]: https://github.com/Artui/django-data-shape/compare/v0.11.0...v0.12.0
|
|
887
1026
|
[0.11.0]: https://github.com/Artui/django-data-shape/compare/v0.10.0...v0.11.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: django-data-shape
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: A realistically shaped test database from Django models: declare cardinality, skew and fan-out, load by COPY, and make the query planner believe it.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Artui/django-data-shape
|
|
6
6
|
Project-URL: Repository, https://github.com/Artui/django-data-shape
|
|
@@ -39,6 +39,7 @@ from django_data_shape.keys.md5_keys import Md5Keys
|
|
|
39
39
|
from django_data_shape.keys.sequential_keys import SequentialKeys
|
|
40
40
|
from django_data_shape.keys.sql_keys import SqlKeys
|
|
41
41
|
from django_data_shape.keys.uuid_keys import UuidKeys
|
|
42
|
+
from django_data_shape.paired import Paired
|
|
42
43
|
from django_data_shape.projection import Projection
|
|
43
44
|
from django_data_shape.require_clone_strategy import require_clone_strategy
|
|
44
45
|
from django_data_shape.scale_protocol import ScaleProtocol
|
|
@@ -46,6 +47,7 @@ from django_data_shape.scaled_shape import scaled_shape
|
|
|
46
47
|
from django_data_shape.scaled_world import scaled_world
|
|
47
48
|
from django_data_shape.shape import Shape
|
|
48
49
|
from django_data_shape.shape_digest import shape_digest
|
|
50
|
+
from django_data_shape.shape_from_factory import shape_from_factory
|
|
49
51
|
from django_data_shape.shape_not_empty import ShapeNotEmpty
|
|
50
52
|
from django_data_shape.table import Table
|
|
51
53
|
from django_data_shape.table_result import TableResult
|
|
@@ -79,6 +81,7 @@ __all__ = [
|
|
|
79
81
|
"KeyFunction",
|
|
80
82
|
"Md5Keys",
|
|
81
83
|
"KeyStrategy",
|
|
84
|
+
"Paired",
|
|
82
85
|
"PerParent",
|
|
83
86
|
"Projection",
|
|
84
87
|
"ScaleProtocol",
|
|
@@ -106,6 +109,7 @@ __all__ = [
|
|
|
106
109
|
"fan_out_sizes",
|
|
107
110
|
"require_clone_strategy",
|
|
108
111
|
"scaled_shape",
|
|
112
|
+
"shape_from_factory",
|
|
109
113
|
"scaled_world",
|
|
110
114
|
"shape_digest",
|
|
111
115
|
"template_database",
|
|
@@ -21,10 +21,13 @@ from django_data_shape.generate_rows import generate_rows
|
|
|
21
21
|
from django_data_shape.invalid_shape import InvalidShape
|
|
22
22
|
from django_data_shape.keys.disjoint import Disjoint
|
|
23
23
|
from django_data_shape.order_tables import order_tables
|
|
24
|
+
from django_data_shape.paired import Paired
|
|
25
|
+
from django_data_shape.paired_plan import PairedPlan
|
|
24
26
|
from django_data_shape.projection import Projection
|
|
25
27
|
from django_data_shape.refuse_queries import refuse_queries
|
|
26
28
|
from django_data_shape.require_postgres import require_postgres
|
|
27
29
|
from django_data_shape.resolve_fan_out import resolve_fan_out
|
|
30
|
+
from django_data_shape.resolve_paired import resolve_paired
|
|
28
31
|
from django_data_shape.shape import Shape
|
|
29
32
|
from django_data_shape.shape_not_empty import ShapeNotEmpty
|
|
30
33
|
from django_data_shape.table import Table
|
|
@@ -115,8 +118,8 @@ def build(
|
|
|
115
118
|
if isinstance(table, Projection):
|
|
116
119
|
loaded = _project(connection, table, shape.seed)
|
|
117
120
|
else:
|
|
118
|
-
plans = _resolve(connection, table, shape.seed)
|
|
119
|
-
loaded = _load(connection, table, shape.seed, plans)
|
|
121
|
+
plans, pairings = _resolve(connection, table, shape.seed)
|
|
122
|
+
loaded = _load(connection, table, shape.seed, plans, pairings)
|
|
120
123
|
_reset_sequence(connection, table.model)
|
|
121
124
|
_analyze(connection, table.db_table)
|
|
122
125
|
results.append(TableResult(table=table.db_table, rows=loaded))
|
|
@@ -163,11 +166,21 @@ def _project(connection: Any, projection: Projection, seed: int) -> int:
|
|
|
163
166
|
return inserted
|
|
164
167
|
|
|
165
168
|
|
|
166
|
-
def _resolve(
|
|
167
|
-
|
|
169
|
+
def _resolve(
|
|
170
|
+
connection: Any, table: Table, seed: int
|
|
171
|
+
) -> tuple[dict[str, FanOutPlan], dict[str, PairedPlan]]:
|
|
172
|
+
"""Partition each declared relation over the parent keys that exist.
|
|
173
|
+
|
|
174
|
+
Fan-outs first and pairings second, because a pairing chooses partners
|
|
175
|
+
*inside* the groups a fan-out produced -- so the partition it reads has to
|
|
176
|
+
exist before it can be resolved. That ordering is the mechanism rather than
|
|
177
|
+
a detail of this function: it is what makes an edge table one pass.
|
|
178
|
+
"""
|
|
168
179
|
parent_fields = table.parent_fields()
|
|
169
180
|
plans: dict[str, FanOutPlan] = {}
|
|
170
181
|
for name, field in table.relations():
|
|
182
|
+
if isinstance(table.fields[name], Paired):
|
|
183
|
+
continue
|
|
171
184
|
plans[name] = resolve_fan_out(
|
|
172
185
|
cast("FanOut", table.fields[name]),
|
|
173
186
|
cast("type[Model]", field.related_model),
|
|
@@ -178,7 +191,22 @@ def _resolve(connection: Any, table: Table, seed: int) -> dict[str, FanOutPlan]:
|
|
|
178
191
|
connection,
|
|
179
192
|
parent_fields.get(name, ()),
|
|
180
193
|
)
|
|
181
|
-
|
|
194
|
+
pairings: dict[str, PairedPlan] = {}
|
|
195
|
+
for name, field in table.relations():
|
|
196
|
+
declared = table.fields[name]
|
|
197
|
+
if not isinstance(declared, Paired):
|
|
198
|
+
continue
|
|
199
|
+
pairings[name] = resolve_paired(
|
|
200
|
+
declared,
|
|
201
|
+
cast("type[Model]", field.related_model),
|
|
202
|
+
plans[declared.relation],
|
|
203
|
+
table.rows,
|
|
204
|
+
seed,
|
|
205
|
+
table.db_table,
|
|
206
|
+
name,
|
|
207
|
+
connection,
|
|
208
|
+
)
|
|
209
|
+
return plans, pairings
|
|
182
210
|
|
|
183
211
|
|
|
184
212
|
def _require_empty(connection: Any, table: Table | Projection) -> None:
|
|
@@ -228,7 +256,13 @@ def _require_empty(connection: Any, table: Table | Projection) -> None:
|
|
|
228
256
|
# API reached through Django's cursor wrapper, and it appears on no Django base
|
|
229
257
|
# class, so annotating the real wrapper type would mean asserting the checker
|
|
230
258
|
# out of the way on every line that uses it.
|
|
231
|
-
def _load(
|
|
259
|
+
def _load(
|
|
260
|
+
connection: Any,
|
|
261
|
+
table: Table,
|
|
262
|
+
seed: int,
|
|
263
|
+
plans: dict[str, FanOutPlan],
|
|
264
|
+
pairings: dict[str, PairedPlan],
|
|
265
|
+
) -> int:
|
|
232
266
|
"""Stream generated rows into the table, by the fastest route the backend has.
|
|
233
267
|
|
|
234
268
|
Each declared value is passed through its field's ``get_db_prep_save``
|
|
@@ -254,7 +288,7 @@ def _load(connection: Any, table: Table, seed: int, plans: dict[str, FanOutPlan]
|
|
|
254
288
|
prepare = [pk_field.get_db_prep_save] + [field.get_db_prep_save for _, field in table.columns()]
|
|
255
289
|
rows = (
|
|
256
290
|
tuple(prep(value, connection) for prep, value in zip(prepare, row, strict=True))
|
|
257
|
-
for row in generate_rows(table, seed, plans)
|
|
291
|
+
for row in generate_rows(table, seed, plans, pairings)
|
|
258
292
|
)
|
|
259
293
|
|
|
260
294
|
# The guard is built here rather than inside each route because it belongs
|
{django_data_shape-0.13.0 → django_data_shape-0.15.0}/django_data_shape/check_constraints.py
RENAMED
|
@@ -163,8 +163,12 @@ def _check_independent_fan_outs(
|
|
|
163
163
|
That is why this is otherwise a refusal and not a wider capacity
|
|
164
164
|
calculation: no arithmetic over the two marginals decides it, because the
|
|
165
165
|
failure is that nothing enumerates the combinations at all. Deduplicated edges are a
|
|
166
|
-
different algorithm rather than a bigger loop
|
|
167
|
-
|
|
166
|
+
different algorithm rather than a bigger loop. **That algorithm now exists**
|
|
167
|
+
as :class:`~django_data_shape.paired.Paired`, which is the remedy this
|
|
168
|
+
refusal names: one side stays a fan-out and the other takes distinct
|
|
169
|
+
partners inside its groups, so a duplicate pair is impossible rather than
|
|
170
|
+
removed. A statement of your own is still the answer for an edge set that
|
|
171
|
+
has to be a particular one --
|
|
168
172
|
:class:`~django_data_shape.projection.Projection` with ``columns=`` and
|
|
169
173
|
``sql=`` fills the table from a select that can deduplicate.
|
|
170
174
|
"""
|
|
@@ -191,10 +195,11 @@ def _check_independent_fan_outs(
|
|
|
191
195
|
"pairs come out together is an artefact of that index, and two rows sharing a pair is a "
|
|
192
196
|
"matter of the seed. The combinations do fit, which is why the pigeonhole arithmetic "
|
|
193
197
|
"lets this through -- what is missing is anything that enumerates them, and the load "
|
|
194
|
-
"then fails inside COPY at a row number that moves when the seed does.
|
|
195
|
-
f"
|
|
196
|
-
"
|
|
197
|
-
"
|
|
198
|
+
"then fails inside COPY at a row number that moves when the seed does. An edge table "
|
|
199
|
+
f"declares one side and pairs the other: {fanned[1]}=Paired({fanned[0]!r}, Zipf()) takes "
|
|
200
|
+
f"distinct partners inside each {fanned[0]} group, so no pair can repeat and the edge "
|
|
201
|
+
"count is still exactly the row count. An edge set that has to be a particular one is "
|
|
202
|
+
f"filled by a statement instead -- Projection({where}, columns=(...), sql=...)."
|
|
198
203
|
)
|
|
199
204
|
|
|
200
205
|
|
|
@@ -101,6 +101,15 @@ class FanOutPlan:
|
|
|
101
101
|
end = self._starts[index + 1] if index + 1 < len(self._starts) else self._rows
|
|
102
102
|
return slot - start, end - start
|
|
103
103
|
|
|
104
|
+
def group_of(self, row: int) -> int:
|
|
105
|
+
"""Which parent this row belongs to, as an index into the parent keys.
|
|
106
|
+
|
|
107
|
+
Public because a paired column needs it: the partner is chosen inside
|
|
108
|
+
this row's group, so "which group" and "where in it" are both asked of
|
|
109
|
+
the partition rather than recomputed beside it.
|
|
110
|
+
"""
|
|
111
|
+
return self._slot(row)
|
|
112
|
+
|
|
104
113
|
def _slot(self, row: int) -> int:
|
|
105
114
|
"""Which parent owns this child row, as an index into ``keys``."""
|
|
106
115
|
slot = (row * self._stride) % self._rows
|
|
@@ -10,6 +10,8 @@ from django_data_shape.derivations.derivation import Derivation
|
|
|
10
10
|
from django_data_shape.derivations.scope import Scope
|
|
11
11
|
from django_data_shape.distributions.distribution import Distribution
|
|
12
12
|
from django_data_shape.fan_out_plan import FanOutPlan
|
|
13
|
+
from django_data_shape.paired import Paired
|
|
14
|
+
from django_data_shape.paired_plan import PairedPlan
|
|
13
15
|
from django_data_shape.table import Table
|
|
14
16
|
from django_data_shape.utils import draw, field_stream
|
|
15
17
|
|
|
@@ -20,7 +22,10 @@ _Source = Callable[[int, list[object]], object]
|
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
def generate_rows(
|
|
23
|
-
table: Table,
|
|
25
|
+
table: Table,
|
|
26
|
+
seed: int,
|
|
27
|
+
plans: Mapping[str, FanOutPlan] | None = None,
|
|
28
|
+
pairings: Mapping[str, PairedPlan] | None = None,
|
|
24
29
|
) -> Iterator[tuple[Any, ...]]:
|
|
25
30
|
"""Yield one tuple per row: the primary key, then each declared column.
|
|
26
31
|
|
|
@@ -54,6 +59,7 @@ def generate_rows(
|
|
|
54
59
|
nothing but peak RSS.
|
|
55
60
|
"""
|
|
56
61
|
plans = plans or {}
|
|
62
|
+
pairings = pairings or {}
|
|
57
63
|
keys = table.keys
|
|
58
64
|
key_stream = field_stream(seed, table.db_table, ":key")
|
|
59
65
|
columns = table.columns()
|
|
@@ -64,7 +70,7 @@ def generate_rows(
|
|
|
64
70
|
# of column this is, and where each of its sources comes from, are decided
|
|
65
71
|
# once rather than a million times.
|
|
66
72
|
steps: list[_Step] = [
|
|
67
|
-
(slot_of[name], _producer(table, seed, plans, name))
|
|
73
|
+
(slot_of[name], _producer(table, seed, plans, pairings, name))
|
|
68
74
|
for name, _field in columns
|
|
69
75
|
if name not in table.computation_order()
|
|
70
76
|
]
|
|
@@ -81,12 +87,23 @@ def generate_rows(
|
|
|
81
87
|
|
|
82
88
|
|
|
83
89
|
def _producer(
|
|
84
|
-
table: Table,
|
|
90
|
+
table: Table,
|
|
91
|
+
seed: int,
|
|
92
|
+
plans: Mapping[str, FanOutPlan],
|
|
93
|
+
pairings: Mapping[str, PairedPlan],
|
|
94
|
+
name: str,
|
|
85
95
|
) -> Callable[[int, list[object]], object]:
|
|
86
96
|
"""The step for a column that depends on nothing already in the row."""
|
|
87
97
|
plan = plans.get(name)
|
|
88
98
|
if plan is not None:
|
|
89
99
|
return partial(_from_plan, plan)
|
|
100
|
+
pairing = pairings.get(name)
|
|
101
|
+
if pairing is not None:
|
|
102
|
+
# Two questions of the fan-out this pairs with -- which group, and where
|
|
103
|
+
# in it -- and one of the pairing. All three are O(1) because both are
|
|
104
|
+
# partitions of the row range rather than per-row draws.
|
|
105
|
+
declared = cast("Paired", table.fields[name])
|
|
106
|
+
return partial(_from_pairing, plans[declared.relation], pairing)
|
|
90
107
|
distribution = cast("Distribution", table.fields[name])
|
|
91
108
|
return partial(_from_distribution, distribution, field_stream(seed, table.db_table, name))
|
|
92
109
|
|
|
@@ -149,6 +166,11 @@ def _from_plan(plan: FanOutPlan, row: int, values: list[object]) -> object:
|
|
|
149
166
|
return plan.key_for(row)
|
|
150
167
|
|
|
151
168
|
|
|
169
|
+
def _from_pairing(plan: FanOutPlan, pairing: PairedPlan, row: int, values: list[object]) -> object:
|
|
170
|
+
position, _size = plan.group_position(row)
|
|
171
|
+
return pairing.partner_for(plan.group_of(row), position)
|
|
172
|
+
|
|
173
|
+
|
|
152
174
|
def _from_derivation(
|
|
153
175
|
derivation: Derivation,
|
|
154
176
|
stream: int,
|
|
@@ -47,17 +47,26 @@ class Md5Keys:
|
|
|
47
47
|
a test computes both for the same rows and compares them, which is the
|
|
48
48
|
only form of agreement that means anything here.
|
|
49
49
|
|
|
50
|
-
``md5`` takes the same eight-byte big-endian pair Python hashes,
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
50
|
+
``md5`` takes the same eight-byte big-endian pair Python hashes, turned
|
|
51
|
+
back into bytes by ``decode``, so neither side is hashing a rendering of
|
|
52
|
+
the other's input. The stamp is two ``overlay`` calls on the hex digest:
|
|
53
|
+
nibble 13 is the version and is always ``4``, and nibble 17 is the
|
|
54
|
+
variant, which keeps its low two bits and takes ``8`` in the high two.
|
|
55
|
+
|
|
56
|
+
**The stream is embedded as hex rather than converted by the server**,
|
|
57
|
+
and that is a fix rather than a preference. It used to be written
|
|
58
|
+
``to_hex(<stream>::bigint)``, which asks PostgreSQL to re-derive a
|
|
59
|
+
number Python produced as *unsigned* -- and ``bigint`` is signed, so any
|
|
60
|
+
stream above 2^63 raised ``NumericValueOutOfRange`` before a row was
|
|
61
|
+
written. That is a coin flip per table, not a property of any schema:
|
|
62
|
+
the stream is a hash of the table and field names, so roughly half of
|
|
63
|
+
all table names land above the limit. It is a constant by the time this
|
|
64
|
+
statement is built, so nothing about it needs deriving at all.
|
|
65
|
+
|
|
66
|
+
``row`` stays a conversion because it is genuinely an expression the
|
|
67
|
+
server evaluates, and a row index is far below the limit.
|
|
56
68
|
"""
|
|
57
|
-
digest = (
|
|
58
|
-
f"md5(decode(lpad(to_hex({stream}::bigint), 16, '0') || "
|
|
59
|
-
f"lpad(to_hex(({row})::bigint), 16, '0'), 'hex'))"
|
|
60
|
-
)
|
|
69
|
+
digest = f"md5(decode('{stream:016x}' || lpad(to_hex(({row})::bigint), 16, '0'), 'hex'))"
|
|
61
70
|
variant = f"to_hex((('x' || substr({digest}, 17, 1))::bit(4)::int & 3) | 8)"
|
|
62
71
|
return (
|
|
63
72
|
f"(overlay(overlay({digest} placing '4' from 13 for 1) "
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The other end of a many-to-many edge, distinct within its partner's group."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from django_data_shape.distributions.distribution import Distribution
|
|
6
|
+
from django_data_shape.invalid_shape import InvalidShape
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Paired:
|
|
10
|
+
"""A relation filled with the opposite end of an edge, never repeating a pair.
|
|
11
|
+
|
|
12
|
+
The declaration a through table needs, and the reason two
|
|
13
|
+
:class:`~django_data_shape.fan_out.FanOut` declarations are refused beside
|
|
14
|
+
one another. Two fan-outs partition the same rows without either seeing the
|
|
15
|
+
other, so nothing enumerates the pairs and whether two rows collide is a
|
|
16
|
+
matter of the seed. ``Paired`` is the half that looks:
|
|
17
|
+
|
|
18
|
+
.. code-block:: python
|
|
19
|
+
|
|
20
|
+
Table(
|
|
21
|
+
Membership,
|
|
22
|
+
rows=50_000,
|
|
23
|
+
company=FanOut(Zipf()),
|
|
24
|
+
person=Paired("company", Zipf()),
|
|
25
|
+
role=Constant("member"),
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
``company`` partitions the rows as any fan-out does -- that is the **declared**
|
|
29
|
+
degree distribution, and it is exact. Within each of its groups, ``person``
|
|
30
|
+
takes that many **distinct** partners, so a duplicate pair is impossible by
|
|
31
|
+
construction rather than removed afterwards: the same company implies a
|
|
32
|
+
different person, and a different company implies a different pair. The edge
|
|
33
|
+
count is therefore exactly ``rows``, which is the rule this package holds to
|
|
34
|
+
everywhere else and which a deduplicating generator would have given up.
|
|
35
|
+
|
|
36
|
+
``relation`` names the fan-out this one is paired with, spelled the way
|
|
37
|
+
:class:`~django_data_shape.derivations.per_parent.PerParent` names the
|
|
38
|
+
relation it groups by, and for the same reason: a table with three foreign
|
|
39
|
+
keys should not make a reader guess which one the grouping is over.
|
|
40
|
+
|
|
41
|
+
``weights`` decides which partners are popular. It is a
|
|
42
|
+
:class:`~django_data_shape.distributions.distribution.Distribution` over the
|
|
43
|
+
partner table the same way a fan-out's ``sizes`` is over the parent table,
|
|
44
|
+
so ``Zipf()`` gives the realistic heavy tail and a flat one gives the
|
|
45
|
+
uniform shape this package exists to argue against.
|
|
46
|
+
|
|
47
|
+
**The second side's marginal is derived, not declared, and that is on
|
|
48
|
+
purpose.** Both marginals plus the edge count is over-determined: fixing all
|
|
49
|
+
three is a constraint satisfaction problem, and a CSP cannot stream into
|
|
50
|
+
``COPY``. So the edge count and one side's distribution are declared and the
|
|
51
|
+
other side follows -- and what follows *approximates* ``weights`` rather than
|
|
52
|
+
reproducing it. Measured against exact weighted sampling without
|
|
53
|
+
replacement, the busiest partner comes out about 1.1 times as busy, the 99th
|
|
54
|
+
percentile about 30% high, and about 7% fewer partners are touched at all.
|
|
55
|
+
Those numbers are in the documentation rather than only here, because a
|
|
56
|
+
derived shape nobody quotes is a shape nobody can check.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, relation: str, weights: Distribution) -> None:
|
|
60
|
+
if not isinstance(relation, str) or not relation:
|
|
61
|
+
raise InvalidShape(
|
|
62
|
+
f"Paired needs the name of the fan-out it is paired with, got {relation!r}. "
|
|
63
|
+
"A table with more than one foreign key cannot be left to guess which one the "
|
|
64
|
+
"pairing is over."
|
|
65
|
+
)
|
|
66
|
+
self._relation = relation
|
|
67
|
+
self._weights = weights
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def relation(self) -> str:
|
|
71
|
+
return self._relation
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def weights(self) -> Distribution:
|
|
75
|
+
return self._weights
|
|
76
|
+
|
|
77
|
+
def canonical(self) -> object:
|
|
78
|
+
"""The fan-out it pairs with and the weights over partners. See ``Canonical``."""
|
|
79
|
+
return (self._relation, self._weights)
|
|
80
|
+
|
|
81
|
+
def __repr__(self) -> str:
|
|
82
|
+
return f"Paired({self._relation!r}, {self._weights!r})"
|