django-data-shape 0.13.0__tar.gz → 0.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/CHANGELOG.md +88 -1
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/PKG-INFO +1 -1
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/__init__.py +2 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/build.py +41 -7
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/check_constraints.py +11 -6
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/fan_out_plan.py +9 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/generate_rows.py +25 -3
- django_data_shape-0.14.0/django_data_shape/paired.py +82 -0
- django_data_shape-0.14.0/django_data_shape/paired_plan.py +145 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/projection.py +165 -27
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/resolve_fan_out.py +23 -2
- django_data_shape-0.14.0/django_data_shape/resolve_paired.py +145 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/table.py +46 -3
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/version.py +1 -1
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/projections.md +35 -1
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/reference.md +2 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/relations.md +71 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/pyproject.toml +1 -1
- django_data_shape-0.14.0/tests/test_build_m2m.py +100 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_build_projection.py +55 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_check_constraints.py +45 -0
- django_data_shape-0.14.0/tests/test_paired.py +34 -0
- django_data_shape-0.14.0/tests/test_paired_plan.py +116 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_projection.py +138 -3
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_resolve_fan_out.py +43 -0
- django_data_shape-0.14.0/tests/test_resolve_paired.py +117 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_table.py +60 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/testapp/models.py +61 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.github/CODEOWNERS +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.github/SECURITY.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.github/dependabot.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.github/workflows/release.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.github/workflows/tests.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.github/workflows/upstream-drift.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.gitignore +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/.pre-commit-config.yaml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/CLAUDE.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/LICENSE +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/Makefile +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/README.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/apply_statistics_targets.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/build_result.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/canonical.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/check_invariants.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/children_per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/clone_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivation_queried_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/after.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/aligned.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/derivation.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/derived.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/given.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/derivations/scope.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/ascending.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/bounded.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/categorical.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/constant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/distinct.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/distribution.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/sequential.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/skew.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/uniform.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/distributions/zipf.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/drop_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/fan_out.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/fan_out_sizes.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/fixtures/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/fixtures/scale_fixture.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/fixtures/shape_fixture.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/fixtures/skip_unless_postgres.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/infer_key_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/invalid_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/invariant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/invariant_violated.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/disjoint.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/key_function.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/key_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/md5_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/sequential_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/sql_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/keys/uuid_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/order_derivations.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/order_tables.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/py.typed +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/refuse_queries.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/require_clone_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/require_postgres.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/scale_protocol.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/scaled_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/scaled_world.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/shape_digest.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/shape_not_empty.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/table_result.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/template_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/unhashable_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/unsupported_backend.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/utils.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/world_changed.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/derivations.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/index.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/invariants.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/keys.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/pytest.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/docs/statistics.md +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/mkdocs.yml +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/scripts/release-publish.sh +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/conftest.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/conftest_settings.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/derivations/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/derivations/test_after.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/derivations/test_aligned.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/derivations/test_derived.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/derivations/test_given.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/derivations/test_per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/distributions/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/distributions/test_constant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/distributions/test_sequential.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/distributions/test_skew.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/distributions/test_uniform.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/distributions/test_zipf.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/keys/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/keys/test_key_function.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/keys/test_md5_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/keys/test_sequential_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/keys/test_uuid_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/scale_protocol_consumers.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/scale_protocol_impostors.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_apply_statistics_targets.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_build.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_build_derivations.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_build_graph.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_build_invariants.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_build_keys.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_build_portable.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_children_per_parent.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_documentation.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_fan_out.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_fan_out_sizes.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_fixtures.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_generate_rows.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_infer_key_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_invariant.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_order_derivations.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_order_tables.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_reference_documents_the_public_surface.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_refuse_queries.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_require_clone_strategy.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_require_postgres.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_scale_protocol.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_scaled_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_scaled_world.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_shape.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_shape_digest.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_template_database.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_utils.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/test_version.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/tests/testapp/__init__.py +0 -0
- {django_data_shape-0.13.0 → django_data_shape-0.14.0}/uv.lock +0 -0
|
@@ -7,6 +7,92 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.14.0] — 2026-09-03
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **`Paired` fills the second key of a many-to-many, and the edge count stays
|
|
15
|
+
exact.** Two fan-outs on one through table are refused because a fan-out is a
|
|
16
|
+
partition computed from the row index alone, so two of them partition the same
|
|
17
|
+
rows without either seeing the other -- nothing enumerates the pairs and a
|
|
18
|
+
collision is a matter of the seed. `person=Paired("company", Zipf())` is the
|
|
19
|
+
half that looks: `company` partitions the rows as any fan-out does, and within
|
|
20
|
+
each of its groups `person` takes that many **distinct** partners. A duplicate
|
|
21
|
+
pair is impossible by construction rather than removed afterwards, so nothing
|
|
22
|
+
is deduplicated and the build never reports an achieved count against a
|
|
23
|
+
requested one. The design notes for this milestone had given that up; they
|
|
24
|
+
were wrong, and the rule that cardinality is declared survives M2M.
|
|
25
|
+
- **What binds is the busiest group, not the product.** Every row of one
|
|
26
|
+
group needs a different partner, so the constraint is
|
|
27
|
+
`largest group <= partners` rather than `rows <= groups x partners` -- and a
|
|
28
|
+
heavy tail puts a large share of every edge on one group, `Zipf(1.2)` over
|
|
29
|
+
five thousand of them putting 21% on the top one. A declaration that looks
|
|
30
|
+
sparse against the product can still be impossible, and it cannot be known
|
|
31
|
+
until the partition is resolved: this is the one structural refusal that
|
|
32
|
+
waits for the build rather than firing where the shape is written.
|
|
33
|
+
- **The second side is derived and approximate, and the numbers are
|
|
34
|
+
published.** Both marginals plus the edge count is over-determined, and
|
|
35
|
+
fixing all three is a constraint satisfaction problem that cannot stream
|
|
36
|
+
into `COPY`. Measured against exact weighted sampling over 200,000 edges,
|
|
37
|
+
the busiest partner comes out 1.09 times as busy, the 99th percentile about
|
|
38
|
+
30% high, and about 7% fewer partners are touched.
|
|
39
|
+
- **Partners are allocated across weight bands and strided within one**, so
|
|
40
|
+
nothing is ever drawn and asked whether it is taken. Rejection sampling
|
|
41
|
+
degenerates exactly where the shape is most interesting -- 245 probes per
|
|
42
|
+
row on that world -- and the usual fix, sampling the ones to leave out when
|
|
43
|
+
a group wants more than half, is ten times faster and **a different sampling
|
|
44
|
+
rule**, so a group one row over the halfway mark would get a materially
|
|
45
|
+
different membership from one row under. The band count is **derived from
|
|
46
|
+
the declared weights** rather than chosen, which systematic allocation is
|
|
47
|
+
what makes possible: it converges as bands get finer where largest-remainder
|
|
48
|
+
runs away, so "enough" is a limit rather than a number somebody tuned.
|
|
49
|
+
|
|
50
|
+
- **`FanOutPlan.group_of`** answers which parent a row belongs to, which a
|
|
51
|
+
pairing needs to choose partners inside this row's group.
|
|
52
|
+
|
|
53
|
+
- **`Projection(..., through=Model)` says which model a derived join runs on.**
|
|
54
|
+
An abstract base carrying `created_by`/`updated_by` to `User` -- a pattern most
|
|
55
|
+
Django schemas have -- makes **any two models in the schema joinable**, so the
|
|
56
|
+
derivation had candidates everywhere and could settle nothing: not for one
|
|
57
|
+
pair but for every pair, which put the derived form out of reach for a whole
|
|
58
|
+
application and left `sql=` as the only route to this feature's own motivating
|
|
59
|
+
example. The refusal now names `through=` first, and names a model that can
|
|
60
|
+
actually resolve the join rather than the first one alphabetically -- an audit
|
|
61
|
+
model is reached by two edges from each side, so it is a real candidate and
|
|
62
|
+
never a usable answer. Where every candidate is reached more than once,
|
|
63
|
+
`through=` cannot narrow anything and the refusal says so instead of
|
|
64
|
+
suggesting it.
|
|
65
|
+
|
|
66
|
+
### Changed
|
|
67
|
+
|
|
68
|
+
- **The two-fan-out refusal names the declaration that replaces it.** Its remedy
|
|
69
|
+
used to be a statement of your own, because nothing else could keep the
|
|
70
|
+
constraint. It now names `Paired` first and keeps `Projection(columns=, sql=)`
|
|
71
|
+
as the escape hatch for an edge set that has to be a particular one.
|
|
72
|
+
|
|
73
|
+
### Fixed
|
|
74
|
+
|
|
75
|
+
- **A rounded `Uniform` can size a fan-out.**
|
|
76
|
+
`FanOut(Uniform(1, 10, places=0))` was refused with "needs numeric fan-out
|
|
77
|
+
sizes, but produced `Decimal('3')`" -- which reads as a contradiction, and was
|
|
78
|
+
one keyword away from `FanOut(Uniform(1, 10))`, the spelling this package's own
|
|
79
|
+
docstring recommends and two of its refusals suggest. `places=0` is the natural
|
|
80
|
+
way to say that a fan-out size is a count. The guard was
|
|
81
|
+
`isinstance(weight, (int, float))` while the next line already did
|
|
82
|
+
`float(weight)`, so it was stricter than the arithmetic it protected. It is now
|
|
83
|
+
the numeric tower plus `Decimal`, which `numbers.Real` deliberately excludes,
|
|
84
|
+
and `Fraction` and numpy scalars come along with it. `bool` stays refused. The
|
|
85
|
+
message also names a remedy now, which it was alone in not doing.
|
|
86
|
+
|
|
87
|
+
- **`columns=` accepts the column an `INSERT` actually lists.** The refusal that
|
|
88
|
+
sends a reader there says "the select's columns are what the insert lists", and
|
|
89
|
+
what an insert lists for a foreign key is `event_id` -- which was then refused
|
|
90
|
+
as "no field named event_id". That is a contradiction rather than a correction,
|
|
91
|
+
and the surrounding documentation reinforced the wrong reading. Both spellings
|
|
92
|
+
are accepted and resolve to the same column, the messages say so, and the same
|
|
93
|
+
dict is read for `statistics=` so the two cannot disagree about what a column
|
|
94
|
+
is called.
|
|
95
|
+
|
|
10
96
|
## [0.13.0] — 2026-09-03
|
|
11
97
|
|
|
12
98
|
### Added
|
|
@@ -881,7 +967,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
881
967
|
into `COPY FROM STDIN`, which psycopg 2 cannot do without materialising them
|
|
882
968
|
first.
|
|
883
969
|
|
|
884
|
-
[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.
|
|
970
|
+
[Unreleased]: https://github.com/Artui/django-data-shape/compare/v0.14.0...HEAD
|
|
971
|
+
[0.14.0]: https://github.com/Artui/django-data-shape/compare/v0.13.0...v0.14.0
|
|
885
972
|
[0.13.0]: https://github.com/Artui/django-data-shape/compare/v0.12.0...v0.13.0
|
|
886
973
|
[0.12.0]: https://github.com/Artui/django-data-shape/compare/v0.11.0...v0.12.0
|
|
887
974
|
[0.11.0]: https://github.com/Artui/django-data-shape/compare/v0.10.0...v0.11.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: django-data-shape
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.14.0
|
|
4
4
|
Summary: A realistically shaped test database from Django models: declare cardinality, skew and fan-out, load by COPY, and make the query planner believe it.
|
|
5
5
|
Project-URL: Homepage, https://github.com/Artui/django-data-shape
|
|
6
6
|
Project-URL: Repository, https://github.com/Artui/django-data-shape
|
|
@@ -39,6 +39,7 @@ from django_data_shape.keys.md5_keys import Md5Keys
|
|
|
39
39
|
from django_data_shape.keys.sequential_keys import SequentialKeys
|
|
40
40
|
from django_data_shape.keys.sql_keys import SqlKeys
|
|
41
41
|
from django_data_shape.keys.uuid_keys import UuidKeys
|
|
42
|
+
from django_data_shape.paired import Paired
|
|
42
43
|
from django_data_shape.projection import Projection
|
|
43
44
|
from django_data_shape.require_clone_strategy import require_clone_strategy
|
|
44
45
|
from django_data_shape.scale_protocol import ScaleProtocol
|
|
@@ -79,6 +80,7 @@ __all__ = [
|
|
|
79
80
|
"KeyFunction",
|
|
80
81
|
"Md5Keys",
|
|
81
82
|
"KeyStrategy",
|
|
83
|
+
"Paired",
|
|
82
84
|
"PerParent",
|
|
83
85
|
"Projection",
|
|
84
86
|
"ScaleProtocol",
|
|
@@ -21,10 +21,13 @@ from django_data_shape.generate_rows import generate_rows
|
|
|
21
21
|
from django_data_shape.invalid_shape import InvalidShape
|
|
22
22
|
from django_data_shape.keys.disjoint import Disjoint
|
|
23
23
|
from django_data_shape.order_tables import order_tables
|
|
24
|
+
from django_data_shape.paired import Paired
|
|
25
|
+
from django_data_shape.paired_plan import PairedPlan
|
|
24
26
|
from django_data_shape.projection import Projection
|
|
25
27
|
from django_data_shape.refuse_queries import refuse_queries
|
|
26
28
|
from django_data_shape.require_postgres import require_postgres
|
|
27
29
|
from django_data_shape.resolve_fan_out import resolve_fan_out
|
|
30
|
+
from django_data_shape.resolve_paired import resolve_paired
|
|
28
31
|
from django_data_shape.shape import Shape
|
|
29
32
|
from django_data_shape.shape_not_empty import ShapeNotEmpty
|
|
30
33
|
from django_data_shape.table import Table
|
|
@@ -115,8 +118,8 @@ def build(
|
|
|
115
118
|
if isinstance(table, Projection):
|
|
116
119
|
loaded = _project(connection, table, shape.seed)
|
|
117
120
|
else:
|
|
118
|
-
plans = _resolve(connection, table, shape.seed)
|
|
119
|
-
loaded = _load(connection, table, shape.seed, plans)
|
|
121
|
+
plans, pairings = _resolve(connection, table, shape.seed)
|
|
122
|
+
loaded = _load(connection, table, shape.seed, plans, pairings)
|
|
120
123
|
_reset_sequence(connection, table.model)
|
|
121
124
|
_analyze(connection, table.db_table)
|
|
122
125
|
results.append(TableResult(table=table.db_table, rows=loaded))
|
|
@@ -163,11 +166,21 @@ def _project(connection: Any, projection: Projection, seed: int) -> int:
|
|
|
163
166
|
return inserted
|
|
164
167
|
|
|
165
168
|
|
|
166
|
-
def _resolve(
|
|
167
|
-
|
|
169
|
+
def _resolve(
|
|
170
|
+
connection: Any, table: Table, seed: int
|
|
171
|
+
) -> tuple[dict[str, FanOutPlan], dict[str, PairedPlan]]:
|
|
172
|
+
"""Partition each declared relation over the parent keys that exist.
|
|
173
|
+
|
|
174
|
+
Fan-outs first and pairings second, because a pairing chooses partners
|
|
175
|
+
*inside* the groups a fan-out produced -- so the partition it reads has to
|
|
176
|
+
exist before it can be resolved. That ordering is the mechanism rather than
|
|
177
|
+
a detail of this function: it is what makes an edge table one pass.
|
|
178
|
+
"""
|
|
168
179
|
parent_fields = table.parent_fields()
|
|
169
180
|
plans: dict[str, FanOutPlan] = {}
|
|
170
181
|
for name, field in table.relations():
|
|
182
|
+
if isinstance(table.fields[name], Paired):
|
|
183
|
+
continue
|
|
171
184
|
plans[name] = resolve_fan_out(
|
|
172
185
|
cast("FanOut", table.fields[name]),
|
|
173
186
|
cast("type[Model]", field.related_model),
|
|
@@ -178,7 +191,22 @@ def _resolve(connection: Any, table: Table, seed: int) -> dict[str, FanOutPlan]:
|
|
|
178
191
|
connection,
|
|
179
192
|
parent_fields.get(name, ()),
|
|
180
193
|
)
|
|
181
|
-
|
|
194
|
+
pairings: dict[str, PairedPlan] = {}
|
|
195
|
+
for name, field in table.relations():
|
|
196
|
+
declared = table.fields[name]
|
|
197
|
+
if not isinstance(declared, Paired):
|
|
198
|
+
continue
|
|
199
|
+
pairings[name] = resolve_paired(
|
|
200
|
+
declared,
|
|
201
|
+
cast("type[Model]", field.related_model),
|
|
202
|
+
plans[declared.relation],
|
|
203
|
+
table.rows,
|
|
204
|
+
seed,
|
|
205
|
+
table.db_table,
|
|
206
|
+
name,
|
|
207
|
+
connection,
|
|
208
|
+
)
|
|
209
|
+
return plans, pairings
|
|
182
210
|
|
|
183
211
|
|
|
184
212
|
def _require_empty(connection: Any, table: Table | Projection) -> None:
|
|
@@ -228,7 +256,13 @@ def _require_empty(connection: Any, table: Table | Projection) -> None:
|
|
|
228
256
|
# API reached through Django's cursor wrapper, and it appears on no Django base
|
|
229
257
|
# class, so annotating the real wrapper type would mean asserting the checker
|
|
230
258
|
# out of the way on every line that uses it.
|
|
231
|
-
def _load(
|
|
259
|
+
def _load(
|
|
260
|
+
connection: Any,
|
|
261
|
+
table: Table,
|
|
262
|
+
seed: int,
|
|
263
|
+
plans: dict[str, FanOutPlan],
|
|
264
|
+
pairings: dict[str, PairedPlan],
|
|
265
|
+
) -> int:
|
|
232
266
|
"""Stream generated rows into the table, by the fastest route the backend has.
|
|
233
267
|
|
|
234
268
|
Each declared value is passed through its field's ``get_db_prep_save``
|
|
@@ -254,7 +288,7 @@ def _load(connection: Any, table: Table, seed: int, plans: dict[str, FanOutPlan]
|
|
|
254
288
|
prepare = [pk_field.get_db_prep_save] + [field.get_db_prep_save for _, field in table.columns()]
|
|
255
289
|
rows = (
|
|
256
290
|
tuple(prep(value, connection) for prep, value in zip(prepare, row, strict=True))
|
|
257
|
-
for row in generate_rows(table, seed, plans)
|
|
291
|
+
for row in generate_rows(table, seed, plans, pairings)
|
|
258
292
|
)
|
|
259
293
|
|
|
260
294
|
# The guard is built here rather than inside each route because it belongs
|
{django_data_shape-0.13.0 → django_data_shape-0.14.0}/django_data_shape/check_constraints.py
RENAMED
|
@@ -163,8 +163,12 @@ def _check_independent_fan_outs(
|
|
|
163
163
|
That is why this is otherwise a refusal and not a wider capacity
|
|
164
164
|
calculation: no arithmetic over the two marginals decides it, because the
|
|
165
165
|
failure is that nothing enumerates the combinations at all. Deduplicated edges are a
|
|
166
|
-
different algorithm rather than a bigger loop
|
|
167
|
-
|
|
166
|
+
different algorithm rather than a bigger loop. **That algorithm now exists**
|
|
167
|
+
as :class:`~django_data_shape.paired.Paired`, which is the remedy this
|
|
168
|
+
refusal names: one side stays a fan-out and the other takes distinct
|
|
169
|
+
partners inside its groups, so a duplicate pair is impossible rather than
|
|
170
|
+
removed. A statement of your own is still the answer for an edge set that
|
|
171
|
+
has to be a particular one --
|
|
168
172
|
:class:`~django_data_shape.projection.Projection` with ``columns=`` and
|
|
169
173
|
``sql=`` fills the table from a select that can deduplicate.
|
|
170
174
|
"""
|
|
@@ -191,10 +195,11 @@ def _check_independent_fan_outs(
|
|
|
191
195
|
"pairs come out together is an artefact of that index, and two rows sharing a pair is a "
|
|
192
196
|
"matter of the seed. The combinations do fit, which is why the pigeonhole arithmetic "
|
|
193
197
|
"lets this through -- what is missing is anything that enumerates them, and the load "
|
|
194
|
-
"then fails inside COPY at a row number that moves when the seed does.
|
|
195
|
-
f"
|
|
196
|
-
"
|
|
197
|
-
"
|
|
198
|
+
"then fails inside COPY at a row number that moves when the seed does. An edge table "
|
|
199
|
+
f"declares one side and pairs the other: {fanned[1]}=Paired({fanned[0]!r}, Zipf()) takes "
|
|
200
|
+
f"distinct partners inside each {fanned[0]} group, so no pair can repeat and the edge "
|
|
201
|
+
"count is still exactly the row count. An edge set that has to be a particular one is "
|
|
202
|
+
f"filled by a statement instead -- Projection({where}, columns=(...), sql=...)."
|
|
198
203
|
)
|
|
199
204
|
|
|
200
205
|
|
|
@@ -101,6 +101,15 @@ class FanOutPlan:
|
|
|
101
101
|
end = self._starts[index + 1] if index + 1 < len(self._starts) else self._rows
|
|
102
102
|
return slot - start, end - start
|
|
103
103
|
|
|
104
|
+
def group_of(self, row: int) -> int:
|
|
105
|
+
"""Which parent this row belongs to, as an index into the parent keys.
|
|
106
|
+
|
|
107
|
+
Public because a paired column needs it: the partner is chosen inside
|
|
108
|
+
this row's group, so "which group" and "where in it" are both asked of
|
|
109
|
+
the partition rather than recomputed beside it.
|
|
110
|
+
"""
|
|
111
|
+
return self._slot(row)
|
|
112
|
+
|
|
104
113
|
def _slot(self, row: int) -> int:
|
|
105
114
|
"""Which parent owns this child row, as an index into ``keys``."""
|
|
106
115
|
slot = (row * self._stride) % self._rows
|
|
@@ -10,6 +10,8 @@ from django_data_shape.derivations.derivation import Derivation
|
|
|
10
10
|
from django_data_shape.derivations.scope import Scope
|
|
11
11
|
from django_data_shape.distributions.distribution import Distribution
|
|
12
12
|
from django_data_shape.fan_out_plan import FanOutPlan
|
|
13
|
+
from django_data_shape.paired import Paired
|
|
14
|
+
from django_data_shape.paired_plan import PairedPlan
|
|
13
15
|
from django_data_shape.table import Table
|
|
14
16
|
from django_data_shape.utils import draw, field_stream
|
|
15
17
|
|
|
@@ -20,7 +22,10 @@ _Source = Callable[[int, list[object]], object]
|
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
def generate_rows(
|
|
23
|
-
table: Table,
|
|
25
|
+
table: Table,
|
|
26
|
+
seed: int,
|
|
27
|
+
plans: Mapping[str, FanOutPlan] | None = None,
|
|
28
|
+
pairings: Mapping[str, PairedPlan] | None = None,
|
|
24
29
|
) -> Iterator[tuple[Any, ...]]:
|
|
25
30
|
"""Yield one tuple per row: the primary key, then each declared column.
|
|
26
31
|
|
|
@@ -54,6 +59,7 @@ def generate_rows(
|
|
|
54
59
|
nothing but peak RSS.
|
|
55
60
|
"""
|
|
56
61
|
plans = plans or {}
|
|
62
|
+
pairings = pairings or {}
|
|
57
63
|
keys = table.keys
|
|
58
64
|
key_stream = field_stream(seed, table.db_table, ":key")
|
|
59
65
|
columns = table.columns()
|
|
@@ -64,7 +70,7 @@ def generate_rows(
|
|
|
64
70
|
# of column this is, and where each of its sources comes from, are decided
|
|
65
71
|
# once rather than a million times.
|
|
66
72
|
steps: list[_Step] = [
|
|
67
|
-
(slot_of[name], _producer(table, seed, plans, name))
|
|
73
|
+
(slot_of[name], _producer(table, seed, plans, pairings, name))
|
|
68
74
|
for name, _field in columns
|
|
69
75
|
if name not in table.computation_order()
|
|
70
76
|
]
|
|
@@ -81,12 +87,23 @@ def generate_rows(
|
|
|
81
87
|
|
|
82
88
|
|
|
83
89
|
def _producer(
|
|
84
|
-
table: Table,
|
|
90
|
+
table: Table,
|
|
91
|
+
seed: int,
|
|
92
|
+
plans: Mapping[str, FanOutPlan],
|
|
93
|
+
pairings: Mapping[str, PairedPlan],
|
|
94
|
+
name: str,
|
|
85
95
|
) -> Callable[[int, list[object]], object]:
|
|
86
96
|
"""The step for a column that depends on nothing already in the row."""
|
|
87
97
|
plan = plans.get(name)
|
|
88
98
|
if plan is not None:
|
|
89
99
|
return partial(_from_plan, plan)
|
|
100
|
+
pairing = pairings.get(name)
|
|
101
|
+
if pairing is not None:
|
|
102
|
+
# Two questions of the fan-out this pairs with -- which group, and where
|
|
103
|
+
# in it -- and one of the pairing. All three are O(1) because both are
|
|
104
|
+
# partitions of the row range rather than per-row draws.
|
|
105
|
+
declared = cast("Paired", table.fields[name])
|
|
106
|
+
return partial(_from_pairing, plans[declared.relation], pairing)
|
|
90
107
|
distribution = cast("Distribution", table.fields[name])
|
|
91
108
|
return partial(_from_distribution, distribution, field_stream(seed, table.db_table, name))
|
|
92
109
|
|
|
@@ -149,6 +166,11 @@ def _from_plan(plan: FanOutPlan, row: int, values: list[object]) -> object:
|
|
|
149
166
|
return plan.key_for(row)
|
|
150
167
|
|
|
151
168
|
|
|
169
|
+
def _from_pairing(plan: FanOutPlan, pairing: PairedPlan, row: int, values: list[object]) -> object:
|
|
170
|
+
position, _size = plan.group_position(row)
|
|
171
|
+
return pairing.partner_for(plan.group_of(row), position)
|
|
172
|
+
|
|
173
|
+
|
|
152
174
|
def _from_derivation(
|
|
153
175
|
derivation: Derivation,
|
|
154
176
|
stream: int,
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The other end of a many-to-many edge, distinct within its partner's group."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from django_data_shape.distributions.distribution import Distribution
|
|
6
|
+
from django_data_shape.invalid_shape import InvalidShape
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Paired:
|
|
10
|
+
"""A relation filled with the opposite end of an edge, never repeating a pair.
|
|
11
|
+
|
|
12
|
+
The declaration a through table needs, and the reason two
|
|
13
|
+
:class:`~django_data_shape.fan_out.FanOut` declarations are refused beside
|
|
14
|
+
one another. Two fan-outs partition the same rows without either seeing the
|
|
15
|
+
other, so nothing enumerates the pairs and whether two rows collide is a
|
|
16
|
+
matter of the seed. ``Paired`` is the half that looks:
|
|
17
|
+
|
|
18
|
+
.. code-block:: python
|
|
19
|
+
|
|
20
|
+
Table(
|
|
21
|
+
Membership,
|
|
22
|
+
rows=50_000,
|
|
23
|
+
company=FanOut(Zipf()),
|
|
24
|
+
person=Paired("company", Zipf()),
|
|
25
|
+
role=Constant("member"),
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
``company`` partitions the rows as any fan-out does -- that is the **declared**
|
|
29
|
+
degree distribution, and it is exact. Within each of its groups, ``person``
|
|
30
|
+
takes that many **distinct** partners, so a duplicate pair is impossible by
|
|
31
|
+
construction rather than removed afterwards: the same company implies a
|
|
32
|
+
different person, and a different company implies a different pair. The edge
|
|
33
|
+
count is therefore exactly ``rows``, which is the rule this package holds to
|
|
34
|
+
everywhere else and which a deduplicating generator would have given up.
|
|
35
|
+
|
|
36
|
+
``relation`` names the fan-out this one is paired with, spelled the way
|
|
37
|
+
:class:`~django_data_shape.derivations.per_parent.PerParent` names the
|
|
38
|
+
relation it groups by, and for the same reason: a table with three foreign
|
|
39
|
+
keys should not make a reader guess which one the grouping is over.
|
|
40
|
+
|
|
41
|
+
``weights`` decides which partners are popular. It is a
|
|
42
|
+
:class:`~django_data_shape.distributions.distribution.Distribution` over the
|
|
43
|
+
partner table the same way a fan-out's ``sizes`` is over the parent table,
|
|
44
|
+
so ``Zipf()`` gives the realistic heavy tail and a flat one gives the
|
|
45
|
+
uniform shape this package exists to argue against.
|
|
46
|
+
|
|
47
|
+
**The second side's marginal is derived, not declared, and that is on
|
|
48
|
+
purpose.** Both marginals plus the edge count is over-determined: fixing all
|
|
49
|
+
three is a constraint satisfaction problem, and a CSP cannot stream into
|
|
50
|
+
``COPY``. So the edge count and one side's distribution are declared and the
|
|
51
|
+
other side follows -- and what follows *approximates* ``weights`` rather than
|
|
52
|
+
reproducing it. Measured against exact weighted sampling without
|
|
53
|
+
replacement, the busiest partner comes out about 1.1 times as busy, the 99th
|
|
54
|
+
percentile about 30% high, and about 7% fewer partners are touched at all.
|
|
55
|
+
Those numbers are in the documentation rather than only here, because a
|
|
56
|
+
derived shape nobody quotes is a shape nobody can check.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, relation: str, weights: Distribution) -> None:
|
|
60
|
+
if not isinstance(relation, str) or not relation:
|
|
61
|
+
raise InvalidShape(
|
|
62
|
+
f"Paired needs the name of the fan-out it is paired with, got {relation!r}. "
|
|
63
|
+
"A table with more than one foreign key cannot be left to guess which one the "
|
|
64
|
+
"pairing is over."
|
|
65
|
+
)
|
|
66
|
+
self._relation = relation
|
|
67
|
+
self._weights = weights
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def relation(self) -> str:
|
|
71
|
+
return self._relation
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def weights(self) -> Distribution:
|
|
75
|
+
return self._weights
|
|
76
|
+
|
|
77
|
+
def canonical(self) -> object:
|
|
78
|
+
"""The fan-out it pairs with and the weights over partners. See ``Canonical``."""
|
|
79
|
+
return (self._relation, self._weights)
|
|
80
|
+
|
|
81
|
+
def __repr__(self) -> str:
|
|
82
|
+
return f"Paired({self._relation!r}, {self._weights!r})"
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Which partner each row of an edge table points at, answered per row."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections import defaultdict
|
|
7
|
+
|
|
8
|
+
from django_data_shape.utils import draw
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PairedPlan:
|
|
12
|
+
"""Distinct partners inside every group, chosen once and read per row.
|
|
13
|
+
|
|
14
|
+
The construction, in one sentence: **partners are allocated across weight
|
|
15
|
+
bands and then strided within a band**, so nothing is ever drawn and asked
|
|
16
|
+
"is this one taken".
|
|
17
|
+
|
|
18
|
+
That is what separates this from the obvious approach. Choosing partners one
|
|
19
|
+
at a time and rejecting duplicates degenerates exactly where the shape is
|
|
20
|
+
most interesting -- a group needing most of the partner table probes almost
|
|
21
|
+
every slot, measured at 245 probes per row on 200,000 edges -- and the usual
|
|
22
|
+
fix, sampling the ones to *leave out* when a group wants more than half, is
|
|
23
|
+
ten times faster and **a different sampling rule**, so a group one row over
|
|
24
|
+
the halfway mark would get a materially different membership from one row
|
|
25
|
+
under. A cliff at an arbitrary size that no declaration mentions is the kind
|
|
26
|
+
of thing this package exists to refuse, so neither is what runs here.
|
|
27
|
+
|
|
28
|
+
**Allocation across bands is systematic, not largest-remainder**, and that
|
|
29
|
+
distinction is the whole reason the band count can be derived. Largest
|
|
30
|
+
remainder is deterministic: the same heavy bands win the fractional top-up
|
|
31
|
+
in every group, so a band holding one partner is included in nearly every
|
|
32
|
+
group and its degree runs away -- measured at nearly three times what exact
|
|
33
|
+
weighted sampling gives, and it never settles as bands get finer. Systematic
|
|
34
|
+
allocation lays the bands' shares end to end, takes one offset per group and
|
|
35
|
+
counts the integer points each band's interval covers: the expected count
|
|
36
|
+
per band is its share, the total is the group's size by construction, and
|
|
37
|
+
the answer *converges* as bands get finer instead of running away. Because
|
|
38
|
+
it converges, "enough bands" is a limit rather than an optimum, and a band
|
|
39
|
+
rule can be stated in weight ratios rather than in a number somebody tuned.
|
|
40
|
+
|
|
41
|
+
**What it is, honestly.** The result approximates exact weighted sampling
|
|
42
|
+
without replacement rather than reproducing it: the busiest partner comes
|
|
43
|
+
out about 1.1 times as busy, the 99th percentile about 30% high, and about
|
|
44
|
+
7% fewer partners are touched. It is an approximation with **no tuning
|
|
45
|
+
parameter**, which is the bar the alternatives failed, and the side it
|
|
46
|
+
approximates is derived rather than declared -- so nothing here promises a
|
|
47
|
+
marginal and then misses it.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
keys: list[int],
|
|
53
|
+
sizes: list[int],
|
|
54
|
+
weights: list[float],
|
|
55
|
+
bands: list[int],
|
|
56
|
+
stream: int,
|
|
57
|
+
) -> None:
|
|
58
|
+
self._keys = keys
|
|
59
|
+
self._sizes = sizes
|
|
60
|
+
# Three purposes, three streams, rather than one stream and an offset:
|
|
61
|
+
# an offset is an arithmetic relationship between draws that are
|
|
62
|
+
# supposed to be independent.
|
|
63
|
+
self._offsets = stream
|
|
64
|
+
self._strides = stream ^ 0x9E3779B97F4A7C15
|
|
65
|
+
self._starts = stream ^ 0xC2B2AE3D27D4EB4F
|
|
66
|
+
self._members: dict[int, list[int]] = defaultdict(list)
|
|
67
|
+
for index, band in enumerate(bands):
|
|
68
|
+
self._members[band].append(index)
|
|
69
|
+
self._bands = sorted(self._members)
|
|
70
|
+
self._mass = {band: sum(weights[i] for i in self._members[band]) for band in self._bands}
|
|
71
|
+
self._total = sum(self._mass.values())
|
|
72
|
+
# One list per group, built once. The whole partition is O(rows), and a
|
|
73
|
+
# row reads its own slot -- the alternative, recomputing a group's
|
|
74
|
+
# partners per row, would be O(rows x group size).
|
|
75
|
+
self._chosen: list[list[int]] = [self._for_group(g, s) for g, s in enumerate(sizes)]
|
|
76
|
+
|
|
77
|
+
def partner_for(self, group: int, position: int) -> int:
|
|
78
|
+
"""The key for the ``position``-th row of ``group``."""
|
|
79
|
+
return self._keys[self._chosen[group][position]]
|
|
80
|
+
|
|
81
|
+
def sizes(self) -> list[int]:
|
|
82
|
+
"""How many rows each group holds, which is the fan-out's own partition."""
|
|
83
|
+
return list(self._sizes)
|
|
84
|
+
|
|
85
|
+
def _for_group(self, group: int, size: int) -> list[int]:
|
|
86
|
+
chosen: list[int] = []
|
|
87
|
+
for band, wanted in self._allocate(group, size):
|
|
88
|
+
people = self._members[band]
|
|
89
|
+
modulus = len(people)
|
|
90
|
+
# A distinct draw index per (group, band), and per purpose. Mixing
|
|
91
|
+
# them arithmetically -- `group * 31 + band` -- collides as soon as
|
|
92
|
+
# there are more than 31 bands, and two groups sharing a stride is
|
|
93
|
+
# a correlation nothing in the declaration asked for and nothing in
|
|
94
|
+
# the output announces.
|
|
95
|
+
slot = group * len(self._bands) + self._bands.index(band)
|
|
96
|
+
stride = _stride(modulus, draw(self._strides, slot))
|
|
97
|
+
start = int(draw(self._starts, slot) * modulus)
|
|
98
|
+
# A stride coprime with the band size visits every member before
|
|
99
|
+
# repeating, so `wanted <= modulus` picks are distinct without any
|
|
100
|
+
# membership test at all.
|
|
101
|
+
chosen.extend(people[(start + step * stride) % modulus] for step in range(wanted))
|
|
102
|
+
return chosen
|
|
103
|
+
|
|
104
|
+
def _allocate(self, group: int, size: int) -> list[tuple[int, int]]:
|
|
105
|
+
"""How many partners this group takes from each band. Sums to ``size``.
|
|
106
|
+
|
|
107
|
+
Systematic: the shares are laid end to end and one offset per group
|
|
108
|
+
counts the integer points each band covers. A band asked for more than
|
|
109
|
+
it holds spills into the next with room, which is the only step here
|
|
110
|
+
that is not purely statistical and is what keeps the total exact when a
|
|
111
|
+
group is large enough to exhaust the heavy bands.
|
|
112
|
+
"""
|
|
113
|
+
offset = draw(self._offsets, group)
|
|
114
|
+
running = 0.0
|
|
115
|
+
previous = 0
|
|
116
|
+
wanted: dict[int, int] = {}
|
|
117
|
+
for band in self._bands:
|
|
118
|
+
running += size * self._mass[band] / self._total
|
|
119
|
+
boundary = math.floor(running - offset) + 1
|
|
120
|
+
wanted[band] = max(0, boundary - previous)
|
|
121
|
+
previous = boundary
|
|
122
|
+
spill = 0
|
|
123
|
+
for band in self._bands:
|
|
124
|
+
capacity = len(self._members[band])
|
|
125
|
+
if wanted[band] > capacity:
|
|
126
|
+
spill += wanted[band] - capacity
|
|
127
|
+
wanted[band] = capacity
|
|
128
|
+
for band in self._bands:
|
|
129
|
+
if not spill:
|
|
130
|
+
break
|
|
131
|
+
room = len(self._members[band]) - wanted[band]
|
|
132
|
+
taken = min(room, spill)
|
|
133
|
+
wanted[band] += taken
|
|
134
|
+
spill -= taken
|
|
135
|
+
return [(band, count) for band, count in wanted.items() if count]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _stride(modulus: int, drawn: float) -> int:
|
|
139
|
+
"""A step coprime with ``modulus``, so stepping by it is a bijection."""
|
|
140
|
+
if modulus <= 2:
|
|
141
|
+
return 1
|
|
142
|
+
stride = 1 + int(drawn * (modulus - 1))
|
|
143
|
+
while math.gcd(stride, modulus) != 1:
|
|
144
|
+
stride = stride % (modulus - 1) + 1
|
|
145
|
+
return stride
|