policyengine 6.3.4__tar.gz → 6.4.1__tar.gz

{policyengine-6.3.4 → policyengine-6.4.1}/CHANGELOG.md

@@ -1,3 +1,22 @@
+## [6.4.1] - 2026-10-15
+
+### Changed
+
+- The UK Budget reform contract report gives each assertion a unique id, and the population checks declare the rows a full run emits.
+- Certify policyengine-uk 2.130.1 with policyengine-uk-data 1.58.1 (enhanced_frs_2024_25) as the UK default pair, on the publisher's compatibility claim.
+
+
+## [6.4.0] - 2026-10-14
+
+### Added
+
+- Added `policyengine.disclosure`, a self-contained layer for statistical disclosure control and uncertainty on area-level outputs, and `compute_uk_geography_disclosure` for UK regions, constituencies and local authorities. Each published number rests on at least 10 distinct survey households (clones counted once through `source_household_id`), carries a 90% bootstrap interval and is rounded to the order of its standard error. Rows are `reported`, `approximate` or `withheld`. A release check reports numbers a reader could recover by subtraction across geography levels, a package and its components, and earlier publications. Keyed noise per survey household is available behind `DisclosureSettings(noise=True)`. `validate_public` and `python -m policyengine.disclosure validate` check the format of published files. UK runs now keep `source_household_id` and `region` on the household output when the dataset provides them.
+
+### Changed
+
+- Deprecated `compute_uk_constituency_impacts`, `compute_uk_local_authority_impacts` and their `ConstituencyImpact` and `LocalAuthorityImpact` classes. Their output is not disclosure-controlled; use `compute_uk_geography_disclosure` instead. They now log a warning through the `policyengine` logger and emit a `DeprecationWarning`, and otherwise behave as before.
+
+
 ## [6.3.4] - 2026-10-13
 
 ### Changed

{policyengine-6.3.4/src/policyengine.egg-info → policyengine-6.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: policyengine
-Version: 6.3.4
+Version: 6.4.1
 Summary: A package to conduct policy analysis using PolicyEngine tax-benefit models.
 Author-email: PolicyEngine <hello@policyengine.org>
 License:                     GNU AFFERO GENERAL PUBLIC LICENSE
@@ -687,11 +687,11 @@ Requires-Dist: networkx>=3.0; extra == "graph"
 Provides-Extra: models
 Requires-Dist: policyengine-core==3.33.0; extra == "models"
 Requires-Dist: policyengine-us==2.2.1; extra == "models"
-Requires-Dist: policyengine-uk==2.126.12; extra == "models"
+Requires-Dist: policyengine-uk==2.130.1; extra == "models"
 Requires-Dist: spm-calculator==1.0.0; extra == "models"
 Provides-Extra: uk
 Requires-Dist: policyengine-core==3.33.0; extra == "uk"
-Requires-Dist: policyengine-uk==2.126.12; extra == "uk"
+Requires-Dist: policyengine-uk==2.130.1; extra == "uk"
 Provides-Extra: us
 Requires-Dist: policyengine-core==3.33.0; extra == "us"
 Requires-Dist: policyengine-us==2.2.1; extra == "us"
@@ -713,7 +713,7 @@ Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
 Requires-Dist: hypothesis>=6.100.0; extra == "dev"
 Requires-Dist: policyengine-core==3.33.0; extra == "dev"
 Requires-Dist: policyengine-us==2.2.1; extra == "dev"
-Requires-Dist: policyengine-uk==2.126.12; extra == "dev"
+Requires-Dist: policyengine-uk==2.130.1; extra == "dev"
 Requires-Dist: spm-calculator==1.0.0; extra == "dev"
 Dynamic: license-file
 

{policyengine-6.3.4 → policyengine-6.4.1}/docs/_quarto.yml

@@ -39,6 +39,7 @@ website:
           - impact-analysis.md
           - outputs.md
           - regions.md
+          - disclosure.md
           - visualisation.md
       - section: "Methodology"
         contents:

policyengine-6.4.1/docs/disclosure.md

@@ -0,0 +1,239 @@
+---
+title: "Disclosure control"
+---
+
+Local results from survey microdata rest on few households, and a licence
+binds every published number. `policyengine.disclosure` turns per-household
+results into area tables that follow the licence's rules and carry their
+uncertainty. Use it for every UK result below the national level that leaves
+your machine.
+
+```python
+from policyengine.outputs import compute_uk_geography_disclosure
+
+impact = compute_uk_geography_disclosure(
+    baseline_simulation=baseline,
+    reform_simulation=reform,
+    year=2026,
+    levels=("region", "constituency"),
+    reform="budget",
+)
+impact.findings  # what blocks publication, if anything
+impact.public  # the rows that may be published, or None while blocked
+```
+
+The UK datasets derive from surveys under the UK Data Service End User
+Licence. The defaults below follow its disclosure rules. You can raise them
+for a study with stricter conditions; you cannot lower them.
+
+## What a published row holds
+
+Each row is one outcome for one area, reform and year.
+
+| Column | Content |
+|---|---|
+| `level`, `area_code`, `reform`, `year`, `outcome` | The key of the row |
+| `value`, `lo`, `hi` | The rounded estimate and its 90% interval |
+| `direction` | `gain`, `loss`, `too_close_to_call` or `no_change` |
+| `tier` | `reported`, `approximate` or `withheld` |
+| `reason` | `thin_support` for an approximate row, `disclosure_control` for a withheld row |
+| `support_band` | `<10`, `10–29` or `30+` survey households, never an exact count |
+| `noise` | Whether the row carries keyed noise |
+| `estimator` | `direct`: the estimate comes from the area's own records |
+
+The outcomes are `average_change`, `relative_change`, `total_change`,
+`share_gaining`, `share_losing` and `share_no_change`. A withheld row carries
+no value, interval, direction or support band.
+
+## The rules
+
+**Count floor.** A number must rest on at least 10 distinct survey households
+that contribute to it. For a change, a household contributes when its income
+changes by more than `change_tolerance` (£1 a year, which is also the lowest
+value the setting takes). Copies and clones of one survey household count
+once, through the dataset's `source_household_id`. For a share, both sides
+must pass: the households that gain and those that do not.
+
+**Dominance.** While noise is off, a number is withheld in two cases:
+
+- one survey household carries more than half of it, by size;
+- the p% rule: a reader of the number, or one household that knows its own
+  part, could estimate some household's part to within 10%. Every household
+  is a possible target and every other household a possible reader, so the
+  rule also catches a total whose other parts cancel.
+
+These rules protect each household against any reader and against any one
+other household. They do not stop two or more households that pool what they
+know: a number that three households carry almost alone passes, as long as 10
+contribute.
+
+**Effective sample.** Weights that differ a lot make an estimate less precise
+than its household count suggests. Where the data gives one area and one
+weight per record, a low effective sample makes a row `approximate`: it keeps
+its value and interval, but the interval understates the uncertainty. Where
+the data is an area × household weight matrix, the household count passes by
+construction, so the effective sample must also be at least 10. Weights given
+as a matrix keep this rule whatever they hold: one row of a dense matrix has
+one weight per record, yet every household of the nation sits in that area.
+
+**Zero cells.** A cell with no contributing household is published as zero,
+with the interval [0, 0] and the direction `no_change`. It means that no
+surveyed household in the area changed, not that no household there does. Its
+support band counts the households it was measured on.
+
+**Settings.** The layer validates its settings again before it uses them, so
+a copy made with `model_copy(update=...)` cannot lower a floor.
+
+**Linked outcomes.** Average, relative and total change share a numerator, so
+they are published or withheld together. Gaining, losing and no change add to
+one, so the third share is published only when it is safe to derive.
+
+## Uncertainty and rounding
+
+Intervals come from a Poisson bootstrap over survey households with 1,000
+replicates. A ratio resamples its numerator and denominator together. The
+weights stay fixed, so the intervals leave out calibration uncertainty and are
+too narrow, most of all where support is thin.
+
+Values are rounded to the order of magnitude of their standard error. The
+interval is rounded outward on the same base.
+
+## The release check
+
+Published numbers constrain each other. If a region is public and all but one
+of its constituencies are public, subtraction gives the last one. The check
+looks for such remainders:
+
+- child areas against their parent area (constituencies in regions, regions in
+  the nation);
+- the components of a package against the package;
+- when two levels that do not nest are released together, an area against
+  the areas of the other level that lie wholly inside it, and against the
+  areas that meet it (their sum minus the area is what they hold outside it);
+- earlier publications, passed as `prior_public`.
+
+A remainder gets the rules of a cell. Estimates added with `publish=False`
+are a reference: the release does not emit their rows, but it treats the rows
+that pass as public elsewhere and the withheld ones as secret, and checks
+them like its own.
+
+`compute_uk_geography_disclosure` always checks constituencies and local
+authorities against their region, and regions against the nation, also when
+you publish one level only. A level given as a weight matrix needs
+`parent_of=` with the region of each area; without it the function raises,
+because a check against the nation alone nearly always passes.
+
+The check reports; it does not withhold cells by itself. A blocking finding
+names the group and suggests a cell. Withhold it, then check again. When a
+finding suggests no cell, withhold the parent (`finding.group` at
+`finding.group_level`) instead:
+
+```python
+for finding in impact.findings:
+    print(finding.code, finding.group, finding.suggested)
+
+impact.release.withhold(
+    level="constituency",
+    area_code="E14001234",
+    reform="budget",
+    year=2026,
+    outcome="average_change",
+)
+impact.refresh()  # checks again and fills impact.public once nothing blocks
+```
+
+Two findings can be accepted after review, with `acknowledge=[...]`:
+
+- `unchecked_overlap`: levels that overlap without nesting, such as
+  constituencies and local authorities, are released together. The release
+  checks each area against the areas wholly inside it and against the areas
+  that meet it. It does not check other differences between overlapping
+  areas, and on weight-matrix data it checks none.
+- `noise_reference_public`: an earlier table shows the same cells under
+  another noise setting.
+
+## Noise
+
+Noise is off by default. With `DisclosureSettings(noise=True)`, each survey
+household's change is scaled by a fixed factor between 8% and 12% away from 1,
+derived from its id and a secret key. The same household has the same factor
+in every table and every rebuild. Noise replaces the dominance rule; the count
+floor still applies. The range cannot start closer to 1 than 8%.
+
+A number that one household dominates, published with noise, gives that
+household's part to within 8 to 12%. The p% rule treats 10% as the line, so
+noise puts such a number right at that line, not safely past it. Whether the
+UK Data Service accepts noise in place of the dominance rule is a licence
+question, which is why noise stays off by default.
+
+- **Size.** The noise in a total is about a tenth of that total's sampling
+  error.
+- **Key.** Pass `noise_key=` or set `POLICYENGINE_DISCLOSURE_NOISE_KEY`. There
+  is no default key. Keep the key out of repositories and CI logs, and do not
+  change it between releases of one dataset: two noised versions of a number
+  average towards the truth.
+- **Limits.** Denominators carry no noise. The factor cancels in a ratio of
+  two numbers that the same household dominates. A remainder that subtraction
+  recovers is the noised sum of the withheld cells, so it has the protection
+  of a noised cell and no more: the count floor applies to it, and noise
+  stands in for the dominance rule as it does for a cell. An un-noised copy
+  of the same cell gives the factor away.
+
+## Validating published files
+
+`validate_public` checks the format of a table, a CSV file or a directory. It
+needs no key and no microdata, so it can run on every pull request:
+
+```bash
+python -m policyengine.disclosure validate public/data --reviewed notes.csv
+```
+
+It rejects private audit columns, unknown tiers, a value in a withheld row,
+unrounded values, a direction that does not match its interval, and files it
+does not recognise. It cannot tell whether a number rested on enough
+households; only the release that made the file can.
+
+## Private objects
+
+`audit_dir=` writes the exact values, household counts and shares behind every
+row. The directory must be outside any git repository.
+
+`AreaEstimates`, the object that `estimate_change` returns, holds the same
+exact values. With noise on it also holds the noise factor of every household
+that gains or loses, and within one dataset those factors do the same job as
+the key. Keep it in memory and treat it as you treat the key. It cannot be
+pickled or copied.
+
+## What a reader can still derive
+
+`average_change` and `relative_change` together give the area's total
+baseline income to within a few per cent (household count × average ÷
+relative). That total is a denominator: it carries no noise and is not
+checked for dominance.
+
+## Arrays without a simulation
+
+The package works on arrays and imports nothing from the rest of the library:
+
+```python
+from policyengine.disclosure import AreaWeights, Release, estimate_change
+
+seats = estimate_change(
+    level="constituency",
+    reform="budget",
+    year=2026,
+    areas=AreaWeights.from_labels(seat_codes, household_weights),
+    source_ids=source_household_ids,
+    baseline=baseline_net_income,
+    reformed=reformed_net_income,
+)
+release = Release()
+release.add(seats)
+release.check()
+public = release.to_public()
+```
+
+`AreaWeights.from_matrix(matrix, area_codes)` takes an area × household weight
+matrix instead. A dataset without `source_household_id` needs `source_ids=`
+from the caller; the layer does not guess which records share a survey
+household.

{policyengine-6.3.4 → policyengine-6.4.1}/docs/engineering/runbooks/build-m-us-populace-certification.md

@@ -2,8 +2,10 @@
 
 Use this runbook to certify a new national US Populace release together with
 the shared ACS-local release used for state and congressional-district
-simulations. Read the [data certification](../skills/data-certification.md)
-skill first for validation semantics.
+simulations. For now, state and district simulations use the national release
+instead; see the interim exception in Step 2. Read the
+[data certification](../skills/data-certification.md) skill first for
+validation semantics.
 
 This historical filename is retained so existing links remain valid; the
 procedure itself is release-independent.
@@ -13,7 +15,8 @@ procedure itself is release-independent.
 Both release manifests have been published to `policyengine/populace-us`, and
 the model version used by the national release is known. The national release
 will remain the default dataset; the local-area release will remain
-non-default and be selected only for supported regional runs.
+non-default and be selected only for supported regional runs (none while the
+interim exception in Step 2 holds).
 
 ## Prerequisites
 
@@ -60,6 +63,19 @@ This one command:
 - writes the changelog fragment
   `changelog.d/certify-us-$NATIONAL_RELEASE_ID.changed.md`.
 
+**Interim exception (from policyengine 6.3.4; PolicyEngine/policyengine.py#604).**
+State and congressional-district regions filter the national dataset until an
+ACS-local release has no SPM unit without a classified adult
+(PolicyEngine/microcosm#1263). Run the command above unchanged, with
+`--regional-manifest-uri`: without it, certification drops the ACS-local
+dataset and leaves no state or district template. Then, before Step 3, set
+`region_datasets.state` and `region_datasets.congressional_district` in
+`data_releases.us` to `{"path_template": "populace_us_2024.h5"}` by hand. In
+Step 5, expect those two templates to select the national artifact, not the
+local-area one. The data-certification skill
+(`docs/engineering/skills/data-certification.md`) says what must hold before
+the exception ends.
+
 ## Step 3 — regenerate TRACE sidecars
 
 The certify step does not touch TRACE sidecars. Regenerate them so they record
@@ -87,15 +103,16 @@ PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py
 
 ## Step 5 — certify and verify the local-area release
 
-The certification command passes the immutable ACS-local release manifest with
-`--regional-manifest-uri`. Confirm all of the following:
+The certification command passes the immutable ACS-local
+release manifest with `--regional-manifest-uri`. Confirm all of the following:
 
 - `default_dataset` remains the national Populace dataset;
 - the local-area artifact appears in `data_releases.us.datasets` with its
   repository type, immutable revision, and SHA-256;
 - `region_datasets.national` selects the national artifact;
 - `region_datasets.state` and `region_datasets.congressional_district` select
-  the shared local-area artifact;
+  the shared local-area artifact (the national artifact while the interim
+  exception in Step 2 holds);
 - no derived `states/*.h5` or `districts/*.h5` artifacts are runtime inputs.
 
 ```bash

{policyengine-6.3.4 → policyengine-6.4.1}/docs/engineering/skills/data-certification.md

@@ -35,8 +35,33 @@ python scripts/bundle.py certify-data --country us --data-producer populace \
   --model-version "<policyengine-us-version>"
 ```
 
-US state and congressional-district regions are row filters over one shared,
-certified ACS-local Populace dataset. Its release manifest must declare
+**Interim exception (from policyengine 6.3.4; PolicyEngine/policyengine.py#604).**
+State and congressional-district regions currently filter the certified
+national dataset, not the ACS-local one. The certified ACS-local release has
+no source-backed SPM independence, so canonical SPM refuses its SPM units that
+have no classified adult, and with them every state-region run
+(PolicyEngine/microcosm#1263, PolicyEngine/policyengine-sim-api#747). The 6.3.4
+manifest was certified with `--regional-manifest-uri`, as below, and then
+edited by hand. Until the exception ends, do the same:
+
+- still pass `--regional-manifest-uri`. Without it, certification rewrites
+  `data_releases.us` from the national manifest alone, which drops the
+  ACS-local dataset and leaves no state or district template at all;
+- then set `region_datasets.state` and `region_datasets.congressional_district`
+  to `{"path_template": "populace_us_2024.h5"}` by hand, before regenerating
+  the TRACE sidecars. `tests/test_release_manifests.py` fails if they name
+  the ACS-local file, and also if the ACS-local dataset is no longer
+  certified.
+
+Restore the ACS-local regions only with an ACS-local release in which every
+SPM unit has at least one classified adult under the pinned spm-calculator's
+rule (zero unclassified units), and after a state-region run through the
+public wrapper succeeds on it. Then remove this exception, the temporary test
+and the matching notes in `docs/regions.md` and `docs/bundles.md`.
+
+Outside that exception, US state and congressional-district regions are row
+filters over one shared, certified ACS-local Populace dataset. Its release
+manifest must declare
 `dataset_role: non_default_local_area`, `is_default: false`, no default
 datasets, and exactly one pinned H5 microdata artifact. Certification writes:
 

{policyengine-6.3.4 → policyengine-6.4.1}/docs/examples.md

@@ -16,6 +16,13 @@ Applying parametric reforms, comparing baseline and reform with `ChangeAggregate
 ```{.python include="../examples/policy_change_uk.py"}
 ```
 
+## UK geography results under disclosure control
+
+Estimating results by area from per-household arrays, checking the release, withholding a cell and validating the public table. The data is synthetic.
+
+```{.python include="../examples/uk_geography_disclosure.py"}
+```
+
 ## UK income bands
 
 Calculating net income and tax by income decile using representative microdata and `Aggregate` with quantile filters.

{policyengine-6.3.4 → policyengine-6.4.1}/docs/outputs.md

@@ -266,7 +266,28 @@ for row in impacts.district_results:
     print(row["district_geoid"], row["avg_change"], row["winner_percentage"])
 ```
 
-### UK constituencies / local authorities
+### UK regions, constituencies and local authorities
+
+`compute_uk_geography_disclosure` returns results by area under disclosure
+control: each number passes a floor on the survey households behind it and
+carries a 90% interval. Use it for anything you publish. See
+[Disclosure control](disclosure.md).
+
+```python
+from policyengine.outputs import compute_uk_geography_disclosure
+
+impact = compute_uk_geography_disclosure(
+    baseline_simulation=baseline,
+    reform_simulation=reform,
+    year=2026,
+    levels=("region", "constituency"),
+)
+impact.public.to_frame()
+```
+
+The two older functions below are deprecated. Their output is not
+disclosure-controlled, so do not publish it when the data is licensed survey
+microdata.
 
 Constituency and local-authority breakdowns group household output rows by
 dataset-provided longwise geography columns. Constituencies use

{policyengine-6.3.4 → policyengine-6.4.1}/docs/regions.md

@@ -93,6 +93,11 @@ national UK dataset.
 
 ## UK parliamentary constituencies
 
+To publish results by constituency, local authority or region, use
+`compute_uk_geography_disclosure`; see [Disclosure control](disclosure.md).
+The functions in this section and the next are deprecated: their output is
+not disclosure-controlled.
+
 Constituency-level impacts group household output rows by the longwise
 `constituency_code_oa` column carried by the dataset. If the constituency CSV is
 available locally or from the PolicyEngine UK GCS bucket, PolicyEngine uses it

policyengine-6.4.1/examples/uk_geography_disclosure.py

@@ -0,0 +1,84 @@
+"""Example: Publish area results under disclosure control.
+
+This script demonstrates:
+1. Building estimates for two geography levels from per-household arrays
+2. Checking a release for numbers a reader could recover by subtraction
+3. Resolving a finding by withholding the suggested cell
+4. Writing the public table and validating its format
+
+The data is synthetic, so the script needs no dataset.
+
+Run: python examples/uk_geography_disclosure.py
+"""
+
+import tempfile
+from pathlib import Path
+
+import numpy as np
+
+from policyengine.disclosure import (
+    AreaWeights,
+    Release,
+    estimate_change,
+    validate_public,
+)
+
+
+def main() -> Path:
+    rng = np.random.default_rng(0)
+    # Two regions of three seats, 400 households a seat. In seat S1 the
+    # reform reaches only a handful of households.
+    seats = np.repeat(["S1", "S2", "S3", "S4", "S5", "S6"], 400)
+    regions = np.repeat(["North", "South"], 1_200)
+    reach = np.where(seats == "S1", 0.01, np.where(seats == "S2", 0.08, 0.35))
+    affected = rng.random(len(seats)) < reach
+    weights = rng.uniform(150.0, 450.0, len(seats))
+    baseline = rng.lognormal(10.4, 0.5, len(seats))
+    reformed = baseline + np.where(affected, rng.uniform(100.0, 900.0, len(seats)), 0)
+    source_ids = np.arange(len(seats))
+
+    release = Release()
+    for level, codes in (("region", regions), ("constituency", seats)):
+        release.add(
+            estimate_change(
+                level=level,
+                reform="example",
+                year=2026,
+                areas=AreaWeights.from_labels(codes, weights),
+                source_ids=source_ids,
+                baseline=baseline,
+                reformed=reformed,
+            )
+        )
+    release.add_nesting("constituency", "region")
+
+    # S1 is withheld. Its region is public, so subtraction would recover it
+    # unless a second seat is withheld too.
+    for finding in release.check():
+        print(f"{finding.code}: {finding.message} Suggested: {finding.suggested}")
+        release.withhold(
+            level="constituency",
+            area_code=finding.suggested,
+            reform="example",
+            year=2026,
+            outcome="average_change" if finding.family == "change" else finding.family,
+        )
+    assert release.check() == []
+
+    public = release.to_public()
+    frame = public.to_frame()
+    print(
+        frame[frame["outcome"] == "average_change"][
+            ["level", "area_code", "value", "lo", "hi", "tier", "support_band"]
+        ].to_string(index=False)
+    )
+
+    output = Path(tempfile.mkdtemp(prefix="disclosure-example-"))
+    public.to_csv(output / "example.csv")
+    assert validate_public(output) == []
+    print(f"Wrote and validated {output / 'example.csv'}")
+    return output
+
+
+if __name__ == "__main__":
+    main()

{policyengine-6.3.4 → policyengine-6.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "policyengine"
-version = "6.3.4"
+version = "6.4.1"
 description = "A package to conduct policy analysis using PolicyEngine tax-benefit models."
 readme = "README.md"
 authors = [
@@ -42,12 +42,12 @@ graph = [
 models = [
     "policyengine-core==3.33.0",
     "policyengine-us==2.2.1",
-    "policyengine-uk==2.126.12",
+    "policyengine-uk==2.130.1",
     "spm-calculator==1.0.0",
 ]
 uk = [
     "policyengine-core==3.33.0",
-    "policyengine-uk==2.126.12",
+    "policyengine-uk==2.130.1",
 ]
 us = [
     "policyengine-core==3.33.0",
@@ -71,7 +71,7 @@ dev = [
     "hypothesis>=6.100.0",
     "policyengine-core==3.33.0",
     "policyengine-us==2.2.1",
-    "policyengine-uk==2.126.12",
+    "policyengine-uk==2.130.1",
     "spm-calculator==1.0.0",
 ]