policyengine 3.4.2__tar.gz → 3.4.4__tar.gz

policyengine-3.4.4/.github/bump_version.py

@@ -0,0 +1,135 @@
+"""Infer semver bump from towncrier fragment types and update version."""
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+SEMVER_PATTERN = re.compile(r"^(\d+)\.(\d+)\.(\d+)$")
+
+
+def parse_version(version: str) -> tuple[int, int, int]:
+    match = SEMVER_PATTERN.match(version)
+    if not match:
+        raise ValueError(f"Invalid semver: {version}")
+    return tuple(int(part) for part in match.groups())
+
+
+def get_pyproject_version(pyproject_path: Path) -> str:
+    text = pyproject_path.read_text()
+    match = re.search(r'^version\s*=\s*"(\d+\.\d+\.\d+)"', text, re.MULTILINE)
+    if not match:
+        print(
+            "Could not find version in pyproject.toml",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    return match.group(1)
+
+
+def get_changelog_versions(changelog_path: Path) -> list[str]:
+    if not changelog_path.exists():
+        return []
+    return re.findall(
+        r"^## \[(\d+\.\d+\.\d+)\]", changelog_path.read_text(), re.MULTILINE
+    )
+
+
+def get_git_tag_versions(repo_root: Path) -> list[str]:
+    try:
+        result = subprocess.run(
+            ["git", "tag"],
+            cwd=repo_root,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        return []
+
+    versions = []
+    for tag in result.stdout.splitlines():
+        normalized = tag.removeprefix("v").strip()
+        if SEMVER_PATTERN.match(normalized):
+            versions.append(normalized)
+    return versions
+
+
+def get_current_version(
+    pyproject_path: Path,
+    changelog_path: Path,
+    repo_root: Path,
+) -> str:
+    candidates = [get_pyproject_version(pyproject_path)]
+    candidates.extend(get_changelog_versions(changelog_path))
+    candidates.extend(get_git_tag_versions(repo_root))
+    return max(candidates, key=parse_version)
+
+
+def infer_bump(changelog_dir: Path) -> str:
+    fragments = [
+        f for f in changelog_dir.iterdir() if f.is_file() and f.name != ".gitkeep"
+    ]
+    if not fragments:
+        print("No changelog fragments found", file=sys.stderr)
+        sys.exit(1)
+
+    categories = {f.suffix.lstrip(".") for f in fragments}
+    for f in fragments:
+        parts = f.stem.split(".")
+        if len(parts) >= 2:
+            categories.add(parts[-1])
+
+    if "breaking" in categories:
+        return "major"
+    if "added" in categories or "removed" in categories:
+        return "minor"
+    return "patch"
+
+
+def bump_version(version: str, bump: str) -> str:
+    major, minor, patch = (int(x) for x in version.split("."))
+    if bump == "major":
+        return f"{major + 1}.0.0"
+    elif bump == "minor":
+        return f"{major}.{minor + 1}.0"
+    else:
+        return f"{major}.{minor}.{patch + 1}"
+
+
+def update_file(path: Path, new_version: str):
+    text = path.read_text()
+    updated, replacements = re.subn(
+        r'(^version\s*=\s*")(\d+\.\d+\.\d+)(")',
+        rf"\g<1>{new_version}\g<3>",
+        text,
+        flags=re.MULTILINE,
+    )
+    if replacements == 0:
+        print(
+            f"Could not update version in {path}",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    if updated != text:
+        path.write_text(updated)
+        print(f"  Updated {path}")
+
+
+def main():
+    root = Path(__file__).resolve().parent.parent
+    pyproject = root / "pyproject.toml"
+    changelog = root / "CHANGELOG.md"
+    changelog_dir = root / "changelog.d"
+
+    current = get_current_version(pyproject, changelog, root)
+    bump = infer_bump(changelog_dir)
+    new = bump_version(current, bump)
+
+    print(f"Version: {current} -> {new} ({bump})")
+
+    update_file(pyproject, new)
+
+
+if __name__ == "__main__":
+    main()

{policyengine-3.4.2 → policyengine-3.4.4}/.github/workflows/push.yaml

@@ -81,10 +81,8 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 18.x
-      - name: Install MyST
-        run: npm install -g mystmd
       - name: Build HTML Assets
-        run: cd docs && myst build --html
+        run: make docs
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
@@ -108,6 +106,9 @@ jobs:
         uses: actions/checkout@v4
         with:
           token: ${{ steps.app-token.outputs.token }}
+          fetch-depth: 0
+      - name: Fetch tags
+        run: git fetch --tags --force
       - name: Setup Python
         uses: actions/setup-python@v5
         with:

{policyengine-3.4.2 → policyengine-3.4.4}/CHANGELOG.md

@@ -1,3 +1,24 @@
+## [3.4.4] - 2026-04-13
+
+### Changed
+
+- Add TRACE TRO export helpers for certified runtime bundles and expose them through `policyengine.core`.
+
+
+## [3.4.3] - 2026-04-13
+
+### Fixed
+
+- Fix the release versioning workflow so it bumps from the highest known released version instead of regressing to a stale version from `pyproject.toml`.
+
+
+## [3.4.1] - 2026-04-13
+
+### Changed
+
+- Add certified bundle metadata that records runtime model pins alongside build-time data artifact provenance and compatibility fingerprints.
+
+
 ## [3.4.2] - 2026-04-12
 
 ### Changed

{policyengine-3.4.2 → policyengine-3.4.4}/Makefile

@@ -1,9 +1,15 @@
-.PHONY: docs
+.PHONY: docs docs-serve
+
+MYSTMD_VERSION ?= 1.8.3
+MYST_CMD = npx --yes mystmd@$(MYSTMD_VERSION)
 
 all: build-package
 
 docs:
-	cd docs && jupyter book start
+	cd docs && $(MYST_CMD) build --html
+
+docs-serve:
+	cd docs && $(MYST_CMD) start
 
 install:
 	uv pip install -e .[dev]
@@ -27,4 +33,4 @@ build-package:
 	python -m build
 
 test:
-	pytest tests --cov=policyengine --cov-report=term-missing
+	pytest tests --cov=policyengine --cov-report=term-missing

{policyengine-3.4.2 → policyengine-3.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: policyengine
-Version: 3.4.2
+Version: 3.4.4
 Summary: A package to conduct policy analysis using PolicyEngine tax-benefit models.
 Author-email: PolicyEngine <hello@policyengine.org>
 License:                     GNU AFFERO GENERAL PUBLIC LICENSE
@@ -678,7 +678,7 @@ Requires-Dist: requests>=2.31.0
 Requires-Dist: psutil>=5.9.0
 Provides-Extra: uk
 Requires-Dist: policyengine_core>=3.23.6; extra == "uk"
-Requires-Dist: policyengine-uk==2.78.0; extra == "uk"
+Requires-Dist: policyengine-uk==2.74.0; extra == "uk"
 Provides-Extra: us
 Requires-Dist: policyengine_core>=3.23.6; extra == "us"
 Requires-Dist: policyengine-us==1.602.0; extra == "us"
@@ -693,7 +693,7 @@ Requires-Dist: build; extra == "dev"
 Requires-Dist: pytest-asyncio>=0.26.0; extra == "dev"
 Requires-Dist: ruff>=0.9.0; extra == "dev"
 Requires-Dist: policyengine_core>=3.23.6; extra == "dev"
-Requires-Dist: policyengine-uk==2.78.0; extra == "dev"
+Requires-Dist: policyengine-uk==2.74.0; extra == "dev"
 Requires-Dist: policyengine-us==1.602.0; extra == "dev"
 Requires-Dist: towncrier>=24.8.0; extra == "dev"
 Requires-Dist: mypy>=1.11.0; extra == "dev"
@@ -788,7 +788,8 @@ uv pip install -e .[dev]        # install with dev dependencies (pytest, ruff, m
 ```bash
 make format           # ruff format
 make test             # pytest with coverage
-make docs             # build Jupyter Book documentation
+make docs             # build static MyST/Jupyter Book 2 HTML docs
+make docs-serve       # preview the docs locally
 make clean            # remove caches, build artifacts, .h5 files
 ```
 

{policyengine-3.4.2 → policyengine-3.4.4}/README.md

@@ -86,7 +86,8 @@ uv pip install -e .[dev]        # install with dev dependencies (pytest, ruff, m
 ```bash
 make format           # ruff format
 make test             # pytest with coverage
-make docs             # build Jupyter Book documentation
+make docs             # build static MyST/Jupyter Book 2 HTML docs
+make docs-serve       # preview the docs locally
 make clean            # remove caches, build artifacts, .h5 files
 ```
 

{policyengine-3.4.2 → policyengine-3.4.4}/docs/dev.md

@@ -12,17 +12,19 @@
 ```bash
 git clone https://github.com/PolicyEngine/policyengine.py.git
 cd policyengine.py
-uv pip install -e .[dev]
+uv pip install -e ".[dev]"
 ```
 
-This installs both UK and US country models plus dev dependencies (pytest, ruff, mypy, towncrier).
+This installs the shared analysis layer, both country model extras, and the dev
+dependencies used in CI (pytest, ruff, mypy, towncrier).
 
 ## Common commands
 
 ```bash
 make format           # ruff format
 make test             # pytest with coverage
-make docs             # build documentation site
+make docs             # build static MyST/Jupyter Book 2 HTML docs
+make docs-serve       # preview the docs locally
 make clean            # remove caches, build artifacts, .h5 files
 ```
 
@@ -60,7 +62,7 @@ PRs trigger the following checks:
 | Tests (Python 3.13) | Required | `make test` |
 | Tests (Python 3.14) | Required | `make test` |
 | Mypy | Informational | `mypy src/policyengine` |
-| Docs build | Required | MyST build |
+| Docs build | Required | `make docs` |
 
 ## Versioning and releases
 
@@ -73,6 +75,8 @@ echo "Description of change" > changelog.d/my-change.added
 
 On merge, the versioning workflow bumps the version, builds the changelog, and creates a GitHub Release.
 
+For the target release-bundle architecture, see [Release bundles](release-bundles.md). That document defines the split between country `*-data` build manifests and `policyengine.py` certified runtime bundles.
+
 ## Architecture
 
 ### Package layout

{policyengine-3.4.2 → policyengine-3.4.4}/docs/index.md

@@ -16,4 +16,5 @@ We do this by:
 - [US tax-benefit model](country-models-us.md): Entities, parameters, reform examples
 - [Examples](examples.md): Complete working scripts
 - [Visualisation](visualisation.md): Publication-ready charts with Plotly
+- [Release bundles](release-bundles.md): Reproducible model-plus-data certification and provenance
 - [Development](dev.md): Setup, testing, CI, architecture

{policyengine-3.4.2 → policyengine-3.4.4}/docs/myst.yml

@@ -17,6 +17,7 @@ project:
     - file: country-models-us.md
     - file: examples.md
     - file: visualisation.md
+    - file: release-bundles.md
     - file: dev.md
 
 site:

policyengine-3.4.4/docs/release-bundles.md

@@ -0,0 +1,329 @@
+# Release Bundles
+
+This document defines the intended reproducibility boundary for `policyengine.py`.
+
+The key design decision is:
+
+- country `*-data` repos build and stage immutable data artifacts
+- `policyengine.py` is the only component that certifies supported runtime bundles
+- `policyengine.py` does not rebuild country data itself
+
+This keeps country-specific data construction in the country data repos while still giving users a single top-level version to cite and pin.
+
+## Why this boundary exists
+
+For countries like the UK, the data package is not model-independent. Dataset construction, imputations, and calibration steps call the country model directly. That means a published dataset artifact depends on:
+
+- the country model version used during data construction
+- the calibration targets used during data construction
+- the raw input data used during data construction
+
+If `policyengine.py` only pins a country model version and a data package version without checking that relationship, the provenance boundary is incomplete.
+
+## Roles
+
+### Country model package
+
+Examples: `policyengine-uk`, `policyengine-us`
+
+The country model package owns:
+
+- policy logic
+- variables and parameters
+- reforms
+- a `data_build_fingerprint` for the subset of model logic that affects data construction
+
+It does not own final runtime bundle certification.
+
+### Country data package
+
+Examples: `policyengine-uk-data`, `policyengine-us-data`
+
+The country data package owns:
+
+- data build pipelines
+- raw input acquisition
+- calibration target snapshots
+- expensive dataset construction
+- staging immutable build artifacts with provenance
+
+It does not define the final supported runtime bundle exposed to users.
+
+### `policyengine.py`
+
+`policyengine.py` owns:
+
+- runtime bundle certification
+- user-facing reproducibility boundaries
+- the supported mapping from `policyengine.py` version to country model version and certified data artifact
+
+It does not rebuild microdata artifacts.
+
+## Two manifest layers
+
+The architecture has two manifest layers with different responsibilities.
+
+### 1. Data build manifest
+
+Published by the country `*-data` repo.
+
+This answers:
+
+- what bytes were produced
+- how they were produced
+- which exact model and targets produced them
+
+Suggested schema:
+
+```json
+{
+  "schema_version": 1,
+  "country_id": "uk",
+  "data_package": {
+    "name": "policyengine-uk-data",
+    "version": "1.41.0"
+  },
+  "build": {
+    "build_id": "uk-data-2026-04-12T12-30-00Z",
+    "git_sha": "abc123",
+    "built_at": "2026-04-12T12:30:00Z",
+    "built_with_model_package": {
+      "name": "policyengine-uk",
+      "version": "2.81.0",
+      "git_sha": "def456",
+      "data_build_fingerprint": "sha256:..."
+    },
+    "calibration_targets": {
+      "snapshot_id": "2026-04-10",
+      "sha256": "sha256:..."
+    },
+    "raw_inputs": [
+      {
+        "name": "frs_2023_24",
+        "sha256": "sha256:..."
+      }
+    ],
+    "build_environment": {
+      "python_version": "3.13.3",
+      "lockfile_sha256": "sha256:..."
+    }
+  },
+  "default_datasets": {
+    "national": "enhanced_frs_2023_24",
+    "baseline": "frs_2023_24"
+  },
+  "artifacts": {
+    "enhanced_frs_2023_24": {
+      "kind": "microdata",
+      "repo_id": "policyengine/policyengine-uk-data-private",
+      "path": "builds/uk-data-2026-04-12T12-30-00Z/enhanced_frs_2023_24.h5",
+      "revision": "uk-data-2026-04-12T12-30-00Z",
+      "sha256": "sha256:...",
+      "size_bytes": 123456789
+    }
+  }
+}
+```
+
+Notes:
+
+- `build_id` must be immutable.
+- build artifacts should be staged under a build-specific path or revision, not a floating release tag.
+- the build manifest is the authoritative provenance record for the artifact bytes.
+
+### 2. Certified runtime bundle manifest
+
+Published by `policyengine.py`.
+
+This answers:
+
+- which model and data artifact are supported together at runtime
+- which exact dataset should be used by default
+- which artifact checksum and provenance should be surfaced to users
+
+Suggested schema:
+
+```json
+{
+  "schema_version": 1,
+  "policyengine_version": "3.5.0",
+  "bundle_id": "uk-3.5.0",
+  "published_at": "2026-04-12T13:00:00Z",
+  "country_id": "uk",
+  "model_package": {
+    "name": "policyengine-uk",
+    "version": "2.81.1"
+  },
+  "certified_data_artifact": {
+    "data_package": {
+      "name": "policyengine-uk-data",
+      "version": "1.41.0"
+    },
+    "build_id": "uk-data-2026-04-12T12-30-00Z",
+    "dataset": "enhanced_frs_2023_24",
+    "uri": "hf://policyengine/policyengine-uk-data-private/builds/uk-data-2026-04-12T12-30-00Z/enhanced_frs_2023_24.h5@uk-data-2026-04-12T12-30-00Z",
+    "sha256": "sha256:..."
+  },
+  "certification": {
+    "compatibility_basis": "matching_data_build_fingerprint",
+    "built_with_model_version": "2.81.0",
+    "certified_for_model_version": "2.81.1",
+    "data_build_fingerprint": "sha256:...",
+    "certified_by": "policyengine.py release workflow"
+  },
+  "default_dataset": "enhanced_frs_2023_24",
+  "region_artifacts": {
+    "national": {
+      "dataset": "enhanced_frs_2023_24"
+    }
+  }
+}
+```
+
+Notes:
+
+- this is the user-facing reproducibility boundary
+- apps and APIs should surface this bundle, not only country package versions
+- a bundle may reuse a previously staged data artifact if compatibility is explicitly certified
+
+## TRACE export
+
+The internal build manifest and certified runtime bundle remain the operational source of
+truth.
+
+TRACE sits on top of those manifests as a standards-based export layer.
+
+### What gets exported
+
+Country `*-data` repos should emit a `trace.tro.jsonld` file for each published data
+release. That TRO should cover:
+
+- the release manifest itself
+- each published artifact hash listed in the release manifest
+- the build-time model provenance recorded in the release manifest
+
+`policyengine.py` should emit a separate certified-bundle TRO. That TRO should cover:
+
+- the bundled country release manifest shipped in `policyengine.py`
+- the country data release manifest resolved for the certified data package version
+- the certified dataset artifact hash
+- the certification basis used to allow runtime reuse
+
+### What TRACE does not replace
+
+TRACE is not the source of truth for compatibility policy.
+
+In particular, TRACE does not decide:
+
+- whether a new model version can safely reuse an existing data artifact
+- how `data_build_fingerprint` is computed
+- which staged artifact becomes a supported runtime default
+
+Those decisions still belong to the country data build manifest and the
+`policyengine.py` certified runtime bundle.
+
+### Why we still want it
+
+TRACE adds three things our internal manifests do not provide by themselves:
+
+- a standard declaration format for provenance exchange
+- a composition fingerprint over the exact artifacts in scope
+- a better external surface for papers, audits, and reproducibility reviews
+
+That is why the recommended design is:
+
+- internal manifests for build/certification control
+- generated TRACE TROs for standards-based export
+
+## Compatibility rule
+
+The architecture should avoid forcing a new data build for every harmless country model release.
+
+To do that safely, compatibility must be explicit.
+
+### Data build fingerprint
+
+Each country model package should expose a `data_build_fingerprint` that covers the subset of logic that affects dataset construction or calibration.
+
+Examples of inputs to the fingerprint:
+
+- variables used in imputations
+- variables used in calibration loss matrices
+- parameters referenced during data construction
+- uprating or target-computation logic used during the build
+
+Things that should usually not affect the fingerprint:
+
+- runtime-only outputs that are not used in data construction
+- UI-oriented metadata
+- code paths unrelated to data construction
+
+### Certification rules
+
+`policyengine.py` may certify a staged data artifact for a model version only if one of the following is true:
+
+1. the model version exactly matches the `built_with_model_package.version`
+2. the model version has the same `data_build_fingerprint` as the build-time model version
+
+If neither is true, the bundle release must fail and a new data build is required.
+
+This should be a hard failure, not a warning.
+
+## Artifact states
+
+Artifacts should move through explicit states:
+
+- `staged`: built by the country data repo and available for inspection or later certification
+- `certified`: referenced by a released `policyengine.py` runtime bundle
+- `deprecated`: no longer recommended for new use, but still reproducible
+
+The key point is that `staged` and `certified` are different states. A staged artifact is not automatically part of a supported runtime release.
+
+## UK release workflow
+
+### Case 1: model-only release
+
+1. Cut UK model release candidate `M`.
+2. Compute `data_build_fingerprint(M)`.
+3. Compare it to the fingerprint recorded in the previously certified data build manifest.
+4. If the fingerprint matches, skip the expensive UK data rebuild.
+5. Release `policyengine.py` with a new certified runtime bundle that points to the existing staged UK artifact.
+
+### Case 2: data-affecting release
+
+1. Cut UK model release candidate `M`.
+2. Compute `data_build_fingerprint(M)`.
+3. If the fingerprint changed, build a new UK data artifact in `policyengine-uk-data` against:
+   - exact `policyengine-uk==M`
+   - exact target snapshot
+   - exact raw input hashes
+4. Stage the new artifact under a build-specific immutable path or revision.
+5. Publish the UK data build manifest.
+6. Release `policyengine.py` with a certified runtime bundle that points to the new staged artifact.
+
+## Implementation guidance
+
+The current `release_manifest.json` mechanism in country data repos is a good starting point, but it is not yet enough on its own. The target implementation should add:
+
+- `built_with_model_package.version`
+- `built_with_model_package.git_sha`
+- `built_with_model_package.data_build_fingerprint`
+- calibration target snapshot metadata
+- immutable staged artifact paths or revisions
+
+The target implementation in `policyengine.py` should add:
+
+- hard validation of bundle certification rules
+- explicit runtime bundle metadata on simulations, APIs, and app responses
+- checksum-backed dataset resolution from the certified bundle manifest
+
+## Why not let `policyengine.py` build all country data directly?
+
+Because that would centralise the wrong concerns:
+
+- country-specific private data handling would move into the generic orchestration layer
+- country-specific build logic would move into the generic orchestration layer
+- expensive build failures would block the top-level runtime package more often
+- provenance would still originate in the country data pipeline, so `policyengine.py` would not actually eliminate the need for the country build manifest
+
+`policyengine.py` should be the certification boundary, not the country data build system.

{policyengine-3.4.2 → policyengine-3.4.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "policyengine"
-version = "3.4.2"
+version = "3.4.4"
 description = "A package to conduct policy analysis using PolicyEngine tax-benefit models."
 readme = "README.md"
 authors = [
@@ -28,7 +28,7 @@ dependencies = [
 [project.optional-dependencies]
 uk = [
     "policyengine_core>=3.23.6",
-    "policyengine-uk==2.78.0",
+    "policyengine-uk==2.74.0",
 ]
 us = [
     "policyengine_core>=3.23.6",
@@ -45,7 +45,7 @@ dev = [
     "pytest-asyncio>=0.26.0",
     "ruff>=0.9.0",
     "policyengine_core>=3.23.6",
-    "policyengine-uk==2.78.0",
+    "policyengine-uk==2.74.0",
     "policyengine-us==1.602.0",
     "towncrier>=24.8.0",
     "mypy>=1.11.0",