morpc-census 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- morpc_census-0.1.0/.gitattributes +2 -0
- morpc_census-0.1.0/.github/workflows/ci.yml +61 -0
- morpc_census-0.1.0/.github/workflows/deploy.yml +49 -0
- morpc_census-0.1.0/.github/workflows/publish.yml +38 -0
- morpc_census-0.1.0/.github/workflows/python-publish.yml +73 -0
- morpc_census-0.1.0/.github/workflows/todo_to_issue.yml +38 -0
- morpc_census-0.1.0/.gitignore +12 -0
- morpc_census-0.1.0/CHANGELOG.md +45 -0
- morpc_census-0.1.0/CONTRIBUTING.md +57 -0
- morpc_census-0.1.0/PKG-INFO +153 -0
- morpc_census-0.1.0/README.md +131 -0
- morpc_census-0.1.0/app/app.py +36 -0
- morpc_census-0.1.0/dev/architecture.md +97 -0
- morpc_census-0.1.0/doc/.gitignore +5 -0
- morpc_census-0.1.0/doc/01-morpc-geos-demo.ipynb +1857 -0
- morpc_census-0.1.0/doc/02-morpc-census-demo.ipynb +8541 -0
- morpc_census-0.1.0/doc/03-morpc-poverty-race-demo.ipynb +1725 -0
- morpc_census-0.1.0/doc/assets/HORIZONTAL_LOGOS_PRIMARY_COLOR_V2.png +0 -0
- morpc_census-0.1.0/doc/index.md +18 -0
- morpc_census-0.1.0/doc/myst.yml +13 -0
- morpc_census-0.1.0/morpc.zip +0 -0
- morpc_census-0.1.0/morpc_census/__init__.py +64 -0
- morpc_census-0.1.0/morpc_census/api.py +1330 -0
- morpc_census-0.1.0/morpc_census/constants.py +196 -0
- morpc_census-0.1.0/morpc_census/geos.py +1215 -0
- morpc_census-0.1.0/morpc_census/py.typed +0 -0
- morpc_census-0.1.0/morpc_census/tigerweb.py +296 -0
- morpc_census-0.1.0/morpc_census.egg-info/PKG-INFO +153 -0
- morpc_census-0.1.0/morpc_census.egg-info/SOURCES.txt +42 -0
- morpc_census-0.1.0/morpc_census.egg-info/dependency_links.txt +1 -0
- morpc_census-0.1.0/morpc_census.egg-info/requires.txt +11 -0
- morpc_census-0.1.0/morpc_census.egg-info/top_level.txt +2 -0
- morpc_census-0.1.0/pyproject.toml +51 -0
- morpc_census-0.1.0/reference/dev_notes.md +773 -0
- morpc_census-0.1.0/release_new_package.md +31 -0
- morpc_census-0.1.0/setup.cfg +4 -0
- morpc_census-0.1.0/tests/__init__.py +0 -0
- morpc_census-0.1.0/tests/conftest.py +25 -0
- morpc_census-0.1.0/tests/test_api.py +1162 -0
- morpc_census-0.1.0/tests/test_constants.py +107 -0
- morpc_census-0.1.0/tests/test_geoidfq_class.py +192 -0
- morpc_census-0.1.0/tests/test_geos_classes.py +172 -0
- morpc_census-0.1.0/tests/test_smoke.py +44 -0
- morpc_census-0.1.0/tests/test_tigerweb.py +92 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: ["main", "**"]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: ["main"]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
name: Test (Python ${{ matrix.python-version }})
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.10", "3.11", "3.12"]
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
22
|
+
uses: actions/setup-python@v5
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python-version }}
|
|
25
|
+
|
|
26
|
+
- name: Install morpc-py (private dependency)
|
|
27
|
+
run: |
|
|
28
|
+
git clone https://github.com/morpc/morpc-py.git /tmp/morpc-py
|
|
29
|
+
pip install /tmp/morpc-py/
|
|
30
|
+
|
|
31
|
+
- name: Install morpc-census
|
|
32
|
+
run: pip install -e ".[dev]"
|
|
33
|
+
|
|
34
|
+
- name: Run offline tests
|
|
35
|
+
run: pytest -m "not network" --tb=short
|
|
36
|
+
|
|
37
|
+
build:
|
|
38
|
+
name: Build check
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
|
|
41
|
+
steps:
|
|
42
|
+
- uses: actions/checkout@v4
|
|
43
|
+
|
|
44
|
+
- name: Set up Python
|
|
45
|
+
uses: actions/setup-python@v5
|
|
46
|
+
with:
|
|
47
|
+
python-version: "3.12"
|
|
48
|
+
|
|
49
|
+
- name: Install build tools
|
|
50
|
+
run: pip install build twine
|
|
51
|
+
|
|
52
|
+
- name: Install morpc-py (private dependency)
|
|
53
|
+
run: |
|
|
54
|
+
git clone https://github.com/morpc/morpc-py.git /tmp/morpc-py
|
|
55
|
+
pip install /tmp/morpc-py/
|
|
56
|
+
|
|
57
|
+
- name: Build package
|
|
58
|
+
run: python -m build
|
|
59
|
+
|
|
60
|
+
- name: Check distribution
|
|
61
|
+
run: twine check dist/*
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# This file was created automatically with `myst init --gh-pages` 🪄 💚
|
|
2
|
+
# Ensure your GitHub Pages settings for this repository are set to deploy with **GitHub Actions**.
|
|
3
|
+
|
|
4
|
+
name: MyST GitHub Pages Deploy
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
# Runs on pushes targeting the default branch
|
|
8
|
+
branches: [main]
|
|
9
|
+
env:
|
|
10
|
+
# `BASE_URL` determines, relative to the root of the domain, the URL that your site is served from.
|
|
11
|
+
# E.g., if your site lives at `https://mydomain.org/myproject`, set `BASE_URL=/myproject`.
|
|
12
|
+
# If, instead, your site lives at the root of the domain, at `https://mydomain.org`, set `BASE_URL=''`.
|
|
13
|
+
BASE_URL: /${{ github.event.repository.name }}
|
|
14
|
+
|
|
15
|
+
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
|
|
16
|
+
permissions:
|
|
17
|
+
contents: read
|
|
18
|
+
pages: write
|
|
19
|
+
id-token: write
|
|
20
|
+
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
|
|
21
|
+
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
|
|
22
|
+
concurrency:
|
|
23
|
+
group: 'pages'
|
|
24
|
+
cancel-in-progress: false
|
|
25
|
+
jobs:
|
|
26
|
+
deploy:
|
|
27
|
+
environment:
|
|
28
|
+
name: github-pages
|
|
29
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v4
|
|
33
|
+
- name: Setup Pages
|
|
34
|
+
uses: actions/configure-pages@v3
|
|
35
|
+
- uses: actions/setup-node@v4
|
|
36
|
+
with:
|
|
37
|
+
node-version: 18.x
|
|
38
|
+
- name: Install MyST Markdown
|
|
39
|
+
run: npm install -g mystmd
|
|
40
|
+
- name: Build HTML Assets
|
|
41
|
+
run: myst build --html
|
|
42
|
+
working-directory: ./doc
|
|
43
|
+
- name: Upload artifact
|
|
44
|
+
uses: actions/upload-pages-artifact@v3
|
|
45
|
+
with:
|
|
46
|
+
path: './doc/_build/html'
|
|
47
|
+
- name: Deploy to GitHub Pages
|
|
48
|
+
id: deployment
|
|
49
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
publish:
|
|
9
|
+
name: Build and publish
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
environment: pypi
|
|
12
|
+
permissions:
|
|
13
|
+
id-token: write
|
|
14
|
+
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.12"
|
|
22
|
+
|
|
23
|
+
- name: Install build tools
|
|
24
|
+
run: pip install build twine
|
|
25
|
+
|
|
26
|
+
- name: Install morpc-py (private dependency)
|
|
27
|
+
run: |
|
|
28
|
+
git clone https://github.com/morpc/morpc-py.git /tmp/morpc-py
|
|
29
|
+
pip install /tmp/morpc-py/
|
|
30
|
+
|
|
31
|
+
- name: Build package
|
|
32
|
+
run: python -m build
|
|
33
|
+
|
|
34
|
+
- name: Check distribution
|
|
35
|
+
run: twine check dist/*
|
|
36
|
+
|
|
37
|
+
- name: Publish to PyPI
|
|
38
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# This workflow will upload a Python Package to PyPI when a release is created
|
|
2
|
+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
|
3
|
+
|
|
4
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
5
|
+
# They are provided by a third-party and are governed by
|
|
6
|
+
# separate terms of service, privacy policy, and support
|
|
7
|
+
# documentation.
|
|
8
|
+
|
|
9
|
+
name: Upload Python Package
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
release-build:
|
|
20
|
+
runs-on: windows-latest
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
with:
|
|
25
|
+
persist-credentials: false
|
|
26
|
+
- name: Set up Python
|
|
27
|
+
uses: actions/setup-python@v5
|
|
28
|
+
with:
|
|
29
|
+
python-version: "3.x"
|
|
30
|
+
- name: Install pypa/build
|
|
31
|
+
run: >-
|
|
32
|
+
python3 -m
|
|
33
|
+
pip install
|
|
34
|
+
build
|
|
35
|
+
--user
|
|
36
|
+
- name: Build a binary wheel and a source tarball
|
|
37
|
+
run: python3 -m build
|
|
38
|
+
- name: Store the distribution packages
|
|
39
|
+
uses: actions/upload-artifact@v4
|
|
40
|
+
with:
|
|
41
|
+
name: python-package-distributions
|
|
42
|
+
path: dist/
|
|
43
|
+
|
|
44
|
+
pypi-publish:
|
|
45
|
+
runs-on: ubuntu-latest
|
|
46
|
+
needs:
|
|
47
|
+
- release-build
|
|
48
|
+
permissions:
|
|
49
|
+
# IMPORTANT: this permission is mandatory for trusted publishing
|
|
50
|
+
id-token: write
|
|
51
|
+
|
|
52
|
+
# Dedicated environments with protections for publishing are strongly recommended.
|
|
53
|
+
# For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
|
|
54
|
+
environment:
|
|
55
|
+
name: pypi
|
|
56
|
+
# OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status:
|
|
57
|
+
url: https://pypi.org/p/morpc
|
|
58
|
+
#
|
|
59
|
+
# ALTERNATIVE: if your GitHub Release name is the PyPI project version string
|
|
60
|
+
# ALTERNATIVE: exactly, uncomment the following line instead:
|
|
61
|
+
# url: https://pypi.org/project/morpc/${{ github.event.release.name }}
|
|
62
|
+
|
|
63
|
+
steps:
|
|
64
|
+
- name: Retrieve release distributions
|
|
65
|
+
uses: actions/download-artifact@v4
|
|
66
|
+
with:
|
|
67
|
+
name: python-package-distributions
|
|
68
|
+
path: dist/
|
|
69
|
+
|
|
70
|
+
- name: Publish release distributions to PyPI
|
|
71
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
72
|
+
with:
|
|
73
|
+
packages-dir: dist/
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: "Run TODO to Issue"
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
inputs:
|
|
6
|
+
MANUAL_COMMIT_REF:
|
|
7
|
+
description: "The SHA of the commit to get the diff for"
|
|
8
|
+
required: true
|
|
9
|
+
MANUAL_BASE_REF:
|
|
10
|
+
description: "By default, the commit entered above is compared to the one directly before it; to go back further, enter an earlier SHA here"
|
|
11
|
+
required: false
|
|
12
|
+
jobs:
|
|
13
|
+
build:
|
|
14
|
+
runs-on: "ubuntu-latest"
|
|
15
|
+
permissions:
|
|
16
|
+
contents: write
|
|
17
|
+
issues: write
|
|
18
|
+
pull-requests: write
|
|
19
|
+
steps:
|
|
20
|
+
- uses: "actions/checkout@v4"
|
|
21
|
+
- name: "TODO to Issue"
|
|
22
|
+
uses: "alstr/todo-to-issue-action@v5"
|
|
23
|
+
with:
|
|
24
|
+
INSERT_ISSUE_URLS: "true"
|
|
25
|
+
CLOSE_ISSUES: "true"
|
|
26
|
+
- name: Set Git user
|
|
27
|
+
run: |
|
|
28
|
+
git config --global user.name "github-actions[bot]"
|
|
29
|
+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
|
|
30
|
+
- name: Commit and Push Changes
|
|
31
|
+
run: |
|
|
32
|
+
git add -A
|
|
33
|
+
if [[ `git status --porcelain` ]]; then
|
|
34
|
+
git commit -m "Automatically added GitHub issue links to TODOs"
|
|
35
|
+
git push origin main
|
|
36
|
+
else
|
|
37
|
+
echo "No changes to commit"
|
|
38
|
+
fi
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
|
6
|
+
This project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## [Unreleased]
|
|
11
|
+
|
|
12
|
+
## [0.1.0] — 2026-05-16
|
|
13
|
+
|
|
14
|
+
Initial beta release.
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- **`CensusAPI`** — fetches ACS/Decennial data by group or variable list, melts wide API response to a long-format DataFrame with `estimate`/`moe` columns, `concept`, `universe`, `survey`, `reference_period`, `geoidfq`, and `name`.
|
|
19
|
+
- **`DimensionTable`** — reshapes a `CensusAPI.long` DataFrame into a MultiIndex wide table (`wide()`), computes column percentages with Census Bureau derived proportion MOEs (`percent()`), remaps variable labels (`remap()`), and drops dimension levels by name, integer index, or list (`drop()`). Dimension columns are ordered categoricals preserving Census API hierarchy order.
|
|
20
|
+
- **`RaceDimensionTable`** — subclass of `DimensionTable` for analysing racial-iteration groups (e.g. B17020A–I). Normalises variable codes, concept, and universe across races, adds an ordered `race` column level, and computes within-race percentages.
|
|
21
|
+
- **`Endpoint` / `Group`** — lazy-cached wrappers around the Census API metadata endpoints (vintages, groups, variables).
|
|
22
|
+
- **`GeoIDFQ`** — parses, inspects, and constructs Census fully-qualified geographic IDs (e.g. `1400000US39049010100`).
|
|
23
|
+
- **`Scope` / `SumLevel` / `SCOPES`** — named geographic extents and summary-level utilities for building Census API `for`/`in` query parameters.
|
|
24
|
+
- **`fetch_geos_from_scope_sumlevel`** — fetches TIGERweb boundary GeoDataFrames for a named scope and summary level.
|
|
25
|
+
- **`get_tigerweb_layers_map`** — returns a live layer-name → MapServer-ID mapping for ACS, Decennial, or Current TIGERweb services.
|
|
26
|
+
- **`morpc_census.constants`** — lookup tables for age groups, race, education, income-to-poverty, and NTD categories.
|
|
27
|
+
- Frictionless schema and resource file generation via `CensusAPI.save()`.
|
|
28
|
+
- `@pytest.mark.network` test suite for live API calls; full offline suite runnable without a Census API key.
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- `HIGHLEVEL_DESC_FROM_ID` renamed to `HIGHLEVEL_DESC_TO_ID` (the dict maps descriptions *to* IDs, not from them).
|
|
33
|
+
- `DimensionTable.variable_type` renamed to `value_cols`.
|
|
34
|
+
- `find_replace_variable_map` parameter `map` renamed to `label_map` (avoids shadowing the Python builtin).
|
|
35
|
+
- `CensusAPI.melt()` refactored into focused private helpers (`_melt_wide_to_long`, `_attach_dataset_metadata`, `_pivot_and_coerce`).
|
|
36
|
+
- `wide()` column MultiIndex now uses canonical level order (`concept > universe > survey > geoidfq > name > [race] > reference_period > value_type`) and names the value-type level `'value_type'` instead of `None`.
|
|
37
|
+
- `percent()` MOE now uses the Census Bureau derived proportion formula.
|
|
38
|
+
- `_get_api_key()` and `get_all_avail_endpoints()` are `@functools.cache`-decorated (no repeated filesystem/network calls).
|
|
39
|
+
- `get_tigerweb_layers_map` accepts `survey='current'` (no year required).
|
|
40
|
+
|
|
41
|
+
### Fixed
|
|
42
|
+
|
|
43
|
+
- Python 3.10/3.11 syntax error in `geos.py` (`Scope.sql` f-string nesting).
|
|
44
|
+
- `NAME` column missing from variable-batch fetch results.
|
|
45
|
+
- `set_levels` race-ordering bug that swapped column labels without moving data.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Contributing to morpc-census
|
|
2
|
+
|
|
3
|
+
## Setup
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
# 1. Install the private morpc dependency
|
|
7
|
+
git clone https://github.com/morpc/morpc-py.git
|
|
8
|
+
pip install /path/to/morpc-py/
|
|
9
|
+
|
|
10
|
+
# 2. Clone and install morpc-census in editable mode
|
|
11
|
+
git clone https://github.com/jinskeep-morpc/morpc-census.git
|
|
12
|
+
pip install -e /path/to/morpc-census/[dev]
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Running tests
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
# Offline tests only (no Census API key required)
|
|
19
|
+
pytest
|
|
20
|
+
|
|
21
|
+
# Include live network tests (requires CENSUS_API_KEY in environment or .env)
|
|
22
|
+
pytest -m network
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Tests are split by the `network` marker. The default `pytest` run (`-m 'not network'`) is safe to run locally without credentials.
|
|
26
|
+
|
|
27
|
+
## Making changes
|
|
28
|
+
|
|
29
|
+
1. Create a GitHub issue describing the problem and intended solution.
|
|
30
|
+
2. Create a branch (one branch per roadmap phase or per logical feature).
|
|
31
|
+
3. Make changes and commit in logical chunks.
|
|
32
|
+
4. Write tests for all changed behavior.
|
|
33
|
+
5. Append a dated entry to `reference/dev_notes.md` summarising the change.
|
|
34
|
+
6. Open a pull request and request review.
|
|
35
|
+
|
|
36
|
+
## Versioning
|
|
37
|
+
|
|
38
|
+
This project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html): `MAJOR.MINOR.PATCH`.
|
|
39
|
+
|
|
40
|
+
| Increment | When |
|
|
41
|
+
|-----------|------|
|
|
42
|
+
| `PATCH` | Backwards-compatible bug fixes |
|
|
43
|
+
| `MINOR` | New backwards-compatible functionality |
|
|
44
|
+
| `MAJOR` | Breaking changes to the public API |
|
|
45
|
+
|
|
46
|
+
**Breaking changes** include:
|
|
47
|
+
- Removing or renaming public functions, classes, or parameters
|
|
48
|
+
- Changing return types or shapes in a way callers must update for
|
|
49
|
+
- Dropping support for a previously supported Python version
|
|
50
|
+
|
|
51
|
+
Breaking changes must be noted in `CHANGELOG.md` under the relevant version entry before the release is tagged.
|
|
52
|
+
|
|
53
|
+
## Releasing
|
|
54
|
+
|
|
55
|
+
1. Update `CHANGELOG.md` — move items from `[Unreleased]` to a new `[X.Y.Z]` section.
|
|
56
|
+
2. Create and push a git tag: `git tag vX.Y.Z && git push origin vX.Y.Z`.
|
|
57
|
+
3. Create a GitHub release from the tag — this triggers the publish workflow.
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: morpc-census
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Census data tools used by MORPC
|
|
5
|
+
Author-email: MORPC data team <dataandmaps@morpc.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Typing :: Typed
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
Requires-Dist: morpc>=0.4.3
|
|
13
|
+
Requires-Dist: numpy>=1.24
|
|
14
|
+
Requires-Dist: pandas>=2.0
|
|
15
|
+
Requires-Dist: geopandas>=0.14
|
|
16
|
+
Requires-Dist: frictionless>=5.0
|
|
17
|
+
Requires-Dist: requests>=2.28
|
|
18
|
+
Requires-Dist: shapely>=2.0
|
|
19
|
+
Requires-Dist: python-dotenv>=1.0
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
22
|
+
|
|
23
|
+
# morpc-census
|
|
24
|
+
|
|
25
|
+
## Introduction
|
|
26
|
+
|
|
27
|
+
`morpc-census` is a Python package maintained by the MORPC data team for working with US Census Bureau data. It provides tools for connecting to the Census API, retrieving survey data, and structuring results as long-format tables with [frictionless](https://github.com/frictionlessdata/frictionless-py) metadata.
|
|
28
|
+
|
|
29
|
+
This package depends on [morpc-py](https://github.com/morpc/morpc-py) for shared MORPC utilities.
|
|
30
|
+
|
|
31
|
+
### Modules
|
|
32
|
+
|
|
33
|
+
- **morpc_census.api** — Census API client (`Endpoint`, `Group`, `CensusAPI`) and data structuring classes (`DimensionTable`, `RaceDimensionTable`) for reshaping results into long- and wide-format DataFrames with frictionless metadata.
|
|
34
|
+
- **morpc_census.geos** — Geography utilities: `Scope` (named query extents), `SumLevel` (summary level codes), `GeoIDFQ` (GEOID parser/builder), and functions for translating between Census GEOIDs and MORPC geography definitions.
|
|
35
|
+
- **morpc_census.tigerweb** — Fetches GeoDataFrames from the Census TIGERweb REST API.
|
|
36
|
+
- **morpc_census.constants** — Domain lookup tables for age groups, race, education, income-to-poverty, and NTD categories.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
`morpc-census` depends on [morpc-py](https://github.com/morpc/morpc-py), a private MORPC utility package that is not on PyPI. Install it manually before installing `morpc-census`:
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
# 1. Install the private morpc dependency first
|
|
44
|
+
git clone https://github.com/morpc/morpc-py.git
|
|
45
|
+
pip install /path/to/morpc-py/
|
|
46
|
+
|
|
47
|
+
# 2. Install morpc-census
|
|
48
|
+
pip install morpc-census
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Dev Install
|
|
52
|
+
|
|
53
|
+
To install an editable version for development:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
# 1. Install the private morpc dependency
|
|
57
|
+
git clone https://github.com/morpc/morpc-py.git
|
|
58
|
+
pip install /path/to/morpc-py/
|
|
59
|
+
|
|
60
|
+
# 2. Clone and install morpc-census in editable mode
|
|
61
|
+
git clone https://github.com/jinskeep-morpc/morpc-census.git
|
|
62
|
+
pip install -e /path/to/morpc-census/
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Then import as:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import morpc_census
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Usage
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from morpc_census import Endpoint, Group, CensusAPI, DimensionTable, RaceDimensionTable, SCOPES, SumLevel
|
|
75
|
+
|
|
76
|
+
ep = Endpoint('acs/acs5', 2023)
|
|
77
|
+
grp = Group(ep, 'B01001')
|
|
78
|
+
|
|
79
|
+
# Fetch ACS 5-year age/sex data for counties in the 15-county region
|
|
80
|
+
api = CensusAPI(ep, SCOPES['region15'], group=grp, sumlevel=SumLevel('county'))
|
|
81
|
+
|
|
82
|
+
# Long-format DataFrame
|
|
83
|
+
print(api.long.head())
|
|
84
|
+
|
|
85
|
+
# Reshape into wide MultiIndex table and compute percentages
|
|
86
|
+
table = DimensionTable(api.long)
|
|
87
|
+
wide = table.wide()
|
|
88
|
+
pct = table.percent()
|
|
89
|
+
|
|
90
|
+
# Save data + frictionless schema + resource to disk
|
|
91
|
+
api.save('./output')
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Demos and Documentation
|
|
95
|
+
|
|
96
|
+
See [demos](https://jinskeep-morpc.github.io/morpc-census/) for examples and documentation.
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
## Roadmap — Code Improvements
|
|
101
|
+
|
|
102
|
+
- [x] Fix Python 3.10/3.11 syntax bug in `geos.py`
|
|
103
|
+
- [x] Cache `_get_api_key()`
|
|
104
|
+
- [x] Replace global `_avail_endpoints_cache` with `@functools.cache`
|
|
105
|
+
- [x] Avoid double-computing `wide()` inside `percent()`
|
|
106
|
+
- [x] Rename `DimensionTable.variable_type` → `value_cols`
|
|
107
|
+
- [x] Rename `map` parameter in `find_replace_variable_map` → `label_map`
|
|
108
|
+
- [x] Split `CensusAPI.melt()` into focused private helpers
|
|
109
|
+
- [x] Pin `numpy` as an explicit dependency
|
|
110
|
+
- [x] Add minimum version pins to all dependencies
|
|
111
|
+
- [x] Add module docstrings to `geos.py` and `tigerweb.py`
|
|
112
|
+
- [x] Validate or auto-fetch `tigerweb.py` `current_endpoints`
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Roadmap — Production Readiness & PyPI Release
|
|
117
|
+
|
|
118
|
+
### Phase 1 — Pre-release cleanup
|
|
119
|
+
|
|
120
|
+
- [x] Apply all code improvement items above
|
|
121
|
+
- [x] Fix the README usage example
|
|
122
|
+
- [x] Expand test coverage for offline paths (`wide()`, `percent()`, `remap()`, `drop()`, `melt()`)
|
|
123
|
+
- [x] Update `pyproject.toml` classifier from `Development Status :: 1 - Planning` to `4 - Beta`
|
|
124
|
+
- [x] Add a `CHANGELOG.md`
|
|
125
|
+
- [x] Add a `py.typed` marker file
|
|
126
|
+
|
|
127
|
+
### Phase 2 — Dependency audit
|
|
128
|
+
|
|
129
|
+
- [x] Assess whether `morpc` can be published to PyPI or replaced
|
|
130
|
+
- [x] Document installation order if `morpc` remains a private dependency
|
|
131
|
+
- [x] Pin `morpc` to a minimum version in `pyproject.toml`
|
|
132
|
+
|
|
133
|
+
### Phase 3 — CI/CD
|
|
134
|
+
|
|
135
|
+
- [x] Add GitHub Actions CI workflow (`pytest -m "not network"` on push/PR, Python 3.10–3.12)
|
|
136
|
+
- [x] Add build verification step (`python -m build` + `twine check`)
|
|
137
|
+
- [x] Add publish workflow triggered on release tags
|
|
138
|
+
|
|
139
|
+
### Phase 4 — Versioning & release
|
|
140
|
+
|
|
141
|
+
- [x] Switch to dynamic versioning via `setuptools-scm`
|
|
142
|
+
- [x] Tag `v0.1.0` and publish first release
|
|
143
|
+
- [x] Document versioning policy and breaking-change rules in `CONTRIBUTING.md`
|
|
144
|
+
|
|
145
|
+
### Phase 5 — Documentation
|
|
146
|
+
|
|
147
|
+
- [ ] Auto-generate API reference docs from docstrings
|
|
148
|
+
- [ ] Add `CONTRIBUTING.md` with setup instructions and PR process
|
|
149
|
+
- [ ] Add usage examples to docstrings for commonly called functions
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
This product uses the Census Bureau Data API but is not endorsed or certified by the Census Bureau.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# morpc-census
|
|
2
|
+
|
|
3
|
+
## Introduction
|
|
4
|
+
|
|
5
|
+
`morpc-census` is a Python package maintained by the MORPC data team for working with US Census Bureau data. It provides tools for connecting to the Census API, retrieving survey data, and structuring results as long-format tables with [frictionless](https://github.com/frictionlessdata/frictionless-py) metadata.
|
|
6
|
+
|
|
7
|
+
This package depends on [morpc-py](https://github.com/morpc/morpc-py) for shared MORPC utilities.
|
|
8
|
+
|
|
9
|
+
### Modules
|
|
10
|
+
|
|
11
|
+
- **morpc_census.api** — Census API client (`Endpoint`, `Group`, `CensusAPI`) and data structuring classes (`DimensionTable`, `RaceDimensionTable`) for reshaping results into long- and wide-format DataFrames with frictionless metadata.
|
|
12
|
+
- **morpc_census.geos** — Geography utilities: `Scope` (named query extents), `SumLevel` (summary level codes), `GeoIDFQ` (GEOID parser/builder), and functions for translating between Census GEOIDs and MORPC geography definitions.
|
|
13
|
+
- **morpc_census.tigerweb** — Fetches GeoDataFrames from the Census TIGERweb REST API.
|
|
14
|
+
- **morpc_census.constants** — Domain lookup tables for age groups, race, education, income-to-poverty, and NTD categories.
|
|
15
|
+
|
|
16
|
+
## Installation
|
|
17
|
+
|
|
18
|
+
`morpc-census` depends on [morpc-py](https://github.com/morpc/morpc-py), a private MORPC utility package that is not on PyPI. Install it manually before installing `morpc-census`:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
# 1. Install the private morpc dependency first
|
|
22
|
+
git clone https://github.com/morpc/morpc-py.git
|
|
23
|
+
pip install /path/to/morpc-py/
|
|
24
|
+
|
|
25
|
+
# 2. Install morpc-census
|
|
26
|
+
pip install morpc-census
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### Dev Install
|
|
30
|
+
|
|
31
|
+
To install an editable version for development:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
# 1. Install the private morpc dependency
|
|
35
|
+
git clone https://github.com/morpc/morpc-py.git
|
|
36
|
+
pip install /path/to/morpc-py/
|
|
37
|
+
|
|
38
|
+
# 2. Clone and install morpc-census in editable mode
|
|
39
|
+
git clone https://github.com/jinskeep-morpc/morpc-census.git
|
|
40
|
+
pip install -e /path/to/morpc-census/
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Then import as:
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import morpc_census
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Usage
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from morpc_census import Endpoint, Group, CensusAPI, DimensionTable, RaceDimensionTable, SCOPES, SumLevel
|
|
53
|
+
|
|
54
|
+
ep = Endpoint('acs/acs5', 2023)
|
|
55
|
+
grp = Group(ep, 'B01001')
|
|
56
|
+
|
|
57
|
+
# Fetch ACS 5-year age/sex data for counties in the 15-county region
|
|
58
|
+
api = CensusAPI(ep, SCOPES['region15'], group=grp, sumlevel=SumLevel('county'))
|
|
59
|
+
|
|
60
|
+
# Long-format DataFrame
|
|
61
|
+
print(api.long.head())
|
|
62
|
+
|
|
63
|
+
# Reshape into wide MultiIndex table and compute percentages
|
|
64
|
+
table = DimensionTable(api.long)
|
|
65
|
+
wide = table.wide()
|
|
66
|
+
pct = table.percent()
|
|
67
|
+
|
|
68
|
+
# Save data + frictionless schema + resource to disk
|
|
69
|
+
api.save('./output')
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Demos and Documentation
|
|
73
|
+
|
|
74
|
+
See [demos](https://jinskeep-morpc.github.io/morpc-census/) for examples and documentation.
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Roadmap — Code Improvements
|
|
79
|
+
|
|
80
|
+
- [x] Fix Python 3.10/3.11 syntax bug in `geos.py`
|
|
81
|
+
- [x] Cache `_get_api_key()`
|
|
82
|
+
- [x] Replace global `_avail_endpoints_cache` with `@functools.cache`
|
|
83
|
+
- [x] Avoid double-computing `wide()` inside `percent()`
|
|
84
|
+
- [x] Rename `DimensionTable.variable_type` → `value_cols`
|
|
85
|
+
- [x] Rename `map` parameter in `find_replace_variable_map` → `label_map`
|
|
86
|
+
- [x] Split `CensusAPI.melt()` into focused private helpers
|
|
87
|
+
- [x] Pin `numpy` as an explicit dependency
|
|
88
|
+
- [x] Add minimum version pins to all dependencies
|
|
89
|
+
- [x] Add module docstrings to `geos.py` and `tigerweb.py`
|
|
90
|
+
- [x] Validate or auto-fetch `tigerweb.py` `current_endpoints`
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Roadmap — Production Readiness & PyPI Release
|
|
95
|
+
|
|
96
|
+
### Phase 1 — Pre-release cleanup
|
|
97
|
+
|
|
98
|
+
- [x] Apply all code improvement items above
|
|
99
|
+
- [x] Fix the README usage example
|
|
100
|
+
- [x] Expand test coverage for offline paths (`wide()`, `percent()`, `remap()`, `drop()`, `melt()`)
|
|
101
|
+
- [x] Update `pyproject.toml` classifier from `Development Status :: 1 - Planning` to `4 - Beta`
|
|
102
|
+
- [x] Add a `CHANGELOG.md`
|
|
103
|
+
- [x] Add a `py.typed` marker file
|
|
104
|
+
|
|
105
|
+
### Phase 2 — Dependency audit
|
|
106
|
+
|
|
107
|
+
- [x] Assess whether `morpc` can be published to PyPI or replaced
|
|
108
|
+
- [x] Document installation order if `morpc` remains a private dependency
|
|
109
|
+
- [x] Pin `morpc` to a minimum version in `pyproject.toml`
|
|
110
|
+
|
|
111
|
+
### Phase 3 — CI/CD
|
|
112
|
+
|
|
113
|
+
- [x] Add GitHub Actions CI workflow (`pytest -m "not network"` on push/PR, Python 3.10–3.12)
|
|
114
|
+
- [x] Add build verification step (`python -m build` + `twine check`)
|
|
115
|
+
- [x] Add publish workflow triggered on release tags
|
|
116
|
+
|
|
117
|
+
### Phase 4 — Versioning & release
|
|
118
|
+
|
|
119
|
+
- [x] Switch to dynamic versioning via `setuptools-scm`
|
|
120
|
+
- [x] Tag `v0.1.0` and publish first release
|
|
121
|
+
- [x] Document versioning policy and breaking-change rules in `CONTRIBUTING.md`
|
|
122
|
+
|
|
123
|
+
### Phase 5 — Documentation
|
|
124
|
+
|
|
125
|
+
- [ ] Auto-generate API reference docs from docstrings
|
|
126
|
+
- [ ] Add `CONTRIBUTING.md` with setup instructions and PR process
|
|
127
|
+
- [ ] Add usage examples to docstrings for commonly called functions
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
This product uses the Census Bureau Data API but is not endorsed or certified by the Census Bureau.
|