census-loader 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- census_loader-0.1.0/LICENSE +21 -0
- census_loader-0.1.0/PKG-INFO +295 -0
- census_loader-0.1.0/README.md +267 -0
- census_loader-0.1.0/pyproject.toml +42 -0
- census_loader-0.1.0/setup.cfg +4 -0
- census_loader-0.1.0/src/census_loader/__init__.py +19 -0
- census_loader-0.1.0/src/census_loader/load.py +42 -0
- census_loader-0.1.0/src/census_loader/py.typed +0 -0
- census_loader-0.1.0/src/census_loader/series.py +847 -0
- census_loader-0.1.0/src/census_loader/utils.py +938 -0
- census_loader-0.1.0/src/census_loader.egg-info/PKG-INFO +295 -0
- census_loader-0.1.0/src/census_loader.egg-info/SOURCES.txt +19 -0
- census_loader-0.1.0/src/census_loader.egg-info/dependency_links.txt +1 -0
- census_loader-0.1.0/src/census_loader.egg-info/requires.txt +8 -0
- census_loader-0.1.0/src/census_loader.egg-info/top_level.txt +1 -0
- census_loader-0.1.0/tests/test_catalog.py +57 -0
- census_loader-0.1.0/tests/test_clean_frame_and_guards.py +141 -0
- census_loader-0.1.0/tests/test_discovery_and_clean.py +347 -0
- census_loader-0.1.0/tests/test_geography_and_loader.py +102 -0
- census_loader-0.1.0/tests/test_pull_loop.py +383 -0
- census_loader-0.1.0/tests/test_utils_uncovered.py +398 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ian Solberg
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: census-loader
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Python wrapper around the U.S. Census Bureau API that abstracts away variable codes, dataset endpoints, and FIPS identifiers behind a clean, human-readable interface.
|
|
5
|
+
Author: Ian Solberg
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: pandas
|
|
21
|
+
Requires-Dist: numpy
|
|
22
|
+
Requires-Dist: census
|
|
23
|
+
Requires-Dist: python-dotenv
|
|
24
|
+
Requires-Dist: requests
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest; extra == "dev"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# Census Loader
|
|
30
|
+
|
|
31
|
+
A Python wrapper around the U.S. Census Bureau API that abstracts away
|
|
32
|
+
variable codes, dataset endpoints, and FIPS identifiers behind a clean,
|
|
33
|
+
human-readable interface. Installs as `census-loader`, for anyone who pulls
|
|
34
|
+
Census data into pandas.
|
|
35
|
+
|
|
36
|
+
Instead of this:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
data = census.acs5.get(
|
|
40
|
+
("B19013_001E",),
|
|
41
|
+
{"for": "county:*", "in": "state:25"},
|
|
42
|
+
year=2022
|
|
43
|
+
)
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
You write this:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
cfg = Config(
|
|
50
|
+
"output.pkl", "./data",
|
|
51
|
+
year=2022,
|
|
52
|
+
geo="county_in_state",
|
|
53
|
+
state="Massachusetts",
|
|
54
|
+
series=["MEDIAN_HH_INCOME", "TOTAL_POP", "MEDIAN_RENT"],
|
|
55
|
+
)
|
|
56
|
+
result = pull_census(cfg)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Setup
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install census-loader
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Or from a checkout of this repository: `pip install -e .`.
|
|
66
|
+
|
|
67
|
+
Dependencies (installed automatically by pip; `requirements.txt` is kept
|
|
68
|
+
for reference): `pandas`, `numpy`, `census`, `python-dotenv`, `requests`.
|
|
69
|
+
|
|
70
|
+
Get a free key at
|
|
71
|
+
[api.census.gov/data/key_signup.html](https://api.census.gov/data/key_signup.html),
|
|
72
|
+
then add it to a `.env` file in your project root:
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
CENSUS_API_KEY=your_key_here
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Quickstart
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from census_loader import Config, pull_census
|
|
82
|
+
|
|
83
|
+
cfg = Config(
|
|
84
|
+
"ma_counties.pkl", "./output",
|
|
85
|
+
year=2022,
|
|
86
|
+
geo="county_in_state",
|
|
87
|
+
state="Massachusetts",
|
|
88
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "MEDIAN_RENT"],
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
result = pull_census(cfg) # dict of DataFrames keyed by friendly name
|
|
92
|
+
result["Median_Household_Income"]
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Exploring the catalog
|
|
96
|
+
|
|
97
|
+
You never need to open `series.py`. Three discovery functions browse,
|
|
98
|
+
search, and inspect everything from a REPL or notebook:
|
|
99
|
+
|
|
100
|
+
- `available()`: browse categories; `available("INCOME")` drills into a
|
|
101
|
+
category or subcategory.
|
|
102
|
+
- `search("poverty")`: search by keyword.
|
|
103
|
+
- `info("MEDIAN_HH_INCOME")`: full details (name, category, dataset,
|
|
104
|
+
underlying variable codes).
|
|
105
|
+
- `geos()`: browse geography templates.
|
|
106
|
+
|
|
107
|
+
## Output format
|
|
108
|
+
|
|
109
|
+
`pull_census` returns a `dict[str, pd.DataFrame]` and saves a pickle to
|
|
110
|
+
your output path. Each DataFrame has geo columns on the left (FIPS codes +
|
|
111
|
+
human-readable names) and data columns on the right. Column naming depends
|
|
112
|
+
on the series type (single variable, multi-variable, group table); labels
|
|
113
|
+
are fetched automatically from the Census metadata API. `pickle_loader`
|
|
114
|
+
flattens a saved pickle to one DataFrame.
|
|
115
|
+
|
|
116
|
+
| Series type | Example key | Column names |
|
|
117
|
+
|---|---|---|
|
|
118
|
+
| Single variable | `TOTAL_POP` | `Total_Population` |
|
|
119
|
+
| Multi-variable | `HOMEOWNERSHIP_RATE` | `Homeownership_Rate__Owner occupied`, `Homeownership_Rate__Total` |
|
|
120
|
+
| Group table | `HH_INCOME_BRACKETS` | `Household_Income_Distribution__Less than $10,000`, `Household_Income_Distribution__$10,000 to $14,999`, ... |
|
|
121
|
+
|
|
122
|
+
## API limits
|
|
123
|
+
|
|
124
|
+
Free Census API keys allow 500 requests per day with a batch size of 50
|
|
125
|
+
variables per request. The loader sleeps 0.5s between calls to stay well
|
|
126
|
+
under rate limits. Metadata label fetches are not rate-limited and are
|
|
127
|
+
cached per session.
|
|
128
|
+
|
|
129
|
+
## Reference
|
|
130
|
+
|
|
131
|
+
The sections below hold the full reference: Config options, series
|
|
132
|
+
selection, geography templates, flattening into a single DataFrame,
|
|
133
|
+
worked examples, and the complete category table.
|
|
134
|
+
|
|
135
|
+
### Project layout
|
|
136
|
+
|
|
137
|
+
```
|
|
138
|
+
Census_Loader/
|
|
139
|
+
pyproject.toml
|
|
140
|
+
requirements.txt
|
|
141
|
+
README.md
|
|
142
|
+
.env-example
|
|
143
|
+
src/census_loader/
|
|
144
|
+
__init__.py # public API re-exports
|
|
145
|
+
utils.py # Config, loader, discovery tools
|
|
146
|
+
load.py # pull_census entry point
|
|
147
|
+
series.py # series catalog, GEO templates, FIPS codes
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### Config Reference
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
Config(
|
|
154
|
+
filename, # Output filename (auto-corrects to .pkl)
|
|
155
|
+
output_path, # Directory for output files
|
|
156
|
+
year=2022, # ACS/PEP vintage year
|
|
157
|
+
geo=..., # Geography level (see below)
|
|
158
|
+
state=..., # State name or FIPS code
|
|
159
|
+
county=..., # County FIPS (3-digit)
|
|
160
|
+
tract=..., # Tract code
|
|
161
|
+
series=..., # What to pull (see below)
|
|
162
|
+
batch_size=50, # Max series per API batch
|
|
163
|
+
)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### Series selection
|
|
167
|
+
|
|
168
|
+
The `series` parameter accepts any granularity: individual series,
|
|
169
|
+
subcategories, categories, or a mix:
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
series=None # everything (169 series)
|
|
173
|
+
series="TOTAL_POP" # one series
|
|
174
|
+
series="INCOME" # whole category (18 series)
|
|
175
|
+
series="HOUSEHOLD_INCOME" # subcategory (9 series)
|
|
176
|
+
series=["TOTAL_POP", "INCOME", "MEDIAN_RENT"] # mix & match
|
|
177
|
+
series=["POVERTY", "HOUSEHOLD_INCOME", "TOTAL_POP"] # category + subcategory + series
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Case-insensitive. Overlapping selections deduplicate automatically.
|
|
181
|
+
|
|
182
|
+
### Geography
|
|
183
|
+
|
|
184
|
+
Pass a template name and fill in the required parameters:
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
# All states (default)
|
|
188
|
+
Config(..., geo="state_all")
|
|
189
|
+
|
|
190
|
+
# All counties in one state
|
|
191
|
+
Config(..., geo="county_in_state", state="Massachusetts")
|
|
192
|
+
|
|
193
|
+
# Tracts in one county
|
|
194
|
+
Config(..., geo="tract_in_county", state="Massachusetts", county="017")
|
|
195
|
+
|
|
196
|
+
# School districts in a state
|
|
197
|
+
Config(..., geo="school_district_in_state", state="Massachusetts")
|
|
198
|
+
|
|
199
|
+
# All ZCTAs
|
|
200
|
+
Config(..., geo="zcta_all")
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
State names resolve automatically, so `"Massachusetts"` and `"25"` both work.
|
|
204
|
+
For edge cases not covered by templates, pass a raw dict:
|
|
205
|
+
`Config(..., geo={"for": "county:017", "in": "state:25"})`.
|
|
206
|
+
|
|
207
|
+
### Flattening into a single DataFrame
|
|
208
|
+
|
|
209
|
+
Since every series in a pull shares the same geography, you can merge on the
|
|
210
|
+
shared geo columns:
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
from functools import reduce
|
|
214
|
+
import pandas as pd
|
|
215
|
+
|
|
216
|
+
frames = list(data.values())
|
|
217
|
+
shared = set(frames[0].columns)
|
|
218
|
+
for df in frames[1:]:
|
|
219
|
+
shared &= set(df.columns)
|
|
220
|
+
|
|
221
|
+
merge_keys = [c for c in shared
|
|
222
|
+
if all(not pd.api.types.is_numeric_dtype(df[c]) for df in frames)]
|
|
223
|
+
|
|
224
|
+
flat = reduce(
|
|
225
|
+
lambda left, right: pd.merge(left, right, on=merge_keys, how="outer"),
|
|
226
|
+
frames,
|
|
227
|
+
)
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
### Examples
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
# County-level dashboard data for one state
|
|
234
|
+
cfg = Config(
|
|
235
|
+
"ma_dashboard.pkl", "./output",
|
|
236
|
+
year=2022, geo="county_in_state", state="Massachusetts",
|
|
237
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "MEDIAN_RENT", "MEDIAN_HOME_VALUE",
|
|
238
|
+
"HOMEOWNERSHIP_RATE", "HH_INCOME_BRACKETS"],
|
|
239
|
+
)
|
|
240
|
+
pull_census(cfg)
|
|
241
|
+
|
|
242
|
+
# Tract-level deep dive
|
|
243
|
+
cfg = Config(
|
|
244
|
+
"middlesex_tracts.pkl", "./output",
|
|
245
|
+
year=2022, geo="tract_in_county", state="Massachusetts", county="017",
|
|
246
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "POP_65_PLUS"],
|
|
247
|
+
)
|
|
248
|
+
pull_census(cfg)
|
|
249
|
+
|
|
250
|
+
# Full category pull, all states
|
|
251
|
+
cfg = Config("national_income.pkl", "./output", year=2022, series="INCOME")
|
|
252
|
+
pull_census(cfg)
|
|
253
|
+
|
|
254
|
+
# School district comparison
|
|
255
|
+
cfg = Config(
|
|
256
|
+
"ma_schools.pkl", "./output",
|
|
257
|
+
year=2022, geo="school_district_in_state", state="Massachusetts",
|
|
258
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "HH_INCOME_BRACKETS"],
|
|
259
|
+
)
|
|
260
|
+
pull_census(cfg)
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
### Available Categories
|
|
264
|
+
|
|
265
|
+
| Category | Series | Subcategories |
|
|
266
|
+
|---|---|---|
|
|
267
|
+
| POPULATION | 13 | TOTAL_POPULATION, AGE_DETAIL |
|
|
268
|
+
| RACE_ETHNICITY | 13 | RACE, HISPANIC_ORIGIN |
|
|
269
|
+
| NATIVITY_MIGRATION | 11 | NATIVITY, MIGRATION |
|
|
270
|
+
| LANGUAGE | 6 | none |
|
|
271
|
+
| EDUCATION | 10 | SCHOOL_ENROLLMENT, EDUCATIONAL_ATTAINMENT |
|
|
272
|
+
| HOUSEHOLDS | 12 | HOUSEHOLD_TYPE, MARITAL_STATUS, FERTILITY |
|
|
273
|
+
| INCOME | 18 | HOUSEHOLD_INCOME, EARNINGS, PUBLIC_ASSISTANCE |
|
|
274
|
+
| POVERTY | 12 | ACS_POVERTY, SAIPE_POVERTY |
|
|
275
|
+
| HEALTH_INSURANCE | 6 | ACS_HEALTH_INSURANCE, SAHIE_HEALTH_INSURANCE |
|
|
276
|
+
| EMPLOYMENT | 15 | EMPLOYMENT_STATUS, OCCUPATION_INDUSTRY, COMMUTING |
|
|
277
|
+
| HOUSING | 25 | HOUSING_UNITS, TENURE, HOME_VALUE, HOUSING_INFRASTRUCTURE |
|
|
278
|
+
| DISABILITY | 5 | none |
|
|
279
|
+
| VETERANS | 4 | none |
|
|
280
|
+
| PEP_POPULATION | 6 | none |
|
|
281
|
+
| DECENNIAL | 9 | DECENNIAL_REDISTRICTING, DECENNIAL_DHC |
|
|
282
|
+
| DATA_PROFILES | 4 | none |
|
|
283
|
+
| **Total** | **169** | |
|
|
284
|
+
|
|
285
|
+
## License
|
|
286
|
+
|
|
287
|
+
[MIT](LICENSE)
|
|
288
|
+
|
|
289
|
+
## Start here
|
|
290
|
+
|
|
291
|
+
`src/census_loader/series.py` is the heart of this project: the catalog that
|
|
292
|
+
maps every friendly series name onto its Census variable codes, and the
|
|
293
|
+
reason the wrapper exists at all. `src/census_loader/load.py` then shows how
|
|
294
|
+
a `Config` becomes batched API calls and labelled DataFrames, and
|
|
295
|
+
`tests/test_catalog.py` shows the catalog's invariants.
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
# Census Loader
|
|
2
|
+
|
|
3
|
+
A Python wrapper around the U.S. Census Bureau API that abstracts away
|
|
4
|
+
variable codes, dataset endpoints, and FIPS identifiers behind a clean,
|
|
5
|
+
human-readable interface. Installs as `census-loader`, for anyone who pulls
|
|
6
|
+
Census data into pandas.
|
|
7
|
+
|
|
8
|
+
Instead of this:
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
data = census.acs5.get(
|
|
12
|
+
("B19013_001E",),
|
|
13
|
+
{"for": "county:*", "in": "state:25"},
|
|
14
|
+
year=2022
|
|
15
|
+
)
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
You write this:
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
cfg = Config(
|
|
22
|
+
"output.pkl", "./data",
|
|
23
|
+
year=2022,
|
|
24
|
+
geo="county_in_state",
|
|
25
|
+
state="Massachusetts",
|
|
26
|
+
series=["MEDIAN_HH_INCOME", "TOTAL_POP", "MEDIAN_RENT"],
|
|
27
|
+
)
|
|
28
|
+
result = pull_census(cfg)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Setup
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install census-loader
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Or from a checkout of this repository: `pip install -e .`.
|
|
38
|
+
|
|
39
|
+
Dependencies (installed automatically by pip; `requirements.txt` is kept
|
|
40
|
+
for reference): `pandas`, `numpy`, `census`, `python-dotenv`, `requests`.
|
|
41
|
+
|
|
42
|
+
Get a free key at
|
|
43
|
+
[api.census.gov/data/key_signup.html](https://api.census.gov/data/key_signup.html),
|
|
44
|
+
then add it to a `.env` file in your project root:
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
CENSUS_API_KEY=your_key_here
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Quickstart
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from census_loader import Config, pull_census
|
|
54
|
+
|
|
55
|
+
cfg = Config(
|
|
56
|
+
"ma_counties.pkl", "./output",
|
|
57
|
+
year=2022,
|
|
58
|
+
geo="county_in_state",
|
|
59
|
+
state="Massachusetts",
|
|
60
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "MEDIAN_RENT"],
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
result = pull_census(cfg) # dict of DataFrames keyed by friendly name
|
|
64
|
+
result["Median_Household_Income"]
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Exploring the catalog
|
|
68
|
+
|
|
69
|
+
You never need to open `series.py`. Three discovery functions browse,
|
|
70
|
+
search, and inspect everything from a REPL or notebook:
|
|
71
|
+
|
|
72
|
+
- `available()`: browse categories; `available("INCOME")` drills into a
|
|
73
|
+
category or subcategory.
|
|
74
|
+
- `search("poverty")`: search by keyword.
|
|
75
|
+
- `info("MEDIAN_HH_INCOME")`: full details (name, category, dataset,
|
|
76
|
+
underlying variable codes).
|
|
77
|
+
- `geos()`: browse geography templates.
|
|
78
|
+
|
|
79
|
+
## Output format
|
|
80
|
+
|
|
81
|
+
`pull_census` returns a `dict[str, pd.DataFrame]` and saves a pickle to
|
|
82
|
+
your output path. Each DataFrame has geo columns on the left (FIPS codes +
|
|
83
|
+
human-readable names) and data columns on the right. Column naming depends
|
|
84
|
+
on the series type (single variable, multi-variable, group table); labels
|
|
85
|
+
are fetched automatically from the Census metadata API. `pickle_loader`
|
|
86
|
+
flattens a saved pickle to one DataFrame.
|
|
87
|
+
|
|
88
|
+
| Series type | Example key | Column names |
|
|
89
|
+
|---|---|---|
|
|
90
|
+
| Single variable | `TOTAL_POP` | `Total_Population` |
|
|
91
|
+
| Multi-variable | `HOMEOWNERSHIP_RATE` | `Homeownership_Rate__Owner occupied`, `Homeownership_Rate__Total` |
|
|
92
|
+
| Group table | `HH_INCOME_BRACKETS` | `Household_Income_Distribution__Less than $10,000`, `Household_Income_Distribution__$10,000 to $14,999`, ... |
|
|
93
|
+
|
|
94
|
+
## API limits
|
|
95
|
+
|
|
96
|
+
Free Census API keys allow 500 requests per day with a batch size of 50
|
|
97
|
+
variables per request. The loader sleeps 0.5s between calls to stay well
|
|
98
|
+
under rate limits. Metadata label fetches are not rate-limited and are
|
|
99
|
+
cached per session.
|
|
100
|
+
|
|
101
|
+
## Reference
|
|
102
|
+
|
|
103
|
+
The sections below hold the full reference: Config options, series
|
|
104
|
+
selection, geography templates, flattening into a single DataFrame,
|
|
105
|
+
worked examples, and the complete category table.
|
|
106
|
+
|
|
107
|
+
### Project layout
|
|
108
|
+
|
|
109
|
+
```
|
|
110
|
+
Census_Loader/
|
|
111
|
+
pyproject.toml
|
|
112
|
+
requirements.txt
|
|
113
|
+
README.md
|
|
114
|
+
.env-example
|
|
115
|
+
src/census_loader/
|
|
116
|
+
__init__.py # public API re-exports
|
|
117
|
+
utils.py # Config, loader, discovery tools
|
|
118
|
+
load.py # pull_census entry point
|
|
119
|
+
series.py # series catalog, GEO templates, FIPS codes
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Config Reference
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
Config(
|
|
126
|
+
filename, # Output filename (auto-corrects to .pkl)
|
|
127
|
+
output_path, # Directory for output files
|
|
128
|
+
year=2022, # ACS/PEP vintage year
|
|
129
|
+
geo=..., # Geography level (see below)
|
|
130
|
+
state=..., # State name or FIPS code
|
|
131
|
+
county=..., # County FIPS (3-digit)
|
|
132
|
+
tract=..., # Tract code
|
|
133
|
+
series=..., # What to pull (see below)
|
|
134
|
+
batch_size=50, # Max series per API batch
|
|
135
|
+
)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Series selection
|
|
139
|
+
|
|
140
|
+
The `series` parameter accepts any granularity: individual series,
|
|
141
|
+
subcategories, categories, or a mix:
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
series=None # everything (169 series)
|
|
145
|
+
series="TOTAL_POP" # one series
|
|
146
|
+
series="INCOME" # whole category (18 series)
|
|
147
|
+
series="HOUSEHOLD_INCOME" # subcategory (9 series)
|
|
148
|
+
series=["TOTAL_POP", "INCOME", "MEDIAN_RENT"] # mix & match
|
|
149
|
+
series=["POVERTY", "HOUSEHOLD_INCOME", "TOTAL_POP"] # category + subcategory + series
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Case-insensitive. Overlapping selections deduplicate automatically.
|
|
153
|
+
|
|
154
|
+
### Geography
|
|
155
|
+
|
|
156
|
+
Pass a template name and fill in the required parameters:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
# All states (default)
|
|
160
|
+
Config(..., geo="state_all")
|
|
161
|
+
|
|
162
|
+
# All counties in one state
|
|
163
|
+
Config(..., geo="county_in_state", state="Massachusetts")
|
|
164
|
+
|
|
165
|
+
# Tracts in one county
|
|
166
|
+
Config(..., geo="tract_in_county", state="Massachusetts", county="017")
|
|
167
|
+
|
|
168
|
+
# School districts in a state
|
|
169
|
+
Config(..., geo="school_district_in_state", state="Massachusetts")
|
|
170
|
+
|
|
171
|
+
# All ZCTAs
|
|
172
|
+
Config(..., geo="zcta_all")
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
State names resolve automatically, so `"Massachusetts"` and `"25"` both work.
|
|
176
|
+
For edge cases not covered by templates, pass a raw dict:
|
|
177
|
+
`Config(..., geo={"for": "county:017", "in": "state:25"})`.
|
|
178
|
+
|
|
179
|
+
### Flattening into a single DataFrame
|
|
180
|
+
|
|
181
|
+
Since every series in a pull shares the same geography, you can merge on the
|
|
182
|
+
shared geo columns:
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from functools import reduce
|
|
186
|
+
import pandas as pd
|
|
187
|
+
|
|
188
|
+
frames = list(data.values())
|
|
189
|
+
shared = set(frames[0].columns)
|
|
190
|
+
for df in frames[1:]:
|
|
191
|
+
shared &= set(df.columns)
|
|
192
|
+
|
|
193
|
+
merge_keys = [c for c in shared
|
|
194
|
+
if all(not pd.api.types.is_numeric_dtype(df[c]) for df in frames)]
|
|
195
|
+
|
|
196
|
+
flat = reduce(
|
|
197
|
+
lambda left, right: pd.merge(left, right, on=merge_keys, how="outer"),
|
|
198
|
+
frames,
|
|
199
|
+
)
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Examples
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
# County-level dashboard data for one state
|
|
206
|
+
cfg = Config(
|
|
207
|
+
"ma_dashboard.pkl", "./output",
|
|
208
|
+
year=2022, geo="county_in_state", state="Massachusetts",
|
|
209
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "MEDIAN_RENT", "MEDIAN_HOME_VALUE",
|
|
210
|
+
"HOMEOWNERSHIP_RATE", "HH_INCOME_BRACKETS"],
|
|
211
|
+
)
|
|
212
|
+
pull_census(cfg)
|
|
213
|
+
|
|
214
|
+
# Tract-level deep dive
|
|
215
|
+
cfg = Config(
|
|
216
|
+
"middlesex_tracts.pkl", "./output",
|
|
217
|
+
year=2022, geo="tract_in_county", state="Massachusetts", county="017",
|
|
218
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "POP_65_PLUS"],
|
|
219
|
+
)
|
|
220
|
+
pull_census(cfg)
|
|
221
|
+
|
|
222
|
+
# Full category pull, all states
|
|
223
|
+
cfg = Config("national_income.pkl", "./output", year=2022, series="INCOME")
|
|
224
|
+
pull_census(cfg)
|
|
225
|
+
|
|
226
|
+
# School district comparison
|
|
227
|
+
cfg = Config(
|
|
228
|
+
"ma_schools.pkl", "./output",
|
|
229
|
+
year=2022, geo="school_district_in_state", state="Massachusetts",
|
|
230
|
+
series=["TOTAL_POP", "MEDIAN_HH_INCOME", "HH_INCOME_BRACKETS"],
|
|
231
|
+
)
|
|
232
|
+
pull_census(cfg)
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
### Available Categories
|
|
236
|
+
|
|
237
|
+
| Category | Series | Subcategories |
|
|
238
|
+
|---|---|---|
|
|
239
|
+
| POPULATION | 13 | TOTAL_POPULATION, AGE_DETAIL |
|
|
240
|
+
| RACE_ETHNICITY | 13 | RACE, HISPANIC_ORIGIN |
|
|
241
|
+
| NATIVITY_MIGRATION | 11 | NATIVITY, MIGRATION |
|
|
242
|
+
| LANGUAGE | 6 | none |
|
|
243
|
+
| EDUCATION | 10 | SCHOOL_ENROLLMENT, EDUCATIONAL_ATTAINMENT |
|
|
244
|
+
| HOUSEHOLDS | 12 | HOUSEHOLD_TYPE, MARITAL_STATUS, FERTILITY |
|
|
245
|
+
| INCOME | 18 | HOUSEHOLD_INCOME, EARNINGS, PUBLIC_ASSISTANCE |
|
|
246
|
+
| POVERTY | 12 | ACS_POVERTY, SAIPE_POVERTY |
|
|
247
|
+
| HEALTH_INSURANCE | 6 | ACS_HEALTH_INSURANCE, SAHIE_HEALTH_INSURANCE |
|
|
248
|
+
| EMPLOYMENT | 15 | EMPLOYMENT_STATUS, OCCUPATION_INDUSTRY, COMMUTING |
|
|
249
|
+
| HOUSING | 25 | HOUSING_UNITS, TENURE, HOME_VALUE, HOUSING_INFRASTRUCTURE |
|
|
250
|
+
| DISABILITY | 5 | none |
|
|
251
|
+
| VETERANS | 4 | none |
|
|
252
|
+
| PEP_POPULATION | 6 | none |
|
|
253
|
+
| DECENNIAL | 9 | DECENNIAL_REDISTRICTING, DECENNIAL_DHC |
|
|
254
|
+
| DATA_PROFILES | 4 | none |
|
|
255
|
+
| **Total** | **169** | |
|
|
256
|
+
|
|
257
|
+
## License
|
|
258
|
+
|
|
259
|
+
[MIT](LICENSE)
|
|
260
|
+
|
|
261
|
+
## Start here
|
|
262
|
+
|
|
263
|
+
`src/census_loader/series.py` is the heart of this project: the catalog that
|
|
264
|
+
maps every friendly series name onto its Census variable codes, and the
|
|
265
|
+
reason the wrapper exists at all. `src/census_loader/load.py` then shows how
|
|
266
|
+
a `Config` becomes batched API calls and labelled DataFrames, and
|
|
267
|
+
`tests/test_catalog.py` shows the catalog's invariants.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "census-loader"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A Python wrapper around the U.S. Census Bureau API that abstracts away variable codes, dataset endpoints, and FIPS identifiers behind a clean, human-readable interface."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Ian Solberg" }]
|
|
13
|
+
dependencies = ["pandas", "numpy", "census", "python-dotenv", "requests"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Topic :: Scientific/Engineering",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
dev = ["pytest"]
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
where = ["src"]
|
|
32
|
+
|
|
33
|
+
# SHIP THE TYPES (PEP 561). Without py.typed inside the installed package, a
|
|
34
|
+
# type checker and an editor both IGNORE every annotation in this library and
|
|
35
|
+
# treat it as untyped, so autocomplete gives you nothing and mypy reports it as
|
|
36
|
+
# missing stubs. The annotations were already here; nothing told anyone to use
|
|
37
|
+
# them.
|
|
38
|
+
#
|
|
39
|
+
# No .pyi stub files, deliberately: they would restate every signature in a
|
|
40
|
+
# second file that then drifts from the first.
|
|
41
|
+
[tool.setuptools.package-data]
|
|
42
|
+
census_loader = ["py.typed"]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""A readable Python interface to the U.S. Census Bureau API."""
|
|
2
|
+
|
|
3
|
+
from .load import pull_census
|
|
4
|
+
from .series import ALL_SERIES, CATEGORIES, PREDICATES, SUBCATEGORIES
|
|
5
|
+
from .utils import Config, available, geos, info, pickle_loader, search
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"ALL_SERIES",
|
|
9
|
+
"CATEGORIES",
|
|
10
|
+
"Config",
|
|
11
|
+
"PREDICATES",
|
|
12
|
+
"SUBCATEGORIES",
|
|
13
|
+
"available",
|
|
14
|
+
"geos",
|
|
15
|
+
"info",
|
|
16
|
+
"pickle_loader",
|
|
17
|
+
"pull_census",
|
|
18
|
+
"search",
|
|
19
|
+
]
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from .utils import Config, _load_census_bureau
|
|
4
|
+
from .series import ALL_SERIES, CATEGORIES, SUBCATEGORIES
|
|
5
|
+
|
|
6
|
+
# PUBLIC API
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def pull_census(config: Config) -> dict[str, pd.DataFrame] | None:
|
|
10
|
+
"""
|
|
11
|
+
Pull Census Bureau data and optionally apply scoring.
|
|
12
|
+
|
|
13
|
+
Returns a dict of DataFrames keyed by friendly name, or None on failure.
|
|
14
|
+
Each DataFrame has rows = geographies and columns = variables + geo IDs.
|
|
15
|
+
The result is persisted as a pickle for lossless round-tripping.
|
|
16
|
+
"""
|
|
17
|
+
cfg = config if isinstance(config, Config) else None
|
|
18
|
+
if cfg is None:
|
|
19
|
+
print("Incorrect Configuration Format.")
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
if cfg._series_input is not None:
|
|
23
|
+
print(f"Custom series selection: {len(cfg.SERIES)} series to pull.")
|
|
24
|
+
|
|
25
|
+
result = _load_census_bureau(config=config)
|
|
26
|
+
if result is None:
|
|
27
|
+
return None
|
|
28
|
+
else:
|
|
29
|
+
output = result
|
|
30
|
+
# ── Persist ──────────────────────────────────────────────────────────
|
|
31
|
+
config.OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
|
|
33
|
+
fname = config.FILENAME
|
|
34
|
+
if not fname.endswith(".pkl"):
|
|
35
|
+
fname = fname.rsplit(".", 1)[0] + ".pkl" if "." in fname else fname + ".pkl"
|
|
36
|
+
pkl_file = config.OUTPUT_PATH / fname
|
|
37
|
+
|
|
38
|
+
with open(pkl_file, "wb") as f:
|
|
39
|
+
pickle.dump(output, f)
|
|
40
|
+
print(f"\n Saved → {pkl_file}")
|
|
41
|
+
|
|
42
|
+
return output
|
|
File without changes
|