scfpublic 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scfpublic-1.0.1/LICENSE.md +21 -0
- scfpublic-1.0.1/MANIFEST.in +10 -0
- scfpublic-1.0.1/PKG-INFO +114 -0
- scfpublic-1.0.1/README.md +74 -0
- scfpublic-1.0.1/docs/cpi.md +51 -0
- scfpublic-1.0.1/docs/scfcombo/scfcombo.md +228 -0
- scfpublic-1.0.1/docs/scfcombo/stats.md +142 -0
- scfpublic-1.0.1/docs/scfpublic.md +77 -0
- scfpublic-1.0.1/docs/util.md +59 -0
- scfpublic-1.0.1/examples/SCFPublic.ipynb +710 -0
- scfpublic-1.0.1/examples/historicalTable5.py +89 -0
- scfpublic-1.0.1/examples/standarderr_doc.py +41 -0
- scfpublic-1.0.1/pyproject.toml +59 -0
- scfpublic-1.0.1/setup.cfg +4 -0
- scfpublic-1.0.1/src/scfpublic/__init__.py +38 -0
- scfpublic-1.0.1/src/scfpublic/cpi.py +89 -0
- scfpublic-1.0.1/src/scfpublic/scfcombo/__init__.py +21 -0
- scfpublic-1.0.1/src/scfpublic/scfcombo/scfcombo.py +849 -0
- scfpublic-1.0.1/src/scfpublic/scfcombo/stats.py +290 -0
- scfpublic-1.0.1/src/scfpublic/scfpublic.py +585 -0
- scfpublic-1.0.1/src/scfpublic/util.py +73 -0
- scfpublic-1.0.1/src/scfpublic.egg-info/PKG-INFO +114 -0
- scfpublic-1.0.1/src/scfpublic.egg-info/SOURCES.txt +31 -0
- scfpublic-1.0.1/src/scfpublic.egg-info/dependency_links.txt +1 -0
- scfpublic-1.0.1/src/scfpublic.egg-info/requires.txt +15 -0
- scfpublic-1.0.1/src/scfpublic.egg-info/top_level.txt +1 -0
- scfpublic-1.0.1/tests/__init__.py +127 -0
- scfpublic-1.0.1/tests/conftest.py +518 -0
- scfpublic-1.0.1/tests/test_cpi.py +143 -0
- scfpublic-1.0.1/tests/test_scfcombo.py +384 -0
- scfpublic-1.0.1/tests/test_scfcombo_stats.py +72 -0
- scfpublic-1.0.1/tests/test_scfpublic.py +481 -0
- scfpublic-1.0.1/tests/test_util.py +160 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Payton Crawford.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
include README.md
|
|
2
|
+
include LICENSE.md
|
|
3
|
+
include pyproject.toml
|
|
4
|
+
recursive-include examples *.ipynb *.py
|
|
5
|
+
recursive-include docs *.md *.rst
|
|
6
|
+
recursive-include tests *.py
|
|
7
|
+
recursive-include src/scfpublic *.json *.csv *.txt
|
|
8
|
+
exclude .gitignore
|
|
9
|
+
global-exclude __pycache__
|
|
10
|
+
global-exclude *.py[cod]
|
scfpublic-1.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scfpublic
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: A package for downloading the public files for the Federal Reserve's Survey of Consumers Finances (SCF) 1989-2025 and properly apply multiple imputation and sampling variance calculations.
|
|
5
|
+
Author-email: Payton Crawford <paytonscrawford@outlook.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/CrawfordPayton/scfpublic
|
|
8
|
+
Project-URL: Repository, https://github.com/CrawfordPayton/scfpublic
|
|
9
|
+
Keywords: SCF,Survey,Consumer,Finances,Wealth,Economics,Imputation,Replicates
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.8
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE.md
|
|
25
|
+
Requires-Dist: pandas>=1.5.0
|
|
26
|
+
Requires-Dist: numpy>=1.23.0
|
|
27
|
+
Requires-Dist: requests>=2.28.0
|
|
28
|
+
Requires-Dist: psutil>=5.9.0
|
|
29
|
+
Requires-Dist: pydantic>=2.0.0
|
|
30
|
+
Requires-Dist: scipy>=1.9.0
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest-cov>=3.0.0; extra == "dev"
|
|
34
|
+
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
35
|
+
Requires-Dist: flake8>=5.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: mypy>=0.990; extra == "dev"
|
|
37
|
+
Requires-Dist: jupyter>=1.0.0; extra == "dev"
|
|
38
|
+
Requires-Dist: ipython>=8.0.0; extra == "dev"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# Python Compiler for Public SCF Files (scfpublic)
|
|
42
|
+
#### Author: Payton Crawford
|
|
43
|
+
#### Last Updated: November 19th, 2026
|
|
44
|
+
|
|
45
|
+
## Table of Contents
|
|
46
|
+
* [Introduction](#introduction)
|
|
47
|
+
* [Installation](#installation)
|
|
48
|
+
* [Directory Structure](#directory-structure)
|
|
49
|
+
* [Docs](#docs)
|
|
50
|
+
* [Examples](#examples)
|
|
51
|
+
* [Src](#srcscfpublic)
|
|
52
|
+
* [Tests](#tests)
|
|
53
|
+
* [Packages and Functions](#packages-and-functions)
|
|
54
|
+
|
|
55
|
+
### Introduction
|
|
56
|
+
The purpose of these Python package is to provide a centralized package for downloading the Survey of Consumer Finances files and computing standard errors that are corrected for multiple imputation and the sample design. By doing so, I aim to make it easier and clearer how to properly use SCF data. Please refer to the official Federal Reserve Board website for the [Survey of Consumer Finances](https://www.federalreserve.gov/econres/scfindex.htm). Their website includes the datasets, codebooks (for example, [2025](https://www.federalreserve.gov/econres/files/codebk2025.txt)) containing information by each year for each of the x-variables, and an updated [SAS bulletin macro](https://www.federalreserve.gov/econres/files/bulletin.macro.txt) to define each of the bulletin variables. Additionally, they have a [codebook search](https://www.federalreserve.gov/apps/scfcb), a [net worth flowchat](https://www.federalreserve.gov/econres/files/Networth%20Flowchart.pdf), materials from [previous surveys](https://www.federalreserve.gov/econres/scf-previous-surveys.htm), and other relevant materials.
|
|
57
|
+
|
|
58
|
+
### Installation
|
|
59
|
+
From anywhere in your terminal, run `pip install scfpublic`. This will run commands from `pyproject.toml` to install the Python package and its minimum dependencies locally in that environment. From there, you will be able to access any of the Python files in `src/scfpublic` by running the command `import scfpublic` at the top of a Python script. <br>
|
|
60
|
+
- The Python dependencies that will be installed are listed in `requirements.txt`.
|
|
61
|
+
- For further in-depth details on how to utilize the package, please refer to the Jupyter Notebook in examples.
|
|
62
|
+
|
|
63
|
+
### Directory Structure
|
|
64
|
+
- #### examples
|
|
65
|
+
- Stores Jupyter notebooks detailing how to use each respective package with examples and base Python files showing examples of usage.
|
|
66
|
+
- #### src/scfpublic
|
|
67
|
+
- Stores the Python files that are used each time the package is imported.
|
|
68
|
+
- scfpublic has the scfpublic package, which is used to load x-data, bulletin, and replicate weights files from their current storages into Python as a Pandas dataframe.
|
|
69
|
+
- cpi.py has the cpi package, which is used for creating the cpiadj and cpilag variables that mimics the [SCF bulletin](https://www.federalreserve.gov/econres/files/bulletin.macro.txt).
|
|
70
|
+
- util.py has the util package, which contains annual and monthly conversion functions following the [SCF bulletin](https://www.federalreserve.gov/econres/files/bulletin.macro.txt).
|
|
71
|
+
- #### /scfcombo
|
|
72
|
+
- scfcombo.py has the scfcombo package, which is used to convert the SCF data and replicate weights files to compute imputation error, sampling error, and combined errors for user-defined analysis.
|
|
73
|
+
- stats.py has the scfcombo.stats package, which contains convenience wrapper functions around scf_estimate.
|
|
74
|
+
- #### tests
|
|
75
|
+
- Stores Python test files for each of the above packages.
|
|
76
|
+
- To run the tests for each program, run the command `python3 -m pytest {tests directory}` in your terminal.
|
|
77
|
+
- #### variances
|
|
78
|
+
- Stores `variances.md` details how the combined standard errors are computed and how to use the respective functions for your programming language of choice.
|
|
79
|
+
|
|
80
|
+
### Packages and Functions
|
|
81
|
+
- `scfpublic`
|
|
82
|
+
- Functions
|
|
83
|
+
- `download_year` - Loads year, variables, and file type requested for public SCF data
|
|
84
|
+
- `download_years` - Loads multiple years, variables, and file type requested for public SCF data
|
|
85
|
+
- `download_main_extr_years` - Loads merged main and extract data using years and variables for public SCF data
|
|
86
|
+
- `cpi` - Package for adding CPI adjustments by year
|
|
87
|
+
- Functions
|
|
88
|
+
- `read_excel_from_url` - Reads CPI-U-RS-ALLITEMS Excel file
|
|
89
|
+
- `cpi_adjust` - Creates cpiadj and cpilag variables by year
|
|
90
|
+
- `scfcombo` - Package for imputation and sampling error corrections
|
|
91
|
+
- DataClass
|
|
92
|
+
- `scfcombo_result` - Data class that stores statistic and regression information
|
|
93
|
+
- Functions
|
|
94
|
+
- `display` - Outputs formatted results to terminal
|
|
95
|
+
- `summary` - Outputs data class information
|
|
96
|
+
- Functions
|
|
97
|
+
- `prepare_scf_data` - Prepares SCF data and replicate weights file into a dataset usable for scf_estimate
|
|
98
|
+
- `scf_estimate` - Takes user functions and outputs pooled estimate, imputation variance, sampling variance, and combined variance into a scfcombo_result object
|
|
99
|
+
- `stats` - Convenience functions for basic statistics applications using `scf_estimate`
|
|
100
|
+
- Functions
|
|
101
|
+
- `scf_mean` - Convenience wrapper for mean estimation around scf_estimate
|
|
102
|
+
- `scf_quantile` - Convenience wrapper for median estimation around scf_estimate
|
|
103
|
+
- `scf_regression` - Convenience wrapper for regression estimation around scf_estimate
|
|
104
|
+
- `scf_total` - Convenience wrapper for aggregate estimation around scf_estimate
|
|
105
|
+
- `util` - Package for basic SCF functions
|
|
106
|
+
- Functions
|
|
107
|
+
- `aconv` - Annual conversion function
|
|
108
|
+
- `mconv` - Monthly conversion function
|
|
109
|
+
|
|
110
|
+
Citation: Payton Crawford (2026). scfpublic. Python package. ver. 1.0.0. https://github.com/CrawfordPayton/scfpublic.
|
|
111
|
+
|
|
112
|
+
Bibliography:
|
|
113
|
+
|
|
114
|
+
 Board of Governors of the Federal Reserve System. (2026). 2025 Survey of Consumer Finances [Dataset]. https://doi.org/10.17016/8799.1.
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Python Compiler for Public SCF Files (scfpublic)
|
|
2
|
+
#### Author: Payton Crawford
|
|
3
|
+
#### Last Updated: November 19th, 2026
|
|
4
|
+
|
|
5
|
+
## Table of Contents
|
|
6
|
+
* [Introduction](#introduction)
|
|
7
|
+
* [Installation](#installation)
|
|
8
|
+
* [Directory Structure](#directory-structure)
|
|
9
|
+
* [Docs](#docs)
|
|
10
|
+
* [Examples](#examples)
|
|
11
|
+
* [Src](#srcscfpublic)
|
|
12
|
+
* [Tests](#tests)
|
|
13
|
+
* [Packages and Functions](#packages-and-functions)
|
|
14
|
+
|
|
15
|
+
### Introduction
|
|
16
|
+
The purpose of these Python package is to provide a centralized package for downloading the Survey of Consumer Finances files and computing standard errors that are corrected for multiple imputation and the sample design. By doing so, I aim to make it easier and clearer how to properly use SCF data. Please refer to the official Federal Reserve Board website for the [Survey of Consumer Finances](https://www.federalreserve.gov/econres/scfindex.htm). Their website includes the datasets, codebooks (for example, [2025](https://www.federalreserve.gov/econres/files/codebk2025.txt)) containing information by each year for each of the x-variables, and an updated [SAS bulletin macro](https://www.federalreserve.gov/econres/files/bulletin.macro.txt) to define each of the bulletin variables. Additionally, they have a [codebook search](https://www.federalreserve.gov/apps/scfcb), a [net worth flowchat](https://www.federalreserve.gov/econres/files/Networth%20Flowchart.pdf), materials from [previous surveys](https://www.federalreserve.gov/econres/scf-previous-surveys.htm), and other relevant materials.
|
|
17
|
+
|
|
18
|
+
### Installation
|
|
19
|
+
From anywhere in your terminal, run `pip install scfpublic`. This will run commands from `pyproject.toml` to install the Python package and its minimum dependencies locally in that environment. From there, you will be able to access any of the Python files in `src/scfpublic` by running the command `import scfpublic` at the top of a Python script. <br>
|
|
20
|
+
- The Python dependencies that will be installed are listed in `requirements.txt`.
|
|
21
|
+
- For further in-depth details on how to utilize the package, please refer to the Jupyter Notebook in examples.
|
|
22
|
+
|
|
23
|
+
### Directory Structure
|
|
24
|
+
- #### examples
|
|
25
|
+
- Stores Jupyter notebooks detailing how to use each respective package with examples and base Python files showing examples of usage.
|
|
26
|
+
- #### src/scfpublic
|
|
27
|
+
- Stores the Python files that are used each time the package is imported.
|
|
28
|
+
- scfpublic has the scfpublic package, which is used to load x-data, bulletin, and replicate weights files from their current storages into Python as a Pandas dataframe.
|
|
29
|
+
- cpi.py has the cpi package, which is used for creating the cpiadj and cpilag variables that mimics the [SCF bulletin](https://www.federalreserve.gov/econres/files/bulletin.macro.txt).
|
|
30
|
+
- util.py has the util package, which contains annual and monthly conversion functions following the [SCF bulletin](https://www.federalreserve.gov/econres/files/bulletin.macro.txt).
|
|
31
|
+
- #### /scfcombo
|
|
32
|
+
- scfcombo.py has the scfcombo package, which is used to convert the SCF data and replicate weights files to compute imputation error, sampling error, and combined errors for user-defined analysis.
|
|
33
|
+
- stats.py has the scfcombo.stats package, which contains convenience wrapper functions around scf_estimate.
|
|
34
|
+
- #### tests
|
|
35
|
+
- Stores Python test files for each of the above packages.
|
|
36
|
+
- To run the tests for each program, run the command `python3 -m pytest {tests directory}` in your terminal.
|
|
37
|
+
- #### variances
|
|
38
|
+
- Stores `variances.md` details how the combined standard errors are computed and how to use the respective functions for your programming language of choice.
|
|
39
|
+
|
|
40
|
+
### Packages and Functions
|
|
41
|
+
- `scfpublic`
|
|
42
|
+
- Functions
|
|
43
|
+
- `download_year` - Loads year, variables, and file type requested for public SCF data
|
|
44
|
+
- `download_years` - Loads multiple years, variables, and file type requested for public SCF data
|
|
45
|
+
- `download_main_extr_years` - Loads merged main and extract data using years and variables for public SCF data
|
|
46
|
+
- `cpi` - Package for adding CPI adjustments by year
|
|
47
|
+
- Functions
|
|
48
|
+
- `read_excel_from_url` - Reads CPI-U-RS-ALLITEMS Excel file
|
|
49
|
+
- `cpi_adjust` - Creates cpiadj and cpilag variables by year
|
|
50
|
+
- `scfcombo` - Package for imputation and sampling error corrections
|
|
51
|
+
- DataClass
|
|
52
|
+
- `scfcombo_result` - Data class that stores statistic and regression information
|
|
53
|
+
- Functions
|
|
54
|
+
- `display` - Outputs formatted results to terminal
|
|
55
|
+
- `summary` - Outputs data class information
|
|
56
|
+
- Functions
|
|
57
|
+
- `prepare_scf_data` - Prepares SCF data and replicate weights file into a dataset usable for scf_estimate
|
|
58
|
+
- `scf_estimate` - Takes user functions and outputs pooled estimate, imputation variance, sampling variance, and combined variance into a scfcombo_result object
|
|
59
|
+
- `stats` - Convenience functions for basic statistics applications using `scf_estimate`
|
|
60
|
+
- Functions
|
|
61
|
+
- `scf_mean` - Convenience wrapper for mean estimation around scf_estimate
|
|
62
|
+
- `scf_quantile` - Convenience wrapper for median estimation around scf_estimate
|
|
63
|
+
- `scf_regression` - Convenience wrapper for regression estimation around scf_estimate
|
|
64
|
+
- `scf_total` - Convenience wrapper for aggregate estimation around scf_estimate
|
|
65
|
+
- `util` - Package for basic SCF functions
|
|
66
|
+
- Functions
|
|
67
|
+
- `aconv` - Annual conversion function
|
|
68
|
+
- `mconv` - Monthly conversion function
|
|
69
|
+
|
|
70
|
+
Citation: Payton Crawford (2026). scfpublic. Python package. ver. 1.0.0. https://github.com/CrawfordPayton/scfpublic.
|
|
71
|
+
|
|
72
|
+
Bibliography:
|
|
73
|
+
|
|
74
|
+
 Board of Governors of the Federal Reserve System. (2026). 2025 Survey of Consumer Finances [Dataset]. https://doi.org/10.17016/8799.1.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# scfpublic.cpi
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
Stores cpi adjustment formulas using data from https://www.bls.gov/cpi/research-series/r-cpi-u-rs-home.htm
|
|
5
|
+
|
|
6
|
+
Author: Payton Crawford
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
## `cpi_adjust`
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
cpi_adjust(df: pandas.core.frame.DataFrame, base_year: int = 2025)
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Appends cpiadj and cpilag variables based on 'year' variable in dataset.
|
|
16
|
+
|
|
17
|
+
**Parameters:**
|
|
18
|
+
|
|
19
|
+
- `df (pd.DataFrame)`: Pandas dataframe to have CPI variables added to
|
|
20
|
+
- `base_year (int)`: Year to use as base for for CPI base value (default: 2022)
|
|
21
|
+
|
|
22
|
+
**Returns:**
|
|
23
|
+
|
|
24
|
+
pd.DataFrame: Original dataframe with new cpiadj and cpilag variables merged on year
|
|
25
|
+
|
|
26
|
+
**Raises:**
|
|
27
|
+
|
|
28
|
+
- `ValueError`: Provided invalid year, DataFrame lacks year variable, or DataFrame lacks correct years
|
|
29
|
+
|
|
30
|
+
## `read_excel_from_url`
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
read_excel_from_url(url: str, sheet_name: int = 0, **kwargs)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Download Excel file from BLS URL and read into DataFrame.
|
|
37
|
+
|
|
38
|
+
**Parameters:**
|
|
39
|
+
|
|
40
|
+
- `url (str)`: URL of excel file
|
|
41
|
+
- `sheet_name (int)`: Sheet name or index (default: 0 = first sheet)
|
|
42
|
+
- `**kwargs`: Additional arguments for pd.read_excel()
|
|
43
|
+
|
|
44
|
+
**Returns:**
|
|
45
|
+
|
|
46
|
+
pd.DataFrame: Pandas dataframe containing Excel file from URL
|
|
47
|
+
|
|
48
|
+
**Raises:**
|
|
49
|
+
|
|
50
|
+
- `IOError`: Failure to download or process BLS file
|
|
51
|
+
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# scfpublic.scfcombo.scfcombo
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
SCF Variance Estimation Module for public dataset
|
|
5
|
+
|
|
6
|
+
This module provides a general-purpose framework for computing statistics with
|
|
7
|
+
proper variance estimation on SCF data, following the methodology in:
|
|
8
|
+
|
|
9
|
+
- SAS macro MEANIT from https://www.federalreserve.gov/econres/files/codebk2022.txt
|
|
10
|
+
- Stata module SCFCOMBO from https://ideas.repec.org/c/boc/bocode/s458017.html
|
|
11
|
+
- R function scf_MIcombine from https://www.convey-r.org/1.6-survey-of-consumer-finances-scf.html#survey-of-consumer-finances-scf
|
|
12
|
+
|
|
13
|
+
Author: Payton Crawford
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## `prepare_scf_data`
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
prepare_scf_data(main_df: pandas.core.frame.DataFrame, replicate_weights_df: pandas.core.frame.DataFrame, weight_col: str = 'wgt', implicate_id_col: str = 'y1', household_id_col: str = 'yy1', year_col: str = 'year', repweight_col: str = 'wt1b', repmultiple_col: str = 'mm', n_implicates: int = 5, n_replicates: int = 999, verbose: bool = False) -> Dict[str, pandas.core.frame.DataFrame]
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Prepare SCF data by splitting into implicates and merging with replicate weights.
|
|
23
|
+
|
|
24
|
+
**Parameters:**
|
|
25
|
+
|
|
26
|
+
- `main_df (pd.DataFrame)`: Pandas dataframe containing dataframe for calculating statistics (contains weight, implicate_id, and household_id columns)
|
|
27
|
+
- `replicate_weights_df (pd.DataFrame)`: Pandas dataframe containing replicate weights for each household (contains household_id column)
|
|
28
|
+
- `weight_col (str)`: Contains column name for weight (e.g. wgt)
|
|
29
|
+
- `implicate_id_col (str)`: Contains column name for unique implicate id (e.g. y1)
|
|
30
|
+
- `household_id_col (str)`: Contains column name for unique household id (e.g. yy1)
|
|
31
|
+
- `year_col (str)`: Contains column name for year variable (e.g. year)
|
|
32
|
+
- `repweight_col (str)`: Contains header for replicate weight variable name (1-999) (default is wt1b)
|
|
33
|
+
- `repmultiple_col (str)`: Contains header for replicate multiplier variable name (1-999) (default is mm)
|
|
34
|
+
- `n_implicates (int)`: Number of implicates in main_df (default is 5)
|
|
35
|
+
- `n_replicates (int)`: Number of replicates in replicate_weights_df (default is 999)
|
|
36
|
+
- `verbose (bool)`: True to output print messages, False for quietly
|
|
37
|
+
|
|
38
|
+
**Returns:**
|
|
39
|
+
|
|
40
|
+
dict: Contains the following:
|
|
41
|
+
|
|
42
|
+
- 'implicates': List of DataFrames (one per implicate)
|
|
43
|
+
- 'replicate_weights': DataFrame with replicate weights
|
|
44
|
+
- 'metadata': Dict with dataset information
|
|
45
|
+
|
|
46
|
+
## `scfcombo_result` (class)
|
|
47
|
+
|
|
48
|
+
Container for SCF variance estimation results with output formatting.
|
|
49
|
+
|
|
50
|
+
**Attributes:**
|
|
51
|
+
|
|
52
|
+
- `estimate` (*float/np.ndarray*): Point estimate
|
|
53
|
+
- `imputation_var` (*float/np.ndarray*): Variance due to multiple imputation
|
|
54
|
+
- `sampling_var` (*float/np.ndarray*): Variance due to sampling
|
|
55
|
+
- `total_var` (*float/np.ndarray*): Combined variance
|
|
56
|
+
- `standard_error` (*float/np.ndarray*): Standard error
|
|
57
|
+
- `n_implicates` (*int*): Number of implicates used
|
|
58
|
+
- `n_replicates` (*int*): Number of replicate weights used
|
|
59
|
+
- `n_obs` (*int*): Number of observations (via first implicate)
|
|
60
|
+
- `coefficient_names` (*list*): Names of coefficients (None for scalar results)
|
|
61
|
+
- `statistic_name` (*str*): Name of the statistic (e.g., 'Mean', 'Median', 'Regression') (None for scalar results)
|
|
62
|
+
- `variable_name` (*str*): Variable analyzed (scalar results) or dependent variable (regressions)
|
|
63
|
+
- `is_regression` (*bool*): True if regression, False otherwise to discriminate between vector results (such as group means) and regressions
|
|
64
|
+
|
|
65
|
+
### `scfcombo_result.__init__`
|
|
66
|
+
|
|
67
|
+
### `scfcombo_result.display`
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
scfcombo_result.display(self, alpha: float = 0.05, decimals: int = 4) -> None
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Display formatted output.
|
|
74
|
+
|
|
75
|
+
**Parameters:**
|
|
76
|
+
|
|
77
|
+
- `alpha (float)`: Significance level for confidence intervals (default: 0.05)
|
|
78
|
+
- `decimals (int)`: Number of decimal places to display (default: 4)
|
|
79
|
+
|
|
80
|
+
### `scfcombo_result.results_table`
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
scfcombo_result.results_table(self, alpha: float = 0.05, decimals: Optional[int] = None) -> pandas.core.frame.DataFrame
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Return a coefficient table as a DataFrame.
|
|
87
|
+
|
|
88
|
+
Columns: Variable, Coef., Std. Err., z, P>|z|, CI Lower, CI Upper,
|
|
89
|
+
Imputation SE, Sampling SE. Model-level information is stored in df.attrs.
|
|
90
|
+
|
|
91
|
+
**Parameters:**
|
|
92
|
+
|
|
93
|
+
- `alpha (float)`: Significance level for confidence intervals (default: 0.05)
|
|
94
|
+
- `decimals (int)`: Round numeric columns if provided (default: None, no rounding)
|
|
95
|
+
|
|
96
|
+
**Returns:**
|
|
97
|
+
|
|
98
|
+
pd.DataFrame: Results table, with model info in df.attrs
|
|
99
|
+
|
|
100
|
+
### `scfcombo_result.summary`
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
scfcombo_result.summary(self, alpha: float = 0.05, decimals: Optional[int] = None) -> pandas.core.frame.DataFrame
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Return a summary DataFrame of the results (alias for results_table).
|
|
107
|
+
|
|
108
|
+
**Parameters:**
|
|
109
|
+
|
|
110
|
+
- `alpha (float)`: Significance level for confidence intervals (default: 0.05)
|
|
111
|
+
- `decimals (int)`: Round numeric columns if provided (default: None)
|
|
112
|
+
|
|
113
|
+
**Returns:**
|
|
114
|
+
|
|
115
|
+
pd.DataFrame: Results table
|
|
116
|
+
|
|
117
|
+
## `scf_estimate`
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
scf_estimate(data: Dict[str, pandas.core.frame.DataFrame], estimator: Callable[[pandas.core.frame.DataFrame, numpy.ndarray], Union[float, numpy.ndarray]], subset: Optional[Callable[[pandas.core.frame.DataFrame], pandas.core.series.Series]] = None, coefficient_names: Optional[list] = None, estimand_type: str = 'mean', **estimator_kwargs) -> scfpublic.scfcombo.scfcombo.scfcombo_result
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
General-purpose SCF variance estimation implements the unified variance estimation procedure that works
|
|
124
|
+
for means, medians, quantiles, regression coefficients, or user-defined statistic.
|
|
125
|
+
|
|
126
|
+
Mathematical Procedure:
|
|
127
|
+
|
|
128
|
+
1. For each implicate i (i=1 to 5), where I is the total number of implicates:
|
|
129
|
+
b_hat_imp{i} = estimator(data{i}, weights{i})
|
|
130
|
+
|
|
131
|
+
2. Pooled estimate:
|
|
132
|
+
b_hat_imp = (1/I) * sum(b_hat_imp{i})
|
|
133
|
+
|
|
134
|
+
3. Imputation variance:
|
|
135
|
+
V_imp = (1/(I-1)) * sum(b_hat_imp{i} - b_hat_imp)**2
|
|
136
|
+
|
|
137
|
+
4. For each replicate r (r=1 to 999, using first implicate (i=1) only), where R is the total number of replicates:
|
|
138
|
+
b_hat_sam{r} = estimator(data{1}, weights{r})
|
|
139
|
+
|
|
140
|
+
5. Pooled estimate:
|
|
141
|
+
b_hat_sam = (1/R) * sum(b_hat_sam{r})
|
|
142
|
+
|
|
143
|
+
6. Sampling variance:
|
|
144
|
+
V_sam = (1/(R-1)) * sum(beta_hat{r} - b_hat_sam)**2
|
|
145
|
+
|
|
146
|
+
7. Total variance:
|
|
147
|
+
V_total = ((I+1)/I) * V_imp + V_sam
|
|
148
|
+
|
|
149
|
+
**Parameters:**
|
|
150
|
+
|
|
151
|
+
- `data`: Dictionary from prepare_scf_data() containing:
|
|
152
|
+
|
|
153
|
+
- 'implicates': List of DataFrames (one per implicate)
|
|
154
|
+
- 'replicate_weights': DataFrame with replicate weights
|
|
155
|
+
- 'metadata': Dict with dataset information
|
|
156
|
+
- `estimator`: Function that computes the statistic of interest.
|
|
157
|
+
Signature: estimator(data: pd.DataFrame, weights: np.ndarray, **kwargs)
|
|
158
|
+
-> Union[float, np.ndarray]
|
|
159
|
+
|
|
160
|
+
- data: DataFrame containing the variables needed
|
|
161
|
+
- weights: Array of weights to use
|
|
162
|
+
- **kwargs: Additional arguments passed through estimator_kwargs
|
|
163
|
+
- Returns: Scalar estimate or array of estimates (e.g., coefficients)
|
|
164
|
+
- `subset`: Optional function that takes a DataFrame and returns a boolean Series
|
|
165
|
+
for subsetting. Example: lambda df: df['age'] > 25
|
|
166
|
+
- `coefficient_names`: Optional list of names for vector results (e.g., regression)
|
|
167
|
+
- `estimand_type (str)`: Type of estimand pooling:
|
|
168
|
+
|
|
169
|
+
- 'mean': Average across implicates (default)
|
|
170
|
+
Use for: means, medians, quantiles, regression coefficients, Gini coefficients
|
|
171
|
+
- 'total': Sum across implicates
|
|
172
|
+
Use for: population totals, counts, aggregates
|
|
173
|
+
- `**estimator_kwargs`: Additional keyword arguments passed to estimator function
|
|
174
|
+
|
|
175
|
+
**Returns:**
|
|
176
|
+
|
|
177
|
+
scfcombo_result object containing estimates and variance components
|
|
178
|
+
|
|
179
|
+
**Examples:**
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
>>> # Example 1: Weighted mean
|
|
183
|
+
>>> def weighted_mean(data, weights, variable):
|
|
184
|
+
... non_missing = data[variable].notna()
|
|
185
|
+
... return np.average(data.loc[non_missing, variable], weights=weights[non_missing])
|
|
186
|
+
>>>
|
|
187
|
+
>>> result = scf_estimate(
|
|
188
|
+
... scf_data,
|
|
189
|
+
... weighted_mean,
|
|
190
|
+
... variable='networth'
|
|
191
|
+
... )
|
|
192
|
+
|
|
193
|
+
>>> # Example 2: Weighted median
|
|
194
|
+
>>> def weighted_quantile(data, weights, variable, q):
|
|
195
|
+
... from statsmodels.stats.weightstats import DescrStatsW
|
|
196
|
+
... notmissing = data[variable].notna()
|
|
197
|
+
... weighted_stats = DescrStatsW(data.loc[notmissing, variable].values, weights=weights[notmissing])
|
|
198
|
+
... return float(weighted_stats.quantile(probs=[q]).iloc[0])
|
|
199
|
+
>>>
|
|
200
|
+
>>> result = scf_estimate(
|
|
201
|
+
... scf_data,
|
|
202
|
+
... weighted_quantile,
|
|
203
|
+
... variable='networth',
|
|
204
|
+
... q=0.5
|
|
205
|
+
... )
|
|
206
|
+
|
|
207
|
+
>>> # Example 3: Regression
|
|
208
|
+
>>> import statsmodels.api as sm
|
|
209
|
+
>>>
|
|
210
|
+
>>> def ols_coefficients(data, weights, formula):
|
|
211
|
+
... import statsmodels.formula.api as smf
|
|
212
|
+
... model = smf.wls(formula, data=data, weights=weights)
|
|
213
|
+
... result = model.fit()
|
|
214
|
+
... return result.params.values
|
|
215
|
+
>>>
|
|
216
|
+
>>> result = scf_estimate(
|
|
217
|
+
... scf_data,
|
|
218
|
+
... ols_coefficients,
|
|
219
|
+
... formula='networth ~ age + income + C(education)',
|
|
220
|
+
... coefficient_names=['Intercept', 'age', 'income', 'education_1', 'education_2']
|
|
221
|
+
... )
|
|
222
|
+
|
|
223
|
+
References:
|
|
224
|
+
- SAS macro MEANIT from https://www.federalreserve.gov/econres/files/codebk2022.txt
|
|
225
|
+
- Stata module SCFCOMBO from https://ideas.repec.org/c/boc/bocode/s458017.html
|
|
226
|
+
- R function scf_MIcombine from https://www.convey-r.org/1.6-survey-of-consumer-finances-scf.html#survey-of-consumer-finances-scf
|
|
227
|
+
```
|
|
228
|
+
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# scfpublic.scfcombo.stats
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
Convenience summary-statistic wrappers around scf_estimate().
|
|
5
|
+
|
|
6
|
+
Author: Payton Crawford
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
## `scf_mean`
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
scf_mean(data: Dict[str, pandas.core.frame.DataFrame], variable: str, subset: Optional[Callable[[pandas.core.frame.DataFrame], pandas.core.series.Series]] = None) -> scfpublic.scfcombo.scfcombo.scfcombo_result
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Compute weighted mean with full variance estimation.
|
|
16
|
+
|
|
17
|
+
This is a convenience wrapper around scf_estimate() for computing means.
|
|
18
|
+
|
|
19
|
+
**Parameters:**
|
|
20
|
+
|
|
21
|
+
- `data (dict)`: Dictionary from prepare_scf_data()
|
|
22
|
+
- `variable (str)`: Name of variable to compute mean for
|
|
23
|
+
- `subset (function)`: Optional function for subsetting data
|
|
24
|
+
|
|
25
|
+
**Returns:**
|
|
26
|
+
|
|
27
|
+
scfcombo_result object
|
|
28
|
+
|
|
29
|
+
**Examples:**
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
>>> result = scf_mean(scf_data, 'networth')
|
|
33
|
+
>>> result = scf_mean(scf_data, 'income', subset=lambda df: df['age'] > 25)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## `scf_quantile`
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
scf_quantile(data: Dict[str, pandas.core.frame.DataFrame], variable: str, quantile: float = 0.5, subset: Optional[Callable[[pandas.core.frame.DataFrame], pandas.core.series.Series]] = None) -> scfpublic.scfcombo.scfcombo.scfcombo_result
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Compute weighted quantile with full variance estimation.
|
|
43
|
+
|
|
44
|
+
This is a convenience wrapper around scf_estimate() for computing quantiles.
|
|
45
|
+
|
|
46
|
+
**Parameters:**
|
|
47
|
+
|
|
48
|
+
- `data (dict)`: Dictionary from prepare_scf_data()
|
|
49
|
+
- `variable (str)`: Name of variable to compute quantile for
|
|
50
|
+
- `quantile (float)`: Quantile to compute (0.5 for median)
|
|
51
|
+
- `subset (function)`: Optional function for subsetting data
|
|
52
|
+
|
|
53
|
+
**Returns:**
|
|
54
|
+
|
|
55
|
+
scfcombo_result object
|
|
56
|
+
|
|
57
|
+
**Examples:**
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
>>> result = scf_quantile(scf_data, 'networth', quantile=0.5) # Median
|
|
61
|
+
>>> result = scf_quantile(scf_data, 'income', quantile=0.9) # 90th percentile
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## `scf_regression`
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
scf_regression(data: Dict[str, pandas.core.frame.DataFrame], formula: str, model_type: str = 'ols', subset: Optional[Callable[[pandas.core.frame.DataFrame], pandas.core.series.Series]] = None) -> scfpublic.scfcombo.scfcombo.scfcombo_result
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Estimate regression model with SCF variance correction.
|
|
71
|
+
|
|
72
|
+
This function properly handles survey weights for regression estimation:
|
|
73
|
+
|
|
74
|
+
Weight Usage:
|
|
75
|
+
|
|
76
|
+
1. Point Estimates (Coefficients):
|
|
77
|
+
- For each implicate i: Run regression using main weight (wgt)
|
|
78
|
+
- Pool coefficients: beta_hat = (1/5) * sum(beta_hat{i})
|
|
79
|
+
|
|
80
|
+
2. Imputation Variance:
|
|
81
|
+
- Variance of coefficients across 5 implicates
|
|
82
|
+
|
|
83
|
+
3. Sampling Variance:
|
|
84
|
+
- For each replicate r: Run regression on first implicate using
|
|
85
|
+
replicate weight r (wgt1, wgt2, ..., wgt999)
|
|
86
|
+
- Variance of coefficients across replicates
|
|
87
|
+
|
|
88
|
+
**Parameters:**
|
|
89
|
+
|
|
90
|
+
- `data (dict)`: Dictionary from prepare_scf_data()
|
|
91
|
+
- `formula (str)`: Regression formula in statsmodels format
|
|
92
|
+
- `model_type (str)`: Type of model ('ols', 'logit', 'probit', 'poisson')
|
|
93
|
+
- `subset (function)`: Optional function for subsetting data
|
|
94
|
+
|
|
95
|
+
**Returns:**
|
|
96
|
+
|
|
97
|
+
scfcombo_result object with coefficient estimates and standard errors
|
|
98
|
+
|
|
99
|
+
References:
|
|
100
|
+
|
|
101
|
+
- SAS macro MEANIT from https://www.federalreserve.gov/econres/files/codebk2022.txt
|
|
102
|
+
- Stata module SCFCOMBO from https://ideas.repec.org/c/boc/bocode/s458017.html
|
|
103
|
+
|
|
104
|
+
**Examples:**
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
>>> result = scf_regression(
|
|
108
|
+
... scf_data,
|
|
109
|
+
... 'networth ~ age + income + C(education)',
|
|
110
|
+
... model_type='ols'
|
|
111
|
+
... )
|
|
112
|
+
>>> result.display()
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## `scf_total`
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
scf_total(data: Dict[str, pandas.core.frame.DataFrame], variable: str, subset: Optional[Callable[[pandas.core.frame.DataFrame], pandas.core.series.Series]] = None) -> scfpublic.scfcombo.scfcombo.scfcombo_result
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Compute weighted total (population-level sum) with full variance estimation.
|
|
122
|
+
NOTE: This is optimized for the 'wgt' variable. If the weight specified in prepare_scf_data represents the whole population
|
|
123
|
+
for each implicate (i.e. wgt0), then you should not specify {estimand_type='total'}. Otherwise, the aggregate will be
|
|
124
|
+
5x its correct size.
|
|
125
|
+
|
|
126
|
+
**Parameters:**
|
|
127
|
+
|
|
128
|
+
- `data (dict)`: Dictionary from prepare_scf_data()
|
|
129
|
+
- `variable (str)`: Name of variable to compute total for
|
|
130
|
+
- `subset (function)`: Optional function for subsetting data
|
|
131
|
+
|
|
132
|
+
**Returns:**
|
|
133
|
+
|
|
134
|
+
scfcombo_result object
|
|
135
|
+
|
|
136
|
+
**Examples:**
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
>>> result = scf_total(scf_data, 'networth')
|
|
140
|
+
>>> result.display() # Total household net worth in population
|
|
141
|
+
```
|
|
142
|
+
|