pseudopeople 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pseudopeople/__about__.py +22 -0
- pseudopeople/__init__.py +11 -0
- pseudopeople/default_configuration.yaml +54 -0
- pseudopeople/entities.py +97 -0
- pseudopeople/entity_types.py +59 -0
- pseudopeople/interface.py +43 -0
- pseudopeople/noise.py +74 -0
- pseudopeople/noise_functions.py +268 -0
- pseudopeople/utilities.py +34 -0
- pseudopeople-0.1.0.dist-info/LICENSE +28 -0
- pseudopeople-0.1.0.dist-info/METADATA +79 -0
- pseudopeople-0.1.0.dist-info/RECORD +14 -0
- pseudopeople-0.1.0.dist-info/WHEEL +5 -0
- pseudopeople-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
__all__ = [
|
|
2
|
+
"__title__",
|
|
3
|
+
"__summary__",
|
|
4
|
+
"__uri__",
|
|
5
|
+
"__version__",
|
|
6
|
+
"__author__",
|
|
7
|
+
"__email__",
|
|
8
|
+
"__license__",
|
|
9
|
+
"__copyright__",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
__title__ = "pseudopeople"
|
|
13
|
+
__summary__ = "pseudopeople is package which adds noise to simulated census-scale data using standard scientific Python tools."
|
|
14
|
+
__uri__ = "https://github.com/ihmeuw/pseudopeople"
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
|
|
18
|
+
__author__ = "The pseudopeople developers"
|
|
19
|
+
__email__ = "vivarium.dev@gmail.com"
|
|
20
|
+
|
|
21
|
+
__license__ = "BSD-3-Clause"
|
|
22
|
+
__copyright__ = f"Copyright 2023 {__author__}"
|
pseudopeople/__init__.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Default noising configuration
|
|
2
|
+
# structure follows:
|
|
3
|
+
# Row noise: `form.noise_type`
|
|
4
|
+
# Column-wise parameters: `form.column.noise_type.noise_parameter`
|
|
5
|
+
|
|
6
|
+
decennial_census:
|
|
7
|
+
omission: 0.0145
|
|
8
|
+
duplication: 0.05
|
|
9
|
+
first_name:
|
|
10
|
+
missing_data:
|
|
11
|
+
row_noise_level: 0.01
|
|
12
|
+
middle_initial:
|
|
13
|
+
missing_data:
|
|
14
|
+
row_noise_level: 0.01
|
|
15
|
+
last_name:
|
|
16
|
+
missing_data:
|
|
17
|
+
row_noise_level: 0.01
|
|
18
|
+
age:
|
|
19
|
+
missing_data:
|
|
20
|
+
row_noise_level: 0.01
|
|
21
|
+
date_of_birth:
|
|
22
|
+
missing_data:
|
|
23
|
+
row_noise_level: 0.01
|
|
24
|
+
street_number:
|
|
25
|
+
missing_data:
|
|
26
|
+
row_noise_level: 0.01
|
|
27
|
+
street_name:
|
|
28
|
+
missing_data:
|
|
29
|
+
row_noise_level: 0.01
|
|
30
|
+
unit_number:
|
|
31
|
+
missing_data:
|
|
32
|
+
row_noise_level: 0.01
|
|
33
|
+
city:
|
|
34
|
+
missing_data:
|
|
35
|
+
row_noise_level: 0.01
|
|
36
|
+
state:
|
|
37
|
+
missing_data:
|
|
38
|
+
row_noise_level: 0.01
|
|
39
|
+
zipcode:
|
|
40
|
+
missing_data:
|
|
41
|
+
row_noise_level: 0.01
|
|
42
|
+
relation_to_household_head:
|
|
43
|
+
missing_data:
|
|
44
|
+
row_noise_level: 0.01
|
|
45
|
+
sex:
|
|
46
|
+
missing_data:
|
|
47
|
+
row_noise_level: 0.01
|
|
48
|
+
race_ethnicity:
|
|
49
|
+
missing_data:
|
|
50
|
+
row_noise_level: 0.01
|
|
51
|
+
housing_type:
|
|
52
|
+
missing_data:
|
|
53
|
+
row_noise_level: 0.01
|
|
54
|
+
|
pseudopeople/entities.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import NamedTuple
|
|
3
|
+
|
|
4
|
+
from pseudopeople import noise_functions
|
|
5
|
+
from pseudopeople.entity_types import ColumnNoiseType, RowNoiseType
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# todo: is "form" the right word? Ask RT
|
|
9
|
+
class Form(Enum):
|
|
10
|
+
CENSUS = "decennial_census"
|
|
11
|
+
ACS = "american_communities_survey"
|
|
12
|
+
CPS = "current_population_survey"
|
|
13
|
+
WIC = "women_infants_and_children"
|
|
14
|
+
SSA = "social_security"
|
|
15
|
+
TAX_W2_1099 = "taxes_w2_and_1099"
|
|
16
|
+
TAX_1040 = "taxes_1040"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class __Columns(NamedTuple):
|
|
20
|
+
FIRST_NAME: str = "first_name"
|
|
21
|
+
MIDDLE_INITIAL: str = "middle_initial"
|
|
22
|
+
LAST_NAME: str = "last_name"
|
|
23
|
+
STREET_NAME: str = "street_name"
|
|
24
|
+
ZIP_CODE: str = "zipcode"
|
|
25
|
+
CITY: str = "city"
|
|
26
|
+
AGE: str = "age"
|
|
27
|
+
# todo finish filling in columns
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
COLUMNS = __Columns()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class __NoiseTypes(NamedTuple):
|
|
34
|
+
"""Container for all noise types in the order in which they should be applied:
|
|
35
|
+
omissions, duplications, missing data, incorrect selection, copy from w/in
|
|
36
|
+
household, month and day swaps, zip code miswriting, age miswriting,
|
|
37
|
+
numeric miswriting, nicknames, fake names, phonetic, OCR, typographic"
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
# todo finish filling in noise types in the correct order per the docs
|
|
41
|
+
OMISSION: RowNoiseType = RowNoiseType("omission", noise_functions.omit_rows)
|
|
42
|
+
DUPLICATION: RowNoiseType = RowNoiseType("duplication", noise_functions.duplicate_rows)
|
|
43
|
+
MISSING_DATA: ColumnNoiseType = ColumnNoiseType(
|
|
44
|
+
"missing_data",
|
|
45
|
+
noise_functions.generate_missing_data,
|
|
46
|
+
)
|
|
47
|
+
INCORRECT_SELECTION: ColumnNoiseType = ColumnNoiseType(
|
|
48
|
+
"incorrect_selection", noise_functions.generate_incorrect_selections
|
|
49
|
+
)
|
|
50
|
+
COPY_FROM_WITHIN_HOUSEHOLD: ColumnNoiseType = ColumnNoiseType(
|
|
51
|
+
"copy_from_within_household", noise_functions.generate_within_household_copies
|
|
52
|
+
)
|
|
53
|
+
MONTH_DAY_SWAP: ColumnNoiseType = ColumnNoiseType(
|
|
54
|
+
"month_day_swap", noise_functions.swap_months_and_days
|
|
55
|
+
)
|
|
56
|
+
ZIP_CODE_MISWRITING: ColumnNoiseType = ColumnNoiseType(
|
|
57
|
+
"zipcode_miswriting", noise_functions.miswrite_zip_codes
|
|
58
|
+
)
|
|
59
|
+
AGE_MISWRITING: ColumnNoiseType = ColumnNoiseType(
|
|
60
|
+
"age_miswriting", noise_functions.miswrite_ages
|
|
61
|
+
)
|
|
62
|
+
NUMERIC_MISWRITING: ColumnNoiseType = ColumnNoiseType(
|
|
63
|
+
"numeric_miswriting", noise_functions.miswrite_numerics
|
|
64
|
+
)
|
|
65
|
+
NICKNAME: ColumnNoiseType = ColumnNoiseType(
|
|
66
|
+
"nickname", noise_functions.generate_nicknames
|
|
67
|
+
)
|
|
68
|
+
FAKE_NAME: ColumnNoiseType = ColumnNoiseType(
|
|
69
|
+
"fake_names", noise_functions.generate_fake_names
|
|
70
|
+
)
|
|
71
|
+
PHONETIC: ColumnNoiseType = ColumnNoiseType(
|
|
72
|
+
"phonetic", noise_functions.generate_phonetic_errors
|
|
73
|
+
)
|
|
74
|
+
OCR: ColumnNoiseType = ColumnNoiseType(
|
|
75
|
+
# todo: implement the noise fn
|
|
76
|
+
"ocr",
|
|
77
|
+
noise_functions.generate_ocr_errors,
|
|
78
|
+
)
|
|
79
|
+
TYPOGRAPHIC: ColumnNoiseType = ColumnNoiseType(
|
|
80
|
+
"typographic",
|
|
81
|
+
noise_functions.generate_typographical_errors,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
NOISE_TYPES = __NoiseTypes()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class NoiseParameter(Enum):
|
|
89
|
+
"""
|
|
90
|
+
Enum containing all additional parameters used to specify column noise.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
ROW_NOISE_LEVEL = "row_noise_level"
|
|
94
|
+
TOKEN_NOISE_LEVEL = "token_noise_level"
|
|
95
|
+
TYPOGRAPHIC_NOISE = "typographic_noise"
|
|
96
|
+
AGE_MISWRITING = "age_miswriting"
|
|
97
|
+
ZIPCODE_MISWRITING = "zipcode_miswriting"
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import Any, Callable
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from vivarium import ConfigTree
|
|
6
|
+
from vivarium.framework.randomness import RandomnessStream
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class RowNoiseType:
|
|
11
|
+
"""
|
|
12
|
+
Defines a type of noise that can be applied to a row.
|
|
13
|
+
|
|
14
|
+
The name is the name of the particular noise function (e.g. "omission" or
|
|
15
|
+
"duplication").
|
|
16
|
+
|
|
17
|
+
The noise function takes as input a DataFrame, the configuration value
|
|
18
|
+
for this RowNoise operation, and a RandomnessStream for controlling
|
|
19
|
+
randomness. It applies the noising operation to the entire DataFrame and
|
|
20
|
+
returns the modified DataFrame.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
name: str
|
|
24
|
+
noise_function: Callable[[pd.DataFrame, float, RandomnessStream, str], pd.DataFrame]
|
|
25
|
+
|
|
26
|
+
def __call__(
|
|
27
|
+
self,
|
|
28
|
+
form_data: pd.DataFrame,
|
|
29
|
+
configuration: float,
|
|
30
|
+
randomness_stream: RandomnessStream,
|
|
31
|
+
) -> pd.DataFrame:
|
|
32
|
+
return self.noise_function(form_data, configuration, randomness_stream)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class ColumnNoiseType:
|
|
37
|
+
"""
|
|
38
|
+
Defines a type of noise that can be applied to a column.
|
|
39
|
+
|
|
40
|
+
The name is the name of the particular noise function (e.g. "nickname" or
|
|
41
|
+
"phonetic").
|
|
42
|
+
|
|
43
|
+
The noise function takes as input a Series, the ConfigTree object for this
|
|
44
|
+
ColumnNoise operation, a RandomnessStream for controlling randomness, and
|
|
45
|
+
an additional key for the RandomnessStream. It applies the noising operation
|
|
46
|
+
to the Series and returns the modified Series.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
name: str
|
|
50
|
+
noise_function: Callable[[pd.Series, ConfigTree, RandomnessStream, Any], pd.Series]
|
|
51
|
+
|
|
52
|
+
def __call__(
|
|
53
|
+
self,
|
|
54
|
+
column: pd.Series,
|
|
55
|
+
configuration: ConfigTree,
|
|
56
|
+
randomness_stream: RandomnessStream,
|
|
57
|
+
additional_key: Any,
|
|
58
|
+
) -> pd.Series:
|
|
59
|
+
return self.noise_function(column, configuration, randomness_stream, additional_key)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Union
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from pseudopeople.entities import Form
|
|
8
|
+
from pseudopeople.noise import noise_form
|
|
9
|
+
from pseudopeople.utilities import get_configuration
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# TODO: add year as parameter to select the year of the decennial census to generate (MIC-3909)
|
|
13
|
+
# TODO: add default path: have the package install the small data in a known location and then
|
|
14
|
+
# to make this parameter optional, with the default being the location of the small data that
|
|
15
|
+
# is installed with the package (MIC-3884)
|
|
16
|
+
def generate_decennial_census(
|
|
17
|
+
path: Union[Path, str], seed: int = 0, configuration: Union[Path, str] = None
|
|
18
|
+
):
|
|
19
|
+
"""
|
|
20
|
+
Generates a noised decennial census data from un-noised data.
|
|
21
|
+
|
|
22
|
+
:param path: A path to the un-noised source census data
|
|
23
|
+
:param seed: An integer seed for randomness
|
|
24
|
+
:param configuration: (optional) A path to a configuration YAML file to modify default values
|
|
25
|
+
:return: A pd.DataFrame of noised census data
|
|
26
|
+
"""
|
|
27
|
+
configuration_tree = get_configuration(configuration)
|
|
28
|
+
data = pd.read_csv(path)
|
|
29
|
+
return noise_form(Form.CENSUS, data, configuration_tree, seed)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# Manual testing helper
|
|
33
|
+
if __name__ == "__main__":
|
|
34
|
+
args = sys.argv[1:]
|
|
35
|
+
if len(args) == 1:
|
|
36
|
+
my_path = Path(args[0])
|
|
37
|
+
src = pd.read_csv(my_path)
|
|
38
|
+
out = generate_decennial_census(my_path)
|
|
39
|
+
diff = src[
|
|
40
|
+
~src.astype(str).apply(tuple, 1).isin(out.astype(str).apply(tuple, 1))
|
|
41
|
+
] # get all changed rows
|
|
42
|
+
print(out.head())
|
|
43
|
+
print(diff)
|
pseudopeople/noise.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""
|
|
2
|
+
================
|
|
3
|
+
Noise Form
|
|
4
|
+
================
|
|
5
|
+
|
|
6
|
+
A function that will noise data based on a user's configuration.
|
|
7
|
+
|
|
8
|
+
This function is the core of Pseudopeople, which takes in a pd.DataFrame, or form
|
|
9
|
+
data and will add noise to both rows and columns based on the configured values
|
|
10
|
+
provided by the user. First, the Form will be noised for missing data and have
|
|
11
|
+
rows in each columns changed to null values. Then, the form data will be noised
|
|
12
|
+
by column and row for each type of additional noise type.
|
|
13
|
+
"""
|
|
14
|
+
import pandas as pd
|
|
15
|
+
from vivarium import ConfigTree
|
|
16
|
+
|
|
17
|
+
from pseudopeople.entities import NOISE_TYPES, Form
|
|
18
|
+
from pseudopeople.entity_types import ColumnNoiseType, RowNoiseType
|
|
19
|
+
from pseudopeople.utilities import get_randomness_stream
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def noise_form(
|
|
23
|
+
form: Form,
|
|
24
|
+
form_data: pd.DataFrame,
|
|
25
|
+
configuration: ConfigTree,
|
|
26
|
+
seed: int,
|
|
27
|
+
) -> pd.DataFrame:
|
|
28
|
+
"""
|
|
29
|
+
Adds noise to the input form data. Noise functions are executed in the order
|
|
30
|
+
defined by :py:const: `.NOISE_TYPES`. Row noise functions are applied to the
|
|
31
|
+
whole DataFrame. Column noise functions will be applied to each column that
|
|
32
|
+
is pertinent to it.
|
|
33
|
+
|
|
34
|
+
Noise levels are determined by the noise_config.
|
|
35
|
+
:param form:
|
|
36
|
+
Form needing to be noised
|
|
37
|
+
:param form_data:
|
|
38
|
+
Clean data input which needs to be noised.
|
|
39
|
+
:param configuration:
|
|
40
|
+
Object to configure noise levels
|
|
41
|
+
:param seed:
|
|
42
|
+
Seed for controlling randomness
|
|
43
|
+
:return:
|
|
44
|
+
Noised form data
|
|
45
|
+
"""
|
|
46
|
+
randomness = get_randomness_stream(form, seed)
|
|
47
|
+
|
|
48
|
+
noise_configuration = configuration[form.value]
|
|
49
|
+
for noise_type in NOISE_TYPES:
|
|
50
|
+
if isinstance(noise_type, RowNoiseType):
|
|
51
|
+
# Apply row noise
|
|
52
|
+
form_data = noise_type(form_data, noise_configuration, randomness)
|
|
53
|
+
|
|
54
|
+
elif isinstance(noise_type, ColumnNoiseType):
|
|
55
|
+
columns_to_noise = [
|
|
56
|
+
col
|
|
57
|
+
for col in noise_configuration
|
|
58
|
+
if col in form_data.columns and noise_type.name in noise_configuration[col]
|
|
59
|
+
]
|
|
60
|
+
# Apply column noise to each column as appropriate
|
|
61
|
+
for column in columns_to_noise:
|
|
62
|
+
form_data[column] = noise_type(
|
|
63
|
+
form_data[column],
|
|
64
|
+
noise_configuration[column][noise_type.name],
|
|
65
|
+
randomness,
|
|
66
|
+
column,
|
|
67
|
+
)
|
|
68
|
+
else:
|
|
69
|
+
raise TypeError(
|
|
70
|
+
f"Invalid noise type. Allowed types are {RowNoiseType} and "
|
|
71
|
+
f"{ColumnNoiseType}. Provided {type(noise_type)}."
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
return form_data
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
from vivarium import ConfigTree
|
|
5
|
+
from vivarium.framework.randomness import RandomnessStream
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def omit_rows(
|
|
9
|
+
form_data: pd.DataFrame,
|
|
10
|
+
configuration: float,
|
|
11
|
+
randomness_stream: RandomnessStream,
|
|
12
|
+
) -> pd.DataFrame:
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
:param form_data:
|
|
16
|
+
:param configuration:
|
|
17
|
+
:param randomness_stream:
|
|
18
|
+
:return:
|
|
19
|
+
"""
|
|
20
|
+
# todo actually omit rows
|
|
21
|
+
return form_data
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def duplicate_rows(
|
|
25
|
+
form_data: pd.DataFrame,
|
|
26
|
+
configuration: float,
|
|
27
|
+
randomness_stream: RandomnessStream,
|
|
28
|
+
) -> pd.DataFrame:
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
:param form_data:
|
|
32
|
+
:param configuration:
|
|
33
|
+
:param randomness_stream:
|
|
34
|
+
:return:
|
|
35
|
+
"""
|
|
36
|
+
# todo actually duplicate rows
|
|
37
|
+
return form_data
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def generate_incorrect_selections(
|
|
41
|
+
form_data: pd.DataFrame,
|
|
42
|
+
configuration: float,
|
|
43
|
+
randomness_stream: RandomnessStream,
|
|
44
|
+
additional_key: Any,
|
|
45
|
+
) -> pd.DataFrame:
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
:param form_data:
|
|
49
|
+
:param configuration:
|
|
50
|
+
:param randomness_stream:
|
|
51
|
+
:param additional_key: Key for RandomnessStream
|
|
52
|
+
:return:
|
|
53
|
+
"""
|
|
54
|
+
# todo actually duplicate rows
|
|
55
|
+
return form_data
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def generate_within_household_copies(
|
|
59
|
+
form_data: pd.DataFrame,
|
|
60
|
+
configuration: float,
|
|
61
|
+
randomness_stream: RandomnessStream,
|
|
62
|
+
additional_key: Any,
|
|
63
|
+
) -> pd.DataFrame:
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
:param form_data:
|
|
67
|
+
:param configuration:
|
|
68
|
+
:param randomness_stream:
|
|
69
|
+
:param additional_key: Key for RandomnessStream
|
|
70
|
+
:return:
|
|
71
|
+
"""
|
|
72
|
+
# todo actually duplicate rows
|
|
73
|
+
return form_data
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def swap_months_and_days(
|
|
77
|
+
form_data: pd.DataFrame,
|
|
78
|
+
configuration: float,
|
|
79
|
+
randomness_stream: RandomnessStream,
|
|
80
|
+
additional_key: Any,
|
|
81
|
+
) -> pd.DataFrame:
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
:param form_data:
|
|
85
|
+
:param configuration:
|
|
86
|
+
:param randomness_stream:
|
|
87
|
+
:param additional_key: Key for RandomnessStream
|
|
88
|
+
:return:
|
|
89
|
+
"""
|
|
90
|
+
# todo actually duplicate rows
|
|
91
|
+
return form_data
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def miswrite_zip_codes(
|
|
95
|
+
form_data: pd.DataFrame,
|
|
96
|
+
configuration: float,
|
|
97
|
+
randomness_stream: RandomnessStream,
|
|
98
|
+
additional_key: Any,
|
|
99
|
+
) -> pd.DataFrame:
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
:param form_data:
|
|
103
|
+
:param configuration:
|
|
104
|
+
:param randomness_stream:
|
|
105
|
+
:param additional_key: Key for RandomnessStream
|
|
106
|
+
:return:
|
|
107
|
+
"""
|
|
108
|
+
# todo actually duplicate rows
|
|
109
|
+
return form_data
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def miswrite_ages(
|
|
113
|
+
form_data: pd.DataFrame,
|
|
114
|
+
configuration: float,
|
|
115
|
+
randomness_stream: RandomnessStream,
|
|
116
|
+
additional_key: Any,
|
|
117
|
+
) -> pd.DataFrame:
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
:param form_data:
|
|
121
|
+
:param configuration:
|
|
122
|
+
:param randomness_stream:
|
|
123
|
+
:param additional_key: Key for RandomnessStream
|
|
124
|
+
:return:
|
|
125
|
+
"""
|
|
126
|
+
# todo actually duplicate rows
|
|
127
|
+
return form_data
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def miswrite_numerics(
|
|
131
|
+
form_data: pd.DataFrame,
|
|
132
|
+
configuration: float,
|
|
133
|
+
randomness_stream: RandomnessStream,
|
|
134
|
+
additional_key: Any,
|
|
135
|
+
) -> pd.DataFrame:
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
:param form_data:
|
|
139
|
+
:param configuration:
|
|
140
|
+
:param randomness_stream:
|
|
141
|
+
:param additional_key: Key for RandomnessStream
|
|
142
|
+
:return:
|
|
143
|
+
"""
|
|
144
|
+
# todo actually duplicate rows
|
|
145
|
+
return form_data
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def generate_nicknames(
|
|
149
|
+
column: pd.Series,
|
|
150
|
+
configuration: ConfigTree,
|
|
151
|
+
randomness_stream: RandomnessStream,
|
|
152
|
+
additional_key: Any,
|
|
153
|
+
) -> pd.Series:
|
|
154
|
+
"""
|
|
155
|
+
|
|
156
|
+
:param column:
|
|
157
|
+
:param configuration:
|
|
158
|
+
:param randomness_stream:
|
|
159
|
+
:param additional_key: Key for RandomnessStream
|
|
160
|
+
:return:
|
|
161
|
+
"""
|
|
162
|
+
# todo actually generate nicknames
|
|
163
|
+
return column
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def generate_fake_names(
|
|
167
|
+
column: pd.Series,
|
|
168
|
+
configuration: ConfigTree,
|
|
169
|
+
randomness_stream: RandomnessStream,
|
|
170
|
+
additional_key: Any,
|
|
171
|
+
) -> pd.Series:
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
:param column:
|
|
175
|
+
:param configuration:
|
|
176
|
+
:param randomness_stream:
|
|
177
|
+
:param additional_key: Key for RandomnessStream
|
|
178
|
+
:return:
|
|
179
|
+
"""
|
|
180
|
+
# todo actually generate fake names
|
|
181
|
+
return column
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def generate_phonetic_errors(
|
|
185
|
+
column: pd.Series,
|
|
186
|
+
configuration: ConfigTree,
|
|
187
|
+
randomness_stream: RandomnessStream,
|
|
188
|
+
additional_key: Any,
|
|
189
|
+
) -> pd.Series:
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
:param column:
|
|
193
|
+
:param configuration:
|
|
194
|
+
:param randomness_stream:
|
|
195
|
+
:param additional_key: Key for RandomnessStream
|
|
196
|
+
:return:
|
|
197
|
+
"""
|
|
198
|
+
# todo actually generate fake names
|
|
199
|
+
return column
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def generate_missing_data(
|
|
203
|
+
column: pd.Series,
|
|
204
|
+
configuration: ConfigTree,
|
|
205
|
+
randomness_stream: RandomnessStream,
|
|
206
|
+
additional_key: Any,
|
|
207
|
+
) -> pd.Series:
|
|
208
|
+
"""
|
|
209
|
+
Function that takes a column and blanks out a configurable portion of its data to be missing.
|
|
210
|
+
|
|
211
|
+
:param column: pd.Series of data
|
|
212
|
+
:param configuration: ConfigTree with rate at which to blank the data in column.
|
|
213
|
+
:param randomness_stream: RandomnessStream to utilize Vivarium CRN.
|
|
214
|
+
:param additional_key: Key for RandomnessStream
|
|
215
|
+
:returns: pd.Series of column with configured amount of data missing as an empty string.
|
|
216
|
+
"""
|
|
217
|
+
|
|
218
|
+
# Avoid SettingWithCopyWarning
|
|
219
|
+
column = column.copy()
|
|
220
|
+
noise_level = configuration.row_noise_level
|
|
221
|
+
# Get rows to noise
|
|
222
|
+
to_noise_idx = randomness_stream.filter_for_probability(
|
|
223
|
+
column.index,
|
|
224
|
+
probability=noise_level,
|
|
225
|
+
additional_key=f"{additional_key}_missing_data_filter",
|
|
226
|
+
)
|
|
227
|
+
column.loc[to_noise_idx] = ""
|
|
228
|
+
|
|
229
|
+
return column
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def generate_typographical_errors(
|
|
233
|
+
column: pd.Series,
|
|
234
|
+
configuration: ConfigTree,
|
|
235
|
+
randomness_stream: RandomnessStream,
|
|
236
|
+
additional_key: Any,
|
|
237
|
+
) -> pd.Series:
|
|
238
|
+
"""
|
|
239
|
+
|
|
240
|
+
:param column:
|
|
241
|
+
:param configuration:
|
|
242
|
+
:param randomness_stream:
|
|
243
|
+
:param additional_key: Key for RandomnessStream
|
|
244
|
+
:return:
|
|
245
|
+
"""
|
|
246
|
+
# todo actually generate typographical errors
|
|
247
|
+
return column
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def generate_ocr_errors(
|
|
251
|
+
column: pd.Series,
|
|
252
|
+
configuration: ConfigTree,
|
|
253
|
+
randomness_stream: RandomnessStream,
|
|
254
|
+
additional_key: Any,
|
|
255
|
+
) -> pd.Series:
|
|
256
|
+
"""
|
|
257
|
+
|
|
258
|
+
:param column:
|
|
259
|
+
:param configuration:
|
|
260
|
+
:param randomness_stream:
|
|
261
|
+
:param additional_key: Key for RandomnessStream
|
|
262
|
+
:return:
|
|
263
|
+
"""
|
|
264
|
+
# todo actually generate OCR errors
|
|
265
|
+
return column
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
# todo add noise functions
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Union
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from vivarium.framework.configuration import ConfigTree
|
|
6
|
+
from vivarium.framework.randomness import RandomnessStream
|
|
7
|
+
|
|
8
|
+
from pseudopeople.entities import Form
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def get_randomness_stream(form: Form, seed: int) -> RandomnessStream:
|
|
12
|
+
return RandomnessStream(form.value, lambda: pd.Timestamp("2020-04-01"), seed)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_configuration(user_yaml_path: Union[Path, str] = None) -> ConfigTree:
|
|
16
|
+
"""
|
|
17
|
+
Gets a noising configuration ConfigTree, optionally overridden by a user-provided YAML.
|
|
18
|
+
|
|
19
|
+
:param user_yaml_path: A path to the YAML file defining user overrides for the defaults
|
|
20
|
+
:return: a ConfigTree object of the noising configuration
|
|
21
|
+
"""
|
|
22
|
+
import pseudopeople
|
|
23
|
+
|
|
24
|
+
default_config_layers = [
|
|
25
|
+
"base",
|
|
26
|
+
"user",
|
|
27
|
+
]
|
|
28
|
+
noising_configuration = ConfigTree(
|
|
29
|
+
data=Path(pseudopeople.__file__).resolve().parent / "default_configuration.yaml",
|
|
30
|
+
layers=default_config_layers,
|
|
31
|
+
)
|
|
32
|
+
if user_yaml_path:
|
|
33
|
+
noising_configuration.update(user_yaml_path, layer="user")
|
|
34
|
+
return noising_configuration
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023, Institute for Health Metrics and Evaluation
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: pseudopeople
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: pseudopeople is package which adds noise to simulated census-scale data using standard scientific Python tools.
|
|
5
|
+
Home-page: https://github.com/ihmeuw/pseudopeople
|
|
6
|
+
Author: The pseudopeople developers
|
|
7
|
+
Author-email: vivarium.dev@gmail.com
|
|
8
|
+
License: BSD-3-Clause
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Education
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
|
|
13
|
+
Classifier: Natural Language :: English
|
|
14
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
15
|
+
Classifier: Operating System :: POSIX
|
|
16
|
+
Classifier: Operating System :: POSIX :: BSD
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
19
|
+
Classifier: Programming Language :: Python
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
21
|
+
Classifier: Topic :: Education
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Life
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: pandas
|
|
28
|
+
Requires-Dist: pyyaml (>=5.1)
|
|
29
|
+
Requires-Dist: vivarium
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: sphinx (>=4.0) ; extra == 'dev'
|
|
32
|
+
Requires-Dist: sphinx-rtd-theme ; extra == 'dev'
|
|
33
|
+
Requires-Dist: sphinx-click ; extra == 'dev'
|
|
34
|
+
Requires-Dist: IPython ; extra == 'dev'
|
|
35
|
+
Requires-Dist: matplotlib ; extra == 'dev'
|
|
36
|
+
Requires-Dist: pytest ; extra == 'dev'
|
|
37
|
+
Requires-Dist: pytest-mock ; extra == 'dev'
|
|
38
|
+
Requires-Dist: ipywidgets ; extra == 'dev'
|
|
39
|
+
Requires-Dist: jupyter ; extra == 'dev'
|
|
40
|
+
Provides-Extra: docs
|
|
41
|
+
Requires-Dist: sphinx (>=4.0) ; extra == 'docs'
|
|
42
|
+
Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
|
|
43
|
+
Requires-Dist: sphinx-click ; extra == 'docs'
|
|
44
|
+
Requires-Dist: IPython ; extra == 'docs'
|
|
45
|
+
Requires-Dist: matplotlib ; extra == 'docs'
|
|
46
|
+
Provides-Extra: interactive
|
|
47
|
+
Requires-Dist: IPython ; extra == 'interactive'
|
|
48
|
+
Requires-Dist: ipywidgets ; extra == 'interactive'
|
|
49
|
+
Requires-Dist: jupyter ; extra == 'interactive'
|
|
50
|
+
Provides-Extra: test
|
|
51
|
+
Requires-Dist: pytest ; extra == 'test'
|
|
52
|
+
Requires-Dist: pytest-mock ; extra == 'test'
|
|
53
|
+
|
|
54
|
+
============
|
|
55
|
+
Pseudopeople
|
|
56
|
+
============
|
|
57
|
+
|
|
58
|
+
Pseudopeople is a package intended to add configurable noise to a simulated
|
|
59
|
+
census-scale data-set written using standard scientific Python tools.
|
|
60
|
+
|
|
61
|
+
**Pseudopeople requires Python 3.7-3.10 to run**
|
|
62
|
+
|
|
63
|
+
You can install ``pseudopeople`` from PyPI with pip:
|
|
64
|
+
|
|
65
|
+
``> pip install pseudopeople``
|
|
66
|
+
|
|
67
|
+
or build it from source with
|
|
68
|
+
|
|
69
|
+
``> git clone https://github.com/ihmeuw/pseudopeople.git``
|
|
70
|
+
|
|
71
|
+
``> cd pseudopeople``
|
|
72
|
+
|
|
73
|
+
``> python setup.py install``
|
|
74
|
+
|
|
75
|
+
This will make the ``pseudopeople`` library available to python and install a
|
|
76
|
+
command-line executable called ``...`` that you can use to verify your
|
|
77
|
+
installation with
|
|
78
|
+
|
|
79
|
+
``> ... test``
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
pseudopeople/__about__.py,sha256=QAMJhT9idR8fOcV-i3aaSkE0QWCqpaEDMxx4_bUQrlY,547
|
|
2
|
+
pseudopeople/__init__.py,sha256=wnGxaqs2MD1LmLDR41xDf8ImNcN7RVvY416p43AopKc,229
|
|
3
|
+
pseudopeople/default_configuration.yaml,sha256=pm5YDg848k8-FqFABMcnM94NQ3chvuafjgYjXeBnrd4,1303
|
|
4
|
+
pseudopeople/entities.py,sha256=V21mNvmXU1pQGJjkvZFPlyFeyAfDm-RvlqPtd9BP_qQ,3306
|
|
5
|
+
pseudopeople/entity_types.py,sha256=DzM4fsahY-C_IDkaW45IgPj15bbRkMXq6WZLjKSSn4o,1878
|
|
6
|
+
pseudopeople/interface.py,sha256=ECVeDIcwV7cmpiRWoGa7xMZZtkTSP_RsbiKwWjcoruI,1557
|
|
7
|
+
pseudopeople/noise.py,sha256=ouafbkDmYEOUcMan6OuirOIHyNEU9lR66b08j_s3DNw,2603
|
|
8
|
+
pseudopeople/noise_functions.py,sha256=IBYiMZgVvDfzQx_H1SGm7b8NRB1XhC2R-laHK0YVirA,5977
|
|
9
|
+
pseudopeople/utilities.py,sha256=5aDfhcjeMS3Si_23cy74GjB6KG7H7Bh6106NoRrutss,1113
|
|
10
|
+
pseudopeople-0.1.0.dist-info/LICENSE,sha256=denaAWcYGw8Pfw3jqedWwEedv18-1AH9vnRyGAEAcbc,1530
|
|
11
|
+
pseudopeople-0.1.0.dist-info/METADATA,sha256=RGhOecx8FehEC1YcEPioCXkHFXAHCCM3VN7RD-H1t1U,2885
|
|
12
|
+
pseudopeople-0.1.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
|
|
13
|
+
pseudopeople-0.1.0.dist-info/top_level.txt,sha256=wnbm1AJWmSOc2-XNvEyovNG4tqmwUBFL3REUFlKV9vY,13
|
|
14
|
+
pseudopeople-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pseudopeople
|