vivarium-validation 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vivarium/validation/__init__.py +12 -0
- vivarium/validation/bundle.py +193 -0
- vivarium/validation/comparison.py +375 -0
- vivarium/validation/constants.py +51 -0
- vivarium/validation/data_loader.py +298 -0
- vivarium/validation/data_transformation/__init__.py +0 -0
- vivarium/validation/data_transformation/age_groups.py +710 -0
- vivarium/validation/data_transformation/calculations.py +284 -0
- vivarium/validation/data_transformation/data_schema.py +44 -0
- vivarium/validation/data_transformation/formatting.py +133 -0
- vivarium/validation/data_transformation/measures.py +594 -0
- vivarium/validation/data_transformation/rate_aggregation.py +30 -0
- vivarium/validation/data_transformation/report.py +117 -0
- vivarium/validation/data_transformation/utils.py +155 -0
- vivarium/validation/interface.py +955 -0
- vivarium/validation/py.typed +0 -0
- vivarium/validation/results.py +17 -0
- vivarium/validation/templates/report_template.html +919 -0
- vivarium/validation/visualization/__init__.py +0 -0
- vivarium/validation/visualization/dataframe_utils.py +120 -0
- vivarium/validation/visualization/plot_utils.py +321 -0
- vivarium_validation-0.1.0.dist-info/METADATA +97 -0
- vivarium_validation-0.1.0.dist-info/RECORD +26 -0
- vivarium_validation-0.1.0.dist-info/WHEEL +5 -0
- vivarium_validation-0.1.0.dist-info/licenses/LICENSE +27 -0
- vivarium_validation-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
__version__ = version("vivarium-validation")
|
|
5
|
+
except PackageNotFoundError:
|
|
6
|
+
__version__ = "0.0.0+not-installed"
|
|
7
|
+
|
|
8
|
+
import numpy
|
|
9
|
+
|
|
10
|
+
numpy.seterr(all="raise")
|
|
11
|
+
|
|
12
|
+
from vivarium.validation.interface import ValidationContext
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
from abc import ABC
|
|
2
|
+
from collections.abc import Collection
|
|
3
|
+
from typing import Any, Literal
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from vivarium.validation.constants import DRAW_INDEX, SEED_INDEX, DataSource
|
|
8
|
+
from vivarium.validation.data_loader import DataLoader
|
|
9
|
+
from vivarium.validation.data_transformation import age_groups, calculations
|
|
10
|
+
from vivarium.validation.data_transformation.measures import (
|
|
11
|
+
CategoricalRelativeRisk,
|
|
12
|
+
Measure,
|
|
13
|
+
RatioMeasure,
|
|
14
|
+
RiskExposure,
|
|
15
|
+
)
|
|
16
|
+
from vivarium.validation.visualization import dataframe_utils
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class MeasureDataBundle(ABC):
|
|
20
|
+
measure: Measure
|
|
21
|
+
source: DataSource
|
|
22
|
+
data_loader: DataLoader
|
|
23
|
+
scenarios: dict[str, str] | None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class RatioMeasureDataBundle:
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
measure: RatioMeasure,
|
|
30
|
+
source: DataSource,
|
|
31
|
+
data_loader: DataLoader,
|
|
32
|
+
age_group_df: pd.DataFrame,
|
|
33
|
+
scenarios: dict[str, str] | None = None,
|
|
34
|
+
) -> None:
|
|
35
|
+
self.measure = measure
|
|
36
|
+
self.source = source
|
|
37
|
+
self.scenarios = scenarios if scenarios is not None else {}
|
|
38
|
+
self.datasets = self._get_formatted_datasets(data_loader, age_group_df)
|
|
39
|
+
self.weights = self._get_aggregated_weights(data_loader, age_group_df)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def dataset_names(self) -> dict[str, str]:
|
|
43
|
+
"""Return a dictionary of required datasets for the specified source."""
|
|
44
|
+
if self.source == DataSource.SIM:
|
|
45
|
+
return self.measure.sim_output_datasets
|
|
46
|
+
elif self.source in ([DataSource.ARTIFACT, DataSource.GBD]):
|
|
47
|
+
return self.measure.sim_input_datasets
|
|
48
|
+
else:
|
|
49
|
+
raise ValueError(f"Unsupported data source: {self.source}")
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def index_names(self) -> set[str]:
|
|
53
|
+
return {
|
|
54
|
+
index_name
|
|
55
|
+
for key in self.datasets
|
|
56
|
+
for index_name in self.datasets[key].index.names
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
def get_metadata(self) -> dict[str, Any]:
|
|
60
|
+
"""Organize the data information into a dictionary for display by a styled pandas DataFrame.
|
|
61
|
+
Apply formatting to values that need special handling.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
--------
|
|
65
|
+
A dictionary containing the formatted data information.
|
|
66
|
+
|
|
67
|
+
"""
|
|
68
|
+
dataframe = self.get_measure_data("all")
|
|
69
|
+
data_info: dict[str, Any] = {}
|
|
70
|
+
|
|
71
|
+
# Source as string
|
|
72
|
+
data_info["source"] = self.source.value
|
|
73
|
+
|
|
74
|
+
# Index columns as comma-separated string
|
|
75
|
+
data_info["index_columns"] = list(dataframe.index.names)
|
|
76
|
+
|
|
77
|
+
# Size as formatted string
|
|
78
|
+
size = dataframe.shape
|
|
79
|
+
data_info["size"] = f"{size[0]:,} rows × {size[1]:,} columns"
|
|
80
|
+
|
|
81
|
+
# Draw information
|
|
82
|
+
if DRAW_INDEX in dataframe.index.names:
|
|
83
|
+
num_draws = dataframe.index.get_level_values(DRAW_INDEX).nunique()
|
|
84
|
+
data_info["num_draws"] = f"{num_draws:,}"
|
|
85
|
+
draw_values = list(dataframe.index.get_level_values(DRAW_INDEX).unique())
|
|
86
|
+
data_info[DRAW_INDEX + "s"] = dataframe_utils.format_draws_sample(
|
|
87
|
+
draw_values, self.source
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# Seeds information
|
|
91
|
+
if SEED_INDEX in dataframe.index.names:
|
|
92
|
+
num_seeds = dataframe.index.get_level_values(SEED_INDEX).nunique()
|
|
93
|
+
data_info["num_seeds"] = f"{num_seeds:,}"
|
|
94
|
+
|
|
95
|
+
return data_info
|
|
96
|
+
|
|
97
|
+
def _get_formatted_datasets(
|
|
98
|
+
self, data_loader: DataLoader, age_group_data: pd.DataFrame
|
|
99
|
+
) -> dict[str, pd.DataFrame]:
|
|
100
|
+
"""Formats measure datasets depending on the source."""
|
|
101
|
+
raw_datasets = data_loader._get_raw_data_from_source(self.dataset_names, self.source)
|
|
102
|
+
if self.source == DataSource.SIM:
|
|
103
|
+
datasets = self.measure.get_ratio_datasets_from_sim(
|
|
104
|
+
**raw_datasets,
|
|
105
|
+
)
|
|
106
|
+
elif self.source in [DataSource.ARTIFACT, DataSource.GBD]:
|
|
107
|
+
data = self.measure.get_measure_data_from_sim_inputs(**raw_datasets)
|
|
108
|
+
datasets = {"data": data}
|
|
109
|
+
elif self.source == DataSource.CUSTOM:
|
|
110
|
+
raise NotImplementedError
|
|
111
|
+
else:
|
|
112
|
+
raise ValueError(f"Unsupported data source: {self.source}")
|
|
113
|
+
|
|
114
|
+
datasets = {
|
|
115
|
+
dataset_name: age_groups.format_dataframe_from_age_bin_df(dataset, age_group_data)
|
|
116
|
+
for dataset_name, dataset in datasets.items()
|
|
117
|
+
}
|
|
118
|
+
datasets = {
|
|
119
|
+
key: calculations.filter_data(dataset, self.scenarios, drop_singles=True)
|
|
120
|
+
for key, dataset in datasets.items()
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
return datasets
|
|
124
|
+
|
|
125
|
+
def _get_aggregated_weights(
|
|
126
|
+
self, data_loader: DataLoader, age_group_data: pd.DataFrame
|
|
127
|
+
) -> pd.DataFrame | None:
|
|
128
|
+
"""Fetches and aggregates weights if required by the measure."""
|
|
129
|
+
if self.source not in [DataSource.ARTIFACT, DataSource.GBD]:
|
|
130
|
+
return None
|
|
131
|
+
|
|
132
|
+
raw_weights = data_loader._get_raw_data_from_source(
|
|
133
|
+
self.measure.rate_aggregation_weights.weight_keys, self.source
|
|
134
|
+
)
|
|
135
|
+
weights = self.measure.rate_aggregation_weights.get_weights(**raw_weights)
|
|
136
|
+
return age_groups.format_dataframe_from_age_bin_df(weights, age_group_data)
|
|
137
|
+
|
|
138
|
+
def get_measure_data(
|
|
139
|
+
self, stratifications: Collection[str] | Literal["all"]
|
|
140
|
+
) -> pd.DataFrame:
|
|
141
|
+
"""Get the measure data, optionally aggregated over specified stratifications."""
|
|
142
|
+
if self.source == DataSource.SIM:
|
|
143
|
+
return self._aggregate_scenario_stratifications(self.datasets, stratifications)
|
|
144
|
+
elif self.source in [DataSource.ARTIFACT, DataSource.GBD]:
|
|
145
|
+
return self._aggregate_sim_input_stratifications(stratifications)
|
|
146
|
+
elif self.source == DataSource.CUSTOM:
|
|
147
|
+
raise NotImplementedError
|
|
148
|
+
else:
|
|
149
|
+
raise ValueError(f"Unsupported data source: {self.source}")
|
|
150
|
+
|
|
151
|
+
def _aggregate_scenario_stratifications(
|
|
152
|
+
self,
|
|
153
|
+
datasets: dict[str, pd.DataFrame],
|
|
154
|
+
stratifications: Collection[str] | Literal["all"],
|
|
155
|
+
) -> pd.DataFrame:
|
|
156
|
+
"""This will remove index levels corresponding to the specified stratifications"""
|
|
157
|
+
datasets = {
|
|
158
|
+
key: calculations.stratify(datasets[key], stratifications) for key in datasets
|
|
159
|
+
}
|
|
160
|
+
return self.measure.get_measure_data_from_ratio(**datasets)
|
|
161
|
+
|
|
162
|
+
def _aggregate_sim_input_stratifications(
|
|
163
|
+
self, stratifications: Collection[str] | Literal["all"]
|
|
164
|
+
) -> pd.DataFrame:
|
|
165
|
+
"""Aggregate the artifact data over specified stratifications. Stratifactions will be retained
|
|
166
|
+
in the returned data."""
|
|
167
|
+
data = self.datasets["data"].copy()
|
|
168
|
+
if stratifications != "all":
|
|
169
|
+
stratifications = list(stratifications)
|
|
170
|
+
# Retain input_draw, comparison._aggregate_over_draws is the only place we should aggregate over draws.
|
|
171
|
+
if DRAW_INDEX in data.index.names and DRAW_INDEX not in stratifications:
|
|
172
|
+
stratifications.append(DRAW_INDEX)
|
|
173
|
+
if self.weights is None:
|
|
174
|
+
raise ValueError("Weights are required for aggregating artifact data.")
|
|
175
|
+
|
|
176
|
+
# Update scenario columns to retain during aggregation
|
|
177
|
+
scenario_cols = []
|
|
178
|
+
# NOTE: This is a hack to handle alignment of index levels in weighted_average. Risk
|
|
179
|
+
# stratification column is treated as a scenario column and the population can be
|
|
180
|
+
# broadcast across each index group since the exposure for each group should sum to 1.
|
|
181
|
+
if isinstance(self.measure, (RiskExposure, CategoricalRelativeRisk)):
|
|
182
|
+
scenario_cols.append(self.measure.risk_stratification_column)
|
|
183
|
+
scenario_cols.extend(list(self.scenarios.keys()))
|
|
184
|
+
weighted_avg = calculations.weighted_average(
|
|
185
|
+
data, self.weights, stratifications, scenario_cols
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Reference data can be a float or dataframe. Convert floats so dataframes are aligned
|
|
189
|
+
if not isinstance(weighted_avg, pd.DataFrame):
|
|
190
|
+
weighted_avg = pd.DataFrame(
|
|
191
|
+
{"value": [weighted_avg]}, index=pd.Index([0], name="index")
|
|
192
|
+
)
|
|
193
|
+
return weighted_avg
|
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any, Collection, Literal
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from loguru import logger
|
|
6
|
+
|
|
7
|
+
# StratValue and TargetIntervalConfig now live in vivarium-fuzzy-checker (they
|
|
8
|
+
# configure FuzzyChecker's target intervals). Re-exported here so the historical
|
|
9
|
+
# ``vivarium.validation.comparison`` import path keeps resolving.
|
|
10
|
+
from vivarium.fuzzy_checker import FuzzyChecker, StratValue, TargetIntervalConfig, TestResult
|
|
11
|
+
|
|
12
|
+
from vivarium.validation.bundle import RatioMeasureDataBundle
|
|
13
|
+
from vivarium.validation.constants import DRAW_INDEX, DataSource
|
|
14
|
+
from vivarium.validation.data_transformation.calculations import stratify
|
|
15
|
+
from vivarium.validation.data_transformation.measures import Measure
|
|
16
|
+
from vivarium.validation.visualization import dataframe_utils
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Comparison(ABC):
|
|
20
|
+
"""A Comparison is the basic testing unit to compare two datasets, a "test" dataset and a
|
|
21
|
+
"reference" dataset. The test dataset is the one that is being validated, while the reference
|
|
22
|
+
dataset is the one that is used as a benchmark. The comparison operates on a *measure* of the two datasets,
|
|
23
|
+
typically a derived quantity of the test data such as incidence rate or prevalence."""
|
|
24
|
+
|
|
25
|
+
test_bundle: RatioMeasureDataBundle
|
|
26
|
+
reference_bundle: RatioMeasureDataBundle
|
|
27
|
+
proportion_test_results: dict[str, Any]
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def comparison_key(self) -> str:
|
|
31
|
+
"""A key to indentify a comparison of the form 'entity_type.entity.measure'."""
|
|
32
|
+
if self.test_bundle.measure.measure_key != self.reference_bundle.measure.measure_key:
|
|
33
|
+
raise ValueError("Test and reference bundle measure keys must be the same.")
|
|
34
|
+
return self.test_bundle.measure.measure_key
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
@abstractmethod
|
|
38
|
+
def metadata(self) -> pd.DataFrame:
|
|
39
|
+
"""A summary of the test data and reference data, including:
|
|
40
|
+
- the measure key
|
|
41
|
+
- source
|
|
42
|
+
- index columns
|
|
43
|
+
- size
|
|
44
|
+
- number of draws
|
|
45
|
+
- a sample of the input draws.
|
|
46
|
+
"""
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def get_frame(
|
|
51
|
+
self,
|
|
52
|
+
stratifications: Collection[str] | Literal["all"] = "all",
|
|
53
|
+
num_rows: int | Literal["all"] = "all",
|
|
54
|
+
sort_by: str = "",
|
|
55
|
+
ascending: bool = False,
|
|
56
|
+
aggregate_draws: bool = False,
|
|
57
|
+
) -> pd.DataFrame:
|
|
58
|
+
"""Get a DataFrame of the comparison data, with naive comparison of the test and reference.
|
|
59
|
+
|
|
60
|
+
Parameters:
|
|
61
|
+
-----------
|
|
62
|
+
stratifications
|
|
63
|
+
The stratifications to use for the comparison
|
|
64
|
+
num_rows
|
|
65
|
+
The number of rows to return. If "all", return all rows.
|
|
66
|
+
sort_by
|
|
67
|
+
The column to sort by. Default is "percent_error".
|
|
68
|
+
ascending
|
|
69
|
+
Whether to sort in ascending order. Default is False.
|
|
70
|
+
aggregate_draws
|
|
71
|
+
If True, aggregate over draws and seeds to show means and 95% uncertainty intervals.
|
|
72
|
+
Returns:
|
|
73
|
+
--------
|
|
74
|
+
A DataFrame of the comparison data.
|
|
75
|
+
"""
|
|
76
|
+
pass
|
|
77
|
+
|
|
78
|
+
@abstractmethod
|
|
79
|
+
def verify(
|
|
80
|
+
self,
|
|
81
|
+
step_size: float | None,
|
|
82
|
+
stratifications: Collection[str] | Literal["all"] = "all",
|
|
83
|
+
) -> None:
|
|
84
|
+
pass
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def verified(self) -> bool | None:
|
|
88
|
+
"""Whether this comparison passes validation.
|
|
89
|
+
|
|
90
|
+
Returns None if verification has not been run yet,
|
|
91
|
+
True if all test results pass, False if any fail.
|
|
92
|
+
"""
|
|
93
|
+
if "overall" not in self.proportion_test_results:
|
|
94
|
+
return None
|
|
95
|
+
overall = self.proportion_test_results["overall"]
|
|
96
|
+
stratified = self.proportion_test_results.get("stratified", {})
|
|
97
|
+
reject_nulls = [overall.reject_null] + [
|
|
98
|
+
tr.reject_null for group in stratified.values() for tr in group.values()
|
|
99
|
+
]
|
|
100
|
+
return not any(reject_nulls)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def stratification_metadata(self) -> dict[str, Any]:
|
|
104
|
+
"""Compile stratification metadata from proportion_test_results.
|
|
105
|
+
|
|
106
|
+
Returns a dict with keys 'dimensions', 'values', 'stratification_groups',
|
|
107
|
+
or an empty dict if not yet verified.
|
|
108
|
+
"""
|
|
109
|
+
if "overall" not in self.proportion_test_results:
|
|
110
|
+
return {}
|
|
111
|
+
stratified = self.proportion_test_results.get("stratified", {})
|
|
112
|
+
|
|
113
|
+
dimensions: set[str] = set()
|
|
114
|
+
values: dict[str, set[str]] = {}
|
|
115
|
+
stratification_groups: list[list[str]] = []
|
|
116
|
+
|
|
117
|
+
for strat_key_tuple, group in stratified.items():
|
|
118
|
+
group_dims = list(strat_key_tuple)
|
|
119
|
+
stratification_groups.append(group_dims)
|
|
120
|
+
dimensions.update(group_dims)
|
|
121
|
+
for test_result in group.values():
|
|
122
|
+
if test_result.index_info:
|
|
123
|
+
for dim, val in test_result.index_info.items():
|
|
124
|
+
if dim not in values:
|
|
125
|
+
values[dim] = set()
|
|
126
|
+
values[dim].add(str(val))
|
|
127
|
+
|
|
128
|
+
return {
|
|
129
|
+
"dimensions": sorted(dimensions),
|
|
130
|
+
"values": {dim: sorted(vals) for dim, vals in values.items()},
|
|
131
|
+
"stratification_groups": sorted(
|
|
132
|
+
stratification_groups, key=lambda group: (len(group), group)
|
|
133
|
+
),
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class FuzzyComparison(Comparison):
|
|
138
|
+
"""A FuzzyComparison is a comparison that requires statistical hypothesis testing
|
|
139
|
+
to determine if the distributions of the datasets are the same. We require both the numerator and
|
|
140
|
+
denominator for the test data, to be able to calculate the statistical power."""
|
|
141
|
+
|
|
142
|
+
def __init__(
|
|
143
|
+
self,
|
|
144
|
+
test_bundle: RatioMeasureDataBundle,
|
|
145
|
+
reference_bundle: RatioMeasureDataBundle,
|
|
146
|
+
):
|
|
147
|
+
self.test_bundle = test_bundle
|
|
148
|
+
self.reference_bundle = reference_bundle
|
|
149
|
+
if self.test_bundle.measure != self.reference_bundle.measure:
|
|
150
|
+
raise ValueError("Test and reference measures must be the same.")
|
|
151
|
+
self.measure: Measure = self.test_bundle.measure
|
|
152
|
+
self.proportion_test_results: dict[
|
|
153
|
+
str, TestResult | dict[tuple[str, ...], dict[str, TestResult]]
|
|
154
|
+
] = {
|
|
155
|
+
"stratified": {},
|
|
156
|
+
}
|
|
157
|
+
self._target_interval_configuration: TargetIntervalConfig | None = None
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def target_interval_configuration(self) -> TargetIntervalConfig | None:
|
|
161
|
+
"""The target interval configuration for this comparison."""
|
|
162
|
+
return self._target_interval_configuration
|
|
163
|
+
|
|
164
|
+
@target_interval_configuration.setter
|
|
165
|
+
def target_interval_configuration(self, value: TargetIntervalConfig | None) -> None:
|
|
166
|
+
if value is not None and not isinstance(value, TargetIntervalConfig):
|
|
167
|
+
raise TypeError(
|
|
168
|
+
f"target_interval_configuration must be a TargetIntervalConfig or None, "
|
|
169
|
+
f"got {type(value).__name__}"
|
|
170
|
+
)
|
|
171
|
+
self._target_interval_configuration = value
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def metadata(self) -> pd.DataFrame:
|
|
175
|
+
"""A summary of the test data and reference data, including:
|
|
176
|
+
- the measure key
|
|
177
|
+
- source
|
|
178
|
+
- shared index columns
|
|
179
|
+
- source specific index columns
|
|
180
|
+
- size
|
|
181
|
+
- number of draws
|
|
182
|
+
- a sample of the input draws.
|
|
183
|
+
"""
|
|
184
|
+
measure_key = self.measure.measure_key
|
|
185
|
+
test_info = self.test_bundle.get_metadata()
|
|
186
|
+
reference_info = self.reference_bundle.get_metadata()
|
|
187
|
+
return dataframe_utils.format_metadata(measure_key, test_info, reference_info)
|
|
188
|
+
|
|
189
|
+
def get_frame(
|
|
190
|
+
self,
|
|
191
|
+
stratifications: Collection[str] | Literal["all"] = "all",
|
|
192
|
+
num_rows: int | Literal["all"] = "all",
|
|
193
|
+
sort_by: str = "",
|
|
194
|
+
ascending: bool = False,
|
|
195
|
+
aggregate_draws: bool = False,
|
|
196
|
+
) -> pd.DataFrame:
|
|
197
|
+
"""Get a DataFrame of the comparison data, with naive comparison of the test and reference.
|
|
198
|
+
|
|
199
|
+
Parameters:
|
|
200
|
+
-----------
|
|
201
|
+
stratifications
|
|
202
|
+
The stratifications to use for the comparison
|
|
203
|
+
num_rows
|
|
204
|
+
The number of rows to return. If "all", return all rows.
|
|
205
|
+
sort_by
|
|
206
|
+
The column to sort by. Default for non-aggregated data is "percent_error", for aggregation default is to not sort.
|
|
207
|
+
ascending
|
|
208
|
+
Whether to sort in ascending order. Default is False.
|
|
209
|
+
aggregate_draws
|
|
210
|
+
If True, aggregate over draws to show means and 95% uncertainty intervals.
|
|
211
|
+
Changes the output columns to show mean, 2.5%, and 97.5 for each dataset.
|
|
212
|
+
Returns:
|
|
213
|
+
--------
|
|
214
|
+
A DataFrame of the comparison data.
|
|
215
|
+
"""
|
|
216
|
+
|
|
217
|
+
test_proportion_data, reference_data = self.align_datasets(stratifications)
|
|
218
|
+
# Renaming and aggregating draws happens here instead of _align datasets because
|
|
219
|
+
# "value" and "input_draw" are needed for comparison plots
|
|
220
|
+
test_proportion_data = test_proportion_data.rename(columns={"value": "rate"})
|
|
221
|
+
reference_data = reference_data.rename(columns={"value": "rate"})
|
|
222
|
+
if aggregate_draws:
|
|
223
|
+
test_proportion_data = self._aggregate_over_draws(test_proportion_data)
|
|
224
|
+
reference_data = self._aggregate_over_draws(reference_data)
|
|
225
|
+
test_proportion_data = test_proportion_data.add_prefix("test_")
|
|
226
|
+
reference_data = reference_data.add_prefix("reference_")
|
|
227
|
+
|
|
228
|
+
test_proportion_data, reference_data = self._cast_across_indexes(
|
|
229
|
+
test_proportion_data, reference_data
|
|
230
|
+
)
|
|
231
|
+
merged_data = pd.merge(
|
|
232
|
+
test_proportion_data, reference_data, left_index=True, right_index=True
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
if not aggregate_draws:
|
|
236
|
+
merged_data["percent_error"] = (
|
|
237
|
+
(merged_data["test_rate"] - merged_data["reference_rate"])
|
|
238
|
+
/ merged_data["reference_rate"]
|
|
239
|
+
) * 100
|
|
240
|
+
|
|
241
|
+
if sort_by:
|
|
242
|
+
sort_key = abs if sort_by == "percent_error" else None
|
|
243
|
+
merged_data = merged_data.sort_values(
|
|
244
|
+
by=sort_by,
|
|
245
|
+
key=sort_key,
|
|
246
|
+
ascending=ascending,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
return merged_data if num_rows == "all" else merged_data.head(n=num_rows)
|
|
250
|
+
|
|
251
|
+
def _aggregate_over_draws(self, data: pd.DataFrame) -> pd.DataFrame:
|
|
252
|
+
"""Aggregate data over draws and seeds, computing mean and 95% uncertainty intervals."""
|
|
253
|
+
# If data doesn't have draws, return data
|
|
254
|
+
if DRAW_INDEX not in data.index.names:
|
|
255
|
+
logger.warning("Data does not have draws. Returning data without aggregating.")
|
|
256
|
+
return data
|
|
257
|
+
# If data only has draws, aggregate and cast single value to a dataframe
|
|
258
|
+
if DRAW_INDEX in data.index.names and len(data.index.names) == 1:
|
|
259
|
+
data = data.describe(percentiles=[0.025, 0.975])
|
|
260
|
+
aggregated_data = data.T
|
|
261
|
+
aggregated_data.index = pd.Index([0], name="index")
|
|
262
|
+
else:
|
|
263
|
+
# Get the levels to group by (everything except draws and seeds)
|
|
264
|
+
group_levels = [level for level in data.index.names if level != DRAW_INDEX]
|
|
265
|
+
# Group by the remaining levels and aggregate
|
|
266
|
+
aggregated_data = data.groupby(group_levels, sort=False, observed=True)[
|
|
267
|
+
"rate"
|
|
268
|
+
].describe(percentiles=[0.025, 0.975])
|
|
269
|
+
|
|
270
|
+
return aggregated_data[["mean", "2.5%", "97.5%"]]
|
|
271
|
+
|
|
272
|
+
def verify(
|
|
273
|
+
self,
|
|
274
|
+
step_size: float | None,
|
|
275
|
+
stratifications: Collection[str] | Literal["all"] = "all",
|
|
276
|
+
) -> None:
|
|
277
|
+
"""Verify test and reference data are statistically indistinguishable according to the fuzzy checker."""
|
|
278
|
+
|
|
279
|
+
if self.test_bundle.source != DataSource.SIM:
|
|
280
|
+
raise NotImplementedError("Verification is only implemented for SIM test data.")
|
|
281
|
+
if self.reference_bundle.source not in [DataSource.ARTIFACT, DataSource.GBD]:
|
|
282
|
+
raise NotImplementedError(
|
|
283
|
+
"Verification is only implemented for ARTIFACT or GBD reference data."
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
fuzzy_checker = FuzzyChecker()
|
|
287
|
+
# Get intersection of stratifications and shared indices
|
|
288
|
+
intersection = self.test_bundle.index_names.intersection(
|
|
289
|
+
self.reference_bundle.index_names
|
|
290
|
+
)
|
|
291
|
+
if stratifications == "all":
|
|
292
|
+
stratify_cols = intersection
|
|
293
|
+
else:
|
|
294
|
+
if not set(stratifications).issubset(intersection):
|
|
295
|
+
raise ValueError("Stratifications must be a subset of the intersection")
|
|
296
|
+
stratify_cols = set(stratifications)
|
|
297
|
+
|
|
298
|
+
test_datasets = {
|
|
299
|
+
key: stratify(data, stratify_cols)
|
|
300
|
+
for key, data in self.test_bundle.datasets.items()
|
|
301
|
+
}
|
|
302
|
+
ref_datasets = {
|
|
303
|
+
key: stratify(data, stratify_cols)
|
|
304
|
+
for key, data in self.reference_bundle.datasets.items()
|
|
305
|
+
}
|
|
306
|
+
# Scale rates to the step size of the simulation
|
|
307
|
+
if step_size is None or "population" in self.measure.measure_key:
|
|
308
|
+
target = ref_datasets["data"]
|
|
309
|
+
else:
|
|
310
|
+
target = ref_datasets["data"] / step_size
|
|
311
|
+
|
|
312
|
+
fuzzy_checker.test_proportion_vectorized(
|
|
313
|
+
name=self.measure.measure_key,
|
|
314
|
+
observed_numerator=test_datasets["numerator_data"],
|
|
315
|
+
observed_denominator=test_datasets["denominator_data"],
|
|
316
|
+
target_proportion=target,
|
|
317
|
+
target_interval_config=self.target_interval_configuration,
|
|
318
|
+
)
|
|
319
|
+
for result in fuzzy_checker.proportion_test_diagnostics:
|
|
320
|
+
if result.name_additional == "overall":
|
|
321
|
+
self.proportion_test_results["overall"] = result
|
|
322
|
+
else:
|
|
323
|
+
stratified = self.proportion_test_results["stratified"]
|
|
324
|
+
if isinstance(stratified, dict):
|
|
325
|
+
strat_key = tuple(result.index_info.keys()) if result.index_info else ()
|
|
326
|
+
if strat_key not in stratified:
|
|
327
|
+
stratified[strat_key] = {}
|
|
328
|
+
stratified[strat_key][result.name_additional] = result
|
|
329
|
+
|
|
330
|
+
def align_datasets(
|
|
331
|
+
self,
|
|
332
|
+
stratifications: Collection[str] | Literal["all"] = "all",
|
|
333
|
+
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
334
|
+
"""Resolve any index mismatches between the test and reference datasets."""
|
|
335
|
+
|
|
336
|
+
intersection = self.test_bundle.index_names.intersection(
|
|
337
|
+
self.reference_bundle.index_names
|
|
338
|
+
)
|
|
339
|
+
if stratifications == "all":
|
|
340
|
+
stratifications = intersection
|
|
341
|
+
else:
|
|
342
|
+
if not set(stratifications).issubset(intersection):
|
|
343
|
+
raise ValueError("Stratifications must be a subset of the intersection")
|
|
344
|
+
|
|
345
|
+
aggregated_reference_data = self.reference_bundle.get_measure_data(
|
|
346
|
+
stratifications=set(stratifications) | {DRAW_INDEX}
|
|
347
|
+
if DRAW_INDEX in self.reference_bundle.index_names
|
|
348
|
+
else stratifications,
|
|
349
|
+
)
|
|
350
|
+
stratified_test_data = self.test_bundle.get_measure_data(
|
|
351
|
+
stratifications=set(stratifications) | {DRAW_INDEX}
|
|
352
|
+
if DRAW_INDEX in self.test_bundle.index_names
|
|
353
|
+
else stratifications,
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
## At this point, the only non-common index level should be draws;
|
|
357
|
+
## scenario levels were already dropped during bundle formatting.
|
|
358
|
+
return stratified_test_data, aggregated_reference_data
|
|
359
|
+
|
|
360
|
+
def _cast_across_indexes(
|
|
361
|
+
self, test_data: pd.DataFrame, reference_data: pd.DataFrame
|
|
362
|
+
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
363
|
+
"""Align dataset indexes if stratifications is an empty list
|
|
364
|
+
One dataset might have a single row, so we cast that row to match the other's length
|
|
365
|
+
If both datasets are one row, they will already have the same index."""
|
|
366
|
+
if len(test_data) == 1 and len(reference_data) != 1:
|
|
367
|
+
test_data = pd.concat(
|
|
368
|
+
[test_data] * len(reference_data), ignore_index=True
|
|
369
|
+
).set_index(reference_data.index)
|
|
370
|
+
elif len(reference_data) == 1 and len(test_data) != 1:
|
|
371
|
+
reference_data = pd.concat(
|
|
372
|
+
[reference_data] * len(test_data), ignore_index=True
|
|
373
|
+
).set_index(test_data.index)
|
|
374
|
+
|
|
375
|
+
return test_data, reference_data
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from enum import Enum
|
|
4
|
+
from typing import NamedTuple
|
|
5
|
+
|
|
6
|
+
DRAW_PREFIX = "draw_"
|
|
7
|
+
|
|
8
|
+
DRAW_INDEX = "input_draw"
|
|
9
|
+
SEED_INDEX = "random_seed"
|
|
10
|
+
DAYS_PER_YEAR = 365.0
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class DataSource(Enum):
|
|
14
|
+
SIM = "sim"
|
|
15
|
+
GBD = "gbd"
|
|
16
|
+
ARTIFACT = "artifact"
|
|
17
|
+
CUSTOM = "custom"
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def from_str(cls, source: str) -> DataSource:
|
|
21
|
+
try:
|
|
22
|
+
return cls(source)
|
|
23
|
+
except ValueError:
|
|
24
|
+
raise ValueError(f"Source {source} not recognized. Must be one of {DataSource}")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
LOCATION_ARTIFACT_KEY = "population.location"
|
|
28
|
+
POPULATION_STRUCTURE_ARTIFACT_KEY = "population.structure"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class InputDataIndexNames(NamedTuple):
|
|
32
|
+
LOCATION_ID: str = "location_id"
|
|
33
|
+
SEX_ID: str = "sex_id"
|
|
34
|
+
AGE_GROUP_ID: str = "age_group_id"
|
|
35
|
+
YEAR_ID: str = "year_id"
|
|
36
|
+
PARAMETER: str = "parameter"
|
|
37
|
+
CAUSE_ID: str = "cause_id"
|
|
38
|
+
AFFECTED_ENTITY: str = "affected_entity"
|
|
39
|
+
AFFECTED_MEASURE: str = "affected_measure"
|
|
40
|
+
LOCATION: str = "location"
|
|
41
|
+
SEX: str = "sex"
|
|
42
|
+
AGE_GROUP: str = "age_group"
|
|
43
|
+
AGE_START: str = "age_start"
|
|
44
|
+
AGE_END: str = "age_end"
|
|
45
|
+
YEAR_START: str = "year_start"
|
|
46
|
+
YEAR_END: str = "year_end"
|
|
47
|
+
ENTITY: str = "entity"
|
|
48
|
+
MEASURE: str = "measure"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
INPUT_DATA_INDEX_NAMES = InputDataIndexNames()
|