popgen3 3.0.1__tar.gz

popgen3-3.0.1/PKG-INFO

@@ -0,0 +1,95 @@
+Metadata-Version: 2.4
+Name: popgen3
+Version: 3.0.1
+Summary: A population synthesis and sample weighting tool for transportation planning.
+Home-page: https://github.com/chnfanyu/PopGen3
+Author: Fan Yu
+Author-email: fanyu4@asu.edu
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: numpy>=1.21.6
+Requires-Dist: pandas>=1.3.0
+Requires-Dist: scipy>=1.10.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# PopGen3
+
+## Introduction
+
+**PopGen** is a state-of-the-art open-source synthetic population generator for advanced travel demand modeling. Developed under the guidance of Professor Ram M. Pendyala at Arizona State University, PopGen utilizes a heuristic algorithm (IPU) to control and match both household-level and person-level attribute distributions.
+
+## Key Features
+
+- Advanced control of household and person variables across multiple geographic resolutions (region and geo).
+- Command-line interface for simplified usage and enhanced computational efficiency.
+- Fully compatible with Python 3, with updated dependencies.
+
+## Run PopGen3 Online
+- [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1j1Stb8IA8OfaoPRh232kId8hqi3dUtur?usp=sharing)
+- [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/chnfanyu/PopGen3/HEAD)
+
+## Contents
+
+### Data Inputs
+
+#### Survey sample data:
+  `household_sample.csv`
+  `person_sample.csv`
+
+#### Population marginal data:
+
+- **geo-level marginals:**  
+     `person_marginals.csv`  
+     `household_marginals.csv`  
+
+- **region-level marginals:**  
+  `region_person_marginals.csv`  
+  `region_household_marginals.csv`  
+
+#### Multi-Geographic Resolution Mapping Data:
+
+  `region_geo_mapping.csv`  
+  `geo_sample_mapping.csv`
+
+Example of Multi-Geographic Resolution Levels:
+- <small><i>Region Level:</i> Set as census county subdivision</small>
+- <small><i>Geo Level:</i> Set as census tracts</small>
+- <small><i>Sample Geo Level:</i> Set as Public Use Microdata Areas (PUMAs)</small>
+
+
+### Configuration File 
+
+The `configuration.yaml` file contains several key sections for the PopGen setup. 
+
+#### Designed Input Keys
+
+| **Project Wide Setting** | **Input Data Files** | **Scenario Settings** |
+| --- | --- | --- |
+| - synthesize<br>- name<br>- location | - entities<br>- column_names<br>- location | - description<br>- apply_region_controls<br>- control_variables<br>- parameters<br>- geos_to_synthesize<br>- outputs |
+
+### Key Outputs
+
+#### Sample Weighting Results
+- `weights.csv`
+#### Synthetic Population Generated
+- `housing_synthetic.csv`
+- `person_synthetic.csv`
+

popgen3-3.0.1/README.md

@@ -0,0 +1,63 @@
+# PopGen3
+
+## Introduction
+
+**PopGen** is a state-of-the-art open-source synthetic population generator for advanced travel demand modeling. Developed under the guidance of Professor Ram M. Pendyala at Arizona State University, PopGen utilizes a heuristic algorithm (IPU) to control and match both household-level and person-level attribute distributions.
+
+## Key Features
+
+- Advanced control of household and person variables across multiple geographic resolutions (region and geo).
+- Command-line interface for simplified usage and enhanced computational efficiency.
+- Fully compatible with Python 3, with updated dependencies.
+
+## Run PopGen3 Online
+- [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1j1Stb8IA8OfaoPRh232kId8hqi3dUtur?usp=sharing)
+- [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/chnfanyu/PopGen3/HEAD)
+
+## Contents
+
+### Data Inputs
+
+#### Survey sample data:
+  `household_sample.csv`
+  `person_sample.csv`
+
+#### Population marginal data:
+
+- **geo-level marginals:**  
+     `person_marginals.csv`  
+     `household_marginals.csv`  
+
+- **region-level marginals:**  
+  `region_person_marginals.csv`  
+  `region_household_marginals.csv`  
+
+#### Multi-Geographic Resolution Mapping Data:
+
+  `region_geo_mapping.csv`  
+  `geo_sample_mapping.csv`
+
+Example of Multi-Geographic Resolution Levels:
+- <small><i>Region Level:</i> Set as census county subdivision</small>
+- <small><i>Geo Level:</i> Set as census tracts</small>
+- <small><i>Sample Geo Level:</i> Set as Public Use Microdata Areas (PUMAs)</small>
+
+
+### Configuration File 
+
+The `configuration.yaml` file contains several key sections for the PopGen setup. 
+
+#### Designed Input Keys
+
+| **Project Wide Setting** | **Input Data Files** | **Scenario Settings** |
+| --- | --- | --- |
+| - synthesize<br>- name<br>- location | - entities<br>- column_names<br>- location | - description<br>- apply_region_controls<br>- control_variables<br>- parameters<br>- geos_to_synthesize<br>- outputs |
+
+### Key Outputs
+
+#### Sample Weighting Results
+- `weights.csv`
+#### Synthetic Population Generated
+- `housing_synthetic.csv`
+- `person_synthetic.csv`
+

popgen3-3.0.1/popgen/__init__.py

@@ -0,0 +1,7 @@
+from .project import popgen_run
+
+def run(config_path):
+    """Run PopGen from Python."""
+    popgen_run(config_path)
+
+

popgen3-3.0.1/popgen/cli.py

@@ -0,0 +1,23 @@
+import os
+import argparse
+import logging
+from popgen.project import popgen_run
+
+def main():
+    """Command-line interface for running PopGen."""
+    parser = argparse.ArgumentParser(description="Run PopGen with a specified configuration file.")
+    parser.add_argument(
+        "config_path",
+        type=str,
+        help="Path to the configuration YAML file (e.g., configuration_arizona.yaml)"
+    )
+    args = parser.parse_args()
+
+    if not os.path.exists(args.config_path):
+        logging.error(f"Configuration file '{args.config_path}' not found.")
+        exit(1)
+
+    popgen_run(args.config_path)
+
+if __name__ == "__main__":
+    main()

popgen3-3.0.1/popgen/config.py

@@ -0,0 +1,121 @@
+import yaml
+import logging
+
+#logging.basicConfig(level=logging.INFO)
+
+
+class ConfigError(Exception):
+    pass
+
+
+def wrap_config_value(value):
+    """Wrap YAML elements as Config objects to allow attribute access.
+
+    Example:
+    If YAML defines:
+        attribute1:
+            attribute2: 'Value'
+
+    Then, x.attribute1.attribute2 can be used to access "Value".
+    """
+    if isinstance(value, str):
+        return value
+    try:
+        return value + 0
+    except TypeError:
+        pass
+
+    return Config(value)
+
+
+class Config:
+    """Config class for handling YAML configuration as attribute-accessible objects."""
+
+    DEFAULT_PARAMETERS = {
+        "ipf": {
+            "tolerance": 0.0001,
+            "iterations": 250,
+            "zero_marginal_correction": 0.00001,
+            "rounding_procedure": "bucket",
+            "archive_performance_frequency": 1
+        },
+        "reweighting": {
+            "procedure": "ipu",
+            "tolerance": 0.0001,
+            "inner_iterations": 1,
+            "outer_iterations": 50,
+            "archive_performance_frequency": 1
+        },
+        "draws": {
+            "pvalue_tolerance": 0.9999,
+            "iterations": 25,
+            "seed": 0
+        }
+    }
+
+    def __init__(self, data):
+        self._data = data or {}
+
+    def __setitem__(self, key, value):
+        self._data[key] = value
+
+    def __setattr__(self, key, value):
+        if key == "_data":
+            super().__setattr__(key, value)
+        else:
+            self._data[key] = value
+
+    def __getattr__(self, key):
+        value = self.return_value(key)
+        return wrap_config_value(value) if value is not None else None
+
+    def __getitem__(self, key):
+        value = self.return_value(key)
+        return wrap_config_value(value)
+
+    def __getstate__(self):
+        """Ensure `yaml.dump()` works correctly with Config objects."""
+        return self.__dict__
+
+    def write_to_file(self, filepath):
+        with open(filepath, 'w') as file:
+            yaml.dump(self._data, file, default_flow_style=False)
+
+    def return_value(self, key):
+        """Retrieve value from config data safely."""
+        try:
+            return self._data[key]
+        except KeyError:
+            logging.warning(f"Key '{key}' not found in configuration.")
+            return None  # Return None instead of raising an error
+
+    def __len__(self):
+        return len(self._data)
+
+    def __repr__(self):
+        return repr(self._data)
+
+    def return_list(self):
+        """Return a list of top-level keys in the configuration."""
+        data_list = []
+        for i in self._data:
+            data_list.append(i)
+        return data_list
+
+    def return_dict(self):
+        """Convert nested Config objects to pure dictionaries."""
+
+        def convert(value):
+            if isinstance(value, Config):
+                return value.return_dict()
+            elif isinstance(value, list):
+                return [convert(v) for v in value]
+            elif isinstance(value, dict):
+                return {k: convert(v) for k, v in value.items()}
+            return value
+
+        return convert(self._data)
+
+    def write_to_open(self, filepath):
+        with open(filepath, 'w') as outfile:
+            yaml.dump(self._data, outfile, default_flow_style=False)

popgen3-3.0.1/popgen/data.py

@@ -0,0 +1,132 @@
+import os
+import pandas as pd
+import numpy as np
+import logging
+from .config import ConfigError
+
+
+class DB:
+    """Handles all PopGen input data files and maintains necessary mappings and datasets."""
+
+    def __init__(self, config):
+        self.config = config
+        self.sample = {}
+        self.geo_marginals = {}
+        self.region_marginals = {}
+        self.geo = {}
+        self.geo_ids = None
+        self.region_ids = None
+        self.sample_geo_ids = None
+        self._inputs_config = self.config.project.inputs
+        self.location = os.path.abspath(self.config.project.location)
+
+    def load_data(self):
+        """Loads all required data while handling missing data gracefully."""
+        logging.info("Loading database data...")
+
+        # Load geo correspondence mapping
+        geo_corr_mapping_config = getattr(self._inputs_config.location, "geo_corr_mapping", None)
+        self.geo = self.get_data(geo_corr_mapping_config) if geo_corr_mapping_config else {}
+
+        # Load sample data
+        sample_config = getattr(self._inputs_config.location, "sample", None)
+        self.sample = self.get_data(sample_config) if sample_config else {}
+
+        # Load marginals
+        marginals_config = getattr(self._inputs_config.location, "marginals", None)
+        self.geo_marginals = self.get_data(getattr(marginals_config, "geo", None),
+                                           header=[0, 1]) if marginals_config and hasattr(marginals_config,
+                                                                                          "geo") else {}
+        self.region_marginals = self.get_data(getattr(marginals_config, "region", None),
+                                              header=[0, 1]) if marginals_config and hasattr(marginals_config,
+                                                                                             "region") else {}
+
+        try:
+            self._enumerate_geo_ids()
+        except Exception as e:
+            logging.warning(f"_enumerate_geo_ids failed due to {e}. Defaulting to empty lists.")
+            self.geo_ids = []
+            self.region_ids = []
+
+    def get_data(self, config, header=0):
+        """Loads data while handling missing files gracefully."""
+        config_dict = config.return_dict() if config else {}
+        data_dict = {}
+
+        for item, filename in config_dict.items():
+            if filename is None:
+                logging.warning(f"{item} has no filename specified. Skipping.")
+                continue
+
+            full_location = os.path.join(self.location, filename)
+            if os.path.exists(full_location):
+                try:
+                    data_dict[item] = pd.read_csv(full_location, index_col=0, header=header)
+                    if data_dict[item].index.name:
+                        data_dict[item].loc[:, data_dict[item].index.name] = data_dict[item].index.values
+                except Exception as e:
+                    logging.warning(f"Failed to load {filename} due to {e}. Skipping {item}.")
+            else:
+                logging.warning(f"{filename} not found. Skipping {item}.")
+
+        return data_dict
+
+    def _enumerate_geo_ids(self):
+        """Ensures proper initialization of geo and region IDs."""
+        self.geo_ids_all = []
+        self.region_ids_all = []
+
+        try:
+            if "geo_to_sample" in self.geo:
+                self.geo_ids_all = self.geo["geo_to_sample"].index.tolist()
+
+            if "region_to_geo" in self.geo:
+                self.region_ids_all = np.unique(self.geo["region_to_geo"].index.values).tolist()
+            elif "region_to_sample" in self.geo:
+                self.region_ids_all = np.unique(self.geo["region_to_sample"].index.values).tolist()
+        except Exception as e:
+            logging.warning(f"Failed to enumerate geo IDs due to {e}. Defaulting to empty lists.")
+            self.geo_ids_all = []
+            self.region_ids_all = []
+
+    def get_geo_ids_for_region(self, region_id):
+        """Retrieves geo IDs corresponding to a given region ID."""
+        geo_name = self._inputs_config.column_names.geo
+        geo_list = self.geo["region_to_geo"].loc[region_id, geo_name]
+        return [int(geo_list)] if isinstance(geo_list, (int, np.integer)) else list(geo_list)
+
+    def enumerate_geo_ids_for_scenario(self, scenario_config):
+        """Ensures geo enumeration is handled even when missing data is encountered."""
+        try:
+            self.region_ids = getattr(scenario_config.geos_to_synthesize.region, "ids", self.region_ids_all)
+            self.geo_ids = []
+
+            if "region_to_geo" in self.geo:
+                for region_id in self.region_ids:
+                    geo_list = self.get_geo_ids_for_region(region_id)
+                    if geo_list:
+                        self.geo_ids += geo_list
+            else:
+                print("⚠️ No region_to_geo mapping. Using only region data.")
+                self.geo_ids = self.region_ids if self.region_ids else []
+        except ConfigError as e:
+            print(f"⚠️ KeyError: {e}. Defaulting to all geo and region IDs.")
+            self.geo_ids = self.geo_ids_all if self.geo_ids_all else []
+            self.region_ids = self.region_ids_all if self.region_ids_all else []
+
+    def return_variables_cats(self, entity, variable_names):
+        """Returns unique categories for each variable in an entity dataset."""
+        return {var: self.return_variable_cats(entity, var) for var in variable_names}
+
+    def return_variable_cats(self, entity, variable_name):
+        """Returns unique values for a specific variable in an entity dataset."""
+        return np.unique(self.sample[entity][variable_name].values).tolist()
+
+    def check_data(self):
+        """Runs data consistency checks."""
+        self.check_sample_marginals_consistency()
+        self.check_marginals()
+
+    def check(self):
+        """Placeholder for additional data consistency checks."""
+        pass

popgen3-3.0.1/popgen/draw.py

@@ -0,0 +1,101 @@
+import pandas as pd
+import numpy as np
+import logging
+from scipy import stats
+
+
+class Draw_Population:
+    """Handles the process of drawing synthetic populations based on weighted sampling."""
+
+    def __init__(self, scenario_config, geo_ids, geo_row_idx, geo_frequencies,
+                 geo_constraints, geo_stacked, region_sample_weights):
+        self.scenario_config = scenario_config
+        self.geo_ids = geo_ids
+        self.geo_row_idx = geo_row_idx
+        self.geo_frequencies = geo_frequencies
+        self.geo_constraints = geo_constraints
+        self.geo_stacked = geo_stacked
+        self.region_sample_weights = region_sample_weights
+
+        self.iterations = self.scenario_config.parameters.draws.iterations
+        self.seed = self.scenario_config.parameters.draws.seed
+        self.pvalue_tolerance = self.scenario_config.parameters.draws.pvalue_tolerance
+
+        self.geo_id_rows_syn_dict = {}
+        self.performance_columns = ["p_value", "iterations", "chi_sq_stat"]
+        self.draws_performance = pd.DataFrame(index=self.geo_ids, columns=self.performance_columns)
+
+    def draw_population(self):
+        """Executes the drawing of households for each geographical ID."""
+        np.random.seed(self.seed)
+        logging.info("Starting synthetic population drawing.")
+
+        for geo_id in self.geo_ids:
+            logging.info(f"Processing geo ID: {geo_id}")
+            geo_sample_weights = self.region_sample_weights.loc[:, geo_id]
+            geo_cumulative_weights = self._return_cumulative_probability(geo_sample_weights)
+            geo_id_frequencies = self.geo_frequencies.loc[geo_id, :]
+            geo_id_constraints = self.geo_constraints.loc[geo_id, :]
+
+            if geo_id_frequencies.sum() == 0:
+                logging.warning(f"Skipping geo_id {geo_id}: Zero frequency sum.")
+                continue
+
+            p_value_max, geo_id_rows_syn_max, iter_max, stat_max = -1, None, None, None
+
+            for iter in range(self.iterations):
+                seed = self.seed + iter
+                geo_id_rows_syn = self._pick_households(geo_id_frequencies, geo_cumulative_weights)
+                stat, p_value = self._measure_match(geo_id_rows_syn, geo_id_constraints)
+
+                if p_value > self.pvalue_tolerance:
+                    logging.info(f"Converged at iteration {iter} for geo ID {geo_id}.")
+                    p_value_max, geo_id_rows_syn_max, iter_max, stat_max = p_value, geo_id_rows_syn, iter, stat
+                    break
+                elif p_value > p_value_max:
+                    p_value_max, geo_id_rows_syn_max, iter_max, stat_max = p_value, geo_id_rows_syn, iter, stat
+
+            self.draws_performance.loc[geo_id, self.performance_columns] = (p_value_max, iter_max, stat_max)
+            self.geo_id_rows_syn_dict[geo_id] = geo_id_rows_syn_max
+
+    def _return_cumulative_probability(self, geo_sample_weights):
+        """Computes cumulative probabilities for weighted sampling."""
+        geo_cumulative_weights = {}
+
+        for column in self.geo_frequencies.columns:
+            rows = self.geo_row_idx[column]
+            weights = geo_sample_weights.take(rows)
+            geo_cumulative_weights[column] = (weights / weights.sum()).cumsum()
+
+        return geo_cumulative_weights
+
+    def _pick_households(self, geo_id_frequencies, geo_cumulative_weights):
+        """Selects households based on cumulative probabilities."""
+        last = 0
+        rand_numbers = np.random.random(int(geo_id_frequencies.sum()))
+        list_rows_syn_subpop = []
+
+        for column in self.geo_frequencies.columns:
+            rows = self.geo_row_idx[column]
+            column_frequency = int(geo_id_frequencies[column])
+            column_bins = np.searchsorted(geo_cumulative_weights[column], rand_numbers[last:last + column_frequency],
+                                          side="right")
+            last += column_frequency
+            list_rows_syn_subpop.append(rows.take(column_bins))
+
+        return np.sort(np.concatenate(list_rows_syn_subpop))
+
+    def _measure_match(self, geo_id_rows_syn, geo_id_constraints):
+        """Evaluates the match between synthetic and expected constraints using chi-square test."""
+        geo_id_constraints.name = "constraint"
+        geo_id_synthetic = self.geo_stacked.take(geo_id_rows_syn).sum()
+        geo_id_synthetic = pd.DataFrame(geo_id_synthetic, columns=["synthetic_count"])
+        geo_id_synthetic = geo_id_synthetic.join(geo_id_constraints, how="inner")
+
+        observed, expected = geo_id_synthetic["synthetic_count"], geo_id_synthetic["constraint"]
+        observed_sum, expected_sum = observed.sum(), expected.sum()
+
+        if abs(observed_sum - expected_sum) / expected_sum > 1e-08:
+            expected *= observed_sum / expected_sum
+
+        return stats.chisquare(observed, expected)