guts-base 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

guts_base/__init__.py

@@ -4,7 +4,7 @@ from . import data
 from . import prob
 from . import plot
 
-__version__ = "1.0.5"
+__version__ = "1.0.7"
 
 from .sim import (
     GutsBase,

guts_base/data/openguts.py

@@ -251,6 +251,8 @@ def import_data_to_database(path, database, preprocessing: Optional[Callable] =
     
     experiment.to_database(database=database)
 
+    print("Import to database successful.")
+
 
 def create_database_and_import_data_main(datasets_path, database_path, preprocessing=None, preprocessing_out=None):
     print("\n")

guts_base/data/time_of_death.py

@@ -225,7 +225,7 @@ def make_openguts_observation_table(
     experiment_start = get_unique_value(df["time_start_experiment"])
     experiment_end = get_unique_value(df["time_end_experiment"])
 
-    time = pd.date_range(experiment_start, experiment_end, freq=observation_schedule)
+    times_nominal = pd.date_range(experiment_start, experiment_end, freq=observation_schedule)
     timecol_name = f"time [{observation_schedule.lower()}]"
 
 
@@ -234,13 +234,17 @@ def make_openguts_observation_table(
     # calculate survival time
     df[timecol_name] = df["time_death"] - df["time_start_experiment"]
 
-    time_remainder = df[timecol_name] % pd.Timedelta(1, observation_schedule)
-    if (time_remainder > pd.Timedelta(0)).any():
-        raise ValueError(
-            "Observations should be entered at the same time as the experiment start "+
-            "df['time_death] - df['time_experiment_start'] should be a multiple of "+
-            f"the time resolution of the observation schedule. Here: 1{observation_schedule}"
-        )
+    # this seems to have been necessary, because reindexing removed times smaller than
+    # the observation_schedule interval. This is now resolved by concatenating true
+    # times and nominal times
+    # TODO: remove this commented block when there appear no more errors
+    # time_remainder = df[timecol_name] % pd.Timedelta(1, observation_schedule)
+    # if (time_remainder > pd.Timedelta(0)).any():
+    #     raise ValueError(
+    #         "Observations should be entered at the same time as the experiment start "+
+    #         "df['time_death] - df['time_experiment_start'] should be a multiple of "+
+    #         f"the time resolution of the observation schedule. Here: 1{observation_schedule}"
+    #     )
 
     if observation == "censored":
         # sum IDs that were marked as censored at time t
@@ -258,8 +262,12 @@ def make_openguts_observation_table(
     # df to wide frame
     df_wide = long_to_wide(df_long.reset_index(), id_columns, timecol_name, observation)
     
+    # get a time vector that contains all nominal observation times and also actually
+    # occurred days 
+    observation_times = np.unique(np.concatenate([df_wide.index, times_nominal-experiment_start]))
+
     # reindex wide dataframe on time
-    df_wide = df_wide.reindex(index=time-experiment_start, method=None)
+    df_wide = df_wide.reindex(index=observation_times, method=None)
     df_wide.index = df_wide.index.set_names(timecol_name)
     df_wide = df_wide.fillna(0)
 
@@ -299,8 +307,10 @@ class TimeOfDeathIO:
         self.data = self.from_file()
 
 def main(file: str, sheet: str, out:str, intervention_columns: List[str], 
+         extra_observation_columns: List[str] = [],
          observation_schedule="d", rect_interpolate=False):
     intervention_columns = clean_column_names(list(intervention_columns))
+    extra_observation_columns = clean_column_names(list(extra_observation_columns))
     processed_file = f"{out}/openguts_{os.path.basename(file)}"
     
     print("\n")
@@ -454,6 +464,17 @@ def main(file: str, sheet: str, out:str, intervention_columns: List[str],
         observation_schedule=observation_schedule,
     )
 
+    _extra_observations = []
+    for eob in extra_observation_columns:
+        ob_wide = make_openguts_observation_table(
+            data,
+            observation=eob,
+            observation_schedule=observation_schedule
+        )
+        _extra_observations.append(ob_wide)
+        excel_writer(ob_wide, file=processed_file, sheet=eob)
+
+
     deaths = lethality - censored
 
     # excel export
@@ -537,6 +558,7 @@ def main(file: str, sheet: str, out:str, intervention_columns: List[str],
     else:
         warnings.warn("No metadata found in sheets 'meta' or 'Info'.")
 
+    return processed_file
 
 @click.command()
 @click.option("--file", "-f", help="Path to the xlsx file")
@@ -544,12 +566,14 @@ def main(file: str, sheet: str, out:str, intervention_columns: List[str],
 @click.option("--out", "-o", help="Output directory", default="processed_data")
 @click.option("--observation_schedule", help="Schedule of the observations: d - daily, h - hourly", default="d")
 @click.option("--intervention_columns", "-c", multiple=True, type=str, help="Names of the columns that carry the exposure information")
-def time_of_death_to_openguts(file, sheet, out, observation_schedule, intervention_columns):
-    main(
+@click.option("--extra_observation_columns", "-e", multiple=True, type=str, default=[], help="Names of the columns that carry additional observations beside time-of-death and censoring")
+def time_of_death_to_openguts(file, sheet, out, observation_schedule, intervention_columns, extra_observation_columns):
+    _ = main(
         file=file, 
         sheet=sheet, 
         out=out, 
         intervention_columns=intervention_columns,
+        extra_observation_columns=extra_observation_columns,
         observation_schedule=observation_schedule
     )
 

guts_base/sim/base.py

@@ -49,6 +49,10 @@ class GutsBase(SimulationBase):
     results_interpolation: Tuple[float,float,int] = (np.nan, np.nan, 100)
     _skip_data_processing: bool = False
     ecx_mode: Literal["mean", "draws"] = "mean"
+    exposure_scenarios = {
+        "acute_1day": {"start": 0.0, "end": 1.0},
+        "chronic": {"start": 0.0, "end": None},
+    }
 
     def initialize(self, input: Optional[Dict] = None):
         """Initiaization goes through a couple of steps:
@@ -101,7 +105,12 @@ class GutsBase(SimulationBase):
             self.unit_time = self.config.simulation.unit_time  # type: ignore
 
         if hasattr(self.config.simulation, "skip_data_processing"):
-            self._skip_data_processing = bool(self.config.simulation.skip_data_processing)  # type: ignore
+            self._skip_data_processing = not (
+                self.config.simulation.skip_data_processing == "False"  or
+                self.config.simulation.skip_data_processing == "false" or  # type: ignore
+                self.config.simulation.skip_data_processing == "" or # type: ignore
+                self.config.simulation.skip_data_processing == 0  # type: ignore
+            )
 
         if hasattr(self.config.simulation, "results_interpolation"):
             results_interpolation_string = string_to_list(self.config.simulation.results_interpolation)
@@ -249,8 +258,9 @@ class GutsBase(SimulationBase):
                 observations=self.observations
             )
 
-        if "exposure" in self.observations:
-            self.config.data_structure.exposure.observed=False
+        if "exposure" not in self.observations:
+            self.observations["exposure"] = self.observations[self.config.simulation.substance]
+        self.config.data_structure.exposure.observed=False
 
     def _convert_exposure_units(self):
         """
@@ -418,8 +428,6 @@ class GutsBase(SimulationBase):
             
         self.dispatch_constructor()
         _ = self._prob.posterior_predictions(self, self.inferer.idata) # type: ignore
-        self.inferer.store_results(output=f"{self.output_path}/numpyro_posterior_interp.nc") # type: ignore
-        self.logger.info("Recomputed posterior and storing in `numpyro_posterior_interp.nc`")
 
 
     def prior_predictive_checks(self, **plot_kwargs):
@@ -430,9 +438,12 @@ class GutsBase(SimulationBase):
     def posterior_predictive_checks(self, **plot_kwargs):
         super().posterior_predictive_checks(**plot_kwargs)
 
-        self.recompute_posterior()
+        sim_copy = self.copy()
+        sim_copy.recompute_posterior()
         # TODO: Include posterior_predictive group once the survival predictions are correctly working
-        self._plot.plot_posterior_predictions(self, data_vars=["survival"], groups=["posterior_model_fits"])
+        sim_copy._plot.plot_posterior_predictions(
+            sim_copy, data_vars=["survival"], groups=["posterior_model_fits"]
+        )
 
 
     def plot(self, results):
@@ -479,21 +490,26 @@ class GutsBase(SimulationBase):
         for coord in exposure_coordinates:
             concentrations = np.where(coord == exposure_coordinates, 1.0, 0.0)
 
-            exposure_dict = {
-                coord: ExposureDataDict(start=0, end=1, concentration=conc)
-                for coord, conc in zip(exposure_coordinates, concentrations)
-            }
-
-            scenario = design_exposure_scenario(
-                exposures=exposure_dict,
-                t_max=time_max,
-                dt=1/24,
-                exposure_dimension=exposure_dimension
-            )
+            for _name, _expo_scenario in self.exposure_scenarios.items():
+                exposure_dict = {
+                    coord: ExposureDataDict(
+                        start=_expo_scenario["start"], 
+                        end=_expo_scenario["end"], 
+                        concentration=conc
+                    )
+                    for coord, conc in zip(exposure_coordinates, concentrations)
+                }
+
+                scenario = design_exposure_scenario(
+                    exposures=exposure_dict,
+                    t_max=time_max,
+                    dt=1/24,
+                    exposure_dimension=exposure_dimension
+                )
 
-            scenarios.update({
-                f"1day_exposure_{coord}": scenario
-            })
+                scenarios.update({
+                    f"1day_exposure_{coord}": scenario
+                })
 
         return scenarios
 
@@ -802,27 +818,19 @@ class GutsBase(SimulationBase):
 
         self.dispatch_constructor()
 
-    def export(self, directory: Optional[str] = None):
-        self.config.simulation.skip_data_processing = False
-        super().export(directory=directory)
-
-    def copy(self):
-        """Creates a copy of a SimulationBase object by deepcopying all loose references
-        TODO: If this works out well integrate into pymob. I have the feeling there will
-        still be some problems down the line.
-        """
-        with warnings.catch_warnings(action="ignore"):
-            # create the tempdir in the output path, because on the remote cluster
-            # the default temporary directory may not have enough space. Using the output
-            # path here resolves any path issues.
-            tmp_basedir = os.path.join(self.output_path, "tmp")
-            os.makedirs(tmp_basedir, exist_ok=True)
-            with tempfile.TemporaryDirectory(dir=tmp_basedir) as name:
-                self.export(directory=name)
-                print(f"Exported files ({name}):", os.listdir(name))
-                sim_copy = type(self).from_directory(name)
-
-        return sim_copy
+    def export(self, directory: Optional[str] = None, mode="export", skip_data_processing=True):
+        self.config.simulation.skip_data_processing = skip_data_processing
+        super().export(directory=directory, mode=mode)
+
+    def export_to_scenario(self, scenario, force=False):
+        """Exports a case study as a new scenario for running inference"""
+        self.config.case_study.scenario = scenario
+        self.config.case_study.data = None
+        self.config.case_study.output = None
+        self.config.case_study.scenario_path_override = None
+        self.config.simulation.skip_data_processing = True
+        self.save_observations(filename=f"observations_{scenario}.nc", force=force)
+        self.config.save(force=force)
 
     @staticmethod
     def _condition_posterior(

guts_base/sim/report.py

@@ -1,12 +1,15 @@
+from functools import partial
 import os
 import itertools as it
-from typing import List, Dict
+from typing import List, Dict, Literal, Optional, Union
 import numpy as np
 import pandas as pd
 import xarray as xr
+import arviz as az
 
 from pymob import SimulationBase
 from pymob.sim.report import Report, reporting
+from pymob.inference.analysis import round_to_sigfig, format_parameter
 
 from guts_base.plot import plot_survival_multipanel, plot_exposure_multipanel
 from guts_base.sim.ecx import ECxEstimator
@@ -17,6 +20,12 @@ class GutsReport(Report):
     ecx_draws: int = 250
     ecx_force_draws: bool = False
     set_background_mortality_to_zero = True
+    table_parameter_stat_focus = "mean"
+    units = xr.Dataset({
+        "metric": ["unit"],
+        "k_d": ("metric", ["1/t"])
+    })
+
 
     def additional_reports(self, sim: "SimulationBase"):
         super().additional_reports(sim=sim)
@@ -113,6 +122,164 @@ class GutsReport(Report):
         return out
 
 
+    @reporting
+    def table_parameter_estimates(self, posterior, indices):
+
+        if self.rc.table_parameter_estimates_with_batch_dim_vars:
+            var_names = {
+                k: k for k, v in self.config.model_parameters.free.items()
+            }
+        else:
+            var_names = {
+                k: k for k, v in self.config.model_parameters.free.items()
+                if self.config.simulation.batch_dimension not in v.dims
+            }
+
+        var_names.update(self.rc.table_parameter_estimates_override_names)
+
+        if len(self.rc.table_parameter_estimates_exclude_vars) > 0:
+            self._write(f"Excluding parameters: {self.rc.table_parameter_estimates_exclude_vars} for meaningful visualization")
+
+            var_names = {
+                k: k for k, v in var_names.items() 
+                if k not in self.rc.table_parameter_estimates_exclude_vars
+            }
+
+        tab_report = create_table(
+            posterior=posterior,
+            vars=var_names,
+            error_metric=self.rc.table_parameter_estimates_error_metric,
+            units=self.units,
+            significant_figures=self.rc.table_parameter_estimates_significant_figures,
+            nesting_dimension=indices.keys(),
+            parameters_as_rows=self.rc.table_parameter_estimates_parameters_as_rows,
+        )
+
+        # rewrite table in the desired output format
+        tab = create_table(
+            posterior=posterior,
+            vars=var_names,
+            error_metric=self.rc.table_parameter_estimates_error_metric,
+            units=self.units,
+            significant_figures=self.rc.table_parameter_estimates_significant_figures,
+            fmt=self.rc.table_parameter_estimates_format,
+            nesting_dimension=indices.keys(),
+            parameters_as_rows=self.rc.table_parameter_estimates_parameters_as_rows,
+        )
+
+        self._write_table(tab=tab, tab_report=tab_report, label_insert="Parameter estimates")
+
+
+def create_table(
+    posterior, 
+    error_metric: Literal["hdi","sd"] = "hdi", 
+    vars: Dict = {}, 
+    nesting_dimension: Optional[Union[List,str]] = None,
+    units: xr.Dataset = xr.Dataset(),
+    fmt: Literal["csv", "tsv", "latex"] = "csv",
+    significant_figures: int = 3,
+    parameters_as_rows: bool = True,
+) -> pd.DataFrame:
+    """The function is not ready to deal with any nesting dimensionality
+    and currently expects the 2-D case
+    """
+    tab = az.summary(
+        posterior, var_names=list(vars.keys()), 
+        fmt="xarray", kind="stats", stat_focus="mean", 
+        hdi_prob=0.94
+    )
+
+    tab = tab.rename(vars)
+
+    _units = flatten_coords(
+        dataset=create_units(dataset=tab, defined_units=units), 
+        keep_dims=["metric"]
+    )
+    tab = flatten_coords(dataset=tab, keep_dims=["metric"])
+
+    tab = tab.apply(np.vectorize(
+        partial(round_to_sigfig, sig_fig=significant_figures)
+    ))
+
+
+    if error_metric == "sd":
+        arrays = []
+        for _, data_var in tab.data_vars.items():
+            par_formatted = data_var.sel(metric=["mean", "sd"])\
+                .astype(str).str\
+                .join("metric", sep=" ± ")
+            arrays.append(par_formatted)
+
+
+        table = xr.combine_by_coords(arrays)
+        table = table.assign_coords(metric="mean ± std").expand_dims("metric")
+        table = table.to_dataframe().T
+
+    elif error_metric == "hdi":
+        stacked_tab = tab.sel(metric=["mean", "hdi_3%", "hdi_97%"])\
+            .assign_coords(metric=["mean", "hdi 3%", "hdi 97%"])
+        table = stacked_tab.to_dataframe().T
+
+    else:
+        raise NotImplementedError("Must use one of 'sd' or 'hdi'")
+
+
+    if fmt == "latex":
+        table.columns.names = [c.replace('_',' ') for c in table.columns.names]
+        table.index = [format_parameter(i) for i in list(table.index)]
+        table = table.rename(
+            columns={"hdi 3%": "hdi 3\\%", "hdi 97%": "hdi 97\\%"}
+        )
+    else: 
+        pass
+
+    table["unit"] = _units.to_pandas().T
+    
+
+    if parameters_as_rows:
+        return table
+    else:
+        return table.T
+
+def flatten_coords(dataset: xr.Dataset, keep_dims):
+    """flattens extra coordinates beside the keep_dim dimension for all data variables
+    producing a array with harmonized dimensions
+    """
+    ds = dataset.copy()
+    for var_name, data_var in ds.data_vars.items():
+        extra_coords = [k for k in list(data_var.coords.keys()) if k not in keep_dims]
+        if len(extra_coords) == 0:
+            continue
+        
+        data_var_ = data_var.stack(index=extra_coords)
+        
+        # otherwise
+        for idx in data_var_["index"].values:
+            new_var_name = f"{var_name}[{','.join([str(e) for e in idx])}]"
+            # reset coordinates to move non-dim index coords from coordinates to the
+            # data variables and then select only the var_name from the data vars
+            new_data_var = data_var_.sel({"index": idx}).reset_coords()[var_name]
+            ds[new_var_name] = new_data_var
+
+        ds = ds.drop(var_name)
+
+    # drop any coordinates that should not be in the dataset at this stage
+    extra_coords = [k for k in list(ds.coords.keys()) if k not in keep_dims]
+    ds = ds.drop(extra_coords)
+
+    return ds
+
+def create_units(dataset: xr.Dataset, defined_units: xr.Dataset):
+    units = dataset.sel(metric=["mean"]).astype(str)
+    units = units.assign_coords({"metric": ("metric", ["unit"])})
+    for k, u in units.data_vars.items():
+        if k in defined_units:
+            units = units.assign({k: defined_units[k].astype(units[k].dtype)})
+        else:
+            units[k].values = np.full_like(u.values, "")
+
+    return units
+
 class ParameterConverter:
     def __init__(
         self,

{guts_base-1.0.5.dist-info → guts_base-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: guts_base
-Version: 1.0.5
+Version: 1.0.7
 Summary: Basic GUTS model implementation in pymob
 Author-email: Florian Schunck <fluncki@protonmail.com>
 License:                     GNU GENERAL PUBLIC LICENSE

{guts_base-1.0.5.dist-info → guts_base-1.0.7.dist-info}/RECORD

@@ -1,25 +1,25 @@
-guts_base/__init__.py,sha256=h51lqbwq425YF40jI1Cw6dT0CXtcO_WSrvxbmb1KA3U,227
+guts_base/__init__.py,sha256=W9LH6glLwockPYaR5rX90_p39kfNP3de363E83g-5es,227
 guts_base/mod.py,sha256=AzOCg1A8FP5EtVfp-66HT7G7h_wnHkenieaxTc9qCyk,5796
 guts_base/plot.py,sha256=Sr_d0sXHNajPLPelcGl72yCOEEqB7NGNWhViKYAiTng,6692
 guts_base/prob.py,sha256=ITwo5dAGMHr5xTldilHMbKU6AFsWo4_ZwbfaXh97Gew,5443
 guts_base/data/__init__.py,sha256=JBgft1DTledwvB5hRZnyGiKWv-RXo1OIpb5kJXloOmo,826
 guts_base/data/expydb.py,sha256=Kcc6CeZMl3oEelk5UBN9VEfwgNF3CzTh13ooVkufAjE,8218
 guts_base/data/generator.py,sha256=rGOZU3B0Ho8V6KtfjcAmely8lnlqNFV8cRyGboayTRc,2910
-guts_base/data/openguts.py,sha256=WvhYl_AOdvNgzrcVS2f_PYbXNH_wSAz2uIBSR6BMSh0,11078
+guts_base/data/openguts.py,sha256=7C2wknBQBVxOYVgBL9UzuC9PEoqI2xaVp2pOyZwaIqY,11123
 guts_base/data/preprocessing.py,sha256=qggYkx2x62ingU1BNhJFyL1eQdFQsDJR2lefVfVWW2U,1732
 guts_base/data/survival.py,sha256=U-Ehloo8vnD81VeIglXLEUHX9lt7SjtEs2YEB0D9FHE,5096
-guts_base/data/time_of_death.py,sha256=hwngUwfRP3u8WmD3dHyXrphuu5d8ZJTKyBovGRwAHNQ,21014
+guts_base/data/time_of_death.py,sha256=dHG5jhK--WkemTxJtHVgfdo-FWlFHslBQ0PcYape9fI,22296
 guts_base/data/utils.py,sha256=u3gGDJK15MfRUP4iIxsS-I1oqxD2qH_ugsT7o_Eac18,236
 guts_base/sim/__init__.py,sha256=sbHmT1p2saN0MJ-iYnCDOHjkHtTcKgm7X-dHX5o0tYA,435
-guts_base/sim/base.py,sha256=mYHRDS6KKOrTzgsTAy5IKcVn6zIhqJH9EY9dFrsY0So,37739
+guts_base/sim/base.py,sha256=PJhBFzbs7TB2ewjE2myhEQNvcDBG4DXDols4qamhoYI,38040
 guts_base/sim/constructors.py,sha256=Kz9FHIH3EHsSIKd9sQgHa3eveniFifFlk1Hf-QR69Pg,875
 guts_base/sim/ecx.py,sha256=PeX8UVF1HMMNHaIU-jL6dml4JGezhWwiGSqPFJbOFM4,20808
 guts_base/sim/mempy.py,sha256=IHd87UrmdXpC7y7q0IjYQJH075frjbp2a-dMVBeqZ0U,10164
-guts_base/sim/report.py,sha256=Omm7czcnl3K-eWva_080uQi5RWzUrDxi7gYPjkNO_QM,6787
+guts_base/sim/report.py,sha256=o19MBhKcwty2auPjYWoz4QY91jjJFkA80UTzUuZo1oE,12720
 guts_base/sim/utils.py,sha256=Qj_FPH6kywVxOwgCerS7w5CyuYR9HKmvBWFpmxwDFgk,256
-guts_base-1.0.5.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-guts_base-1.0.5.dist-info/METADATA,sha256=TKQepymlj9INzwGyhPh3jZ37Zcm7BpULGR9L7FvhNR8,45426
-guts_base-1.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-guts_base-1.0.5.dist-info/entry_points.txt,sha256=icsHzG2jQ90ZS7XvLsI5Qj0-qGuPv2T0RBVN5daGCPU,183
-guts_base-1.0.5.dist-info/top_level.txt,sha256=PxhBgUd4r39W_VI4FyJjARwKbV5_glgCVnd6v_zAGdE,10
-guts_base-1.0.5.dist-info/RECORD,,
+guts_base-1.0.7.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+guts_base-1.0.7.dist-info/METADATA,sha256=JlsmLRB8h8yzgCUIVNxSRQulFm2bM8cNCNytGziFKjg,45426
+guts_base-1.0.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+guts_base-1.0.7.dist-info/entry_points.txt,sha256=icsHzG2jQ90ZS7XvLsI5Qj0-qGuPv2T0RBVN5daGCPU,183
+guts_base-1.0.7.dist-info/top_level.txt,sha256=PxhBgUd4r39W_VI4FyJjARwKbV5_glgCVnd6v_zAGdE,10
+guts_base-1.0.7.dist-info/RECORD,,