guts-base 0.8.2__py3-none-any.whl

guts_base/__init__.py

@@ -0,0 +1,14 @@
+from . import sim
+from . import mod
+from . import data
+from . import prob
+from . import plot
+
+__version__ = "0.8.2"
+
+from .sim import (
+    GutsBase,
+    PymobSimulator,
+    ECxEstimator,
+    LPxEstimator, 
+)

guts_base/data/__init__.py

@@ -0,0 +1,34 @@
+from typing import Callable
+
+from . import utils
+from . import openguts
+from . import expydb
+from . import survival
+from . import generator
+from . import time_of_death
+from . import preprocessing
+
+from .openguts import (
+    OpenGutsIO,
+    create_new_columns_and_test_integrity_of_replicates,
+    create_database_and_import_data,
+    create_database_and_import_data_main,
+    import_data_to_database,
+)
+from .survival import (
+    prepare_survival_data_for_conditional_binomial, 
+    survivors_at_start_of_interval,
+    generate_survival_repeated_observations
+)
+
+from .generator import create_artificial_data, design_exposure_scenario
+
+from .expydb import (
+    to_dataset,
+    combine_coords_to_multiindex,
+    reduce_multiindex_to_flat_index
+)
+
+from .time_of_death import (
+    time_of_death_to_openguts
+)

guts_base/data/expydb.py

@@ -0,0 +1,247 @@
+from typing import List, Optional, Literal
+import numpy as np
+import xarray as xr
+import pandas as pd
+import arviz as az
+import datetime
+
+import pandas as pd
+import numpy as np
+
+from expyDB.intervention_model import (
+    Experiment,
+    Treatment, 
+    Timeseries,
+    TsData,
+    from_expydb
+)
+
+
+def prepare_dataset(
+    idata, 
+    variable="survival", 
+    unit_time: Literal["day", "hour", "minute", "second"] = "day"
+):
+    """Get interventions from idata storage with respect ot the treatment 
+    ids of the observations and move non indexing-related metadata (unique metadata)
+    to the attrs container.
+    """
+    # this test is guaranteed when prepare dataset is used together with from_expydb
+    # because from_expydb organizes the data into datasets with 1 variable,
+    # which is the timeseries variable with the coordinates timeseries_id and time
+    # for treatments and replicates. Other variables receive their 'own' dataset
+    assert len(idata[variable].data_vars) == 1
+    array: xr.DataArray = idata[variable][variable]
+    array = array.swap_dims(timeseries_id="treatment_id")
+    array = array.drop_vars("id")
+    # assuming that each timeseries of one variable in each treatment has 
+    # a unique name the resulting index should be unique
+    array = array.set_index(id=("treatment_id", "timeseries_name"))
+    array = array.drop_vars("timeseries_id")
+    assert array.indexes["id"].is_unique
+
+    # format time to h and set as float
+    time_h = array.time.values / pd.Timedelta(1, unit_time)
+    array = array.assign_coords(time=time_h)
+
+    array = move_unique_coordinates_to_attrs(array)
+
+    array.attrs["unit_time"] = unit_time
+
+    # add a unique id for the selected dataset which is only relevant for
+    # the scope of modelling
+    return array
+
+def move_unique_coordinates_to_attrs(array:xr.DataArray) -> xr.DataArray:
+    key: str
+    for key, coord in array.coords.items(): # type:ignore
+        if key in ["id", "treatment_id", "timeseries_id", "experiment_id", "subject_count", "timeseries_name"]:
+            continue
+        
+        if coord.isnull().all():
+            unique_values = [None]
+        else:
+            unique_values = np.unique(coord.data)
+
+        if len(unique_values) == 1:
+            array.attrs.update({key: unique_values[0]})
+            array = array.drop_vars(key)
+    return array
+
+# def prepare_interventions_dataset(interventions_idata, observations, ivs:Optional[List[str]]=None):
+#     """Get interventions from idata storage with respect ot the treatment 
+#     ids of the observations"""
+#     if ivs is None:
+#         ivs = list(interventions_idata.keys())
+#     ds_ivs = get_interventions(
+#         interventions_idata,
+#         observations=observations,
+#         ivs=ivs
+#     )
+
+#     time_h = ds_ivs.time.values / np.timedelta64(1, "h")
+#     ds_ivs = ds_ivs.assign_coords(time=time_h)
+#     ds_ivs.attrs["unit_time"] = "hours (h)"
+
+#     return ds_ivs
+
+def to_dataset(
+    observations_idata, 
+    interventions_idata,
+    unit_time: Literal["day", "hour", "minute", "second"] = "hour"
+) -> xr.Dataset:
+    """Combines intervention and observation datasets, assuming that there is 
+    a unique multiindex that can be constructed from 
+      - treatment_id
+      - timeseries_name
+
+    This way interventions and observations can be combined into a single dataset,
+    """
+    data_arrays = {}
+    for variable in observations_idata.groups():
+        # prepare observations
+        da = prepare_dataset(
+            idata=observations_idata,
+            variable=variable,
+            unit_time=unit_time,
+        )
+        data_arrays.update({variable: da})
+
+    # prepare interventions
+    for variable in interventions_idata.groups():
+        da = prepare_dataset(
+            idata=interventions_idata,
+            variable=variable,
+            unit_time=unit_time,
+        )
+        data_arrays.update({variable: da})
+
+    return xr.combine_by_coords(data_arrays.values())  # type: ignore
+
+
+def reduce_multiindex_to_flat_index(dataset):
+    multi_index = dataset.id.indexes["id"]
+    
+    # create a flat index from the multi index
+    flat_index = multi_index.map(lambda x: "__".join([str(x_) for x_ in x]))
+    
+    # remove multi index from dimension 'id'
+    dataset = dataset.reset_index("id")
+    
+    # assign flat index to dimension 'id'
+    dataset = dataset.assign_coords(id=flat_index)
+
+    return dataset    
+
+def combine_coords_to_multiindex(
+        dataset: xr.Dataset, 
+        coordinates: List[str], 
+        index_name: str, 
+        sep: str = "__"
+    ) -> xr.Dataset:
+    """Simply combines a list of coordinates into a joint string
+
+    Parameters
+    ----------
+    dataset : xr.Dataset
+        The observations dataset
+    coordinates : List[str]
+        The coordinates that should be joined
+    index_name : str
+        The name of the new, joined, coordinate
+    sep : str, optional
+        The string to separate the coordinate components, by default "__"
+
+    Returns
+    -------
+    xr.Dataset
+        Dataset with a new coordinate composed of the listed coordinates
+    """
+    try:
+        multi_index = pd.MultiIndex.from_arrays([dataset[c].values for c in coordinates])
+    except KeyError as err:
+        raise KeyError(
+            f"Did not find key {err} in the dataset. "
+            f"This is probably because the key {err} is equal for all treatments."
+        )
+    multi_index = multi_index.map(lambda x: "__".join([str(x_) for x_ in x]))
+    return dataset.assign_coords({index_name: ("id", multi_index)})
+
+# def get_interventions(interventions_idata, observations, ivs: List[str]) -> xr.Dataset:
+#     """Get the interventions according to the treatment ids of the observation
+#     dataset.
+    
+#     Works only for single interventions
+#     """
+#     X_in = {}
+#     for data_var in ivs:
+#         x_in = interventions_idata[data_var]\
+#             .swap_dims(timeseries_id="treatment_id")\
+#             .sel(treatment_id=observations.treatment_id.values)
+        
+#         x_in = x_in.assign_coords(
+#             _id=("treatment_id", range(x_in.sizes["treatment_id"]))
+#         )
+#         x_in = x_in.swap_dims(treatment_id="_id")
+#         X_in.update({data_var: x_in[data_var]})
+
+
+#     X_in_dataset = xr.concat(X_in.values(), dim="variable")\
+#         .assign_coords(variable=ivs)\
+#         .to_dataset(dim="variable")
+    
+#     if "variable" in X_in_dataset.dims:
+#         X_in_dataset = X_in_dataset.drop_dims("variable")
+
+#     return X_in_dataset
+
+
+
+
+# def combine_interventions(
+#     interventions: az.InferenceData, 
+#     force: bool=False
+# ) -> xr.DataArray:
+#     """Combining interventions into a single dataset is only possible,
+#     if there is only a single timeseries for each intervention.
+
+#     Parameters
+#     ----------
+#     interventions : az.InferenceData
+#         Interventions InferenceData. Contains multiple datasets with at
+#         least one timeseries
+#     force : bool, optional
+#         Override restrictions to combine interventions only when the number
+#         of timeseries is 1, by default False
+
+#     Returns
+#     -------
+#     xr.DataArray
+#         Interventions, combined into a single dataset
+
+#     Raises
+#     ------
+#     ValueError
+#         If the number of timeseries is larger than 1 and force is not True
+#     """
+#     assert isinstance(interventions, az.InferenceData)
+#     arrays = []
+#     for variable, dataset in interventions.items():
+#         if dataset.sizes["timeseries_id"] > 1:
+#             if force:
+#                 arr = dataset.to_array()
+#             else:
+#                 raise ValueError(
+#                     "Combining interventions is only allowed when the number of "
+#                     "Timeseries for each variable is 1. This is to avoid blowing "
+#                     "Up the size of the dataset with nans, because timeseries ids "
+#                     "are different for each variable. You can override this error "
+#                     "By using `force=True`"
+#                 )
+#         else:
+#             arr = dataset.squeeze("timeseries_id").to_array()
+        
+#         arrays.append(arr)
+
+#     return xr.concat(arrays, dim="variable")
+    

guts_base/data/generator.py

@@ -0,0 +1,96 @@
+import numpy as np
+import xarray as xr
+from typing import TypedDict, Dict, Optional, Sequence
+
+class ExposureDataDict(TypedDict):
+    start: int
+    end: int
+    exposure: float|Sequence[float]
+
+def create_artificial_data(
+    t_max, 
+    dt, 
+    exposure_paths=["oral", "topical", "contact"],
+    intensity=[0.1, 0.5, 0.05]
+):
+    rng = np.random.default_rng(1)
+    time = np.arange(0, t_max, step=dt)  # daily time resolution
+
+    # calculate potential exposure based on a lognormal distribution
+    oral = rng.lognormal(mean=np.log(intensity[0]), sigma=0.5, size=len(time))
+    # and include a random exposure days
+    oral *= rng.binomial(n=1, p=1, size=len(time))
+
+
+    # calculate potential exposure based on a lognormal distribution
+    topical = rng.lognormal(mean=np.log(intensity[1]), sigma=1, size=len(time))
+    # and include a random exposure days
+    topical *= rng.binomial(n=1, p=0.25, size=len(time))
+
+
+    # calculate potential exposure based on a lognormal distribution
+    contact = rng.lognormal(mean=np.log(intensity[2]), sigma=0.1, size=len(time))
+    # and include a random exposure days
+    contact *= rng.binomial(n=1, p=0.8, size=len(time))
+
+
+
+    exposures = xr.Dataset(
+        data_vars={
+            "exposure": (("time", "exposure_path"), np.column_stack([oral, topical, contact])),
+        },
+        coords={"time": time, "exposure_path": ["oral", "topical", "contact"]}
+    )
+
+    return exposures.sel(exposure_path=exposure_paths)
+
+
+def design_exposure_timeseries(time: Sequence[float], exposure: ExposureDataDict, eps: float):
+    if exposure is None:
+        return
+    
+    exposure["end"] = time[-1] if exposure["end"] is None else exposure["end"]
+
+    return np.where(
+        np.logical_and(time >= exposure["start"], time < exposure["end"]),
+        exposure["concentration"],
+        0
+    )
+
+def design_exposure_scenario(
+    t_max: float, 
+    dt: float, 
+    exposures: Dict[str,ExposureDataDict],
+    eps: float = 1e-8,
+    exposure_dimension: str = "exposure_type",
+):
+    """
+    TODO: tmax, dt and eps are probably not necessary
+    """
+    time = np.arange(0, t_max, step=dt)  # daily time resolution
+    time = np.unique(np.concatenate([time] + [
+        np.array([time[-1] if vals["end"] is None else vals["end"]])
+        for key, vals in exposures.items()
+
+    ]))
+
+    treatments = {}
+    for key, expo in exposures.items():
+        treat = design_exposure_timeseries(time, expo, eps)
+        treatments.update({key: treat})
+
+    data = np.column_stack(list(treatments.values())).squeeze()
+    data = np.expand_dims(data, axis=0)
+
+    coords = {"id": [0], "time": time}
+    
+    if len(exposures) > 1:
+        coords.update({exposure_dimension: list(treatments.keys())})
+
+    exposures_dataset = xr.Dataset(
+        data_vars={"exposure": (tuple(coords.keys()), data)},
+        coords=coords
+    )
+
+    return exposures_dataset
+

guts_base/data/openguts.py

@@ -0,0 +1,294 @@
+from typing import List, Optional, Callable, Dict, Any, Tuple
+from importlib import import_module
+import warnings
+
+import numpy as np
+import pandas as pd
+from expyDB.database_operations import create_database, experiment_to_db
+from pymob.sim.config import dict_to_string
+
+import glob
+import os
+
+import click
+import pandas as pd
+
+from expyDB.intervention_model import to_expydb, Experiment, PandasConverter
+from guts_base.data.utils import datalad_locked_file_warning
+
+def test_equality_of_exposure_patterns_in_treatment(df):
+    for _, group in df.groupby("treatment_id"):
+        exposures = group.pivot_table(
+            # values="value", 
+            index=["time", "treatment_id"], 
+            columns="replicate_id"
+        )
+
+        equal_expo = exposures.values == exposures.values[:, ].reshape((-1, 1))
+        if not np.all(equal_expo):
+            raise RuntimeError(
+                "Replicates in the same treatment ID have different exposure patterns."
+            )
+        
+def create_new_columns_and_test_integrity_of_replicates(
+    exposure, survival, n_reps, path
+):
+    assert np.all(exposure.columns == survival.columns)
+    
+    columns_new, treatment_reps = identify_replicates(frame=exposure)
+
+    if not np.all(np.array(list(treatment_reps.values())) == n_reps):
+        warnings.warn(
+            f"Actual treatment replicates are different from "
+            f"replicates ({n_reps}), given in Info sheet in file: "
+            f"{path}"
+        )
+
+    return columns_new
+
+def identify_replicates(frame):
+    df = frame.drop(columns="time")
+
+    # Find identical columns and assign group labels
+    group_labels = {}
+    used_cols = set()
+    treatment_map = {}
+
+    for col in df.columns:
+        if col not in used_cols:
+            # compare the column to each column of the dataframe with 
+            # df.apply(func, axis=0) and get the column names
+            identical_cols = df.columns[df.apply(lambda x: x.equals(df[col]), axis=0)].tolist()
+            group_label = f'{len(group_labels) + 1}'
+            group_labels[group_label] = identical_cols
+            used_cols.update(identical_cols)
+
+            for icol in identical_cols:
+                treatment_map.update({icol: group_label})
+
+    columns_new = [f"{treatment_map[col]}__{col}" for col in df.columns]
+    treatment_reps = {key: len(cols) for key, cols in group_labels.items()}
+
+    return columns_new, treatment_reps
+
+
+
+def read_timeseries_sheet(path, sheet, sep=None):
+    ts = pd.read_excel(path, sheet_name=sheet, index_col=0)  # type: ignore
+    multi_index = pd.MultiIndex.from_tuples(
+        [tuple(c.split(sep)) for c in ts.columns], names=["treatment_id", "timeseries_id"]
+    )
+    ts.columns = multi_index
+    return ts
+
+
+class OpenGutsIO:
+    # TODO: Use preprocessing here and use map as a class attribute
+    def __init__(self, file):
+        self._file = file
+        self.data = self.from_file(file)
+
+    def _openguts_wide_to_long(self, frame, columns_new):
+        frame_wide = frame.copy()
+
+        frame_wide.columns = ["time"] + columns_new
+        frame_long = pd.melt(
+            frame=frame_wide, 
+            id_vars=["time"], 
+            value_vars=columns_new,
+            var_name="exposure_id"
+        )
+        # create new index columns from new column names
+        frame_long[["treatment_id", "replicate_id"]] = frame_long\
+            .exposure_id.str.split("__", n=1, expand=True)
+        frame_long = frame_long.drop(columns="exposure_id")
+        
+        return frame_long
+
+
+    def _merge_tables(self, tables: List):
+        data = tables.pop(0).set_index(["time", "treatment_id", "replicate_id"])
+
+        for expo in tables:
+            rdata =expo.set_index(["time", "treatment_id", "replicate_id"])
+            data = pd.merge(
+                left=data, 
+                right=rdata, 
+                how="left", 
+                left_index=True, 
+                right_index=True
+            )
+
+        return data
+
+    def _read_timeseries(self, path, sheets):
+        # design new columns based on the information about replicates and treatments
+        timeseries_long_list = []
+        timeseries_column_list = []
+        time_units = {}
+        for iv in sheets:
+            timeseries_df = pd.read_excel(path, sheet_name=f"{iv}") 
+
+            time_column = timeseries_df.columns[0]
+            time_unit = time_column.lower().replace("time", "").strip(" []")
+
+            # define replicates based on equality of columns
+            timeseries_columns = [c for c in timeseries_df.columns[1:]]
+            timeseries_long = self._openguts_wide_to_long(
+                frame=timeseries_df, columns_new=timeseries_columns
+            )
+            intervention_long = timeseries_long.rename(columns={"value": iv})
+            timeseries_long_list.append(intervention_long)
+            timeseries_column_list.append(timeseries_columns)
+            time_units.update({iv: time_unit})
+
+        return self._merge_tables(timeseries_long_list).reset_index(), time_units
+
+
+    def _read_openguts(self, path, metadata_sheetname="meta"):
+        meta = pd.read_excel(path, sheet_name=metadata_sheetname, index_col=0).dropna(how="all")
+        interventions = meta.loc["experiment__interventions","Value"]
+        if interventions is None:
+            raise ValueError("'experiment__interventions' must be defined in metadata")
+        else:
+            intervention_sheets = [i.strip("[]' ") for i in interventions.split(",")]  # type: ignore
+        
+        observations = meta.loc["experiment__observations","Value"]
+        if observations is None:
+            raise ValueError("'experiment__observations' must be defined in metadata")
+        else:
+            observation_sheets = [i.strip("[]' ") for i in observations.split(",")]  # type: ignore
+        
+        # survival_df = pd.read_excel(path, sheet_name="survival") 
+        # survival_df = survival_df.rename(columns={"time [d]": "time"})
+
+        # design new columns based on the information about replicates and treatments
+        interventions_long, interventions_time_units = self._read_timeseries(path, intervention_sheets)
+        observations_long, observations_time_units = self._read_timeseries(path, observation_sheets)
+        time_unit = {
+            "interventions": interventions_time_units,
+            "observations": observations_time_units
+        }
+
+        # TODO test if all exposures within a treatment (replicates) were nominally the same 
+        # test_equality_of_exposure_patterns_in_treatment(df=exposures_long)
+
+        return interventions_long, observations_long, meta, time_unit
+
+    def from_file(self, file) -> None:
+        (
+            interventions_long, 
+            observations_long, 
+            meta, 
+            time_unit
+        ) = self._read_openguts(path=file)
+
+        self.interventions = interventions_long
+        self.observations = observations_long
+        self.time_unit = time_unit
+        self.meta = meta
+
+    def to_file(self, file):
+        raise NotImplementedError(
+            "This method should implement writing an excel file that corresponds"
+            "to the original input file."
+        )
+    
+    def to_experiment(self) -> Experiment:
+        return Experiment.from_dict(data=dict(
+            interventions=self.interventions, 
+            observations=self.observations,
+            meta=self.meta,
+            time_units=self.time_unit
+        ))
+
+    def from_experiment(self, experiment: Experiment) -> None:
+        data = experiment.to_dict()
+        self.interventions=data["interventions"],
+        self.observations=data["observations"],
+        self.meta=data["meta"],
+        self.time_units=data["time_units"],
+
+    def to_xarray(self):
+        return self.to_experiment().to_xarray()
+
+
+def import_data_to_database(path, database, preprocessing: Optional[Callable] = None, preprocessing_out: Optional[str] = None):
+    """This script takes raw data, preprocesses it to contain all 
+    necessary metadata for expyDB. Then it creates an experiment Model and 
+    processes adds it to the database
+    """
+    # preprocess path
+    if preprocessing is not None:
+        if preprocessing_out is None:
+            filename = os.path.dirname(path)
+            directory = os.path.basename(filename)
+            new_path = path.replace(directory, f"processed_{directory}")
+        else:
+            filename = os.path.basename(path)
+            new_path = preprocessing_out.format(filename=filename)
+
+        os.makedirs(os.path.dirname(new_path), exist_ok=True)
+        
+        processed_path = preprocessing(path, new_path)
+    else:
+        processed_path = path
+
+    # Preprocess excel to interventions and observations in Long form and a 
+    # metadata Series as well as a default time unit
+    openguts = OpenGutsIO(processed_path)
+    
+    # From excel to an Experiment Model instance
+    experiment = openguts.to_experiment()
+
+    # from the Model to the Database
+    if not os.access(database, os.W_OK):
+        warnings.warn(
+            f"Did not write to database. The file '{database}' does "
+            "not have write access."
+        )
+        return
+    
+    experiment.to_database(database=database)
+
+
+def create_database_and_import_data_main(datasets_path, database_path, preprocessing=None, preprocessing_out=None):
+    print("\n")
+    print(f"Creating a database and importing data")
+    print(f"======================================")
+
+    if preprocessing is not None:
+        module, func = preprocessing.rsplit(".", 1)
+        mod = import_module(module)
+        preprocessing_func = getattr(mod, func)
+    else:
+        preprocessing_func = None
+
+    paths = []
+    for p in datasets_path:
+        if os.path.isfile(p):
+            paths.append(p)
+        else:
+            paths.extend(glob.glob(os.path.join(p, "*.xlsx")))
+
+    create_database(database=database_path, force=True)
+    for p in paths:
+        print(f"\nPreprocessing and importing file: {p}")
+        import_data_to_database(
+            path=p, database=database_path, 
+            preprocessing=preprocessing_func,
+            preprocessing_out=preprocessing_out
+        )
+
+@click.command()
+@click.option("--datasets_path", type=str, multiple=True, help="The path to the directory where the excel files are located. Alternatively, use multiple times with paths to files")
+@click.option("--database_path", type=str, help="The path to the database (should end with .db)")
+@click.option("--preprocessing", type=str, help="Function used to preprocess the data", default=None)
+@click.option("--preprocessing-out", type=str, help="A pattern that uses {filename} as a placeholder e.g. 'data/processed_data/{filename}. If unset, preprends 'processes_' to the dirname", default=None)
+def create_database_and_import_data(datasets_path, database_path, preprocessing, preprocessing_out):
+    create_database_and_import_data_main(
+        datasets_path=datasets_path,
+        database_path=database_path,
+        preprocessing=preprocessing,
+        preprocessing_out=preprocessing_out
+    )