pySEQTarget 0.10.0__py3-none-any.whl

pySEQTarget/SEQuential.py

@@ -0,0 +1,375 @@
+import datetime
+import time
+from collections import Counter
+from dataclasses import asdict
+from typing import List, Literal, Optional
+
+import numpy as np
+import polars as pl
+
+from .analysis import (_calculate_hazard, _calculate_survival, _outcome_fit,
+                       _pred_risk, _risk_estimates, _subgroup_fit)
+from .error import _datachecker, _param_checker
+from .expansion import _binder, _diagnostics, _dynamic, _random_selection
+from .helpers import _col_string, _format_time, bootstrap_loop
+from .initialization import (_cense_denominator, _cense_numerator,
+                             _denominator, _numerator, _outcome)
+from .plot import _survival_plot
+from .SEQopts import SEQopts
+from .SEQoutput import SEQoutput
+from .weighting import (_fit_denominator, _fit_LTFU, _fit_numerator,
+                        _fit_visit, _weight_bind, _weight_predict,
+                        _weight_setup, _weight_stats)
+
+
+class SEQuential:
+    """
+    Primary class initializer for SEQuentially nested target trial emulation
+
+    :param data: Data for analysis
+    :type data: pl.DataFrame
+    :param id_col: Column name for unique patient IDs
+    :type id_col: str
+    :param time_col: Column name for observational time points
+    :type time_col: str
+    :param eligible_col: Column name for analytical eligibility
+    :type eligible_col: str
+    :param treatment_col: Column name specifying treatment per time_col
+    :type treatment_col: str
+    :param outcome_col: Column name specifying outcome per time_col
+    :type outcome_col: str
+    :param time_varying_cols: Time-varying column names as covariates (BMI, Age, etc.)
+    :type time_varying_cols: Optional[List[str]] or None
+    :param fixed_cols: Fixed column names as covariates (Sex, YOB, etc.)
+    :type fixed_cols: Optional[List[str]] or None
+    :param method: Method for analysis ['ITT', 'dose-response', or 'censoring']
+    :type method: str
+    :param parameters: Parameters to augment analysis, specified with ``pySEQTarget.SEQopts``
+    :type parameters: Optional[SEQopts] or None
+    """
+
+    def __init__(
+        self,
+        data: pl.DataFrame,
+        id_col: str,
+        time_col: str,
+        eligible_col: str,
+        treatment_col: str,
+        outcome_col: str,
+        time_varying_cols: Optional[List[str]] = None,
+        fixed_cols: Optional[List[str]] = None,
+        method: Literal["ITT", "dose-response", "censoring"] = "ITT",
+        parameters: Optional[SEQopts] = None,
+    ) -> None:
+        self.data = data
+        self.id_col = id_col
+        self.time_col = time_col
+        self.eligible_col = eligible_col
+        self.treatment_col = treatment_col
+        self.outcome_col = outcome_col
+        self.time_varying_cols = time_varying_cols
+        self.fixed_cols = fixed_cols
+        self.method = method
+
+        self._time_initialized = datetime.datetime.now()
+
+        if parameters is None:
+            parameters = SEQopts()
+
+        for name, value in asdict(parameters).items():
+            setattr(self, name, value)
+
+        self._rng = (
+            np.random.RandomState(self.seed) if self.seed is not None else np.random
+        )
+
+        if self.covariates is None:
+            self.covariates = _outcome(self)
+
+        if self.weighted:
+            if self.numerator is None:
+                self.numerator = _numerator(self)
+
+            if self.denominator is None:
+                self.denominator = _denominator(self)
+
+            if self.cense_colname is not None or self.visit_colname is not None:
+                if self.cense_numerator is None:
+                    self.cense_numerator = _cense_numerator(self)
+
+                if self.cense_denominator is None:
+                    self.cense_denominator = _cense_denominator(self)
+
+        _param_checker(self)
+        _datachecker(self)
+
+    def expand(self) -> None:
+        """
+        Creates the sequentially nested, emulated target trial structure
+        """
+        start = time.perf_counter()
+        kept = [
+            self.cense_colname,
+            self.cense_eligible_colname,
+            self.compevent_colname,
+            self.visit_colname,
+            *self.weight_eligible_colnames,
+            *self.excused_colnames,
+        ]
+
+        self.data = self.data.with_columns(
+            [
+                pl.when(pl.col(self.treatment_col).is_in(self.treatment_level))
+                .then(self.eligible_col)
+                .otherwise(0)
+                .alias(self.eligible_col),
+                pl.col(self.treatment_col).shift(1).over([self.id_col]).alias("tx_lag"),
+                pl.lit(False).alias("switch"),
+            ]
+        ).with_columns(
+            [
+                pl.when(pl.col(self.time_col) == 0)
+                .then(pl.lit(False))
+                .otherwise(
+                    (pl.col("tx_lag").is_not_null())
+                    & (pl.col("tx_lag") != pl.col(self.treatment_col))
+                )
+                .cast(pl.Int8)
+                .alias("switch")
+            ]
+        )
+
+        self.DT = _binder(
+            self,
+            kept_cols=_col_string(
+                [
+                    self.covariates,
+                    self.numerator,
+                    self.denominator,
+                    self.cense_numerator,
+                    self.cense_denominator,
+                ]
+            ).union(kept),
+        ).with_columns(pl.col(self.id_col).cast(pl.Utf8).alias(self.id_col))
+
+        self.data = self.data.with_columns(
+            pl.col(self.id_col).cast(pl.Utf8).alias(self.id_col)
+        )
+
+        if self.method != "ITT":
+            _dynamic(self)
+        if self.selection_random:
+            _random_selection(self)
+        _diagnostics(self)
+
+        end = time.perf_counter()
+        self._expansion_time = _format_time(start, end)
+
+    def bootstrap(self, **kwargs) -> None:
+        """
+        Internally sets up bootstrapping - creating a list of IDs to use per iteration
+        """
+        allowed = {
+            "bootstrap_nboot",
+            "bootstrap_sample",
+            "bootstrap_CI",
+            "bootstrap_method",
+        }
+        for key, value in kwargs.items():
+            if key in allowed:
+                setattr(self, key, value)
+            else:
+                raise ValueError(f"Unknown argument: {key}")
+        UIDs = self.DT.select(pl.col(self.id_col)).unique().to_series().to_list()
+        NIDs = len(UIDs)
+
+        self._boot_samples = []
+        for _ in range(self.bootstrap_nboot):
+            sampled_IDs = self._rng.choice(
+                UIDs, size=int(self.bootstrap_sample * NIDs), replace=True
+            )
+            id_counts = Counter(sampled_IDs)
+            self._boot_samples.append(id_counts)
+        return self
+
+    @bootstrap_loop
+    def fit(self) -> None:
+        """
+        Fits weight models (numerator, denominator, censoring) and outcome models (outcome, competing event)
+        """
+        if self.bootstrap_nboot > 0 and not hasattr(self, "_boot_samples"):
+            raise ValueError(
+                "Bootstrap sampling not found. Please run the 'bootstrap' method before fitting with bootstrapping."
+            )
+
+        if self.weighted:
+            WDT = _weight_setup(self)
+            if not self.weight_preexpansion and not self.excused:
+                WDT = WDT.filter(pl.col("followup") > 0)
+
+            WDT = WDT.to_pandas()
+            for col in self.fixed_cols:
+                if col in WDT.columns:
+                    WDT[col] = WDT[col].astype("category")
+
+            _fit_LTFU(self, WDT)
+            _fit_visit(self, WDT)
+            _fit_numerator(self, WDT)
+            _fit_denominator(self, WDT)
+
+            WDT = pl.from_pandas(WDT)
+            WDT = _weight_predict(self, WDT)
+            _weight_bind(self, WDT)
+            self.weight_stats = _weight_stats(self)
+
+        if self.subgroup_colname is not None:
+            return _subgroup_fit(self)
+
+        models = {
+            "outcome": _outcome_fit(
+                self,
+                self.DT,
+                self.outcome_col,
+                self.covariates,
+                self.weighted,
+                "weight",
+            )
+        }
+        if self.compevent_colname is not None:
+            models["compevent"] = _outcome_fit(
+                self,
+                self.DT,
+                self.compevent_colname,
+                self.covariates,
+                self.weighted,
+                "weight",
+            )
+        return models
+
+    def survival(self, **kwargs) -> None:
+        """
+        Uses fit outcome models (outcome, competing event) to estimate risk, survival, and incidence curves
+        """
+        allowed = {"bootstrap_CI", "bootstrap_CI_method"}
+        for key, val in kwargs.items():
+            if key in allowed:
+                setattr(self, key, val)
+            else:
+                raise ValueError(f"Unknown or misplaced arugment: {key}")
+
+        if not hasattr(self, "outcome_model") or not self.outcome_model:
+            raise ValueError(
+                "Outcome model not found. Please run the 'fit' method before calculating survival."
+            )
+
+        start = time.perf_counter()
+
+        risk_data = _pred_risk(self)
+        surv_data = _calculate_survival(self, risk_data)
+        self.km_data = pl.concat([risk_data, surv_data])
+        self.risk_estimates = _risk_estimates(self)
+
+        end = time.perf_counter()
+        self._survival_time = _format_time(start, end)
+
+    def hazard(self) -> None:
+        """
+        Uses fit outcome models (outcome, competing event) to estimate hazard ratios
+        """
+        start = time.perf_counter()
+
+        if not hasattr(self, "outcome_model") or not self.outcome_model:
+            raise ValueError(
+                "Outcome model not found. Please run the 'fit' method before calculating hazard ratio."
+            )
+        self.hazard_ratio = _calculate_hazard(self)
+
+        end = time.perf_counter()
+        self._hazard_time = _format_time(start, end)
+
+    def plot(self, **kwargs) -> None:
+        """
+        Shows a plot specific to plot_type
+        """
+        allowed = {"plot_type", "plot_colors", "plot_title", "plot_labels"}
+        for key, val in kwargs.items():
+            if key in allowed:
+                setattr(self, key, val)
+            else:
+                raise ValueError(f"Unknown or misplaced arugment: {key}")
+        self.km_graph = _survival_plot(self)
+
+    def collect(self) -> SEQoutput:
+        """
+        Collects all results current created into ``SEQoutput`` class
+        """
+        self._time_collected = datetime.datetime.now()
+
+        generated = [
+            "numerator_model",
+            "denominator_model",
+            "outcome_model",
+            "hazard_ratio",
+            "risk_estimates",
+            "km_data",
+            "km_graph",
+            "diagnostics",
+            "_survival_time",
+            "_hazard_time",
+            "_model_time",
+            "_expansion_time",
+            "weight_stats",
+        ]
+        for attr in generated:
+            if not hasattr(self, attr):
+                setattr(self, attr, None)
+
+        # Options ==========================
+        base = SEQopts()
+
+        for name, value in vars(self).items():
+            if name in asdict(base).keys():
+                setattr(base, name, value)
+
+        # Timing =========================
+        time = {
+            "start_time": self._time_initialized,
+            "expansion_time": self._expansion_time,
+            "model_time": self._model_time,
+            "survival_time": self._survival_time,
+            "hazard_time": self._hazard_time,
+            "collection_time": self._time_collected,
+        }
+
+        if self.compevent_colname is not None:
+            compevent_models = [model["compevent"] for model in self.outcome_models]
+        else:
+            compevent_models = None
+
+        if self.outcome_model is not None:
+            outcome_models = [model["outcome"] for model in self.outcome_model]
+
+        if self.risk_estimates is None:
+            risk_ratio = risk_difference = None
+        else:
+            risk_ratio = self.risk_estimates["risk_ratio"]
+            risk_difference = self.risk_estimates["risk_difference"]
+
+        output = SEQoutput(
+            options=base,
+            method=self.method,
+            numerator_models=self.numerator_model,
+            denominator_models=self.denominator_model,
+            outcome_models=outcome_models,
+            compevent_models=compevent_models,
+            weight_statistics=self.weight_stats,
+            hazard=self.hazard_ratio,
+            km_data=self.km_data,
+            km_graph=self.km_graph,
+            risk_ratio=risk_ratio,
+            risk_difference=risk_difference,
+            time=time,
+            diagnostic_tables=self.diagnostics,
+        )
+
+        return output

pySEQTarget/__init__.py

@@ -0,0 +1,5 @@
+from .SEQopts import SEQopts
+from .SEQoutput import SEQoutput
+from .SEQuential import SEQuential
+
+__all__ = ["SEQuential", "SEQopts", "SEQoutput"]

pySEQTarget/analysis/__init__.py

@@ -0,0 +1,8 @@
+from ._hazard import _calculate_hazard as _calculate_hazard
+from ._outcome_fit import _outcome_fit as _outcome_fit
+from ._risk_estimates import _risk_estimates as _risk_estimates
+from ._subgroup_fit import _subgroup_fit as _subgroup_fit
+from ._survival_pred import _calculate_survival as _calculate_survival
+from ._survival_pred import \
+    _get_outcome_predictions as _get_outcome_predictions
+from ._survival_pred import _pred_risk as _pred_risk

pySEQTarget/analysis/_hazard.py

@@ -0,0 +1,211 @@
+import warnings
+
+import numpy as np
+import polars as pl
+from lifelines import CoxPHFitter
+
+
+def _calculate_hazard(self):
+    if self.subgroup_colname is None:
+        return _calculate_hazard_single(self, self.DT, idx=None, val=None)
+
+    all_hazards = []
+    original_DT = self.DT
+
+    for i, val in enumerate(self._unique_subgroups):
+        subgroup_DT = original_DT.filter(pl.col(self.subgroup_colname) == val)
+        hazard = _calculate_hazard_single(self, subgroup_DT, i, val)
+        all_hazards.append(hazard)
+
+    self.DT = original_DT
+    return pl.concat(all_hazards)
+
+
+def _calculate_hazard_single(self, data, idx=None, val=None):
+    full_hr = _hazard_handler(self, data, idx, 0, self._rng)
+
+    if full_hr is None or np.isnan(full_hr):
+        return _create_hazard_output(None, None, None, val, self)
+
+    if self.bootstrap_nboot > 0:
+        boot_hrs = []
+
+        for boot_idx in range(len(self._boot_samples)):
+            id_counts = self._boot_samples[boot_idx]
+
+            boot_data_list = []
+            for id_val, count in id_counts.items():
+                id_data = data.filter(pl.col(self.id_col) == id_val)
+                for _ in range(count):
+                    boot_data_list.append(id_data)
+
+            boot_data = pl.concat(boot_data_list)
+
+            boot_hr = _hazard_handler(self, boot_data, idx, boot_idx + 1, self._rng)
+            if boot_hr is not None and not np.isnan(boot_hr):
+                boot_hrs.append(boot_hr)
+
+        if len(boot_hrs) == 0:
+            return _create_hazard_output(full_hr, None, None, val, self)
+
+        if self.bootstrap_CI_method == "se":
+            from scipy.stats import norm
+
+            z = norm.ppf(1 - (1 - self.bootstrap_CI) / 2)
+            se = np.std(boot_hrs)
+            lci = full_hr - z * se
+            uci = full_hr + z * se
+        else:
+            lci = np.quantile(boot_hrs, (1 - self.bootstrap_CI) / 2)
+            uci = np.quantile(boot_hrs, 1 - (1 - self.bootstrap_CI) / 2)
+    else:
+        lci, uci = None, None
+
+    return _create_hazard_output(full_hr, lci, uci, val, self)
+
+
+def _hazard_handler(self, data, idx, boot_idx, rng):
+    exclude_cols = [
+        "followup",
+        f"followup{self.indicator_squared}",
+        self.treatment_col,
+        f"{self.treatment_col}{self.indicator_baseline}",
+        "period",
+        self.outcome_col,
+    ]
+    if self.compevent_colname:
+        exclude_cols.append(self.compevent_colname)
+    keep_cols = [col for col in data.columns if col not in exclude_cols]
+
+    trials = (
+        data.select(keep_cols)
+        .group_by([self.id_col, "trial"])
+        .first()
+        .with_columns([pl.lit(list(range(self.followup_max + 1))).alias("followup")])
+        .explode("followup")
+        .with_columns(
+            [(pl.col("followup") ** 2).alias(f"followup{self.indicator_squared}")]
+        )
+    )
+
+    if idx is not None:
+        model_dict = self.outcome_model[boot_idx][idx]
+    else:
+        model_dict = self.outcome_model[boot_idx]
+
+    outcome_model = model_dict["outcome"]
+    ce_model = model_dict.get("compevent", None) if self.compevent_colname else None
+
+    all_treatments = []
+    for val in self.treatment_level:
+        tmp = trials.with_columns(
+            [pl.lit(val).alias(f"{self.treatment_col}{self.indicator_baseline}")]
+        )
+
+        tmp_pd = tmp.to_pandas()
+        outcome_prob = outcome_model.predict(tmp_pd)
+        outcome_sim = rng.binomial(1, outcome_prob)
+
+        tmp = tmp.with_columns([pl.Series("outcome", outcome_sim)])
+
+        if ce_model is not None:
+            ce_prob = ce_model.predict(tmp_pd)
+            ce_sim = rng.binomial(1, ce_prob)
+            tmp = tmp.with_columns([pl.Series("ce", ce_sim)])
+
+            tmp = (
+                tmp.with_columns(
+                    [
+                        pl.when((pl.col("outcome") == 1) | (pl.col("ce") == 1))
+                        .then(1)
+                        .otherwise(0)
+                        .alias("any_event")
+                    ]
+                )
+                .with_columns(
+                    [
+                        pl.col("any_event")
+                        .cum_sum()
+                        .over([self.id_col, "trial"])
+                        .alias("event_cumsum")
+                    ]
+                )
+                .filter(pl.col("event_cumsum") <= 1)
+            )
+        else:
+            tmp = tmp.with_columns(
+                [
+                    pl.col("outcome")
+                    .cum_sum()
+                    .over([self.id_col, "trial"])
+                    .alias("event_cumsum")
+                ]
+            ).filter(pl.col("event_cumsum") <= 1)
+
+        tmp = tmp.group_by([self.id_col, "trial"]).last()
+        all_treatments.append(tmp)
+
+    sim_data = pl.concat(all_treatments)
+
+    if ce_model is not None:
+        sim_data = sim_data.with_columns(
+            [
+                pl.when(pl.col("outcome") == 1)
+                .then(pl.lit(1))
+                .when(pl.col("ce") == 1)
+                .then(pl.lit(2))
+                .otherwise(pl.lit(0))
+                .alias("event")
+            ]
+        )
+    else:
+        sim_data = sim_data.with_columns([pl.col("outcome").alias("event")])
+
+    sim_data_pd = sim_data.to_pandas()
+
+    try:
+        # COXPHFITER CURRENTLY HAS DEPRECATED datetime.datetime.utcnow()
+        warnings.filterwarnings("ignore", message=".*datetime.datetime.utcnow.*")
+        if ce_model is not None:
+            cox_data = sim_data_pd[sim_data_pd["event"].isin([0, 1])].copy()
+            cox_data["event_binary"] = (cox_data["event"] == 1).astype(int)
+
+            cph = CoxPHFitter()
+            cph.fit(
+                cox_data,
+                duration_col="followup",
+                event_col="event_binary",
+                formula=f"`{self.treatment_col}{self.indicator_baseline}`",
+            )
+        else:
+            cph = CoxPHFitter()
+            cph.fit(
+                sim_data_pd,
+                duration_col="followup",
+                event_col="event",
+                formula=f"`{self.treatment_col}{self.indicator_baseline}`",
+            )
+
+        hr = np.exp(cph.params_.values[0])
+        return hr
+    except Exception as e:
+        print(f"Cox model fitting failed: {e}")
+        return None
+
+
+def _create_hazard_output(hr, lci, uci, val, self):
+    if lci is not None and uci is not None:
+        output = pl.DataFrame(
+            {
+                "Hazard": [hr if hr is not None else float("nan")],
+                "LCI": [lci],
+                "UCI": [uci],
+            }
+        )
+    else:
+        output = pl.DataFrame({"Hazard": [hr if hr is not None else float("nan")]})
+
+    if val is not None:
+        output = output.with_columns(pl.lit(val).alias(self.subgroup_colname))
+
+    return output

pySEQTarget/analysis/_outcome_fit.py

@@ -0,0 +1,75 @@
+import re
+
+import polars as pl
+import statsmodels.api as sm
+import statsmodels.formula.api as smf
+
+
+def _outcome_fit(
+    self,
+    df: pl.DataFrame,
+    outcome: str,
+    formula: str,
+    weighted: bool = False,
+    weight_col: str = "weight",
+):
+    if weighted:
+        df = df.with_columns(
+            pl.col(weight_col).clip(
+                lower_bound=self.weight_min, upper_bound=self.weight_max
+            )
+        )
+
+    if self.method == "censoring":
+        df = df.filter(pl.col("switch") != 1)
+
+    df_pd = df.to_pandas()
+
+    df_pd[self.treatment_col] = df_pd[self.treatment_col].astype("category")
+    tx_bas = f"{self.treatment_col}{self.indicator_baseline}"
+    df_pd[tx_bas] = df_pd[tx_bas].astype("category")
+
+    if self.followup_class and not self.followup_spline:
+        df_pd["followup"] = df_pd["followup"].astype("category")
+        squared_col = f"followup{self.indicator_squared}"
+        if squared_col in df_pd.columns:
+            df_pd[squared_col] = df_pd[squared_col].astype("category")
+
+    if self.followup_spline:
+        spline = "cr(followup, df=3)"
+
+        formula = re.sub(r"(\w+)\s*\*\s*followup\b", rf"\1*{spline}", formula)
+        formula = re.sub(r"\bfollowup\s*\*\s*(\w+)", rf"{spline}*\1", formula)
+        formula = re.sub(
+            rf"\bfollowup{re.escape(self.indicator_squared)}\b", "", formula
+        )
+        formula = re.sub(r"\bfollowup\b", "", formula)
+
+        formula = re.sub(r"\s+", " ", formula)
+        formula = re.sub(r"\+\s*\+", "+", formula)
+        formula = re.sub(r"^\s*\+\s*|\s*\+\s*$", "", formula).strip()
+
+        if formula:
+            formula = f"{formula} + I({spline}**2)"
+        else:
+            formula = f"I({spline}**2)"
+
+    if self.fixed_cols:
+        for col in self.fixed_cols:
+            if col in df_pd.columns:
+                df_pd[col] = df_pd[col].astype("category")
+
+    full_formula = f"{outcome} ~ {formula}"
+
+    glm_kwargs = {
+        "formula": full_formula,
+        "data": df_pd,
+        "family": sm.families.Binomial(),
+    }
+
+    if weighted:
+        glm_kwargs["var_weights"] = df_pd[weight_col]
+
+    model = smf.glm(**glm_kwargs)
+    model_fit = model.fit()
+    return model_fit