evalrx 0.1.2__py3-none-any.whl

evalrx/agent_assets/skills/outcome-driver-analysis/scripts/fit_outcome_model.py

@@ -0,0 +1,186 @@
+"""
+Steps 5-7 starting template: fit a justified model for a binary outcome, test
+per-variable significance, run goodness-of-fit diagnostics, and visualize results.
+
+Given a dataframe, a binary outcome column, a list of predictor columns, and an
+optional cluster column (for repeated/clustered observations):
+- fits logistic regression (statsmodels Logit) when there's no clustering
+- for clustered data, python's mixed-effects binary GLM support is thin -- this script
+  fits a cluster-robust-SE logistic regression as an approximation and prints a note
+  recommending scripts/fit_outcome_model.R (lme4::glmer / glmmTMB) for a true GLMM
+- reports per-variable Wald and likelihood-ratio significance with odds ratios
+- runs Hosmer-Lemeshow, VIF, ROC/AUC, and a calibration check
+- produces a coefficient/odds-ratio plot, a predicted-probability curve, an ROC curve,
+  and a calibration plot
+
+This is a template to adapt to the real column names and data -- not a black box.
+
+Usage:
+    python fit_outcome_model.py data.csv --outcome success --vars age,region,score \
+        [--cluster cohort] --out output/
+"""
+
+import argparse
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import statsmodels.api as sm
+import statsmodels.formula.api as smf
+from scipy.stats import chi2
+from sklearn.metrics import roc_auc_score, roc_curve
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+
+
+def fit_model(df: pd.DataFrame, outcome: str, predictors: list[str], cluster: str | None):
+    formula = f"{outcome} ~ " + " + ".join(predictors)
+    if cluster is None:
+        model = smf.logit(formula, data=df).fit(disp=0)
+        print("Fit plain logistic regression (observations treated as independent).")
+    else:
+        model = smf.logit(formula, data=df).fit(
+            disp=0, cov_type="cluster", cov_kwds={"groups": df[cluster]}
+        )
+        print(f"Observations are clustered by '{cluster}': fit logistic regression with "
+              f"cluster-robust standard errors as an approximation.")
+        print("For a full mixed-effects logistic regression (random intercept/slope for "
+              f"'{cluster}'), use fit_outcome_model.R instead (lme4::glmer / glmmTMB) -- "
+              "python's support for mixed binary GLMs is limited.")
+    return model
+
+
+def per_variable_significance(df: pd.DataFrame, outcome: str, predictors: list[str],
+                                full_model) -> pd.DataFrame:
+    """Wald p-value/odds ratio per model coefficient (one row per dummy level for a
+    categorical predictor), plus a likelihood-ratio test dropping each *variable* (all
+    of its levels at once) so marginal-screening signal (steps 3-4) can be compared
+    against adjusted significance."""
+    conf_int = full_model.conf_int()
+    rows = []
+    for v in predictors:
+        # a categorical predictor expands to one coefficient per non-reference level,
+        # named like "region[T.south]" by patsy -- match all of them, not just "v" itself
+        coef_names = [c for c in full_model.params.index if c == v or c.startswith(f"{v}[")]
+
+        reduced_predictors = [p for p in predictors if p != v]
+        reduced_formula = f"{outcome} ~ " + (" + ".join(reduced_predictors) if reduced_predictors else "1")
+        reduced_model = smf.logit(reduced_formula, data=df).fit(disp=0)
+        lrt_stat = 2 * (full_model.llf - reduced_model.llf)
+        lrt_p = chi2.sf(lrt_stat, df=len(coef_names))
+
+        for coef_name in coef_names:
+            rows.append({
+                "variable": coef_name,
+                "odds_ratio": np.exp(full_model.params[coef_name]),
+                "ci_low": np.exp(conf_int.loc[coef_name, 0]),
+                "ci_high": np.exp(conf_int.loc[coef_name, 1]),
+                "wald_p": full_model.pvalues[coef_name],
+                "lrt_p": lrt_p,  # shared across all levels of this variable
+            })
+    return pd.DataFrame(rows)
+
+
+def hosmer_lemeshow(y_true, y_pred, n_bins: int = 10) -> tuple[float, float]:
+    df = pd.DataFrame({"y": y_true, "p": y_pred})
+    df["bin"] = pd.qcut(df["p"], n_bins, duplicates="drop")
+    grouped = df.groupby("bin")
+    obs = grouped["y"].sum()
+    exp = grouped["p"].sum()
+    n = grouped["y"].count()
+    stat = (((obs - exp) ** 2) / (exp * (1 - exp / n))).sum()
+    p_value = chi2.sf(stat, df=n_bins - 2)
+    return float(stat), float(p_value)
+
+
+def vif_table(df: pd.DataFrame, predictors: list[str]) -> pd.DataFrame:
+    numeric_predictors = [p for p in predictors if pd.api.types.is_numeric_dtype(df[p])]
+    X = sm.add_constant(df[numeric_predictors].dropna())
+    return pd.DataFrame({
+        "variable": X.columns,
+        "vif": [variance_inflation_factor(X.values, i) for i in range(X.shape[1])],
+    }).query("variable != 'const'")
+
+
+def plot_coefficients(sig_table: pd.DataFrame, out_dir: str) -> None:
+    fig, ax = plt.subplots(figsize=(5, 0.5 * len(sig_table) + 1))
+    y = np.arange(len(sig_table))
+    ax.errorbar(sig_table["odds_ratio"], y,
+                xerr=[sig_table["odds_ratio"] - sig_table["ci_low"], sig_table["ci_high"] - sig_table["odds_ratio"]],
+                fmt="o", color="steelblue")
+    ax.axvline(1, color="gray", linestyle="--", linewidth=1)
+    ax.set_yticks(y)
+    ax.set_yticklabels(sig_table["variable"])
+    ax.set_xlabel("Odds ratio (95% CI)")
+    ax.spines[["top", "right"]].set_visible(False)
+    fig.tight_layout()
+    fig.savefig(os.path.join(out_dir, "coefficient_plot.pdf"))
+    plt.close(fig)
+
+
+def plot_roc(y_true, y_pred, out_dir: str) -> float:
+    fpr, tpr, _ = roc_curve(y_true, y_pred)
+    auc = roc_auc_score(y_true, y_pred)
+    fig, ax = plt.subplots(figsize=(4, 4))
+    ax.plot(fpr, tpr, color="steelblue", label=f"AUC = {auc:.3f}")
+    ax.plot([0, 1], [0, 1], color="gray", linestyle="--")
+    ax.set_xlabel("False positive rate")
+    ax.set_ylabel("True positive rate")
+    ax.legend()
+    ax.spines[["top", "right"]].set_visible(False)
+    fig.tight_layout()
+    fig.savefig(os.path.join(out_dir, "roc_curve.pdf"))
+    plt.close(fig)
+    return float(auc)
+
+
+def plot_calibration(y_true, y_pred, out_dir: str, n_bins: int = 10) -> None:
+    df = pd.DataFrame({"y": y_true, "p": y_pred})
+    df["bin"] = pd.qcut(df["p"], n_bins, duplicates="drop")
+    grouped = df.groupby("bin").agg(observed=("y", "mean"), predicted=("p", "mean"))
+    fig, ax = plt.subplots(figsize=(4, 4))
+    ax.plot(grouped["predicted"], grouped["observed"], "o-", color="steelblue")
+    ax.plot([0, 1], [0, 1], color="gray", linestyle="--")
+    ax.set_xlabel("Mean predicted probability")
+    ax.set_ylabel("Observed proportion")
+    ax.spines[["top", "right"]].set_visible(False)
+    fig.tight_layout()
+    fig.savefig(os.path.join(out_dir, "calibration_plot.pdf"))
+    plt.close(fig)
+
+
+def run(csv_path: str, outcome: str, predictors: list[str], out_dir: str, cluster: str | None) -> None:
+    os.makedirs(out_dir, exist_ok=True)
+    df = pd.read_csv(csv_path).dropna(subset=[outcome] + predictors)
+
+    model = fit_model(df, outcome, predictors, cluster)
+    print(model.summary())
+
+    sig_table = per_variable_significance(df, outcome, predictors, model)
+    sig_table.to_csv(os.path.join(out_dir, "per_variable_significance.csv"), index=False)
+    print("\nPer-variable significance (adjusted for other predictors):\n", sig_table)
+
+    y_pred = model.predict(df)
+    hl_stat, hl_p = hosmer_lemeshow(df[outcome], y_pred)
+    print(f"\nHosmer-Lemeshow: chi2={hl_stat:.3f}, p={hl_p:.4g}")
+
+    vif = vif_table(df, predictors)
+    vif.to_csv(os.path.join(out_dir, "vif.csv"), index=False)
+    print("\nVIF:\n", vif)
+
+    auc = plot_roc(df[outcome], y_pred, out_dir)
+    plot_calibration(df[outcome], y_pred, out_dir)
+    plot_coefficients(sig_table, out_dir)
+    print(f"\nAUC: {auc:.3f}")
+    print(f"Plots written to {out_dir}: coefficient_plot.pdf, roc_curve.pdf, calibration_plot.pdf")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("csv_path")
+    parser.add_argument("--outcome", required=True, help="binary outcome column name")
+    parser.add_argument("--vars", required=True, help="comma-separated predictor column names")
+    parser.add_argument("--cluster", default=None, help="optional clustering/grouping column")
+    parser.add_argument("--out", default="output")
+    args = parser.parse_args()
+    run(args.csv_path, args.outcome, args.vars.split(","), args.out, args.cluster)

evalrx/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.R

@@ -0,0 +1,149 @@
+# Steps 3-4 starting template: per-variable exploration vs. outcome, plus marginal
+# variable screening.
+#
+# Given a dataframe, a binary outcome column, and a list of explanatory variables:
+# - categorical variable -> contingency table vs. outcome + chi-square/Fisher's exact
+# - continuous variable -> side-by-side boxplot vs. outcome + Welch's t-test / Mann-Whitney
+# - optional conditioning variable -> repeat the above within each stratum
+# - marginal screening -> univariate logistic regression per variable, ranked by p-value
+#
+# This is a template to adapt to the real column names and data -- not a black box.
+#
+# Usage:
+#   Rscript univariate_eda.R data.csv outcome "age,region,score" output/ cohort
+
+suppressMessages({
+  library(dplyr)
+  library(ggplot2)
+})
+
+infer_variable_types <- function(df, variables) {
+  sapply(variables, function(v) {
+    if (is.numeric(df[[v]]) && length(unique(df[[v]])) > 10) "continuous" else "categorical"
+  })
+}
+
+categorical_vs_outcome <- function(df, var, outcome) {
+  ct <- table(df[[var]], df[[outcome]])
+  expected <- suppressWarnings(chisq.test(ct)$expected)
+  if (any(expected < 5) && all(dim(ct) == 2)) {
+    test_used <- "fisher_exact"
+    p_value <- fisher.test(ct)$p.value
+  } else {
+    test_used <- "chi_square"
+    p_value <- suppressWarnings(chisq.test(ct)$p.value)
+  }
+  chi2 <- suppressWarnings(chisq.test(ct)$statistic)
+  n <- sum(ct)
+  r <- nrow(ct); k <- ncol(ct)
+  cramers_v <- sqrt((chi2 / n) / min(k - 1, r - 1))
+  list(variable = var, test = test_used, p_value = p_value, cramers_v = as.numeric(cramers_v),
+       contingency_table = ct)
+}
+
+continuous_vs_outcome <- function(df, var, outcome) {
+  groups <- split(df[[var]], df[[outcome]])
+  a <- na.omit(groups[[1]]); b <- na.omit(groups[[2]])
+  normal_a <- shapiro.test(sample(a, min(length(a), 5000)))$p.value > 0.05
+  normal_b <- shapiro.test(sample(b, min(length(b), 5000)))$p.value > 0.05
+  if (normal_a && normal_b) {
+    test_used <- "welch_t"
+    test_result <- t.test(a, b)
+  } else {
+    test_used <- "mann_whitney_u"
+    test_result <- wilcox.test(a, b)
+  }
+  pooled_sd <- sqrt((sd(a)^2 + sd(b)^2) / 2)
+  cohens_d <- (mean(a) - mean(b)) / pooled_sd
+  list(variable = var, test = test_used, statistic = unname(test_result$statistic),
+       p_value = test_result$p.value, cohens_d = cohens_d)
+}
+
+boxplot_vs_outcome <- function(df, var, outcome, out_dir, facet_on = NULL) {
+  p <- ggplot(df, aes(.data[[outcome]], .data[[var]])) + geom_boxplot(fill = "steelblue", alpha = 0.6) +
+    labs(title = var) + theme_minimal()
+  fname <- paste0("box_", var, "_vs_", outcome)
+  if (!is.null(facet_on)) {
+    p <- p + facet_wrap(vars(.data[[facet_on]]))
+    fname <- paste0(fname, "_by_", facet_on)
+  }
+  ggsave(file.path(out_dir, paste0(fname, ".pdf")), p, width = 5, height = 3.5)
+}
+
+conditioned_analysis <- function(df, var, outcome, var_type, condition_on, out_dir) {
+  strata <- unique(na.omit(df[[condition_on]]))
+  rows <- lapply(strata, function(s) {
+    sub <- df[df[[condition_on]] == s, ]
+    if (var_type == "categorical") {
+      r <- categorical_vs_outcome(sub, var, outcome)
+      data.frame(stratum = s, test = r$test, p_value = r$p_value, effect_size = r$cramers_v)
+    } else {
+      r <- continuous_vs_outcome(sub, var, outcome)
+      data.frame(stratum = s, test = r$test, p_value = r$p_value, effect_size = r$cohens_d)
+    }
+  })
+  if (var_type == "continuous") boxplot_vs_outcome(df, var, outcome, out_dir, facet_on = condition_on)
+  do.call(rbind, rows)
+}
+
+marginal_screen <- function(df, outcome, variables) {
+  # the null model is refit on each variable's own non-missing subset below, since
+  # comparing a null model fit on all rows against a univariate model that drops rows
+  # with a missing predictor (mismatched sample sizes) invalidates the LRT
+  rows <- lapply(variables, function(v) {
+    tryCatch({
+      sub <- df[stats::complete.cases(df[, c(outcome, v)]), ]
+      null_model <- glm(as.formula(paste(outcome, "~ 1")), data = sub, family = binomial)
+      model <- glm(as.formula(paste(outcome, "~", v)), data = sub, family = binomial)
+      # likelihood-ratio test: a joint test across all dummy levels for a categorical
+      # variable, not just one coefficient's Wald p-value
+      lrt <- anova(null_model, model, test = "Chisq")
+      p_value <- lrt[["Pr(>Chi)"]][2]
+      aic_improvement <- AIC(null_model) - AIC(model)
+      data.frame(variable = v, p_value = p_value, aic_improvement = aic_improvement)
+    }, error = function(e) data.frame(variable = v, p_value = NA, aic_improvement = NA))
+  })
+  do.call(rbind, rows) %>% arrange(p_value)
+}
+
+run <- function(csv_path, outcome, variables, out_dir, condition_on = NULL) {
+  dir.create(out_dir, showWarnings = FALSE, recursive = TRUE)
+  df <- read.csv(csv_path)
+  var_types <- infer_variable_types(df, variables)
+
+  univariate_results <- list()
+  for (v in variables) {
+    if (var_types[[v]] == "categorical") {
+      result <- categorical_vs_outcome(df, v, outcome)
+      cat("\n", v, "(categorical) vs", outcome, ":\n")
+      print(result$contingency_table)
+      cat(sprintf("  %s: p=%.4g, Cramer's V=%.3f\n", result$test, result$p_value, result$cramers_v))
+      univariate_results[[v]] <- result[c("variable", "test", "p_value", "cramers_v")]
+    } else {
+      boxplot_vs_outcome(df, v, outcome, out_dir)
+      result <- continuous_vs_outcome(df, v, outcome)
+      cat("\n", v, "(continuous) vs", outcome, ":\n")
+      cat(sprintf("  %s: p=%.4g, Cohen's d=%.3f\n", result$test, result$p_value, result$cohens_d))
+      univariate_results[[v]] <- result
+    }
+    if (!is.null(condition_on)) {
+      cond <- conditioned_analysis(df, v, outcome, var_types[[v]], condition_on, out_dir)
+      cat("  conditioned on", condition_on, ":\n"); print(cond)
+      write.csv(cond, file.path(out_dir, paste0("conditioned_", v, "_on_", condition_on, ".csv")), row.names = FALSE)
+    }
+  }
+
+  screen <- marginal_screen(df, outcome, variables)
+  write.csv(screen, file.path(out_dir, "marginal_screening.csv"), row.names = FALSE)
+  cat("\nMarginal screening (ranked by p-value):\n"); print(screen)
+}
+
+if (sys.nframe() == 0) {
+  args <- commandArgs(trailingOnly = TRUE)
+  csv_path <- args[1]
+  outcome <- args[2]
+  variables <- strsplit(args[3], ",")[[1]]
+  out_dir <- ifelse(length(args) >= 4, args[4], "output")
+  condition_on <- if (length(args) >= 5) args[5] else NULL
+  run(csv_path, outcome, variables, out_dir, condition_on)
+}

evalrx/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.py

@@ -0,0 +1,177 @@
+"""
+Steps 3-4 starting template: per-variable exploration vs. outcome, plus marginal
+variable screening.
+
+Given a dataframe, a binary outcome column, and a list of explanatory variables:
+- categorical variable -> contingency table vs. outcome + chi-square/Fisher's exact
+- continuous variable -> side-by-side boxplot vs. outcome + Welch's t-test / Mann-Whitney
+- optional conditioning variable -> repeat the above within each stratum
+- marginal screening -> univariate logistic regression per variable, ranked by p-value
+
+This is a template to adapt to the real column names and data -- not a black box.
+
+Usage:
+    python univariate_eda.py data.csv --outcome success --vars age,region,score \
+        --condition_on cohort --out output/
+"""
+
+import argparse
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import statsmodels.formula.api as smf
+from scipy import stats
+
+
+def infer_variable_types(df: pd.DataFrame, variables: list[str]) -> dict[str, str]:
+    types = {}
+    for v in variables:
+        if pd.api.types.is_numeric_dtype(df[v]) and df[v].nunique() > 10:
+            types[v] = "continuous"
+        else:
+            types[v] = "categorical"
+    return types
+
+
+def categorical_vs_outcome(df: pd.DataFrame, var: str, outcome: str) -> dict:
+    ct = pd.crosstab(df[var], df[outcome])
+    expected = stats.contingency.expected_freq(ct)
+    if (expected < 5).any():
+        _, p, _ = stats.fisher_exact(ct) if ct.shape == (2, 2) else (None, np.nan, None)
+        test_used = "fisher_exact"
+    else:
+        chi2, p, _, _ = stats.chi2_contingency(ct)
+        test_used = "chi_square"
+    n = ct.sum().sum()
+    phi2 = stats.chi2_contingency(ct)[0] / n
+    r, k = ct.shape
+    cramers_v = float(np.sqrt(phi2 / min(k - 1, r - 1))) if min(k - 1, r - 1) > 0 else np.nan
+    return {"variable": var, "test": test_used, "p_value": p, "cramers_v": cramers_v,
+            "contingency_table": ct}
+
+
+def continuous_vs_outcome(df: pd.DataFrame, var: str, outcome: str) -> dict:
+    groups = df.groupby(outcome)[var]
+    labels = list(groups.groups.keys())
+    a, b = groups.get_group(labels[0]).dropna(), groups.get_group(labels[1]).dropna()
+    normal_a = stats.shapiro(a.sample(min(len(a), 5000)))[1] > 0.05
+    normal_b = stats.shapiro(b.sample(min(len(b), 5000)))[1] > 0.05
+    if normal_a and normal_b:
+        stat, p = stats.ttest_ind(a, b, equal_var=False)
+        test_used = "welch_t"
+    else:
+        stat, p = stats.mannwhitneyu(a, b)
+        test_used = "mann_whitney_u"
+    pooled_sd = np.sqrt(((a.std() ** 2) + (b.std() ** 2)) / 2)
+    cohens_d = (a.mean() - b.mean()) / pooled_sd if pooled_sd > 0 else np.nan
+    return {"variable": var, "test": test_used, "statistic": stat, "p_value": p, "cohens_d": cohens_d}
+
+
+def boxplot_vs_outcome(df: pd.DataFrame, var: str, outcome: str, out_dir: str, facet_on: str | None = None) -> None:
+    if facet_on is None:
+        fig, ax = plt.subplots(figsize=(4, 3.5))
+        df.boxplot(column=var, by=outcome, ax=ax)
+        ax.set_title(var)
+        plt.suptitle("")
+        fig.tight_layout()
+        fig.savefig(os.path.join(out_dir, f"box_{var}_vs_{outcome}.pdf"))
+        plt.close(fig)
+    else:
+        strata = df[facet_on].dropna().unique()
+        fig, axes = plt.subplots(1, len(strata), figsize=(4 * len(strata), 3.5), sharey=True)
+        axes = np.atleast_1d(axes)
+        for ax, s in zip(axes, strata):
+            sub = df[df[facet_on] == s]
+            sub.boxplot(column=var, by=outcome, ax=ax)
+            ax.set_title(f"{facet_on} = {s}")
+        plt.suptitle(var)
+        fig.tight_layout()
+        fig.savefig(os.path.join(out_dir, f"box_{var}_vs_{outcome}_by_{facet_on}.pdf"))
+        plt.close(fig)
+
+
+def conditioned_analysis(df: pd.DataFrame, var: str, outcome: str, var_type: str,
+                          condition_on: str, out_dir: str) -> pd.DataFrame:
+    """Repeat the marginal test within each stratum of `condition_on` to check for
+    confounding/interaction (does the relationship appear, vanish, or reverse?)."""
+    rows = []
+    for stratum, sub in df.groupby(condition_on):
+        if var_type == "categorical":
+            result = categorical_vs_outcome(sub, var, outcome)
+            rows.append({"stratum": stratum, "test": result["test"],
+                         "p_value": result["p_value"], "effect_size": result["cramers_v"]})
+        else:
+            result = continuous_vs_outcome(sub, var, outcome)
+            rows.append({"stratum": stratum, "test": result["test"],
+                         "p_value": result["p_value"], "effect_size": result["cohens_d"]})
+    if var_type == "continuous":
+        boxplot_vs_outcome(df, var, outcome, out_dir, facet_on=condition_on)
+    return pd.DataFrame(rows)
+
+
+def marginal_screen(df: pd.DataFrame, outcome: str, variables: list[str]) -> pd.DataFrame:
+    """Univariate logistic regression per variable, ranked by likelihood-ratio p-value
+    (a joint test across all dummy levels for a categorical variable, not just one
+    coefficient) and AIC improvement over the null model.
+
+    The null model is refit on each variable's own non-missing subset, since comparing
+    a null model fit on all rows against a univariate model that silently drops rows
+    with a missing predictor (mismatched sample sizes) invalidates the LRT."""
+    rows = []
+    for v in variables:
+        try:
+            sub = df[[outcome, v]].dropna()
+            null_model = smf.logit(f"{outcome} ~ 1", data=sub).fit(disp=0)
+            model = smf.logit(f"{outcome} ~ Q('{v}')", data=sub).fit(disp=0)
+            df_diff = model.df_model - null_model.df_model
+            lrt_stat = 2 * (model.llf - null_model.llf)
+            p_value = stats.chi2.sf(lrt_stat, df=df_diff) if df_diff > 0 else np.nan
+            aic_improvement = null_model.aic - model.aic
+        except Exception:
+            p_value, aic_improvement = np.nan, np.nan
+        rows.append({"variable": v, "p_value": p_value, "aic_improvement": aic_improvement})
+    return pd.DataFrame(rows).sort_values("p_value")
+
+
+def run(csv_path: str, outcome: str, variables: list[str], out_dir: str, condition_on: str | None) -> None:
+    os.makedirs(out_dir, exist_ok=True)
+    df = pd.read_csv(csv_path)
+    var_types = infer_variable_types(df, variables)
+
+    univariate_results = []
+    for v in variables:
+        if var_types[v] == "categorical":
+            result = categorical_vs_outcome(df, v, outcome)
+            print(f"\n{v} (categorical) vs {outcome}:\n{result['contingency_table']}")
+            print(f"  {result['test']}: p={result['p_value']:.4g}, Cramer's V={result['cramers_v']:.3f}")
+            univariate_results.append({k: v2 for k, v2 in result.items() if k != "contingency_table"})
+        else:
+            boxplot_vs_outcome(df, v, outcome, out_dir)
+            result = continuous_vs_outcome(df, v, outcome)
+            print(f"\n{v} (continuous) vs {outcome}:")
+            print(f"  {result['test']}: p={result['p_value']:.4g}, Cohen's d={result['cohens_d']:.3f}")
+            univariate_results.append(result)
+
+        if condition_on:
+            cond = conditioned_analysis(df, v, outcome, var_types[v], condition_on, out_dir)
+            print(f"  conditioned on {condition_on}:\n{cond}")
+            cond.to_csv(os.path.join(out_dir, f"conditioned_{v}_on_{condition_on}.csv"), index=False)
+
+    pd.DataFrame(univariate_results).to_csv(os.path.join(out_dir, "univariate_vs_outcome.csv"), index=False)
+
+    screen = marginal_screen(df, outcome, variables)
+    screen.to_csv(os.path.join(out_dir, "marginal_screening.csv"), index=False)
+    print("\nMarginal screening (ranked by p-value):\n", screen)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("csv_path")
+    parser.add_argument("--outcome", required=True, help="binary outcome column name")
+    parser.add_argument("--vars", required=True, help="comma-separated explanatory variable column names")
+    parser.add_argument("--condition_on", default=None, help="optional column to stratify/facet on")
+    parser.add_argument("--out", default="output")
+    args = parser.parse_args()
+    run(args.csv_path, args.outcome, args.vars.split(","), args.out, args.condition_on)

evalrx/agent_assets/skills.py

@@ -0,0 +1,27 @@
+"""Bundled Agent Skills shipped with the package.
+
+Bundled skills (eval-chart-style, nature-figure, evalrx-report-ui) are
+applied BY DEFAULT on every skill-capable coding backend: claude/agy discover
+them via the vendored ``<workdir>/.claude/skills/`` + the ``Skill`` tool, and
+codex is pointed at the same vendored ``SKILL.md`` files through the workdir's
+``AGENTS.md`` (it has no Skill tool). Pass ``--no-skills`` / empty ``skills``
+to opt out.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+BUNDLED_SKILLS_DIR = Path(__file__).resolve().parent / "skills"
+SKILL_BACKENDS = {"claude_code", "antigravity", "codex"}
+
+
+def bundled_skill_paths() -> list[str]:
+    """Absolute paths of bundled Agent-Skill directories containing ``SKILL.md``."""
+    if not BUNDLED_SKILLS_DIR.is_dir():
+        return []
+    return [
+        str(path)
+        for path in sorted(BUNDLED_SKILLS_DIR.iterdir())
+        if (path / "SKILL.md").is_file()
+    ]

evalrx/agent_runtime/__init__.py

@@ -0,0 +1,78 @@
+"""CLI-agent runtime: sandboxing, code generation, providers, judge models, and skills.
+
+Shared by ``evalrx.analysis`` and ``evalrx.eval_agent`` — this package
+imports nothing from either, so it is safe for both to depend on it.
+
+Submodules:
+  cli_types      CliAgentConfig / CliAgentResult and BINARY_DEFAULTS
+  cli_runtime    SubprocessRunner, ProcessRun, collect_py_files
+  cli_transcript CLI stdout post-processing (e.g. Claude stream-json rendering)
+  sandbox        ExperimentSandbox — safe subprocess execution of generated code
+  factory        create_sandbox() — subprocess/Docker sandbox backend selection
+  codegen        CodegenRunner — the single boundary stages use to invoke a CLI
+                 coding agent and harvest generated files
+  providers      CLI coding-provider adapters (claude_code, codex, antigravity, ...)
+  judges         CLI-backed judge model wrappers (ClaudeModel, AgyModel) +
+                 liveness-probe autodetection (resolve_cli_judge)
+  skills         Agent Skills resolution, sandbox installation, and prompt policy
+"""
+
+from __future__ import annotations
+
+from evalrx.agent_runtime.cli_types import (
+    BINARY_DEFAULTS,
+    CliAgentConfig,
+    CliAgentResult,
+)
+from evalrx.agent_runtime.codegen import CodegenCodeResult, CodegenRunner
+from evalrx.agent_runtime.factory import (
+    SandboxConfig,
+    SandboxFactoryConfig,
+    create_sandbox,
+)
+from evalrx.agent_runtime.json_shape import validate_json_shape
+from evalrx.agent_runtime.judges import (
+    AgyModel,
+    ClaudeModel,
+    CodexModel,
+    ResolvedJudge,
+    resolve_cli_judge,
+)
+from evalrx.agent_runtime.providers import create_cli_agent
+from evalrx.agent_runtime.sandbox import (
+    ExperimentSandbox,
+    SandboxProtocol,
+    SandboxResult,
+    parse_metrics,
+    validate_entry_point,
+    validate_entry_point_resolved,
+)
+from evalrx.agent_runtime.skills.prompt_policy import fences_hint, skills_hint
+from evalrx.agent_runtime.skills.resolver import resolve_skill_paths
+
+__all__ = [
+    "BINARY_DEFAULTS",
+    "CliAgentConfig",
+    "CliAgentResult",
+    "CodegenCodeResult",
+    "CodegenRunner",
+    "SandboxConfig",
+    "SandboxFactoryConfig",
+    "create_sandbox",
+    "AgyModel",
+    "ClaudeModel",
+    "CodexModel",
+    "ResolvedJudge",
+    "resolve_cli_judge",
+    "create_cli_agent",
+    "ExperimentSandbox",
+    "SandboxProtocol",
+    "SandboxResult",
+    "parse_metrics",
+    "validate_entry_point",
+    "validate_entry_point_resolved",
+    "resolve_skill_paths",
+    "skills_hint",
+    "fences_hint",
+    "validate_json_shape",
+]