causarray 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- causarray/DR_estimation.py +12 -12
- causarray/DR_inference.py +11 -17
- causarray/DR_learner.py +44 -16
- causarray/__about__.py +1 -1
- causarray/gcate.py +27 -18
- causarray/gcate_glm.py +24 -14
- causarray/gcate_opt.py +12 -16
- causarray/utils.py +5 -4
- {causarray-0.0.1.dist-info → causarray-0.0.3.dist-info}/METADATA +42 -8
- causarray-0.0.3.dist-info/RECORD +14 -0
- causarray-0.0.1.dist-info/RECORD +0 -14
- {causarray-0.0.1.dist-info → causarray-0.0.3.dist-info}/WHEEL +0 -0
- {causarray-0.0.1.dist-info → causarray-0.0.3.dist-info}/licenses/LICENSE +0 -0
causarray/DR_estimation.py
CHANGED
|
@@ -69,8 +69,8 @@ def cross_fitting(
|
|
|
69
69
|
pi_hat : array
|
|
70
70
|
Estimated propensity score.
|
|
71
71
|
'''
|
|
72
|
-
func_ps, params_ps = _get_func_ps(ps_model, **kwargs)
|
|
73
|
-
params_glm = _filter_params(fit_glm, kwargs)
|
|
72
|
+
func_ps, params_ps = _get_func_ps(ps_model, verbose=False, **kwargs)
|
|
73
|
+
params_glm = _filter_params(fit_glm, {**kwargs, 'verbose': verbose})
|
|
74
74
|
|
|
75
75
|
if verbose:
|
|
76
76
|
pprint.pprint(params_ps)
|
|
@@ -86,7 +86,8 @@ def cross_fitting(
|
|
|
86
86
|
# Initialize lists to store results
|
|
87
87
|
fit_pi = True if pi_hat is None else False
|
|
88
88
|
pi_hat = np.zeros_like(A, dtype=float) if fit_pi else pi_hat
|
|
89
|
-
|
|
89
|
+
fit_Y = True if Y_hat is None else False
|
|
90
|
+
Y_hat = np.zeros((Y.shape[0],Y.shape[1],A.shape[1],2), dtype=float) if fit_Y else Y_hat
|
|
90
91
|
|
|
91
92
|
# Perform cross-fitting
|
|
92
93
|
for train_index, test_index in folds:
|
|
@@ -97,6 +98,7 @@ def cross_fitting(
|
|
|
97
98
|
Y_train, Y_test = Y[train_index], Y[test_index]
|
|
98
99
|
|
|
99
100
|
if fit_pi:
|
|
101
|
+
if verbose: pprint.pprint('Fit propensity score models...')
|
|
100
102
|
i_ctrl = (np.sum(A_train, axis=1) == 0.)
|
|
101
103
|
|
|
102
104
|
pi = np.zeros_like(A_test, dtype=float)
|
|
@@ -110,17 +112,15 @@ def cross_fitting(
|
|
|
110
112
|
pi[A_train[:,j] == 0., j] = 1 - prob
|
|
111
113
|
else:
|
|
112
114
|
pi[:,j] = func_ps(XA_train[i_cells], A_train[i_cells][:,j], XA_test)
|
|
113
|
-
|
|
114
|
-
# Fit GLM on training data and predict on test data
|
|
115
|
-
res = fit_glm(Y_train, X_train, A_train, family=family, alpha=glm_alpha,
|
|
116
|
-
impute=X_test, **params_glm)
|
|
117
|
-
|
|
118
|
-
# Store results
|
|
119
|
-
if fit_pi:
|
|
120
115
|
pi_hat[test_index] = pi
|
|
121
116
|
|
|
122
|
-
|
|
123
|
-
|
|
117
|
+
if fit_Y:
|
|
118
|
+
if verbose: pprint.pprint('Fit outcome models...')
|
|
119
|
+
# Fit GLM on training data and predict on test data
|
|
120
|
+
res = fit_glm(Y_train, X_train, A_train, family=family, alpha=glm_alpha,
|
|
121
|
+
impute=X_test, **params_glm)
|
|
122
|
+
Y_hat[test_index,:,:,0] = res[1][0]
|
|
123
|
+
Y_hat[test_index,:,:,1] = res[1][1]
|
|
124
124
|
|
|
125
125
|
pi_hat = np.clip(pi_hat, 0.01, 0.99)
|
|
126
126
|
Y_hat = np.clip(Y_hat, None, 1e5)
|
causarray/DR_inference.py
CHANGED
|
@@ -5,16 +5,14 @@ from scipy.stats import norm
|
|
|
5
5
|
from statsmodels.stats.multitest import multipletests, fdrcorrection
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
def multiplier_bootstrap(
|
|
8
|
+
def multiplier_bootstrap(eta, B):
|
|
9
9
|
'''
|
|
10
10
|
Multiplier bootstrap for inference.
|
|
11
11
|
|
|
12
12
|
Parameters
|
|
13
13
|
----------
|
|
14
|
-
|
|
15
|
-
[n, p]
|
|
16
|
-
var_est : array-like
|
|
17
|
-
[p,] Variance of the parameter estimates.
|
|
14
|
+
eta : array-like
|
|
15
|
+
[n, p] Influence function values scaled by inversion of standard deviation.
|
|
18
16
|
B : int
|
|
19
17
|
Number of bootstrap samples.
|
|
20
18
|
|
|
@@ -23,17 +21,13 @@ def multiplier_bootstrap(resid, var_est, B):
|
|
|
23
21
|
z_init : array-like
|
|
24
22
|
[B, p] Bootstrap statistics for each hypothesis.
|
|
25
23
|
'''
|
|
26
|
-
n, p =
|
|
24
|
+
n, p = eta.shape
|
|
27
25
|
B = int(B)
|
|
28
26
|
z_init = np.zeros((B,p))
|
|
29
|
-
eta = resid / np.sqrt(n * var_est[None,:])
|
|
30
27
|
for b in range(B):
|
|
31
28
|
g = np.random.normal(size=n)
|
|
32
29
|
z_init[b, :] = np.sum(eta * g[:, None], axis=0)
|
|
33
30
|
|
|
34
|
-
# g = np.random.normal(size=(B,n,1))
|
|
35
|
-
# z_init = np.sum(resid[None, :,:] * g, axis=1) / np.sqrt(n * var_est[None,:])
|
|
36
|
-
|
|
37
31
|
return z_init
|
|
38
32
|
|
|
39
33
|
|
|
@@ -106,9 +100,9 @@ def augmentation(V, tvalues, c):
|
|
|
106
100
|
|
|
107
101
|
|
|
108
102
|
def fdx_control(
|
|
109
|
-
tau_est,
|
|
103
|
+
tau_est, tvalues_init, eta_est, std_est,
|
|
110
104
|
fdx, B, alpha, c,
|
|
111
|
-
min_var=1e-
|
|
105
|
+
min_var=1e-8, min_diff=0.1
|
|
112
106
|
):
|
|
113
107
|
'''
|
|
114
108
|
Perform FDX control.
|
|
@@ -117,12 +111,12 @@ def fdx_control(
|
|
|
117
111
|
----------
|
|
118
112
|
tau_est : array-like
|
|
119
113
|
The estimated causal effect.
|
|
120
|
-
var_est : array-like
|
|
121
|
-
The estimated variance.
|
|
122
114
|
tvalues_init : array-like
|
|
123
115
|
The estimated t-values.
|
|
124
116
|
eta_est : array-like
|
|
125
|
-
The estimated influence function values.
|
|
117
|
+
The estimated influence function values, scaled by inversion of standard deviation.
|
|
118
|
+
std_est : array-like
|
|
119
|
+
The estimated standard deviation.
|
|
126
120
|
fdx : bool
|
|
127
121
|
If True, perform FDX control.
|
|
128
122
|
B : int
|
|
@@ -141,8 +135,8 @@ def fdx_control(
|
|
|
141
135
|
n = eta_est.shape[0]
|
|
142
136
|
|
|
143
137
|
if fdx:
|
|
144
|
-
id_test =
|
|
145
|
-
z_init = multiplier_bootstrap(eta_est
|
|
138
|
+
id_test = std_est >= min_var
|
|
139
|
+
z_init = multiplier_bootstrap(eta_est / std_est[None,:], B)
|
|
146
140
|
z_init[:, ~id_test] = 0.
|
|
147
141
|
tvalues = tvalues_init.copy()
|
|
148
142
|
tvalues[~id_test] = 0.
|
causarray/DR_learner.py
CHANGED
|
@@ -11,7 +11,7 @@ from causarray.utils import reset_random_seeds, pprint, tqdm, comp_size_factor,
|
|
|
11
11
|
def compute_causal_estimand(
|
|
12
12
|
estimand,
|
|
13
13
|
Y, W, A, W_A=None, family='nb', offset=False,
|
|
14
|
-
Y_hat=None, pi_hat=None,
|
|
14
|
+
Y_hat=None, pi_hat=None, mask=None,
|
|
15
15
|
fdx=False, fdx_B=1000, fdx_alpha=0.05, fdx_c=0.1,
|
|
16
16
|
verbose=False, random_state=0, **kwargs):
|
|
17
17
|
'''
|
|
@@ -39,6 +39,10 @@ def compute_causal_estimand(
|
|
|
39
39
|
Predicted outcomes under treatment of shape (n, p, a, 2).
|
|
40
40
|
pi_hat : array, optional
|
|
41
41
|
Predicted propensity scores of shape (n, a).
|
|
42
|
+
mask : array, optional
|
|
43
|
+
Boolean mask of shape (n, a) for the treatment, indicating which samples are used for
|
|
44
|
+
the estimation of the estimand. This does not affect the estimation of pseudo-outcomes
|
|
45
|
+
and propensity scores.
|
|
42
46
|
|
|
43
47
|
fdx : bool
|
|
44
48
|
Whether to use FDX control, P(FDP > c) < alpha.
|
|
@@ -87,6 +91,12 @@ def compute_causal_estimand(
|
|
|
87
91
|
W_A = W
|
|
88
92
|
elif isinstance(W_A, pd.DataFrame):
|
|
89
93
|
W_A = W_A.values
|
|
94
|
+
|
|
95
|
+
if mask is not None:
|
|
96
|
+
mask = np.array(mask).astype(bool)
|
|
97
|
+
if len(mask.shape) == 1: mask = mask.reshape(-1,1)
|
|
98
|
+
if mask.shape != A.shape:
|
|
99
|
+
raise ValueError('Mask must have the same shape as the treatment matrix')
|
|
90
100
|
|
|
91
101
|
|
|
92
102
|
if verbose:
|
|
@@ -109,6 +119,7 @@ def compute_causal_estimand(
|
|
|
109
119
|
Y_hat=Y_hat, pi_hat=pi_hat, random_state=random_state, verbose=verbose, **kwargs)
|
|
110
120
|
pi_hat = pi_hat.reshape(*A.shape)
|
|
111
121
|
|
|
122
|
+
if verbose: pprint.pprint('Estimating AIPW mean...')
|
|
112
123
|
# point estimation of the treatment effect
|
|
113
124
|
_, etas = AIPW_mean(Y, np.stack([1-A, A], axis=-1),
|
|
114
125
|
Y_hat, np.stack([1-pi_hat, pi_hat], axis=-1), positive=True)
|
|
@@ -116,25 +127,25 @@ def compute_causal_estimand(
|
|
|
116
127
|
# normalize the influence function values
|
|
117
128
|
etas /= size_factors[:,None,None,None]
|
|
118
129
|
|
|
119
|
-
|
|
120
|
-
i_ctrl = (np.sum(A, axis=1) == 0.)
|
|
121
|
-
|
|
122
130
|
res = []
|
|
123
131
|
iters = range(A.shape[1]) if A.shape[1]==1 else tqdm(range(A.shape[1]))
|
|
124
132
|
for j in iters:
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
133
|
+
if mask is not None:
|
|
134
|
+
i_cells = mask[:, j]
|
|
135
|
+
else:
|
|
136
|
+
i_ctrl = (np.sum(A, axis=1) == 0.)
|
|
137
|
+
i_case = (A[:,j] == 1.)
|
|
138
|
+
i_cells = i_ctrl | i_case
|
|
139
|
+
eta_est, tau_est, var_est = estimand(etas[i_cells,:,j], A[i_cells,j], **kwargs)
|
|
129
140
|
|
|
130
|
-
std_est = np.sqrt(
|
|
141
|
+
std_est = np.sqrt(var_est)
|
|
131
142
|
tvalues_init = tau_est / std_est
|
|
132
143
|
|
|
133
144
|
# Multiple testing procedure
|
|
134
|
-
V = fdx_control(tau_est,
|
|
145
|
+
V = fdx_control(tau_est, tvalues_init, eta_est, std_est, fdx, fdx_B, fdx_alpha, fdx_c)
|
|
135
146
|
|
|
136
147
|
# BH correction
|
|
137
|
-
tvalues_init[np.isinf(
|
|
148
|
+
tvalues_init[np.isinf(std_est)] = np.nan
|
|
138
149
|
pvals, qvals, pvals_adj, qvals_adj = bh_correction(tvalues_init)
|
|
139
150
|
|
|
140
151
|
df_res = pd.DataFrame({
|
|
@@ -158,7 +169,7 @@ def compute_causal_estimand(
|
|
|
158
169
|
|
|
159
170
|
def LFC(
|
|
160
171
|
Y, W, A, W_A=None, family='nb', offset=False,
|
|
161
|
-
Y_hat=None, pi_hat=None, cross_est=False,
|
|
172
|
+
Y_hat=None, pi_hat=None, cross_est=False, mask=None, usevar='pooled',
|
|
162
173
|
thres_min=1e-4, thres_diff=1e-6, eps_var=1e-3,
|
|
163
174
|
fdx=False, fdx_alpha=0.05, fdx_c=0.1,
|
|
164
175
|
verbose=False, **kwargs):
|
|
@@ -186,12 +197,18 @@ def LFC(
|
|
|
186
197
|
Predicted propensity scores of shape (n, a).
|
|
187
198
|
cross_est : bool
|
|
188
199
|
Whether to use cross-estimation.
|
|
200
|
+
mask : array, optional
|
|
201
|
+
Boolean mask of shape (n, a) for the treatment, indicating which samples are used for
|
|
202
|
+
the estimation of the estimand. This does not affect the estimation of pseudo-outcomes
|
|
203
|
+
and propensity scores.
|
|
189
204
|
|
|
190
205
|
thres_min : float
|
|
191
206
|
The minimum threshold for the treatment effect.
|
|
192
207
|
thres_diff : float
|
|
193
208
|
The minimum threshold for the difference in treatment effect.
|
|
194
|
-
|
|
209
|
+
eps_var : float
|
|
210
|
+
The minimum threshold for the variance of treatment.
|
|
211
|
+
|
|
195
212
|
fdx : bool
|
|
196
213
|
Whether to use FDX control, P(FDP > c) < alpha.
|
|
197
214
|
fdx_alpha : float
|
|
@@ -210,7 +227,7 @@ def LFC(
|
|
|
210
227
|
Dataframe of test results.
|
|
211
228
|
'''
|
|
212
229
|
|
|
213
|
-
def estimand(etas, **kwargs):
|
|
230
|
+
def estimand(etas, A, **kwargs):
|
|
214
231
|
eta_0, eta_1 = etas[..., 0], etas[..., 1]
|
|
215
232
|
tau_0, tau_1 = np.mean(eta_0, axis=0), np.mean(eta_1, axis=0)
|
|
216
233
|
|
|
@@ -218,7 +235,18 @@ def LFC(
|
|
|
218
235
|
tau_0 = np.clip(tau_0, thres_diff, None)
|
|
219
236
|
tau_est = np.log(tau_1/tau_0)
|
|
220
237
|
eta_est = eta_1 / tau_1[None,:] - eta_0 / tau_0[None,:]
|
|
221
|
-
|
|
238
|
+
|
|
239
|
+
if usevar == 'pooled':
|
|
240
|
+
var_est = (np.var(eta_est, axis=0, ddof=1) + eps_var) / eta_est.shape[0]
|
|
241
|
+
elif usevar == 'unequal':
|
|
242
|
+
# Estimate the variance using Welch's t-test
|
|
243
|
+
var_0 = np.var(eta_est[A==0], axis=0, ddof=1)
|
|
244
|
+
var_1 = np.var(eta_est[A==1], axis=0, ddof=1)
|
|
245
|
+
n_0 = np.sum(A==0)
|
|
246
|
+
n_1 = np.sum(A==1)
|
|
247
|
+
var_est = (var_0 + eps_var) / n_0 + (var_1 + eps_var) / n_1
|
|
248
|
+
else:
|
|
249
|
+
raise ValueError('usevar must be either "pooled" or "unequal"')
|
|
222
250
|
|
|
223
251
|
# filter out low-expressed genes
|
|
224
252
|
idx = (np.maximum(tau_0,tau_1)<thres_min) & ((tau_1-tau_0)<thres_diff)
|
|
@@ -228,7 +256,7 @@ def LFC(
|
|
|
228
256
|
|
|
229
257
|
return compute_causal_estimand(
|
|
230
258
|
estimand, Y, W, A, W_A, family, offset,
|
|
231
|
-
Y_hat=Y_hat, pi_hat=pi_hat,
|
|
259
|
+
Y_hat=Y_hat, pi_hat=pi_hat, mask=mask,
|
|
232
260
|
fdx=fdx, fdx_alpha=fdx_alpha, fdx_c=fdx_c, verbose=verbose, **kwargs)
|
|
233
261
|
|
|
234
262
|
|
causarray/__about__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.0.
|
|
1
|
+
__version__ = "0.0.3"
|
causarray/gcate.py
CHANGED
|
@@ -195,8 +195,8 @@ def estimate_r(Y, X, A, r_max, c=1.,
|
|
|
195
195
|
df_r : DataFrame
|
|
196
196
|
Results of the number of latent factors.
|
|
197
197
|
'''
|
|
198
|
-
X = np.hstack((X, A))
|
|
199
198
|
a, d = A.shape[1], X.shape[1]
|
|
199
|
+
X = np.hstack((X, A))
|
|
200
200
|
n, p = Y.shape
|
|
201
201
|
|
|
202
202
|
Y, kwargs_glm, _ = _check_input(Y, X, family, disp_glm, disp_family, offset, None, **kwargs)
|
|
@@ -210,22 +210,31 @@ def estimate_r(Y, X, A, r_max, c=1.,
|
|
|
210
210
|
r_list = np.arange(1, int(r_max)+1)
|
|
211
211
|
else:
|
|
212
212
|
r_list = np.array(r_max, dtype=int)
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
213
|
+
r_max = np.max(r_list)
|
|
214
|
+
|
|
215
|
+
# Estimate the residual deviance
|
|
216
|
+
res_glm = fit_glm(Y, X, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100, verbose=False)
|
|
217
|
+
u, s, vt = svds(res_glm[-1], k=r_max)
|
|
218
|
+
if u.shape[1]<r_max:
|
|
219
|
+
raise ValueError(f'The number of latent factors is larger than the rank of deviance residuals ({u.shape[1]}). Try to decrease the value of r.')
|
|
220
|
+
Q, _ = sp.linalg.qr(X, mode='economic')
|
|
221
|
+
P1 = np.identity(n) - Q @ Q.T
|
|
222
|
+
P1 = P1.astype(type_f)
|
|
223
|
+
A1 = np.c_[X, P1 @ u]
|
|
224
|
+
|
|
225
|
+
logh = log_h(Y, family, nuisance)
|
|
226
|
+
ll = 2 * (
|
|
227
|
+
nll(Y, X, res_glm[0], family, nuisance, size_factor) / p
|
|
228
|
+
- np.sum(logh) / (n*p) )
|
|
229
|
+
nu = (d+a) * np.maximum(n,p) * np.log(n * p / np.maximum(n,p)) / (n*p)
|
|
230
|
+
jic = ll + c * nu
|
|
231
|
+
res.append([0, ll, nu, jic])
|
|
232
|
+
|
|
233
|
+
for r in r_list[::-1]:
|
|
234
|
+
_, res_2 = estimate(Y, X, r, a,
|
|
235
|
+
0, kwargs_glm, kwargs_ls_1, kwargs_es_1, kwargs_ls_2, kwargs_es_2, A=A1[:,:d+a+r], **kwargs)
|
|
236
|
+
A1, A2 = res_2['X_U'], res_2['B_Gamma']
|
|
237
|
+
|
|
229
238
|
ll = 2 * (
|
|
230
239
|
nll(Y, A1, A2, family, nuisance, size_factor) / p
|
|
231
240
|
- np.sum(logh) / (n*p) )
|
|
@@ -233,5 +242,5 @@ def estimate_r(Y, X, A, r_max, c=1.,
|
|
|
233
242
|
jic = ll + c * nu
|
|
234
243
|
res.append([r, ll, nu, jic])
|
|
235
244
|
|
|
236
|
-
df_r = pd.DataFrame(res, columns=['r', 'deviance', 'nu', 'JIC'])
|
|
245
|
+
df_r = pd.DataFrame(res, columns=['r', 'deviance', 'nu', 'JIC']).sort_values(by='r')
|
|
237
246
|
return df_r
|
causarray/gcate_glm.py
CHANGED
|
@@ -49,13 +49,26 @@ def fit_glm(Y, X, A=None, family='gaussian', disp_family='poisson',
|
|
|
49
49
|
impute : bool
|
|
50
50
|
whether to impute potential outcomes and get predicted values
|
|
51
51
|
offset : bool
|
|
52
|
-
whether to use log of sum of Y as offset
|
|
52
|
+
whether to use log of sum of Y as offset
|
|
53
|
+
|
|
54
|
+
Returns
|
|
55
|
+
-------
|
|
56
|
+
B : array
|
|
57
|
+
d x p matrix of coefficients
|
|
58
|
+
Yhat : array
|
|
59
|
+
n x p x a matrix of predicted values
|
|
60
|
+
disp_glm : array
|
|
61
|
+
p x 1 vector of dispersion parameters
|
|
62
|
+
offsets : array
|
|
63
|
+
n x 1 vector of offsets
|
|
64
|
+
resid_deviance : array
|
|
65
|
+
n x p matrix of deviance residuals
|
|
53
66
|
'''
|
|
54
67
|
np.random.seed(random_state)
|
|
55
68
|
|
|
56
69
|
if family not in ['gaussian', 'poisson', 'nb']:
|
|
57
70
|
raise ValueError('Family not recognized')
|
|
58
|
-
|
|
71
|
+
|
|
59
72
|
d = X.shape[1]
|
|
60
73
|
|
|
61
74
|
if A is None:
|
|
@@ -70,8 +83,6 @@ def fit_glm(Y, X, A=None, family='gaussian', disp_family='poisson',
|
|
|
70
83
|
X_test = X
|
|
71
84
|
X_test = np.c_[X,np.zeros_like(A)]
|
|
72
85
|
X = np.c_[X,A]
|
|
73
|
-
# X_test = X.copy()
|
|
74
|
-
|
|
75
86
|
a = A.shape[1]
|
|
76
87
|
|
|
77
88
|
if offset is not None and offset is not False:
|
|
@@ -90,7 +101,7 @@ def fit_glm(Y, X, A=None, family='gaussian', disp_family='poisson',
|
|
|
90
101
|
pprint.pprint('Fitting {} GLM{}...'.format(family, '' if offsets is None else ' with offset'))
|
|
91
102
|
is_constant = np.all(X == X[0, :], axis=0)
|
|
92
103
|
alpha[is_constant] = 0
|
|
93
|
-
|
|
104
|
+
|
|
94
105
|
|
|
95
106
|
families = {
|
|
96
107
|
'gaussian': lambda disp: sm.families.Gaussian(),
|
|
@@ -122,11 +133,11 @@ def fit_glm(Y, X, A=None, family='gaussian', disp_family='poisson',
|
|
|
122
133
|
Yhat_0 = np.zeros((Y.shape[0], a))
|
|
123
134
|
Yhat_1 = np.zeros((Y.shape[0], a))
|
|
124
135
|
if impute is not False:
|
|
125
|
-
for
|
|
136
|
+
for k in range(a):
|
|
126
137
|
X_test_copy = X_test.copy()
|
|
127
|
-
Yhat_0[:,
|
|
128
|
-
X_test_copy[:, d+
|
|
129
|
-
Yhat_1[:,
|
|
138
|
+
Yhat_0[:,k] = mod.predict(X_test_copy, offset=offsets)
|
|
139
|
+
X_test_copy[:, d+k] = 1
|
|
140
|
+
Yhat_1[:,k] = mod.predict(X_test_copy, offset=offsets)
|
|
130
141
|
else:
|
|
131
142
|
Yhat_0[:,:] = Yhat_1[:,:] = mod.predict(X, offset=offsets).reshape(-1, a)
|
|
132
143
|
|
|
@@ -145,10 +156,11 @@ def fit_glm(Y, X, A=None, family='gaussian', disp_family='poisson',
|
|
|
145
156
|
|
|
146
157
|
|
|
147
158
|
results = Parallel(n_jobs=n_jobs)(delayed(fit_model)(
|
|
148
|
-
j, Y, X, offsets, family, disp_glm, impute, alpha) for j in tqdm(range(Y.shape[1])))
|
|
159
|
+
j, Y, X, offsets, family, disp_glm, impute, alpha) for j in tqdm(range(Y.shape[1]), disable=not verbose))
|
|
160
|
+
if verbose: pprint.pprint('Fitting GLM done.')
|
|
149
161
|
|
|
150
162
|
B, Yhat_0, Yhat_1, resid_deviance = zip(*results)
|
|
151
|
-
B = np.array(B)
|
|
163
|
+
B = np.array(B)
|
|
152
164
|
Yhat_0 = np.array(Yhat_0).transpose(1, 0, 2)
|
|
153
165
|
Yhat_1 = np.array(Yhat_1).transpose(1, 0, 2)
|
|
154
166
|
resid_deviance = np.array(resid_deviance).T
|
|
@@ -171,7 +183,6 @@ def estimate_disp(Y, X=None, A=None, Y_hat=None, disp_family='gaussian', offset=
|
|
|
171
183
|
else:
|
|
172
184
|
offsets = None
|
|
173
185
|
sf = 1.
|
|
174
|
-
|
|
175
186
|
|
|
176
187
|
if Y_hat is None:
|
|
177
188
|
if verbose:
|
|
@@ -183,8 +194,7 @@ def estimate_disp(Y, X=None, A=None, Y_hat=None, disp_family='gaussian', offset=
|
|
|
183
194
|
if disp_family=='gaussian':
|
|
184
195
|
Y_norm = Y/sf
|
|
185
196
|
reg = LinearRegression(fit_intercept=False).fit(X, Y_norm)
|
|
186
|
-
Y_hat = reg.predict(X)
|
|
187
|
-
# Y_hat = np.clip(Y_hat, 0, 1) * sf
|
|
197
|
+
Y_hat = reg.predict(X)
|
|
188
198
|
elif disp_family=='poisson':
|
|
189
199
|
Y_hat = fit_glm(Y, X, None, offset=offsets, family='poisson', impute=False, **kwargs)[1]
|
|
190
200
|
Y_hat /= sf
|
causarray/gcate_opt.py
CHANGED
|
@@ -208,19 +208,24 @@ def alter_min(
|
|
|
208
208
|
a = d
|
|
209
209
|
|
|
210
210
|
# initialization for Theta = A @ B^T
|
|
211
|
-
if A is None
|
|
211
|
+
if A is None:
|
|
212
212
|
if verbose:
|
|
213
213
|
pprint.pprint('Estimating initial latent variables with GLMs...')
|
|
214
|
-
res_glm = fit_glm(Y, X, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100)
|
|
214
|
+
res_glm = fit_glm(Y, X, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100, verbose=verbose)
|
|
215
215
|
u, s, vt = svds(res_glm[-1], k=r)
|
|
216
216
|
|
|
217
217
|
if u.shape[1]<r:
|
|
218
218
|
raise ValueError(f'The number of latent factors is larger than the rank of deviance residuals ({u.shape[1]}). Try to decrease the value of r.')
|
|
219
|
+
|
|
220
|
+
A = np.c_[X, P1 @ u]
|
|
221
|
+
else:
|
|
222
|
+
assert A.shape[1] == d+r
|
|
219
223
|
|
|
224
|
+
if B is None:
|
|
220
225
|
if verbose:
|
|
221
226
|
pprint.pprint('Estimating initial coefficients with GLMs...')
|
|
222
|
-
|
|
223
|
-
B = fit_glm(Y, A, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100)[0]
|
|
227
|
+
|
|
228
|
+
B = fit_glm(Y, A, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100, verbose=verbose)[0]
|
|
224
229
|
|
|
225
230
|
E = A[:, -r:] @ B[:, -r:].T
|
|
226
231
|
u, s, vh = sp.sparse.linalg.svds(E, k=r)
|
|
@@ -228,19 +233,10 @@ def alter_min(
|
|
|
228
233
|
B[:, d:] = vh.T * s[None,:]**(1/2)
|
|
229
234
|
del E, u, s, vh
|
|
230
235
|
|
|
231
|
-
# if offset==1:
|
|
232
|
-
# scale = np.sqrt(np.median(np.abs(X[:,0])))
|
|
233
|
-
# B[:, :offset] = scale
|
|
234
|
-
# A[:, :offset] /= scale
|
|
235
236
|
|
|
236
237
|
if P2 is not None:
|
|
237
238
|
P2 = P2.astype(type_f)
|
|
238
|
-
# E = A[:,d-a:] @ B[:,d-a:].T @ (np.identity(p) - P2)
|
|
239
|
-
# u, s, vh = sp.sparse.linalg.svds(E, k=r)
|
|
240
239
|
B[:, d-a:d] = P2 @ B[:, d-a:d]
|
|
241
|
-
# A[:, d:] = u * s[None,:]**(1/2)
|
|
242
|
-
# B[:, d:] = vh.T * s[None,:]**(1/2)
|
|
243
|
-
# del E, u, s, vh
|
|
244
240
|
|
|
245
241
|
|
|
246
242
|
Y = Y.astype(type_f)
|
|
@@ -265,10 +261,10 @@ def alter_min(
|
|
|
265
261
|
kwargs_ls['alpha'] = kwargs_ls['alpha']
|
|
266
262
|
if verbose:
|
|
267
263
|
pprint.pprint({'kwargs_glm':kwargs_glm,'kwargs_ls':kwargs_ls,'kwargs_es':kwargs_es}, compact=True)
|
|
268
|
-
|
|
264
|
+
pprint.pprint(f'Fitting GCATE (step {1 if P1 is None else 2})...')
|
|
269
265
|
hist = [func_val_pre]
|
|
270
266
|
es = Early_Stopping(**kwargs_es)
|
|
271
|
-
with tqdm(np.arange(kwargs_es['max_iters'])) as pbar:
|
|
267
|
+
with tqdm(np.arange(kwargs_es['max_iters']), disable=not verbose) as pbar:
|
|
272
268
|
for t in pbar:
|
|
273
269
|
func_val, A, B = update(
|
|
274
270
|
Y, A, B, d, weights, P1, P2,
|
|
@@ -281,7 +277,7 @@ def alter_min(
|
|
|
281
277
|
pprint.pprint('Encountered large or infinity values. Try to decrease the value of C for the norm constraints.')
|
|
282
278
|
break
|
|
283
279
|
elif es(func_val):
|
|
284
|
-
pbar.set_postfix_str('Early stopped.' + es.info)
|
|
280
|
+
pbar.set_postfix_str('Early stopped. ' + es.info)
|
|
285
281
|
pbar.close()
|
|
286
282
|
break
|
|
287
283
|
else:
|
causarray/utils.py
CHANGED
|
@@ -50,15 +50,16 @@ def prep_causarray_data(Y, A, X=None, X_A=None, intercept=True):
|
|
|
50
50
|
if not isinstance(A, pd.DataFrame):
|
|
51
51
|
A = np.asarray(A)
|
|
52
52
|
|
|
53
|
-
X = np.zeros((Y.shape[0], 0)) if X is None else np.asarray(X)
|
|
54
|
-
intercept_col = np.ones((X.shape[0], 1)) if intercept else np.empty((X.shape[0], 0))
|
|
55
|
-
X = np.hstack((intercept_col, X))
|
|
56
|
-
|
|
53
|
+
X = np.zeros((Y.shape[0], 0)) if X is None else np.asarray(X)
|
|
57
54
|
X_A = X if X_A is None else np.asarray(X_A)
|
|
58
55
|
loglibsize = np.log2(np.sum(np.asarray(Y), axis=1))
|
|
59
56
|
loglibsize = (loglibsize - np.mean(loglibsize)) / np.std(loglibsize, ddof=1)
|
|
60
57
|
X_A = np.hstack((X_A, loglibsize[:, None]))
|
|
61
58
|
|
|
59
|
+
intercept_col = np.ones((X.shape[0], 1)) if intercept else np.empty((X.shape[0], 0))
|
|
60
|
+
X = np.hstack((intercept_col, X))
|
|
61
|
+
X_A = np.hstack((intercept_col, X_A))
|
|
62
|
+
|
|
62
63
|
return Y, A, X, X_A
|
|
63
64
|
|
|
64
65
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: causarray
|
|
3
|
-
Version: 0.0.
|
|
4
|
-
Summary: causarray is a Python module for
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: causarray is a Python module for simultaneous causal inference with an array of outcomes.
|
|
5
5
|
Author-email: Jin-Hong Du <jinhongd@andrew.cmu.com>, Maya Shen <myshen@andrew.cmu.edu>, Hansruedi Mathys <mathysh@pitt.edu>, Kathryn Roeder <jinhongd@andrew.cmu.com>
|
|
6
6
|
Maintainer-email: Jin-Hong Du <jinhongd@andrew.cmu.com>
|
|
7
7
|
License: MIT License
|
|
@@ -23,25 +23,55 @@ Requires-Dist: statsmodels
|
|
|
23
23
|
Requires-Dist: tqdm
|
|
24
24
|
Description-Content-Type: text/markdown
|
|
25
25
|
|
|
26
|
+
[](https://causarray.readthedocs.io/en/latest/?badge=latest)
|
|
27
|
+
[](https://pypi.org/project/causarray)
|
|
28
|
+
[](https://pepy.tech/project/causarray)
|
|
29
|
+
|
|
30
|
+
|
|
26
31
|
# causarray
|
|
27
32
|
|
|
28
33
|
Advances in single-cell sequencing and CRISPR technologies have enabled detailed case-control comparisons and experimental perturbations at single-cell resolution. However, uncovering causal relationships in observational genomic data remains challenging due to selection bias and inadequate adjustment for unmeasured confounders, particularly in heterogeneous datasets. To address these challenges, we introduce `causarray` [Du25], a doubly robust causal inference framework for analyzing array-based genomic data at both bulk-cell and single-cell levels. `causarray` integrates a generalized confounder adjustment method to account for unmeasured confounders and employs semiparametric inference with flexible machine learning techniques to ensure robust statistical estimation of treatment effects.
|
|
29
34
|
|
|
30
35
|
|
|
31
|
-
|
|
36
|
+
## Usage
|
|
32
37
|
|
|
33
|
-
|
|
38
|
+
We recommend using `causarray` in a conda environment:
|
|
39
|
+
```cmd
|
|
40
|
+
# create a new conda environment and install the necessary packages
|
|
41
|
+
conda create -n causarray python=3.12 -y
|
|
42
|
+
|
|
43
|
+
# activate the environment
|
|
44
|
+
conda activate causarray
|
|
45
|
+
```
|
|
34
46
|
|
|
47
|
+
The module can be installed via PyPI:
|
|
35
48
|
```cmd
|
|
36
|
-
|
|
49
|
+
pip install causarray
|
|
37
50
|
```
|
|
38
51
|
|
|
52
|
+
For `R` users, `reticulate` can be used to call `causarray` from `R`.
|
|
53
|
+
The documentation and tutorials using both `Python` and `R` are available at [causarray.readthedocs.io](https://causarray.readthedocs.io/en/latest/).
|
|
54
|
+
|
|
55
|
+
|
|
39
56
|
|
|
57
|
+
## Logs
|
|
58
|
+
|
|
59
|
+
- [x] (2025-01-30) Python package released on PyPI
|
|
60
|
+
- [x] (2025-02-01) code for reproducing figures in paper
|
|
61
|
+
- [x] (2025-02-02) Tutorial for Python and R
|
|
62
|
+
- [ ] Documentation
|
|
40
63
|
|
|
41
64
|
|
|
42
65
|
<!--
|
|
43
66
|
# Development
|
|
44
67
|
|
|
68
|
+
The dependencies for running `causarray` method are listed in `environment.yml` and can be installed by running
|
|
69
|
+
|
|
70
|
+
```cmd
|
|
71
|
+
PIP_NO_DEPS=1 conda env create -f environment.yml
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
|
|
45
75
|
## Build
|
|
46
76
|
```cmd
|
|
47
77
|
git tag 0.0.0
|
|
@@ -59,12 +89,16 @@ python -m pytest tests/test_DR_learner.py
|
|
|
59
89
|
|
|
60
90
|
```cmd
|
|
61
91
|
mkdir docs
|
|
62
|
-
sphinx-quickstart
|
|
63
92
|
cd docs
|
|
93
|
+
sphinx-quickstart
|
|
94
|
+
|
|
64
95
|
make html # sphinx-build source build
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
rmarkdown::render("perturbseq.Rmd", rmarkdown::md_document(variant = "markdown_github"))
|
|
65
99
|
```
|
|
66
100
|
-->
|
|
67
101
|
|
|
68
102
|
|
|
69
|
-
|
|
70
|
-
[Du25] Jin-Hong Du, Maya Shen, Hansruedi Mathys, and Kathryn Roeder (2025). Causal differential expression analysis under unmeasured confounders with causarray. bioRxiv, 2025-01.
|
|
103
|
+
## References
|
|
104
|
+
[Du25] Jin-Hong Du, Maya Shen, Hansruedi Mathys, and Kathryn Roeder (2025). Causal differential expression analysis under unmeasured confounders with causarray. bioRxiv, 2025-01.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
causarray/DR_estimation.py,sha256=MJNZa1L_L77FGcak_zy7EZ4Dyq4yapz5g371YefAVnQ,9274
|
|
2
|
+
causarray/DR_inference.py,sha256=q_FIKNZGrDv1Ukv_vH3Bp6Z0Is4mtaswEe-_GKnZ6Cs,5181
|
|
3
|
+
causarray/DR_learner.py,sha256=I6KHZIygMy4gWAp0LPWn1RJ_r_49gJjWUhkRm91A-g0,10860
|
|
4
|
+
causarray/__about__.py,sha256=4GZKi13lDTD25YBkGakhZyEQZWTER_OWQMNPoH_UM2c,22
|
|
5
|
+
causarray/__init__.py,sha256=bTyCz9EY3NVccFb472w7gqMxrmidBsxHZjlsfMawPsk,657
|
|
6
|
+
causarray/gcate.py,sha256=9sMYh7h8Iq9MVXY4HY4OzhtOVXem8_fthstkHby_z0A,8938
|
|
7
|
+
causarray/gcate_glm.py,sha256=hi9f-7H3LZ7U7MvfGUPpe_fXE09YSoE-aUAbkb9kjb4,9195
|
|
8
|
+
causarray/gcate_likelihood.py,sha256=srs2ql2uEhgn386w1lES41E_Gb0EKHT5YDIW-25YDec,4797
|
|
9
|
+
causarray/gcate_opt.py,sha256=nyx5IRXva_kmF6j0wXiH41KPXquhnyveED0HOp8cLXY,10400
|
|
10
|
+
causarray/utils.py,sha256=mQuyS3qODPer8twgYNRK_orMMOwHZU4_jf8wum7ZvEc,6811
|
|
11
|
+
causarray-0.0.3.dist-info/METADATA,sha256=72ZVoKJziaC7D8RYZh6XouV-zhAF0qdnI4adNE2Krww,3547
|
|
12
|
+
causarray-0.0.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
13
|
+
causarray-0.0.3.dist-info/licenses/LICENSE,sha256=4zFElAgYMtL4dKsu0A1p70cEo2SXRsGEWmJFdFb2hNg,1067
|
|
14
|
+
causarray-0.0.3.dist-info/RECORD,,
|
causarray-0.0.1.dist-info/RECORD
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
causarray/DR_estimation.py,sha256=wc8jsz1XjsXX5Vp6grbwvEYV8j1MHB-VA70wYbyYUEw,9050
|
|
2
|
-
causarray/DR_inference.py,sha256=0r5qxXTgvw-49ItP5TePC7KCDu4b8XrzgnlJfVWRcZo,5323
|
|
3
|
-
causarray/DR_learner.py,sha256=DdgGbVmvNbF-TulF5XGMvf5mf7Exd2uPPOoBeyRf4Ug,9404
|
|
4
|
-
causarray/__about__.py,sha256=sXLh7g3KC4QCFxcZGBTpG2scR7hmmBsMjq6LqRptkRg,22
|
|
5
|
-
causarray/__init__.py,sha256=bTyCz9EY3NVccFb472w7gqMxrmidBsxHZjlsfMawPsk,657
|
|
6
|
-
causarray/gcate.py,sha256=pnx8O-OnJl6VDjCF1RkgCuruQJVymTrGljc1V0jnRsc,8478
|
|
7
|
-
causarray/gcate_glm.py,sha256=lpVvsH2lDsmxvqzrXA9sbRf7bZed4of_rtM10eZHE2s,8956
|
|
8
|
-
causarray/gcate_likelihood.py,sha256=srs2ql2uEhgn386w1lES41E_Gb0EKHT5YDIW-25YDec,4797
|
|
9
|
-
causarray/gcate_opt.py,sha256=w3KWQiBfwek1O8uslFXPoDBol3XWj0U2yBYZ4WmMJxM,10589
|
|
10
|
-
causarray/utils.py,sha256=fTrah_QX1C6FF6p2i-O1Y9_tljdjYcmxLkWE0tCbrJU,6765
|
|
11
|
-
causarray-0.0.1.dist-info/METADATA,sha256=AoarxDPfvp0qQv1PSSkRwZsNGaEfcPTn0lj6DJ1SirY,2425
|
|
12
|
-
causarray-0.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
13
|
-
causarray-0.0.1.dist-info/licenses/LICENSE,sha256=4zFElAgYMtL4dKsu0A1p70cEo2SXRsGEWmJFdFb2hNg,1067
|
|
14
|
-
causarray-0.0.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|