causarray 0.0.2__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {causarray-0.0.2 → causarray-0.0.4}/.gitignore +2 -1
- {causarray-0.0.2 → causarray-0.0.4}/PKG-INFO +1 -1
- {causarray-0.0.2 → causarray-0.0.4}/causarray/DR_estimation.py +27 -17
- {causarray-0.0.2 → causarray-0.0.4}/causarray/DR_inference.py +11 -17
- {causarray-0.0.2 → causarray-0.0.4}/causarray/DR_learner.py +55 -26
- causarray-0.0.4/causarray/__about__.py +1 -0
- {causarray-0.0.2 → causarray-0.0.4}/causarray/gcate.py +31 -19
- {causarray-0.0.2 → causarray-0.0.4}/causarray/gcate_glm.py +22 -9
- {causarray-0.0.2 → causarray-0.0.4}/causarray/gcate_opt.py +12 -16
- {causarray-0.0.2 → causarray-0.0.4}/causarray/utils.py +7 -4
- causarray-0.0.2/causarray/__about__.py +0 -1
- {causarray-0.0.2 → causarray-0.0.4}/LICENSE +0 -0
- {causarray-0.0.2 → causarray-0.0.4}/README.md +0 -0
- {causarray-0.0.2 → causarray-0.0.4}/causarray/__init__.py +0 -0
- {causarray-0.0.2 → causarray-0.0.4}/causarray/gcate_likelihood.py +0 -0
- {causarray-0.0.2 → causarray-0.0.4}/pyproject.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: causarray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: causarray is a Python module for simultaneous causal inference with an array of outcomes.
|
|
5
5
|
Author-email: Jin-Hong Du <jinhongd@andrew.cmu.com>, Maya Shen <myshen@andrew.cmu.edu>, Hansruedi Mathys <mathysh@pitt.edu>, Kathryn Roeder <jinhongd@andrew.cmu.com>
|
|
6
6
|
Maintainer-email: Jin-Hong Du <jinhongd@andrew.cmu.com>
|
|
@@ -33,7 +33,7 @@ def _get_func_ps(ps_model, **kwargs):
|
|
|
33
33
|
def cross_fitting(
|
|
34
34
|
Y, A, X, X_A, family='poisson', K=1, glm_alpha=1e-4,
|
|
35
35
|
ps_model='logistic',
|
|
36
|
-
pi_hat=None,
|
|
36
|
+
Y_hat=None, pi_hat=None, mask=None, verbose=False, **kwargs):
|
|
37
37
|
'''
|
|
38
38
|
Cross-fitting for causal estimands.
|
|
39
39
|
|
|
@@ -55,10 +55,16 @@ def cross_fitting(
|
|
|
55
55
|
The regularization parameter for the generalized linear model. The default is 1e-4.
|
|
56
56
|
ps_model : str, optional
|
|
57
57
|
The propensity score model. The default is 'logistic'.
|
|
58
|
-
|
|
59
|
-
Propensity score of shape (n, a). The default is None.
|
|
58
|
+
|
|
60
59
|
Y_hat : array, optional
|
|
61
60
|
Estimated potential outcome of shape (n, p, a, 2). The default is None.
|
|
61
|
+
pi_hat : array, optional
|
|
62
|
+
Propensity score of shape (n, a). The default is None.
|
|
63
|
+
mask : array, optional
|
|
64
|
+
Boolean mask of shape (n, a) for the treatment, indicating which samples are used for
|
|
65
|
+
the estimation of the estimand. This does not affect the estimation of pseudo-outcomes
|
|
66
|
+
and propensity scores.
|
|
67
|
+
|
|
62
68
|
**kwargs : dict
|
|
63
69
|
Additional arguments to pass to the model.
|
|
64
70
|
|
|
@@ -69,7 +75,7 @@ def cross_fitting(
|
|
|
69
75
|
pi_hat : array
|
|
70
76
|
Estimated propensity score.
|
|
71
77
|
'''
|
|
72
|
-
func_ps, params_ps = _get_func_ps(ps_model, verbose=
|
|
78
|
+
func_ps, params_ps = _get_func_ps(ps_model, verbose=False, **kwargs)
|
|
73
79
|
params_glm = _filter_params(fit_glm, {**kwargs, 'verbose': verbose})
|
|
74
80
|
|
|
75
81
|
if verbose:
|
|
@@ -86,7 +92,8 @@ def cross_fitting(
|
|
|
86
92
|
# Initialize lists to store results
|
|
87
93
|
fit_pi = True if pi_hat is None else False
|
|
88
94
|
pi_hat = np.zeros_like(A, dtype=float) if fit_pi else pi_hat
|
|
89
|
-
|
|
95
|
+
fit_Y = True if Y_hat is None else False
|
|
96
|
+
Y_hat = np.zeros((Y.shape[0],Y.shape[1],A.shape[1],2), dtype=float) if fit_Y else Y_hat
|
|
90
97
|
|
|
91
98
|
# Perform cross-fitting
|
|
92
99
|
for train_index, test_index in folds:
|
|
@@ -103,7 +110,12 @@ def cross_fitting(
|
|
|
103
110
|
pi = np.zeros_like(A_test, dtype=float)
|
|
104
111
|
for j in range(A.shape[1]):
|
|
105
112
|
i_case = (A_train[:,j] == 1.)
|
|
106
|
-
|
|
113
|
+
|
|
114
|
+
if mask is not None:
|
|
115
|
+
i_cells = mask[:, j]
|
|
116
|
+
else:
|
|
117
|
+
i_ctrl = (np.sum(A_train, axis=1) == 0.)
|
|
118
|
+
i_cells = i_ctrl | i_case
|
|
107
119
|
|
|
108
120
|
if ps_model=='logistic' and XA_train.shape[1]==1 and np.all(XA_train==1):
|
|
109
121
|
prob = np.sum(i_case)/np.sum(i_cells)
|
|
@@ -111,17 +123,15 @@ def cross_fitting(
|
|
|
111
123
|
pi[A_train[:,j] == 0., j] = 1 - prob
|
|
112
124
|
else:
|
|
113
125
|
pi[:,j] = func_ps(XA_train[i_cells], A_train[i_cells][:,j], XA_test)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
Y_hat[test_index,:,:,0] = res[1][0]
|
|
124
|
-
Y_hat[test_index,:,:,1] = res[1][1]
|
|
126
|
+
pi_hat[test_index] = pi
|
|
127
|
+
|
|
128
|
+
if fit_Y:
|
|
129
|
+
if verbose: pprint.pprint('Fit outcome models...')
|
|
130
|
+
# Fit GLM on training data and predict on test data
|
|
131
|
+
res = fit_glm(Y_train, X_train, A_train, family=family, alpha=glm_alpha,
|
|
132
|
+
impute=X_test, **params_glm)
|
|
133
|
+
Y_hat[test_index,:,:,0] = res[1][0]
|
|
134
|
+
Y_hat[test_index,:,:,1] = res[1][1]
|
|
125
135
|
|
|
126
136
|
pi_hat = np.clip(pi_hat, 0.01, 0.99)
|
|
127
137
|
Y_hat = np.clip(Y_hat, None, 1e5)
|
|
@@ -5,16 +5,14 @@ from scipy.stats import norm
|
|
|
5
5
|
from statsmodels.stats.multitest import multipletests, fdrcorrection
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
def multiplier_bootstrap(
|
|
8
|
+
def multiplier_bootstrap(eta, B):
|
|
9
9
|
'''
|
|
10
10
|
Multiplier bootstrap for inference.
|
|
11
11
|
|
|
12
12
|
Parameters
|
|
13
13
|
----------
|
|
14
|
-
|
|
15
|
-
[n, p]
|
|
16
|
-
var_est : array-like
|
|
17
|
-
[p,] Variance of the parameter estimates.
|
|
14
|
+
eta : array-like
|
|
15
|
+
[n, p] Influence function values scaled by inversion of standard deviation.
|
|
18
16
|
B : int
|
|
19
17
|
Number of bootstrap samples.
|
|
20
18
|
|
|
@@ -23,17 +21,13 @@ def multiplier_bootstrap(resid, var_est, B):
|
|
|
23
21
|
z_init : array-like
|
|
24
22
|
[B, p] Bootstrap statistics for each hypothesis.
|
|
25
23
|
'''
|
|
26
|
-
n, p =
|
|
24
|
+
n, p = eta.shape
|
|
27
25
|
B = int(B)
|
|
28
26
|
z_init = np.zeros((B,p))
|
|
29
|
-
eta = resid / np.sqrt(n * var_est[None,:])
|
|
30
27
|
for b in range(B):
|
|
31
28
|
g = np.random.normal(size=n)
|
|
32
29
|
z_init[b, :] = np.sum(eta * g[:, None], axis=0)
|
|
33
30
|
|
|
34
|
-
# g = np.random.normal(size=(B,n,1))
|
|
35
|
-
# z_init = np.sum(resid[None, :,:] * g, axis=1) / np.sqrt(n * var_est[None,:])
|
|
36
|
-
|
|
37
31
|
return z_init
|
|
38
32
|
|
|
39
33
|
|
|
@@ -106,9 +100,9 @@ def augmentation(V, tvalues, c):
|
|
|
106
100
|
|
|
107
101
|
|
|
108
102
|
def fdx_control(
|
|
109
|
-
tau_est,
|
|
103
|
+
tau_est, tvalues_init, eta_est, std_est,
|
|
110
104
|
fdx, B, alpha, c,
|
|
111
|
-
min_var=1e-
|
|
105
|
+
min_var=1e-8, min_diff=0.1
|
|
112
106
|
):
|
|
113
107
|
'''
|
|
114
108
|
Perform FDX control.
|
|
@@ -117,12 +111,12 @@ def fdx_control(
|
|
|
117
111
|
----------
|
|
118
112
|
tau_est : array-like
|
|
119
113
|
The estimated causal effect.
|
|
120
|
-
var_est : array-like
|
|
121
|
-
The estimated variance.
|
|
122
114
|
tvalues_init : array-like
|
|
123
115
|
The estimated t-values.
|
|
124
116
|
eta_est : array-like
|
|
125
|
-
The estimated influence function values.
|
|
117
|
+
The estimated influence function values, scaled by inversion of standard deviation.
|
|
118
|
+
std_est : array-like
|
|
119
|
+
The estimated standard deviation.
|
|
126
120
|
fdx : bool
|
|
127
121
|
If True, perform FDX control.
|
|
128
122
|
B : int
|
|
@@ -141,8 +135,8 @@ def fdx_control(
|
|
|
141
135
|
n = eta_est.shape[0]
|
|
142
136
|
|
|
143
137
|
if fdx:
|
|
144
|
-
id_test =
|
|
145
|
-
z_init = multiplier_bootstrap(eta_est
|
|
138
|
+
id_test = std_est >= min_var
|
|
139
|
+
z_init = multiplier_bootstrap(eta_est / std_est[None,:], B)
|
|
146
140
|
z_init[:, ~id_test] = 0.
|
|
147
141
|
tvalues = tvalues_init.copy()
|
|
148
142
|
tvalues[~id_test] = 0.
|
|
@@ -11,7 +11,7 @@ from causarray.utils import reset_random_seeds, pprint, tqdm, comp_size_factor,
|
|
|
11
11
|
def compute_causal_estimand(
|
|
12
12
|
estimand,
|
|
13
13
|
Y, W, A, W_A=None, family='nb', offset=False,
|
|
14
|
-
Y_hat=None, pi_hat=None,
|
|
14
|
+
Y_hat=None, pi_hat=None, mask=None,
|
|
15
15
|
fdx=False, fdx_B=1000, fdx_alpha=0.05, fdx_c=0.1,
|
|
16
16
|
verbose=False, random_state=0, **kwargs):
|
|
17
17
|
'''
|
|
@@ -39,6 +39,10 @@ def compute_causal_estimand(
|
|
|
39
39
|
Predicted outcomes under treatment of shape (n, p, a, 2).
|
|
40
40
|
pi_hat : array, optional
|
|
41
41
|
Predicted propensity scores of shape (n, a).
|
|
42
|
+
mask : array, optional
|
|
43
|
+
Boolean mask of shape (n, a) for the treatment, indicating which samples are used for
|
|
44
|
+
the estimation of the estimand. This does not affect the estimation of pseudo-outcomes
|
|
45
|
+
and propensity scores.
|
|
42
46
|
|
|
43
47
|
fdx : bool
|
|
44
48
|
Whether to use FDX control, P(FDP > c) < alpha.
|
|
@@ -61,19 +65,16 @@ def compute_causal_estimand(
|
|
|
61
65
|
'''
|
|
62
66
|
reset_random_seeds(random_state)
|
|
63
67
|
|
|
64
|
-
|
|
65
|
-
['kwargs_ls_1', 'kwargs_ls_2', 'kwargs_es_1', 'kwargs_es_2', 'c1', 'num_d']
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
+
# check the input data
|
|
68
69
|
if isinstance(Y, pd.DataFrame):
|
|
69
70
|
gene_names = Y.columns
|
|
70
71
|
Y = Y.values
|
|
71
72
|
else:
|
|
72
73
|
gene_names = range(Y.shape[1])
|
|
74
|
+
Y = Y.astype('float')
|
|
73
75
|
n, p = Y.shape
|
|
74
76
|
|
|
75
|
-
if
|
|
76
|
-
A = A.reshape(-1,1)
|
|
77
|
+
if A.ndim == 1: A = A[:, None]
|
|
77
78
|
if isinstance(A, pd.DataFrame):
|
|
78
79
|
trt_names = A.columns
|
|
79
80
|
A = A.values
|
|
@@ -87,7 +88,16 @@ def compute_causal_estimand(
|
|
|
87
88
|
W_A = W
|
|
88
89
|
elif isinstance(W_A, pd.DataFrame):
|
|
89
90
|
W_A = W_A.values
|
|
90
|
-
|
|
91
|
+
|
|
92
|
+
if mask is not None:
|
|
93
|
+
mask = np.array(mask).astype(bool)
|
|
94
|
+
if len(mask.shape) == 1: mask = mask.reshape(-1,1)
|
|
95
|
+
if mask.shape != A.shape:
|
|
96
|
+
raise ValueError('Mask must have the same shape as the treatment matrix')
|
|
97
|
+
|
|
98
|
+
kwargs = {k:v for k,v in kwargs.items() if k not in
|
|
99
|
+
['kwargs_ls_1', 'kwargs_ls_2', 'kwargs_es_1', 'kwargs_es_2', 'c1', 'num_d']
|
|
100
|
+
}
|
|
91
101
|
|
|
92
102
|
if verbose:
|
|
93
103
|
d_A = W_A.shape[1]
|
|
@@ -103,10 +113,9 @@ def compute_causal_estimand(
|
|
|
103
113
|
else:
|
|
104
114
|
offset = None
|
|
105
115
|
size_factors = np.ones(n)
|
|
106
|
-
|
|
107
|
-
Y = Y.astype('float')
|
|
116
|
+
|
|
108
117
|
Y_hat, pi_hat = cross_fitting(Y, A, W, W_A, family=family, offset=offset,
|
|
109
|
-
Y_hat=Y_hat, pi_hat=pi_hat, random_state=random_state, verbose=verbose, **kwargs)
|
|
118
|
+
Y_hat=Y_hat, pi_hat=pi_hat, mask=mask, random_state=random_state, verbose=verbose, **kwargs)
|
|
110
119
|
pi_hat = pi_hat.reshape(*A.shape)
|
|
111
120
|
|
|
112
121
|
if verbose: pprint.pprint('Estimating AIPW mean...')
|
|
@@ -117,25 +126,25 @@ def compute_causal_estimand(
|
|
|
117
126
|
# normalize the influence function values
|
|
118
127
|
etas /= size_factors[:,None,None,None]
|
|
119
128
|
|
|
120
|
-
|
|
121
|
-
i_ctrl = (np.sum(A, axis=1) == 0.)
|
|
122
|
-
|
|
123
129
|
res = []
|
|
124
130
|
iters = range(A.shape[1]) if A.shape[1]==1 else tqdm(range(A.shape[1]))
|
|
125
131
|
for j in iters:
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
132
|
+
if mask is not None:
|
|
133
|
+
i_cells = mask[:, j]
|
|
134
|
+
else:
|
|
135
|
+
i_ctrl = (np.sum(A, axis=1) == 0.)
|
|
136
|
+
i_case = (A[:,j] == 1.)
|
|
137
|
+
i_cells = i_ctrl | i_case
|
|
138
|
+
eta_est, tau_est, var_est = estimand(etas[i_cells,:,j], A[i_cells,j], **kwargs)
|
|
130
139
|
|
|
131
|
-
std_est = np.sqrt(
|
|
140
|
+
std_est = np.sqrt(var_est)
|
|
132
141
|
tvalues_init = tau_est / std_est
|
|
133
142
|
|
|
134
143
|
# Multiple testing procedure
|
|
135
|
-
V = fdx_control(tau_est,
|
|
144
|
+
V = fdx_control(tau_est, tvalues_init, eta_est, std_est, fdx, fdx_B, fdx_alpha, fdx_c)
|
|
136
145
|
|
|
137
146
|
# BH correction
|
|
138
|
-
tvalues_init[np.isinf(
|
|
147
|
+
tvalues_init[np.isinf(std_est)] = np.nan
|
|
139
148
|
pvals, qvals, pvals_adj, qvals_adj = bh_correction(tvalues_init)
|
|
140
149
|
|
|
141
150
|
df_res = pd.DataFrame({
|
|
@@ -159,7 +168,7 @@ def compute_causal_estimand(
|
|
|
159
168
|
|
|
160
169
|
def LFC(
|
|
161
170
|
Y, W, A, W_A=None, family='nb', offset=False,
|
|
162
|
-
Y_hat=None, pi_hat=None, cross_est=False,
|
|
171
|
+
Y_hat=None, pi_hat=None, cross_est=False, mask=None, usevar='pooled',
|
|
163
172
|
thres_min=1e-4, thres_diff=1e-6, eps_var=1e-3,
|
|
164
173
|
fdx=False, fdx_alpha=0.05, fdx_c=0.1,
|
|
165
174
|
verbose=False, **kwargs):
|
|
@@ -187,12 +196,21 @@ def LFC(
|
|
|
187
196
|
Predicted propensity scores of shape (n, a).
|
|
188
197
|
cross_est : bool
|
|
189
198
|
Whether to use cross-estimation.
|
|
199
|
+
mask : array, optional
|
|
200
|
+
Boolean mask of shape (n, a) for the treatment, indicating which samples are used for
|
|
201
|
+
the estimation of the estimand. This does not affect the estimation of pseudo-outcomes
|
|
202
|
+
and propensity scores.
|
|
203
|
+
usevar : str
|
|
204
|
+
The method to use for estimating the variance of treatment effects.
|
|
205
|
+
Options are 'pooled' (default) or 'unequal'.
|
|
190
206
|
|
|
191
207
|
thres_min : float
|
|
192
208
|
The minimum threshold for the treatment effect.
|
|
193
209
|
thres_diff : float
|
|
194
210
|
The minimum threshold for the difference in treatment effect.
|
|
195
|
-
|
|
211
|
+
eps_var : float
|
|
212
|
+
The minimum threshold for the variance of treatment.
|
|
213
|
+
|
|
196
214
|
fdx : bool
|
|
197
215
|
Whether to use FDX control, P(FDP > c) < alpha.
|
|
198
216
|
fdx_alpha : float
|
|
@@ -211,7 +229,7 @@ def LFC(
|
|
|
211
229
|
Dataframe of test results.
|
|
212
230
|
'''
|
|
213
231
|
|
|
214
|
-
def estimand(etas, **kwargs):
|
|
232
|
+
def estimand(etas, A, **kwargs):
|
|
215
233
|
eta_0, eta_1 = etas[..., 0], etas[..., 1]
|
|
216
234
|
tau_0, tau_1 = np.mean(eta_0, axis=0), np.mean(eta_1, axis=0)
|
|
217
235
|
|
|
@@ -219,7 +237,18 @@ def LFC(
|
|
|
219
237
|
tau_0 = np.clip(tau_0, thres_diff, None)
|
|
220
238
|
tau_est = np.log(tau_1/tau_0)
|
|
221
239
|
eta_est = eta_1 / tau_1[None,:] - eta_0 / tau_0[None,:]
|
|
222
|
-
|
|
240
|
+
|
|
241
|
+
if usevar == 'pooled':
|
|
242
|
+
var_est = (np.var(eta_est, axis=0, ddof=1) + eps_var) / eta_est.shape[0]
|
|
243
|
+
elif usevar == 'unequal':
|
|
244
|
+
# Estimate the variance using Welch's t-test
|
|
245
|
+
var_0 = np.var(eta_est[A==0], axis=0, ddof=1)
|
|
246
|
+
var_1 = np.var(eta_est[A==1], axis=0, ddof=1)
|
|
247
|
+
n_0 = np.sum(A==0)
|
|
248
|
+
n_1 = np.sum(A==1)
|
|
249
|
+
var_est = (var_0 + eps_var) / n_0 + (var_1 + eps_var) / n_1
|
|
250
|
+
else:
|
|
251
|
+
raise ValueError('usevar must be either "pooled" or "unequal"')
|
|
223
252
|
|
|
224
253
|
# filter out low-expressed genes
|
|
225
254
|
idx = (np.maximum(tau_0,tau_1)<thres_min) & ((tau_1-tau_0)<thres_diff)
|
|
@@ -229,7 +258,7 @@ def LFC(
|
|
|
229
258
|
|
|
230
259
|
return compute_causal_estimand(
|
|
231
260
|
estimand, Y, W, A, W_A, family, offset,
|
|
232
|
-
Y_hat=Y_hat, pi_hat=pi_hat,
|
|
261
|
+
Y_hat=Y_hat, pi_hat=pi_hat, mask=mask,
|
|
233
262
|
fdx=fdx, fdx_alpha=fdx_alpha, fdx_c=fdx_c, verbose=verbose, **kwargs)
|
|
234
263
|
|
|
235
264
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.4"
|
|
@@ -82,7 +82,8 @@ def fit_gcate(Y, X, A, r, family='nb', disp_glm=None, disp_family=None, offset=T
|
|
|
82
82
|
kwargs : dict
|
|
83
83
|
Additional keyword arguments.
|
|
84
84
|
'''
|
|
85
|
-
|
|
85
|
+
if X.ndim == 1: X = X[:, None]
|
|
86
|
+
if A.ndim == 1: A = A[:, None]
|
|
86
87
|
X = np.hstack((X, A))
|
|
87
88
|
a = A.shape[1]
|
|
88
89
|
Y, kwargs_glm, lam1 = _check_input(Y, X, family, disp_glm, disp_family, offset, c1, **kwargs)
|
|
@@ -195,8 +196,10 @@ def estimate_r(Y, X, A, r_max, c=1.,
|
|
|
195
196
|
df_r : DataFrame
|
|
196
197
|
Results of the number of latent factors.
|
|
197
198
|
'''
|
|
198
|
-
X =
|
|
199
|
+
if X.ndim == 1: X = X[:, None]
|
|
200
|
+
if A.ndim == 1: A = A[:, None]
|
|
199
201
|
a, d = A.shape[1], X.shape[1]
|
|
202
|
+
X = np.hstack((X, A))
|
|
200
203
|
n, p = Y.shape
|
|
201
204
|
|
|
202
205
|
Y, kwargs_glm, _ = _check_input(Y, X, family, disp_glm, disp_family, offset, None, **kwargs)
|
|
@@ -210,22 +213,31 @@ def estimate_r(Y, X, A, r_max, c=1.,
|
|
|
210
213
|
r_list = np.arange(1, int(r_max)+1)
|
|
211
214
|
else:
|
|
212
215
|
r_list = np.array(r_max, dtype=int)
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
216
|
+
r_max = np.max(r_list)
|
|
217
|
+
|
|
218
|
+
# Estimate the residual deviance
|
|
219
|
+
res_glm = fit_glm(Y, X, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100, verbose=False)
|
|
220
|
+
u, s, vt = svds(res_glm[-1], k=r_max)
|
|
221
|
+
if u.shape[1]<r_max:
|
|
222
|
+
raise ValueError(f'The number of latent factors is larger than the rank of deviance residuals ({u.shape[1]}). Try to decrease the value of r.')
|
|
223
|
+
Q, _ = sp.linalg.qr(X, mode='economic')
|
|
224
|
+
P1 = np.identity(n) - Q @ Q.T
|
|
225
|
+
P1 = P1.astype(type_f)
|
|
226
|
+
A1 = np.c_[X, P1 @ u]
|
|
227
|
+
|
|
228
|
+
logh = log_h(Y, family, nuisance)
|
|
229
|
+
ll = 2 * (
|
|
230
|
+
nll(Y, X, res_glm[0], family, nuisance, size_factor) / p
|
|
231
|
+
- np.sum(logh) / (n*p) )
|
|
232
|
+
nu = (d+a) * np.maximum(n,p) * np.log(n * p / np.maximum(n,p)) / (n*p)
|
|
233
|
+
jic = ll + c * nu
|
|
234
|
+
res.append([0, ll, nu, jic])
|
|
235
|
+
|
|
236
|
+
for r in r_list[::-1]:
|
|
237
|
+
_, res_2 = estimate(Y, X, r, a,
|
|
238
|
+
0, kwargs_glm, kwargs_ls_1, kwargs_es_1, kwargs_ls_2, kwargs_es_2, A=A1[:,:d+a+r], **kwargs)
|
|
239
|
+
A1, A2 = res_2['X_U'], res_2['B_Gamma']
|
|
240
|
+
|
|
229
241
|
ll = 2 * (
|
|
230
242
|
nll(Y, A1, A2, family, nuisance, size_factor) / p
|
|
231
243
|
- np.sum(logh) / (n*p) )
|
|
@@ -233,5 +245,5 @@ def estimate_r(Y, X, A, r_max, c=1.,
|
|
|
233
245
|
jic = ll + c * nu
|
|
234
246
|
res.append([r, ll, nu, jic])
|
|
235
247
|
|
|
236
|
-
df_r = pd.DataFrame(res, columns=['r', 'deviance', 'nu', 'JIC'])
|
|
248
|
+
df_r = pd.DataFrame(res, columns=['r', 'deviance', 'nu', 'JIC']).sort_values(by='r')
|
|
237
249
|
return df_r
|
|
@@ -41,15 +41,29 @@ def fit_glm(Y, X, A=None, family='gaussian', disp_family='poisson',
|
|
|
41
41
|
A : array
|
|
42
42
|
n x 1 vector of treatments or None
|
|
43
43
|
family : str
|
|
44
|
-
|
|
44
|
+
Family of GLM to fit, can be one of: 'gaussian', 'poisson', 'nb'
|
|
45
45
|
disp_glm : array or None
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
impute : bool
|
|
50
|
-
whether to impute potential outcomes and get predicted values
|
|
46
|
+
Dispersion parameter for negative binomial GLM.
|
|
47
|
+
impute : bool or None
|
|
48
|
+
Whether to impute missing values in Y.
|
|
51
49
|
offset : bool
|
|
52
|
-
|
|
50
|
+
Whether to use log of sum of Y as offset.
|
|
51
|
+
shrinkage : bool
|
|
52
|
+
Whether to use regularized GLM.
|
|
53
|
+
alpha : float
|
|
54
|
+
Regularization parameter for regularized GLM.
|
|
55
|
+
maxiter : int
|
|
56
|
+
Maximum number of iterations for GLM fitting.
|
|
57
|
+
thres_disp : float
|
|
58
|
+
Threshold for dispersion parameter for negative binomial GLM.
|
|
59
|
+
n_jobs : int
|
|
60
|
+
Number of jobs to run in parallel.
|
|
61
|
+
random_state : int
|
|
62
|
+
Random seed for reproducibility.
|
|
63
|
+
verbose : bool
|
|
64
|
+
Whether to print progress messages.
|
|
65
|
+
kwargs : dict
|
|
66
|
+
Additional arguments to pass to GLM fitting.
|
|
53
67
|
|
|
54
68
|
Returns
|
|
55
69
|
-------
|
|
@@ -156,8 +170,7 @@ def fit_glm(Y, X, A=None, family='gaussian', disp_family='poisson',
|
|
|
156
170
|
|
|
157
171
|
|
|
158
172
|
results = Parallel(n_jobs=n_jobs)(delayed(fit_model)(
|
|
159
|
-
j, Y, X, offsets, family, disp_glm, impute, alpha) for j in tqdm(range(Y.shape[1])))
|
|
160
|
-
pprint.pprint('Fitting GLM done.')
|
|
173
|
+
j, Y, X, offsets, family, disp_glm, impute, alpha) for j in tqdm(range(Y.shape[1]), disable=not verbose))
|
|
161
174
|
if verbose: pprint.pprint('Fitting GLM done.')
|
|
162
175
|
|
|
163
176
|
B, Yhat_0, Yhat_1, resid_deviance = zip(*results)
|
|
@@ -208,19 +208,24 @@ def alter_min(
|
|
|
208
208
|
a = d
|
|
209
209
|
|
|
210
210
|
# initialization for Theta = A @ B^T
|
|
211
|
-
if A is None
|
|
211
|
+
if A is None:
|
|
212
212
|
if verbose:
|
|
213
213
|
pprint.pprint('Estimating initial latent variables with GLMs...')
|
|
214
|
-
res_glm = fit_glm(Y, X, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100)
|
|
214
|
+
res_glm = fit_glm(Y, X, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100, verbose=verbose)
|
|
215
215
|
u, s, vt = svds(res_glm[-1], k=r)
|
|
216
216
|
|
|
217
217
|
if u.shape[1]<r:
|
|
218
218
|
raise ValueError(f'The number of latent factors is larger than the rank of deviance residuals ({u.shape[1]}). Try to decrease the value of r.')
|
|
219
|
+
|
|
220
|
+
A = np.c_[X, P1 @ u]
|
|
221
|
+
else:
|
|
222
|
+
assert A.shape[1] == d+r
|
|
219
223
|
|
|
224
|
+
if B is None:
|
|
220
225
|
if verbose:
|
|
221
226
|
pprint.pprint('Estimating initial coefficients with GLMs...')
|
|
222
|
-
|
|
223
|
-
B = fit_glm(Y, A, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100)[0]
|
|
227
|
+
|
|
228
|
+
B = fit_glm(Y, A, offset=np.log(size_factor[:,0]), family=family, disp_glm=nuisance[0], maxiter=100, verbose=verbose)[0]
|
|
224
229
|
|
|
225
230
|
E = A[:, -r:] @ B[:, -r:].T
|
|
226
231
|
u, s, vh = sp.sparse.linalg.svds(E, k=r)
|
|
@@ -228,19 +233,10 @@ def alter_min(
|
|
|
228
233
|
B[:, d:] = vh.T * s[None,:]**(1/2)
|
|
229
234
|
del E, u, s, vh
|
|
230
235
|
|
|
231
|
-
# if offset==1:
|
|
232
|
-
# scale = np.sqrt(np.median(np.abs(X[:,0])))
|
|
233
|
-
# B[:, :offset] = scale
|
|
234
|
-
# A[:, :offset] /= scale
|
|
235
236
|
|
|
236
237
|
if P2 is not None:
|
|
237
238
|
P2 = P2.astype(type_f)
|
|
238
|
-
# E = A[:,d-a:] @ B[:,d-a:].T @ (np.identity(p) - P2)
|
|
239
|
-
# u, s, vh = sp.sparse.linalg.svds(E, k=r)
|
|
240
239
|
B[:, d-a:d] = P2 @ B[:, d-a:d]
|
|
241
|
-
# A[:, d:] = u * s[None,:]**(1/2)
|
|
242
|
-
# B[:, d:] = vh.T * s[None,:]**(1/2)
|
|
243
|
-
# del E, u, s, vh
|
|
244
240
|
|
|
245
241
|
|
|
246
242
|
Y = Y.astype(type_f)
|
|
@@ -265,10 +261,10 @@ def alter_min(
|
|
|
265
261
|
kwargs_ls['alpha'] = kwargs_ls['alpha']
|
|
266
262
|
if verbose:
|
|
267
263
|
pprint.pprint({'kwargs_glm':kwargs_glm,'kwargs_ls':kwargs_ls,'kwargs_es':kwargs_es}, compact=True)
|
|
268
|
-
|
|
264
|
+
pprint.pprint(f'Fitting GCATE (step {1 if P1 is None else 2})...')
|
|
269
265
|
hist = [func_val_pre]
|
|
270
266
|
es = Early_Stopping(**kwargs_es)
|
|
271
|
-
with tqdm(np.arange(kwargs_es['max_iters'])) as pbar:
|
|
267
|
+
with tqdm(np.arange(kwargs_es['max_iters']), disable=not verbose) as pbar:
|
|
272
268
|
for t in pbar:
|
|
273
269
|
func_val, A, B = update(
|
|
274
270
|
Y, A, B, d, weights, P1, P2,
|
|
@@ -281,7 +277,7 @@ def alter_min(
|
|
|
281
277
|
pprint.pprint('Encountered large or infinity values. Try to decrease the value of C for the norm constraints.')
|
|
282
278
|
break
|
|
283
279
|
elif es(func_val):
|
|
284
|
-
pbar.set_postfix_str('Early stopped.' + es.info)
|
|
280
|
+
pbar.set_postfix_str('Early stopped. ' + es.info)
|
|
285
281
|
pbar.close()
|
|
286
282
|
break
|
|
287
283
|
else:
|
|
@@ -49,16 +49,19 @@ def prep_causarray_data(Y, A, X=None, X_A=None, intercept=True):
|
|
|
49
49
|
Y = np.minimum(Y, np.round(np.quantile(np.max(Y, 0), 0.999)))
|
|
50
50
|
if not isinstance(A, pd.DataFrame):
|
|
51
51
|
A = np.asarray(A)
|
|
52
|
+
if A.ndim == 1:
|
|
53
|
+
A = A[:, None]
|
|
52
54
|
|
|
53
|
-
X = np.zeros((Y.shape[0], 0)) if X is None else np.asarray(X)
|
|
54
|
-
intercept_col = np.ones((X.shape[0], 1)) if intercept else np.empty((X.shape[0], 0))
|
|
55
|
-
X = np.hstack((intercept_col, X))
|
|
56
|
-
|
|
55
|
+
X = np.zeros((Y.shape[0], 0)) if X is None else np.asarray(X)
|
|
57
56
|
X_A = X if X_A is None else np.asarray(X_A)
|
|
58
57
|
loglibsize = np.log2(np.sum(np.asarray(Y), axis=1))
|
|
59
58
|
loglibsize = (loglibsize - np.mean(loglibsize)) / np.std(loglibsize, ddof=1)
|
|
60
59
|
X_A = np.hstack((X_A, loglibsize[:, None]))
|
|
61
60
|
|
|
61
|
+
intercept_col = np.ones((X.shape[0], 1)) if intercept else np.empty((X.shape[0], 0))
|
|
62
|
+
X = np.hstack((intercept_col, X))
|
|
63
|
+
X_A = np.hstack((intercept_col, X_A))
|
|
64
|
+
|
|
62
65
|
return Y, A, X, X_A
|
|
63
66
|
|
|
64
67
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.0.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|