causarray 0.0.7__tar.gz → 0.0.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {causarray-0.0.7 → causarray-0.0.9}/.gitignore +2 -0
- {causarray-0.0.7 → causarray-0.0.9}/PKG-INFO +35 -2
- {causarray-0.0.7 → causarray-0.0.9}/README.md +34 -1
- {causarray-0.0.7 → causarray-0.0.9}/causarray/DR_estimation.py +294 -3
- {causarray-0.0.7 → causarray-0.0.9}/causarray/DR_learner.py +65 -23
- causarray-0.0.9/causarray/__about__.py +1 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/__init__.py +9 -3
- causarray-0.0.9/causarray/diagnostics.py +628 -0
- causarray-0.0.7/causarray/__about__.py +0 -1
- causarray-0.0.7/causarray/diagnostics.py +0 -172
- {causarray-0.0.7 → causarray-0.0.9}/LICENSE +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/DR_inference.py +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/gcate.py +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/gcate_glm.py +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/gcate_likelihood.py +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/gcate_opt.py +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/nb_glm_fast.py +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/causarray/utils.py +0 -0
- {causarray-0.0.7 → causarray-0.0.9}/pyproject.toml +0 -0
|
@@ -184,6 +184,8 @@ docs/source/tutorial/replogle/replogle_subset.h5ad
|
|
|
184
184
|
docs/source/tutorial/replogle/replogle_subset_norm*.h5ad
|
|
185
185
|
docs/source/tutorial/replogle/replogle_normed.h5ad
|
|
186
186
|
docs/source/tutorial/replogle/replogle_results.h5
|
|
187
|
+
docs/source/tutorial/replogle/replogle_supt5h_go_*.csv
|
|
188
|
+
docs/source/tutorial/replogle/replogle_propensity_batch12*
|
|
187
189
|
|
|
188
190
|
# Unrelated tutorial directory (not part of causarray package docs)
|
|
189
191
|
docs/source/tutorial/scp/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: causarray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.9
|
|
4
4
|
Summary: causarray is a Python module for simultaneous causal inference with an array of outcomes.
|
|
5
5
|
Author-email: Jin-Hong Du <jinhongd@hku.hk>, Maya Shen <myshen@andrew.cmu.edu>, Hansruedi Mathys <mathysh@pitt.edu>, Kathryn Roeder <jinhongd@hku.hk>
|
|
6
6
|
Maintainer-email: Jin-Hong Du <jinhongd@hku.hk>
|
|
@@ -54,7 +54,16 @@ For optimal parallel performance, we recommend installing `llvm-openmp` if using
|
|
|
54
54
|
conda install -c conda-forge llvm-openmp
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
-
For `R` users, `reticulate` can be used to call `causarray` from
|
|
57
|
+
For `R` users, `reticulate` can be used to call `causarray` from R while
|
|
58
|
+
keeping NumPy >2 in the Python environment. Create the separate R environment
|
|
59
|
+
with a current NumPy-2-compatible reticulate build:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
conda env create -f environment-r.yaml
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The R tutorial runs from `causarray-r` and connects to the Python package in
|
|
66
|
+
the `causarray` environment.
|
|
58
67
|
The documentation and tutorials using both `Python` and `R` are available at [causarray.readthedocs.io](https://causarray.readthedocs.io/en/latest/).
|
|
59
68
|
|
|
60
69
|
|
|
@@ -89,6 +98,30 @@ df_res = gcate_lfc_batch(
|
|
|
89
98
|
See the [Replogle-E-K562 tutorial](https://causarray.readthedocs.io/en/latest/tutorial/replogle/replogle-py.html)
|
|
90
99
|
for a demonstration on 200 perturbations from a genome-wide CRISPRi screen.
|
|
91
100
|
|
|
101
|
+
### Diagnostic masks without refitting effects
|
|
102
|
+
|
|
103
|
+
Treatment-by-gene support or quality-control rules can be aligned to an
|
|
104
|
+
existing causarray result table by label, even when its rows are reordered:
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from causarray import align_test_mask
|
|
108
|
+
|
|
109
|
+
keep = align_test_mask(
|
|
110
|
+
df_res,
|
|
111
|
+
support_mask, # treatments × genes, Boolean
|
|
112
|
+
treatment_names=perturbations,
|
|
113
|
+
gene_names=genes,
|
|
114
|
+
)
|
|
115
|
+
df_res_flagged = df_res.assign(support_keep=keep)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
This operation only annotates or subsets existing results. It does not refit
|
|
119
|
+
the causarray LFC or change standard errors and p-values. If a diagnostic rule
|
|
120
|
+
was selected after inspecting the outcomes, retain the original adjusted
|
|
121
|
+
p-values rather than redefining the multiple-testing family post hoc. The
|
|
122
|
+
[Replogle tutorial](https://causarray.readthedocs.io/en/latest/tutorial/replogle/replogle-py.html)
|
|
123
|
+
compares several expression-support rules with marginal Wilcoxon results.
|
|
124
|
+
|
|
92
125
|
## Changelog
|
|
93
126
|
|
|
94
127
|
See [CHANGELOG](https://causarray.readthedocs.io/en/latest/changelog.html) for a full version history.
|
|
@@ -29,7 +29,16 @@ For optimal parallel performance, we recommend installing `llvm-openmp` if using
|
|
|
29
29
|
conda install -c conda-forge llvm-openmp
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
-
For `R` users, `reticulate` can be used to call `causarray` from
|
|
32
|
+
For `R` users, `reticulate` can be used to call `causarray` from R while
|
|
33
|
+
keeping NumPy >2 in the Python environment. Create the separate R environment
|
|
34
|
+
with a current NumPy-2-compatible reticulate build:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
conda env create -f environment-r.yaml
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The R tutorial runs from `causarray-r` and connects to the Python package in
|
|
41
|
+
the `causarray` environment.
|
|
33
42
|
The documentation and tutorials using both `Python` and `R` are available at [causarray.readthedocs.io](https://causarray.readthedocs.io/en/latest/).
|
|
34
43
|
|
|
35
44
|
|
|
@@ -64,6 +73,30 @@ df_res = gcate_lfc_batch(
|
|
|
64
73
|
See the [Replogle-E-K562 tutorial](https://causarray.readthedocs.io/en/latest/tutorial/replogle/replogle-py.html)
|
|
65
74
|
for a demonstration on 200 perturbations from a genome-wide CRISPRi screen.
|
|
66
75
|
|
|
76
|
+
### Diagnostic masks without refitting effects
|
|
77
|
+
|
|
78
|
+
Treatment-by-gene support or quality-control rules can be aligned to an
|
|
79
|
+
existing causarray result table by label, even when its rows are reordered:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from causarray import align_test_mask
|
|
83
|
+
|
|
84
|
+
keep = align_test_mask(
|
|
85
|
+
df_res,
|
|
86
|
+
support_mask, # treatments × genes, Boolean
|
|
87
|
+
treatment_names=perturbations,
|
|
88
|
+
gene_names=genes,
|
|
89
|
+
)
|
|
90
|
+
df_res_flagged = df_res.assign(support_keep=keep)
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
This operation only annotates or subsets existing results. It does not refit
|
|
94
|
+
the causarray LFC or change standard errors and p-values. If a diagnostic rule
|
|
95
|
+
was selected after inspecting the outcomes, retain the original adjusted
|
|
96
|
+
p-values rather than redefining the multiple-testing family post hoc. The
|
|
97
|
+
[Replogle tutorial](https://causarray.readthedocs.io/en/latest/tutorial/replogle/replogle-py.html)
|
|
98
|
+
compares several expression-support rules with marginal Wilcoxon results.
|
|
99
|
+
|
|
67
100
|
## Changelog
|
|
68
101
|
|
|
69
102
|
See [CHANGELOG](https://causarray.readthedocs.io/en/latest/changelog.html) for a full version history.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
2
3
|
from sklearn.linear_model import LogisticRegression
|
|
3
4
|
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
|
4
5
|
from sklearn_ensemble_cv import reset_random_seeds, Ensemble, ECV
|
|
@@ -10,6 +11,7 @@ from joblib import Parallel, delayed
|
|
|
10
11
|
from tqdm import tqdm
|
|
11
12
|
import pprint
|
|
12
13
|
import warnings
|
|
14
|
+
from collections.abc import Mapping
|
|
13
15
|
|
|
14
16
|
from sklearn.model_selection import KFold, ShuffleSplit
|
|
15
17
|
|
|
@@ -40,6 +42,27 @@ def _get_func_ps(ps_model, **kwargs):
|
|
|
40
42
|
return func_ps, params_ps
|
|
41
43
|
|
|
42
44
|
|
|
45
|
+
def _validate_clip(clip):
|
|
46
|
+
"""Validate a propensity clipping bound and return it as ``(lower, upper)``.
|
|
47
|
+
|
|
48
|
+
Raises
|
|
49
|
+
------
|
|
50
|
+
ValueError
|
|
51
|
+
If ``clip`` is not a pair of numbers satisfying
|
|
52
|
+
``0 <= lower < upper <= 1``.
|
|
53
|
+
"""
|
|
54
|
+
message = 'clip must be None or a pair 0 <= lower < upper <= 1'
|
|
55
|
+
if isinstance(clip, (str, bytes)) or np.ndim(clip) != 1:
|
|
56
|
+
raise ValueError(message)
|
|
57
|
+
try:
|
|
58
|
+
lower, upper = (float(bound) for bound in clip)
|
|
59
|
+
except (TypeError, ValueError):
|
|
60
|
+
raise ValueError(message) from None
|
|
61
|
+
if not 0 <= lower < upper <= 1:
|
|
62
|
+
raise ValueError(message)
|
|
63
|
+
return lower, upper
|
|
64
|
+
|
|
65
|
+
|
|
43
66
|
def estimate_propensity_scores(
|
|
44
67
|
A, X_A, K=1, ps_model='logistic', mask=None, clip=None,
|
|
45
68
|
random_state=0, verbose=False, class_weight='balanced', **kwargs,
|
|
@@ -172,12 +195,280 @@ def estimate_propensity_scores(
|
|
|
172
195
|
pi_hat[test_index, j] = func_ps(x_train, y_train, XA_test)
|
|
173
196
|
|
|
174
197
|
if clip is not None:
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
pi_hat = np.clip(pi_hat, clip[0], clip[1])
|
|
198
|
+
lower, upper = _validate_clip(clip)
|
|
199
|
+
pi_hat = np.clip(pi_hat, lower, upper)
|
|
178
200
|
return pi_hat
|
|
179
201
|
|
|
180
202
|
|
|
203
|
+
def refit_propensity_scores(
|
|
204
|
+
A, X_A, drop_by_treatment=None, pi_hat=None, treatment_names=None,
|
|
205
|
+
covariate_names=None, penalty_factors_by_treatment=None, K=1,
|
|
206
|
+
ps_model='logistic', mask=None, clip=None, random_state=0, verbose=False,
|
|
207
|
+
class_weight='balanced', **kwargs,
|
|
208
|
+
):
|
|
209
|
+
"""Refit propensity scores with treatment-specific covariate filtering.
|
|
210
|
+
|
|
211
|
+
This helper supports sensitivity analyses in which different treatments
|
|
212
|
+
omit different observed covariates or latent factors, or apply stronger L2
|
|
213
|
+
regularization to selected covariates. When existing scores are supplied,
|
|
214
|
+
only treatments named in either treatment-specific mapping are refitted;
|
|
215
|
+
the remaining columns are carried over from ``pi_hat`` unchanged, up to the
|
|
216
|
+
shared ``clip`` described below. Outcome models are not fit by this function
|
|
217
|
+
and cached ``Y_hat`` values can be reused in :func:`LFC`.
|
|
218
|
+
|
|
219
|
+
Parameters
|
|
220
|
+
----------
|
|
221
|
+
A : array-like, shape (n, a)
|
|
222
|
+
Binary treatment indicator matrix.
|
|
223
|
+
X_A : array-like, shape (n, d_A)
|
|
224
|
+
Full propensity-model design before treatment-specific filtering.
|
|
225
|
+
drop_by_treatment : mapping or None
|
|
226
|
+
Treatment names or indices mapped to covariate names or indices to
|
|
227
|
+
remove for that treatment. Defaults to no removals.
|
|
228
|
+
pi_hat : array-like or None, shape (n, a)
|
|
229
|
+
Existing raw propensity scores. If supplied, treatments absent from
|
|
230
|
+
``drop_by_treatment`` are not refitted.
|
|
231
|
+
treatment_names, covariate_names : sequence, optional
|
|
232
|
+
Column labels, inferred from DataFrames when possible.
|
|
233
|
+
penalty_factors_by_treatment : mapping or None
|
|
234
|
+
Treatment names or indices mapped to ``{covariate: factor}`` mappings.
|
|
235
|
+
A factor greater than one applies that multiple of the ordinary L2
|
|
236
|
+
penalty to the named coefficient. This is implemented by dividing the
|
|
237
|
+
covariate by ``sqrt(factor)`` during both fitting and prediction and is
|
|
238
|
+
available only for ``ps_model='logistic'`` with an L2 penalty. A factor
|
|
239
|
+
of one leaves the covariate unchanged. Interpret relative penalties on
|
|
240
|
+
a common scale; ``prep_causarray_data`` standardizes log-library size.
|
|
241
|
+
K, ps_model, mask, random_state, verbose, class_weight, **kwargs
|
|
242
|
+
Passed to :func:`estimate_propensity_scores` for each refitted model.
|
|
243
|
+
clip : tuple(float, float) or None, optional
|
|
244
|
+
Bounds applied to the **whole** returned matrix, refitted columns and
|
|
245
|
+
carried-over columns alike, so that a single consistent bound reaches
|
|
246
|
+
:func:`LFC`. Pass ``None`` to leave carried-over scores exactly as
|
|
247
|
+
supplied.
|
|
248
|
+
|
|
249
|
+
Returns
|
|
250
|
+
-------
|
|
251
|
+
pi_updated : ndarray, shape (n, a)
|
|
252
|
+
Updated propensity scores.
|
|
253
|
+
report : DataFrame
|
|
254
|
+
Audit table of refitted treatments, retained/dropped covariates, and
|
|
255
|
+
the resulting score spread. ``degenerate_design`` flags a treatment
|
|
256
|
+
whose retained design is constant, and ``score_std`` reports the
|
|
257
|
+
standard deviation of its refitted scores on eligible rows; both make a
|
|
258
|
+
collapsed propensity model visible without reading the warning stream.
|
|
259
|
+
|
|
260
|
+
Warns
|
|
261
|
+
-----
|
|
262
|
+
RuntimeWarning
|
|
263
|
+
If filtering leaves a constant design for some treatment, in which case
|
|
264
|
+
its scores carry no covariate information.
|
|
265
|
+
|
|
266
|
+
Notes
|
|
267
|
+
-----
|
|
268
|
+
A very large penalty factor shrinks a coefficient towards zero without
|
|
269
|
+
making the design constant. That case leaves ``degenerate_design`` False
|
|
270
|
+
but drives ``score_std`` towards zero.
|
|
271
|
+
|
|
272
|
+
.. versionadded:: 0.0.9
|
|
273
|
+
"""
|
|
274
|
+
if drop_by_treatment is None:
|
|
275
|
+
drop_by_treatment = {}
|
|
276
|
+
if penalty_factors_by_treatment is None:
|
|
277
|
+
penalty_factors_by_treatment = {}
|
|
278
|
+
if not isinstance(drop_by_treatment, Mapping):
|
|
279
|
+
raise ValueError('drop_by_treatment must be a mapping')
|
|
280
|
+
if not isinstance(penalty_factors_by_treatment, Mapping):
|
|
281
|
+
raise ValueError('penalty_factors_by_treatment must be a mapping')
|
|
282
|
+
if penalty_factors_by_treatment:
|
|
283
|
+
if ps_model != 'logistic':
|
|
284
|
+
raise ValueError(
|
|
285
|
+
'penalty_factors_by_treatment is supported only for logistic models'
|
|
286
|
+
)
|
|
287
|
+
if kwargs.get('penalty', 'l2') != 'l2':
|
|
288
|
+
raise ValueError(
|
|
289
|
+
'penalty_factors_by_treatment requires logistic penalty="l2"'
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
if isinstance(A, pd.DataFrame):
|
|
293
|
+
if treatment_names is None:
|
|
294
|
+
treatment_names = list(A.columns)
|
|
295
|
+
A_array = A.to_numpy()
|
|
296
|
+
else:
|
|
297
|
+
A_array = np.asarray(A)
|
|
298
|
+
if A_array.ndim == 1:
|
|
299
|
+
A_array = A_array[:, None]
|
|
300
|
+
if A_array.ndim != 2 or not np.all(np.isin(A_array, (0, 1))):
|
|
301
|
+
raise ValueError('A must be a one- or two-dimensional binary matrix')
|
|
302
|
+
if treatment_names is None:
|
|
303
|
+
treatment_names = list(range(A_array.shape[1]))
|
|
304
|
+
treatment_names = list(treatment_names)
|
|
305
|
+
if len(treatment_names) != A_array.shape[1]:
|
|
306
|
+
raise ValueError('treatment_names must match the number of treatments')
|
|
307
|
+
if len(set(treatment_names)) != len(treatment_names):
|
|
308
|
+
raise ValueError('treatment_names must be unique')
|
|
309
|
+
|
|
310
|
+
if isinstance(X_A, pd.DataFrame):
|
|
311
|
+
if covariate_names is None:
|
|
312
|
+
covariate_names = list(X_A.columns)
|
|
313
|
+
X_array = X_A.to_numpy()
|
|
314
|
+
else:
|
|
315
|
+
X_array = np.asarray(X_A)
|
|
316
|
+
if X_array.ndim != 2 or X_array.shape[0] != A_array.shape[0]:
|
|
317
|
+
raise ValueError('X_A must be two-dimensional with the same rows as A')
|
|
318
|
+
try:
|
|
319
|
+
X_array = np.asarray(X_array, dtype=float)
|
|
320
|
+
except (TypeError, ValueError) as exc:
|
|
321
|
+
raise ValueError('X_A must contain numeric covariates') from exc
|
|
322
|
+
if not np.all(np.isfinite(X_array)):
|
|
323
|
+
raise ValueError('X_A must contain only finite values')
|
|
324
|
+
if covariate_names is None:
|
|
325
|
+
covariate_names = [f'covariate_{j + 1}' for j in range(X_array.shape[1])]
|
|
326
|
+
covariate_names = list(covariate_names)
|
|
327
|
+
if len(covariate_names) != X_array.shape[1]:
|
|
328
|
+
raise ValueError('covariate_names must match the number of covariates')
|
|
329
|
+
if len(set(covariate_names)) != len(covariate_names):
|
|
330
|
+
raise ValueError('covariate_names must be unique')
|
|
331
|
+
|
|
332
|
+
treatment_lookup = {name: j for j, name in enumerate(treatment_names)}
|
|
333
|
+
covariate_lookup = {name: j for j, name in enumerate(covariate_names)}
|
|
334
|
+
|
|
335
|
+
def resolve_treatment(value):
|
|
336
|
+
if value in treatment_lookup:
|
|
337
|
+
return treatment_lookup[value]
|
|
338
|
+
if isinstance(value, (int, np.integer)) and 0 <= int(value) < A_array.shape[1]:
|
|
339
|
+
return int(value)
|
|
340
|
+
raise ValueError(f'Unknown treatment: {value}')
|
|
341
|
+
|
|
342
|
+
def resolve_covariate(value):
|
|
343
|
+
if value in covariate_lookup:
|
|
344
|
+
return covariate_lookup[value]
|
|
345
|
+
if isinstance(value, (int, np.integer)) and 0 <= int(value) < X_array.shape[1]:
|
|
346
|
+
return int(value)
|
|
347
|
+
raise ValueError(f'Unknown covariate: {value}')
|
|
348
|
+
|
|
349
|
+
drops = {}
|
|
350
|
+
for treatment, covariates in drop_by_treatment.items():
|
|
351
|
+
j = resolve_treatment(treatment)
|
|
352
|
+
if j in drops:
|
|
353
|
+
raise ValueError(f'Treatment {treatment_names[j]} is specified more than once')
|
|
354
|
+
if isinstance(covariates, (str, bytes)):
|
|
355
|
+
covariates = [covariates]
|
|
356
|
+
indices = [resolve_covariate(value) for value in covariates]
|
|
357
|
+
if len(set(indices)) != len(indices):
|
|
358
|
+
raise ValueError(
|
|
359
|
+
f'Duplicate dropped covariates for treatment {treatment_names[j]}'
|
|
360
|
+
)
|
|
361
|
+
drops[j] = set(indices)
|
|
362
|
+
|
|
363
|
+
penalty_factors = {}
|
|
364
|
+
for treatment, factors in penalty_factors_by_treatment.items():
|
|
365
|
+
j = resolve_treatment(treatment)
|
|
366
|
+
if j in penalty_factors:
|
|
367
|
+
raise ValueError(f'Treatment {treatment_names[j]} is specified more than once')
|
|
368
|
+
if not isinstance(factors, Mapping):
|
|
369
|
+
raise ValueError(
|
|
370
|
+
f'Penalty factors for treatment {treatment_names[j]} must be a mapping'
|
|
371
|
+
)
|
|
372
|
+
resolved = {}
|
|
373
|
+
for covariate, factor in factors.items():
|
|
374
|
+
k = resolve_covariate(covariate)
|
|
375
|
+
try:
|
|
376
|
+
factor = float(factor)
|
|
377
|
+
except (TypeError, ValueError) as exc:
|
|
378
|
+
raise ValueError('Penalty factors must be finite numbers at least 1') from exc
|
|
379
|
+
if not np.isfinite(factor) or factor < 1:
|
|
380
|
+
raise ValueError('Penalty factors must be finite numbers at least 1')
|
|
381
|
+
if k in resolved:
|
|
382
|
+
raise ValueError(
|
|
383
|
+
f'Duplicate penalty factors for covariate {covariate_names[k]}'
|
|
384
|
+
)
|
|
385
|
+
resolved[k] = factor
|
|
386
|
+
penalty_factors[j] = resolved
|
|
387
|
+
|
|
388
|
+
for j in set(drops).intersection(penalty_factors):
|
|
389
|
+
conflict = drops[j].intersection(penalty_factors[j])
|
|
390
|
+
if conflict:
|
|
391
|
+
names = [covariate_names[k] for k in sorted(conflict)]
|
|
392
|
+
raise ValueError(
|
|
393
|
+
f'Covariates cannot be both dropped and penalized for '
|
|
394
|
+
f'{treatment_names[j]}: {names}'
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
if pi_hat is None:
|
|
398
|
+
pi_updated = np.empty(A_array.shape, dtype=float)
|
|
399
|
+
refit_indices = range(A_array.shape[1])
|
|
400
|
+
else:
|
|
401
|
+
pi_updated = np.asarray(pi_hat, dtype=float).copy()
|
|
402
|
+
if pi_updated.shape != A_array.shape:
|
|
403
|
+
raise ValueError('pi_hat must have the same shape as A')
|
|
404
|
+
if (not np.all(np.isfinite(pi_updated))
|
|
405
|
+
or np.any((pi_updated < 0) | (pi_updated > 1))):
|
|
406
|
+
raise ValueError('pi_hat must contain finite probabilities in [0, 1]')
|
|
407
|
+
refit_indices = sorted(set(drops).union(penalty_factors))
|
|
408
|
+
|
|
409
|
+
mask_array = None
|
|
410
|
+
if mask is not None:
|
|
411
|
+
mask_array = np.asarray(mask, dtype=bool)
|
|
412
|
+
if mask_array.ndim == 1:
|
|
413
|
+
mask_array = mask_array[:, None]
|
|
414
|
+
if mask_array.shape != A_array.shape:
|
|
415
|
+
raise ValueError('Mask must have the same shape as the treatment matrix')
|
|
416
|
+
|
|
417
|
+
ctrl = np.sum(A_array, axis=1) == 0
|
|
418
|
+
rows = []
|
|
419
|
+
for j in refit_indices:
|
|
420
|
+
dropped = drops.get(j, set())
|
|
421
|
+
retained = [k for k in range(X_array.shape[1]) if k not in dropped]
|
|
422
|
+
if not retained:
|
|
423
|
+
raise ValueError(
|
|
424
|
+
f'Filtering removes every covariate for treatment {treatment_names[j]}'
|
|
425
|
+
)
|
|
426
|
+
eligible = ctrl | (A_array[:, j] == 1)
|
|
427
|
+
if mask_array is not None:
|
|
428
|
+
eligible &= mask_array[:, j]
|
|
429
|
+
X_treatment = X_array[:, retained].copy()
|
|
430
|
+
treatment_penalties = penalty_factors.get(j, {})
|
|
431
|
+
for position, k in enumerate(retained):
|
|
432
|
+
factor = treatment_penalties.get(k, 1.0)
|
|
433
|
+
X_treatment[:, position] /= np.sqrt(factor)
|
|
434
|
+
degenerate = bool(np.all(np.ptp(X_treatment[eligible], axis=0) == 0))
|
|
435
|
+
if degenerate:
|
|
436
|
+
warnings.warn(
|
|
437
|
+
f'The propensity design for treatment {treatment_names[j]} is '
|
|
438
|
+
'constant after filtering; its scores fall back to a '
|
|
439
|
+
'class-weighted prevalence and carry no covariate information.',
|
|
440
|
+
RuntimeWarning, stacklevel=2,
|
|
441
|
+
)
|
|
442
|
+
scores = estimate_propensity_scores(
|
|
443
|
+
A_array[:, [j]], X_treatment, K=K,
|
|
444
|
+
ps_model=ps_model, mask=eligible[:, None], clip=None,
|
|
445
|
+
random_state=random_state, verbose=verbose,
|
|
446
|
+
class_weight=class_weight, **kwargs,
|
|
447
|
+
)
|
|
448
|
+
pi_updated[:, j] = scores[:, 0]
|
|
449
|
+
rows.append({
|
|
450
|
+
'treatment': treatment_names[j],
|
|
451
|
+
'dropped_covariates': [covariate_names[k] for k in sorted(dropped)],
|
|
452
|
+
'retained_covariates': [covariate_names[k] for k in retained],
|
|
453
|
+
'penalty_factors': {
|
|
454
|
+
covariate_names[k]: treatment_penalties[k]
|
|
455
|
+
for k in retained if treatment_penalties.get(k, 1.0) != 1.0
|
|
456
|
+
},
|
|
457
|
+
'n_retained': len(retained),
|
|
458
|
+
'degenerate_design': degenerate,
|
|
459
|
+
'score_std': float(np.std(scores[eligible, 0])),
|
|
460
|
+
})
|
|
461
|
+
|
|
462
|
+
if clip is not None:
|
|
463
|
+
lower, upper = _validate_clip(clip)
|
|
464
|
+
pi_updated = np.clip(pi_updated, lower, upper)
|
|
465
|
+
report = pd.DataFrame(rows, columns=[
|
|
466
|
+
'treatment', 'dropped_covariates', 'retained_covariates',
|
|
467
|
+
'penalty_factors', 'n_retained', 'degenerate_design', 'score_std',
|
|
468
|
+
])
|
|
469
|
+
return pi_updated, report
|
|
470
|
+
|
|
471
|
+
|
|
181
472
|
def cross_fitting(
|
|
182
473
|
Y, A, X, X_A, family='poisson', K=1, glm_alpha=1e-4,
|
|
183
474
|
ps_model='logistic', ps_class_weight='balanced',
|
|
@@ -10,6 +10,24 @@ from causarray.DR_inference import fdx_control, bh_correction
|
|
|
10
10
|
from causarray.utils import reset_random_seeds, pprint, tqdm, comp_size_factor, _filter_params
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
def _add_log2fc_columns(df_res):
|
|
14
|
+
"""Add base-2 LFC aliases while retaining natural-log result columns."""
|
|
15
|
+
log2 = np.log(2.0)
|
|
16
|
+
log2fc = df_res['tau'].to_numpy() / log2
|
|
17
|
+
log2fc_se = df_res['std'].to_numpy() / log2
|
|
18
|
+
|
|
19
|
+
# Recompute existing aliases as well as missing ones. This normalizes
|
|
20
|
+
# mixed old/new frames loaded from resumable batch caches and guarantees
|
|
21
|
+
# that the aliases remain exact transformations of tau and std.
|
|
22
|
+
for name in ('log2fc', 'log2fc_se'):
|
|
23
|
+
if name in df_res.columns:
|
|
24
|
+
df_res.pop(name)
|
|
25
|
+
insert_at = df_res.columns.get_loc('std') + 1
|
|
26
|
+
df_res.insert(insert_at, 'log2fc', log2fc)
|
|
27
|
+
df_res.insert(insert_at + 1, 'log2fc_se', log2fc_se)
|
|
28
|
+
return df_res
|
|
29
|
+
|
|
30
|
+
|
|
13
31
|
|
|
14
32
|
def compute_causal_estimand(
|
|
15
33
|
estimand,
|
|
@@ -301,19 +319,29 @@ def LFC(
|
|
|
301
319
|
|
|
302
320
|
* ``'unequal'`` (default, v0.0.6+): Welch variance
|
|
303
321
|
``s₀²/n₀ + s₁²/n₁`` with Welch-Satterthwaite degrees of
|
|
304
|
-
freedom; p-values use the t-distribution.
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
322
|
+
freedom; p-values use the t-distribution. Prefer this estimator when
|
|
323
|
+
treatment and control sample sizes or effective sample sizes are
|
|
324
|
+
meaningfully unbalanced, when arm-specific pseudo-outcome variances
|
|
325
|
+
may differ, and for case-control, bulk, and donor-level pseudo-bulk
|
|
326
|
+
analyses. Independence of the rows does not imply equal
|
|
327
|
+
treatment-arm variances, and biological heterogeneity or imbalance
|
|
328
|
+
can make pooled inference anti-conservative.
|
|
310
329
|
* ``'pooled'``: pooled-variance estimator ``(s² + eps_var) / n``.
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
330
|
+
For a small, approximately balanced perturbation comparison, this
|
|
331
|
+
estimator may provide better power when the independent sampling
|
|
332
|
+
units and arm-specific pseudo-outcome variances are reasonably
|
|
333
|
+
comparable. Treat it as an opt-in, empirically justified analysis,
|
|
334
|
+
not as an automatic choice for every small study. Balanced sample
|
|
335
|
+
counts alone are insufficient for case-control data, where biological
|
|
336
|
+
heterogeneity commonly favors ``'unequal'``. Pooled inference can
|
|
337
|
+
produce substantially smaller standard errors and many more
|
|
338
|
+
discoveries.
|
|
339
|
+
|
|
340
|
+
There is no universal arm-size ratio at which the choice should switch.
|
|
341
|
+
Inspect nominal and propensity-weighted effective sample sizes,
|
|
342
|
+
arm-specific pseudo-outcome variability, and sensitivity of the
|
|
343
|
+
discoveries. When those diagnostics are uncertain, retain
|
|
344
|
+
``'unequal'``.
|
|
317
345
|
|
|
318
346
|
``'unequal'`` accommodates arm-specific variance but does not model
|
|
319
347
|
within-donor or within-subject correlation. Repeated cells from the
|
|
@@ -359,9 +387,17 @@ def LFC(
|
|
|
359
387
|
Returns
|
|
360
388
|
-------
|
|
361
389
|
df_res : DataFrame
|
|
362
|
-
Test results with effect
|
|
363
|
-
``
|
|
364
|
-
``
|
|
390
|
+
Test results with natural-log effect estimate ``tau`` and standard
|
|
391
|
+
error ``std``, together with their base-2 equivalents ``log2fc`` and
|
|
392
|
+
``log2fc_se``. The fold change is treatment relative to control, and
|
|
393
|
+
``log2fc_se`` is a standard error (not a sample standard deviation).
|
|
394
|
+
The result also contains inference columns, raw ``mean_control`` and
|
|
395
|
+
``mean_treated`` counterfactual means, and an ``estimable`` flag (plus
|
|
396
|
+
``trt`` for multiple treatments).
|
|
397
|
+
|
|
398
|
+
.. versionadded:: 0.0.8
|
|
399
|
+
Added the ``log2fc`` and ``log2fc_se`` convenience columns. The
|
|
400
|
+
original natural-log ``tau`` and ``std`` columns remain unchanged.
|
|
365
401
|
"""
|
|
366
402
|
|
|
367
403
|
def estimand(etas, A, **kwargs):
|
|
@@ -448,12 +484,13 @@ def LFC(
|
|
|
448
484
|
if K is None:
|
|
449
485
|
K = 2 if cross_est else 1
|
|
450
486
|
|
|
451
|
-
|
|
487
|
+
df_res, estimation = compute_causal_estimand(
|
|
452
488
|
estimand, Y, W, A, W_A, family, offset,
|
|
453
489
|
Y_hat=Y_hat, pi_hat=pi_hat, mask=mask,
|
|
454
490
|
fdx=fdx, fdx_alpha=fdx_alpha, fdx_c=fdx_c,
|
|
455
491
|
verbose=verbose, backend=backend, K=K, ps_clip=ps_clip,
|
|
456
492
|
ps_class_weight=ps_class_weight, **kwargs)
|
|
493
|
+
return _add_log2fc_columns(df_res), estimation
|
|
457
494
|
|
|
458
495
|
|
|
459
496
|
|
|
@@ -634,10 +671,13 @@ def gcate_lfc_batch(
|
|
|
634
671
|
lfc_kwargs : dict or None
|
|
635
672
|
Extra keyword arguments forwarded to :func:`LFC`
|
|
636
673
|
(e.g. ``usevar``, ``fdx``, ``thres_min``). Retain the default
|
|
637
|
-
``usevar='unequal'``
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
674
|
+
``usevar='unequal'`` when arm sizes or effective sample sizes are
|
|
675
|
+
meaningfully unbalanced, when arm-specific variability may differ, and
|
|
676
|
+
for case-control, bulk, or pseudo-bulk analyses. For a small,
|
|
677
|
+
approximately balanced perturbation comparison with comparable
|
|
678
|
+
pseudo-outcome variability, ``lfc_kwargs=dict(usevar='pooled')`` may
|
|
679
|
+
improve power. There is no universal balance threshold; compare the
|
|
680
|
+
relevant diagnostics and retain ``'unequal'`` when uncertain.
|
|
641
681
|
**kwargs
|
|
642
682
|
Additional arguments forwarded to both :func:`fit_gcate_batch` and
|
|
643
683
|
:func:`LFC`. When a key collides with ``gcate_kwargs`` /
|
|
@@ -649,8 +689,10 @@ def gcate_lfc_batch(
|
|
|
649
689
|
Returns
|
|
650
690
|
-------
|
|
651
691
|
df_res : DataFrame
|
|
652
|
-
Concatenated result from all batches.
|
|
653
|
-
|
|
692
|
+
Concatenated result from all batches. Includes natural-log ``tau`` and
|
|
693
|
+
``std`` columns, base-2 ``log2fc`` and ``log2fc_se`` columns, and a
|
|
694
|
+
``'batch'`` column with the 0-based batch index. Older compatible
|
|
695
|
+
caches containing only ``tau`` and ``std`` are upgraded in memory.
|
|
654
696
|
"""
|
|
655
697
|
import gc
|
|
656
698
|
from causarray.gcate import fit_gcate_batch
|
|
@@ -821,7 +863,7 @@ def gcate_lfc_batch(
|
|
|
821
863
|
result = pd.concat(
|
|
822
864
|
[new_dfs[i] for i in all_indices], axis=0
|
|
823
865
|
).reset_index(drop=True)
|
|
824
|
-
return result
|
|
866
|
+
return _add_log2fc_columns(result)
|
|
825
867
|
|
|
826
868
|
|
|
827
869
|
def LFC_batch(*args, **kwargs):
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.9"
|
|
@@ -11,13 +11,19 @@ __all__ = [
|
|
|
11
11
|
'fit_glm', 'fit_glm_fast', 'fit_glm_ondisk',
|
|
12
12
|
'reset_random_seeds', 'fit_gcate', 'fit_gcate_batch',
|
|
13
13
|
'estimate_propensity_scores', 'summarize_propensity_scores',
|
|
14
|
-
'plot_propensity_scores',
|
|
14
|
+
'plot_propensity_scores', 'refit_propensity_scores',
|
|
15
|
+
'summarize_treatment_associations', 'plot_treatment_associations',
|
|
16
|
+
'align_test_mask',
|
|
15
17
|
]
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
from causarray.DR_learner import LFC, gcate_lfc_batch, LFC_batch # ATE, SATE, FC
|
|
19
|
-
from causarray.DR_estimation import estimate_propensity_scores
|
|
20
|
-
from causarray.diagnostics import
|
|
21
|
+
from causarray.DR_estimation import estimate_propensity_scores, refit_propensity_scores
|
|
22
|
+
from causarray.diagnostics import (
|
|
23
|
+
summarize_propensity_scores, plot_propensity_scores,
|
|
24
|
+
summarize_treatment_associations, plot_treatment_associations,
|
|
25
|
+
align_test_mask,
|
|
26
|
+
)
|
|
21
27
|
from causarray.gcate_glm import fit_glm
|
|
22
28
|
from causarray.nb_glm_fast import fit_glm_fast, fit_glm_ondisk
|
|
23
29
|
from causarray.utils import prep_causarray_data, reset_random_seeds, comp_size_factor
|