causarray 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- causarray/DR_estimation.py +294 -3
- causarray/DR_learner.py +65 -23
- causarray/__about__.py +1 -1
- causarray/__init__.py +9 -3
- causarray/diagnostics.py +464 -8
- {causarray-0.0.7.dist-info → causarray-0.0.9.dist-info}/METADATA +35 -2
- causarray-0.0.9.dist-info/RECORD +16 -0
- causarray-0.0.7.dist-info/RECORD +0 -16
- {causarray-0.0.7.dist-info → causarray-0.0.9.dist-info}/WHEEL +0 -0
- {causarray-0.0.7.dist-info → causarray-0.0.9.dist-info}/licenses/LICENSE +0 -0
causarray/DR_estimation.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
2
3
|
from sklearn.linear_model import LogisticRegression
|
|
3
4
|
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
|
4
5
|
from sklearn_ensemble_cv import reset_random_seeds, Ensemble, ECV
|
|
@@ -10,6 +11,7 @@ from joblib import Parallel, delayed
|
|
|
10
11
|
from tqdm import tqdm
|
|
11
12
|
import pprint
|
|
12
13
|
import warnings
|
|
14
|
+
from collections.abc import Mapping
|
|
13
15
|
|
|
14
16
|
from sklearn.model_selection import KFold, ShuffleSplit
|
|
15
17
|
|
|
@@ -40,6 +42,27 @@ def _get_func_ps(ps_model, **kwargs):
|
|
|
40
42
|
return func_ps, params_ps
|
|
41
43
|
|
|
42
44
|
|
|
45
|
+
def _validate_clip(clip):
|
|
46
|
+
"""Validate a propensity clipping bound and return it as ``(lower, upper)``.
|
|
47
|
+
|
|
48
|
+
Raises
|
|
49
|
+
------
|
|
50
|
+
ValueError
|
|
51
|
+
If ``clip`` is not a pair of numbers satisfying
|
|
52
|
+
``0 <= lower < upper <= 1``.
|
|
53
|
+
"""
|
|
54
|
+
message = 'clip must be None or a pair 0 <= lower < upper <= 1'
|
|
55
|
+
if isinstance(clip, (str, bytes)) or np.ndim(clip) != 1:
|
|
56
|
+
raise ValueError(message)
|
|
57
|
+
try:
|
|
58
|
+
lower, upper = (float(bound) for bound in clip)
|
|
59
|
+
except (TypeError, ValueError):
|
|
60
|
+
raise ValueError(message) from None
|
|
61
|
+
if not 0 <= lower < upper <= 1:
|
|
62
|
+
raise ValueError(message)
|
|
63
|
+
return lower, upper
|
|
64
|
+
|
|
65
|
+
|
|
43
66
|
def estimate_propensity_scores(
|
|
44
67
|
A, X_A, K=1, ps_model='logistic', mask=None, clip=None,
|
|
45
68
|
random_state=0, verbose=False, class_weight='balanced', **kwargs,
|
|
@@ -172,12 +195,280 @@ def estimate_propensity_scores(
|
|
|
172
195
|
pi_hat[test_index, j] = func_ps(x_train, y_train, XA_test)
|
|
173
196
|
|
|
174
197
|
if clip is not None:
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
pi_hat = np.clip(pi_hat, clip[0], clip[1])
|
|
198
|
+
lower, upper = _validate_clip(clip)
|
|
199
|
+
pi_hat = np.clip(pi_hat, lower, upper)
|
|
178
200
|
return pi_hat
|
|
179
201
|
|
|
180
202
|
|
|
203
|
+
def refit_propensity_scores(
|
|
204
|
+
A, X_A, drop_by_treatment=None, pi_hat=None, treatment_names=None,
|
|
205
|
+
covariate_names=None, penalty_factors_by_treatment=None, K=1,
|
|
206
|
+
ps_model='logistic', mask=None, clip=None, random_state=0, verbose=False,
|
|
207
|
+
class_weight='balanced', **kwargs,
|
|
208
|
+
):
|
|
209
|
+
"""Refit propensity scores with treatment-specific covariate filtering.
|
|
210
|
+
|
|
211
|
+
This helper supports sensitivity analyses in which different treatments
|
|
212
|
+
omit different observed covariates or latent factors, or apply stronger L2
|
|
213
|
+
regularization to selected covariates. When existing scores are supplied,
|
|
214
|
+
only treatments named in either treatment-specific mapping are refitted;
|
|
215
|
+
the remaining columns are carried over from ``pi_hat`` unchanged, up to the
|
|
216
|
+
shared ``clip`` described below. Outcome models are not fit by this function
|
|
217
|
+
and cached ``Y_hat`` values can be reused in :func:`LFC`.
|
|
218
|
+
|
|
219
|
+
Parameters
|
|
220
|
+
----------
|
|
221
|
+
A : array-like, shape (n, a)
|
|
222
|
+
Binary treatment indicator matrix.
|
|
223
|
+
X_A : array-like, shape (n, d_A)
|
|
224
|
+
Full propensity-model design before treatment-specific filtering.
|
|
225
|
+
drop_by_treatment : mapping or None
|
|
226
|
+
Treatment names or indices mapped to covariate names or indices to
|
|
227
|
+
remove for that treatment. Defaults to no removals.
|
|
228
|
+
pi_hat : array-like or None, shape (n, a)
|
|
229
|
+
Existing raw propensity scores. If supplied, treatments absent from
|
|
230
|
+
``drop_by_treatment`` are not refitted.
|
|
231
|
+
treatment_names, covariate_names : sequence, optional
|
|
232
|
+
Column labels, inferred from DataFrames when possible.
|
|
233
|
+
penalty_factors_by_treatment : mapping or None
|
|
234
|
+
Treatment names or indices mapped to ``{covariate: factor}`` mappings.
|
|
235
|
+
A factor greater than one applies that multiple of the ordinary L2
|
|
236
|
+
penalty to the named coefficient. This is implemented by dividing the
|
|
237
|
+
covariate by ``sqrt(factor)`` during both fitting and prediction and is
|
|
238
|
+
available only for ``ps_model='logistic'`` with an L2 penalty. A factor
|
|
239
|
+
of one leaves the covariate unchanged. Interpret relative penalties on
|
|
240
|
+
a common scale; ``prep_causarray_data`` standardizes log-library size.
|
|
241
|
+
K, ps_model, mask, random_state, verbose, class_weight, **kwargs
|
|
242
|
+
Passed to :func:`estimate_propensity_scores` for each refitted model.
|
|
243
|
+
clip : tuple(float, float) or None, optional
|
|
244
|
+
Bounds applied to the **whole** returned matrix, refitted columns and
|
|
245
|
+
carried-over columns alike, so that a single consistent bound reaches
|
|
246
|
+
:func:`LFC`. Pass ``None`` to leave carried-over scores exactly as
|
|
247
|
+
supplied.
|
|
248
|
+
|
|
249
|
+
Returns
|
|
250
|
+
-------
|
|
251
|
+
pi_updated : ndarray, shape (n, a)
|
|
252
|
+
Updated propensity scores.
|
|
253
|
+
report : DataFrame
|
|
254
|
+
Audit table of refitted treatments, retained/dropped covariates, and
|
|
255
|
+
the resulting score spread. ``degenerate_design`` flags a treatment
|
|
256
|
+
whose retained design is constant, and ``score_std`` reports the
|
|
257
|
+
standard deviation of its refitted scores on eligible rows; both make a
|
|
258
|
+
collapsed propensity model visible without reading the warning stream.
|
|
259
|
+
|
|
260
|
+
Warns
|
|
261
|
+
-----
|
|
262
|
+
RuntimeWarning
|
|
263
|
+
If filtering leaves a constant design for some treatment, in which case
|
|
264
|
+
its scores carry no covariate information.
|
|
265
|
+
|
|
266
|
+
Notes
|
|
267
|
+
-----
|
|
268
|
+
A very large penalty factor shrinks a coefficient towards zero without
|
|
269
|
+
making the design constant. That case leaves ``degenerate_design`` False
|
|
270
|
+
but drives ``score_std`` towards zero.
|
|
271
|
+
|
|
272
|
+
.. versionadded:: 0.0.9
|
|
273
|
+
"""
|
|
274
|
+
if drop_by_treatment is None:
|
|
275
|
+
drop_by_treatment = {}
|
|
276
|
+
if penalty_factors_by_treatment is None:
|
|
277
|
+
penalty_factors_by_treatment = {}
|
|
278
|
+
if not isinstance(drop_by_treatment, Mapping):
|
|
279
|
+
raise ValueError('drop_by_treatment must be a mapping')
|
|
280
|
+
if not isinstance(penalty_factors_by_treatment, Mapping):
|
|
281
|
+
raise ValueError('penalty_factors_by_treatment must be a mapping')
|
|
282
|
+
if penalty_factors_by_treatment:
|
|
283
|
+
if ps_model != 'logistic':
|
|
284
|
+
raise ValueError(
|
|
285
|
+
'penalty_factors_by_treatment is supported only for logistic models'
|
|
286
|
+
)
|
|
287
|
+
if kwargs.get('penalty', 'l2') != 'l2':
|
|
288
|
+
raise ValueError(
|
|
289
|
+
'penalty_factors_by_treatment requires logistic penalty="l2"'
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
if isinstance(A, pd.DataFrame):
|
|
293
|
+
if treatment_names is None:
|
|
294
|
+
treatment_names = list(A.columns)
|
|
295
|
+
A_array = A.to_numpy()
|
|
296
|
+
else:
|
|
297
|
+
A_array = np.asarray(A)
|
|
298
|
+
if A_array.ndim == 1:
|
|
299
|
+
A_array = A_array[:, None]
|
|
300
|
+
if A_array.ndim != 2 or not np.all(np.isin(A_array, (0, 1))):
|
|
301
|
+
raise ValueError('A must be a one- or two-dimensional binary matrix')
|
|
302
|
+
if treatment_names is None:
|
|
303
|
+
treatment_names = list(range(A_array.shape[1]))
|
|
304
|
+
treatment_names = list(treatment_names)
|
|
305
|
+
if len(treatment_names) != A_array.shape[1]:
|
|
306
|
+
raise ValueError('treatment_names must match the number of treatments')
|
|
307
|
+
if len(set(treatment_names)) != len(treatment_names):
|
|
308
|
+
raise ValueError('treatment_names must be unique')
|
|
309
|
+
|
|
310
|
+
if isinstance(X_A, pd.DataFrame):
|
|
311
|
+
if covariate_names is None:
|
|
312
|
+
covariate_names = list(X_A.columns)
|
|
313
|
+
X_array = X_A.to_numpy()
|
|
314
|
+
else:
|
|
315
|
+
X_array = np.asarray(X_A)
|
|
316
|
+
if X_array.ndim != 2 or X_array.shape[0] != A_array.shape[0]:
|
|
317
|
+
raise ValueError('X_A must be two-dimensional with the same rows as A')
|
|
318
|
+
try:
|
|
319
|
+
X_array = np.asarray(X_array, dtype=float)
|
|
320
|
+
except (TypeError, ValueError) as exc:
|
|
321
|
+
raise ValueError('X_A must contain numeric covariates') from exc
|
|
322
|
+
if not np.all(np.isfinite(X_array)):
|
|
323
|
+
raise ValueError('X_A must contain only finite values')
|
|
324
|
+
if covariate_names is None:
|
|
325
|
+
covariate_names = [f'covariate_{j + 1}' for j in range(X_array.shape[1])]
|
|
326
|
+
covariate_names = list(covariate_names)
|
|
327
|
+
if len(covariate_names) != X_array.shape[1]:
|
|
328
|
+
raise ValueError('covariate_names must match the number of covariates')
|
|
329
|
+
if len(set(covariate_names)) != len(covariate_names):
|
|
330
|
+
raise ValueError('covariate_names must be unique')
|
|
331
|
+
|
|
332
|
+
treatment_lookup = {name: j for j, name in enumerate(treatment_names)}
|
|
333
|
+
covariate_lookup = {name: j for j, name in enumerate(covariate_names)}
|
|
334
|
+
|
|
335
|
+
def resolve_treatment(value):
|
|
336
|
+
if value in treatment_lookup:
|
|
337
|
+
return treatment_lookup[value]
|
|
338
|
+
if isinstance(value, (int, np.integer)) and 0 <= int(value) < A_array.shape[1]:
|
|
339
|
+
return int(value)
|
|
340
|
+
raise ValueError(f'Unknown treatment: {value}')
|
|
341
|
+
|
|
342
|
+
def resolve_covariate(value):
|
|
343
|
+
if value in covariate_lookup:
|
|
344
|
+
return covariate_lookup[value]
|
|
345
|
+
if isinstance(value, (int, np.integer)) and 0 <= int(value) < X_array.shape[1]:
|
|
346
|
+
return int(value)
|
|
347
|
+
raise ValueError(f'Unknown covariate: {value}')
|
|
348
|
+
|
|
349
|
+
drops = {}
|
|
350
|
+
for treatment, covariates in drop_by_treatment.items():
|
|
351
|
+
j = resolve_treatment(treatment)
|
|
352
|
+
if j in drops:
|
|
353
|
+
raise ValueError(f'Treatment {treatment_names[j]} is specified more than once')
|
|
354
|
+
if isinstance(covariates, (str, bytes)):
|
|
355
|
+
covariates = [covariates]
|
|
356
|
+
indices = [resolve_covariate(value) for value in covariates]
|
|
357
|
+
if len(set(indices)) != len(indices):
|
|
358
|
+
raise ValueError(
|
|
359
|
+
f'Duplicate dropped covariates for treatment {treatment_names[j]}'
|
|
360
|
+
)
|
|
361
|
+
drops[j] = set(indices)
|
|
362
|
+
|
|
363
|
+
penalty_factors = {}
|
|
364
|
+
for treatment, factors in penalty_factors_by_treatment.items():
|
|
365
|
+
j = resolve_treatment(treatment)
|
|
366
|
+
if j in penalty_factors:
|
|
367
|
+
raise ValueError(f'Treatment {treatment_names[j]} is specified more than once')
|
|
368
|
+
if not isinstance(factors, Mapping):
|
|
369
|
+
raise ValueError(
|
|
370
|
+
f'Penalty factors for treatment {treatment_names[j]} must be a mapping'
|
|
371
|
+
)
|
|
372
|
+
resolved = {}
|
|
373
|
+
for covariate, factor in factors.items():
|
|
374
|
+
k = resolve_covariate(covariate)
|
|
375
|
+
try:
|
|
376
|
+
factor = float(factor)
|
|
377
|
+
except (TypeError, ValueError) as exc:
|
|
378
|
+
raise ValueError('Penalty factors must be finite numbers at least 1') from exc
|
|
379
|
+
if not np.isfinite(factor) or factor < 1:
|
|
380
|
+
raise ValueError('Penalty factors must be finite numbers at least 1')
|
|
381
|
+
if k in resolved:
|
|
382
|
+
raise ValueError(
|
|
383
|
+
f'Duplicate penalty factors for covariate {covariate_names[k]}'
|
|
384
|
+
)
|
|
385
|
+
resolved[k] = factor
|
|
386
|
+
penalty_factors[j] = resolved
|
|
387
|
+
|
|
388
|
+
for j in set(drops).intersection(penalty_factors):
|
|
389
|
+
conflict = drops[j].intersection(penalty_factors[j])
|
|
390
|
+
if conflict:
|
|
391
|
+
names = [covariate_names[k] for k in sorted(conflict)]
|
|
392
|
+
raise ValueError(
|
|
393
|
+
f'Covariates cannot be both dropped and penalized for '
|
|
394
|
+
f'{treatment_names[j]}: {names}'
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
if pi_hat is None:
|
|
398
|
+
pi_updated = np.empty(A_array.shape, dtype=float)
|
|
399
|
+
refit_indices = range(A_array.shape[1])
|
|
400
|
+
else:
|
|
401
|
+
pi_updated = np.asarray(pi_hat, dtype=float).copy()
|
|
402
|
+
if pi_updated.shape != A_array.shape:
|
|
403
|
+
raise ValueError('pi_hat must have the same shape as A')
|
|
404
|
+
if (not np.all(np.isfinite(pi_updated))
|
|
405
|
+
or np.any((pi_updated < 0) | (pi_updated > 1))):
|
|
406
|
+
raise ValueError('pi_hat must contain finite probabilities in [0, 1]')
|
|
407
|
+
refit_indices = sorted(set(drops).union(penalty_factors))
|
|
408
|
+
|
|
409
|
+
mask_array = None
|
|
410
|
+
if mask is not None:
|
|
411
|
+
mask_array = np.asarray(mask, dtype=bool)
|
|
412
|
+
if mask_array.ndim == 1:
|
|
413
|
+
mask_array = mask_array[:, None]
|
|
414
|
+
if mask_array.shape != A_array.shape:
|
|
415
|
+
raise ValueError('Mask must have the same shape as the treatment matrix')
|
|
416
|
+
|
|
417
|
+
ctrl = np.sum(A_array, axis=1) == 0
|
|
418
|
+
rows = []
|
|
419
|
+
for j in refit_indices:
|
|
420
|
+
dropped = drops.get(j, set())
|
|
421
|
+
retained = [k for k in range(X_array.shape[1]) if k not in dropped]
|
|
422
|
+
if not retained:
|
|
423
|
+
raise ValueError(
|
|
424
|
+
f'Filtering removes every covariate for treatment {treatment_names[j]}'
|
|
425
|
+
)
|
|
426
|
+
eligible = ctrl | (A_array[:, j] == 1)
|
|
427
|
+
if mask_array is not None:
|
|
428
|
+
eligible &= mask_array[:, j]
|
|
429
|
+
X_treatment = X_array[:, retained].copy()
|
|
430
|
+
treatment_penalties = penalty_factors.get(j, {})
|
|
431
|
+
for position, k in enumerate(retained):
|
|
432
|
+
factor = treatment_penalties.get(k, 1.0)
|
|
433
|
+
X_treatment[:, position] /= np.sqrt(factor)
|
|
434
|
+
degenerate = bool(np.all(np.ptp(X_treatment[eligible], axis=0) == 0))
|
|
435
|
+
if degenerate:
|
|
436
|
+
warnings.warn(
|
|
437
|
+
f'The propensity design for treatment {treatment_names[j]} is '
|
|
438
|
+
'constant after filtering; its scores fall back to a '
|
|
439
|
+
'class-weighted prevalence and carry no covariate information.',
|
|
440
|
+
RuntimeWarning, stacklevel=2,
|
|
441
|
+
)
|
|
442
|
+
scores = estimate_propensity_scores(
|
|
443
|
+
A_array[:, [j]], X_treatment, K=K,
|
|
444
|
+
ps_model=ps_model, mask=eligible[:, None], clip=None,
|
|
445
|
+
random_state=random_state, verbose=verbose,
|
|
446
|
+
class_weight=class_weight, **kwargs,
|
|
447
|
+
)
|
|
448
|
+
pi_updated[:, j] = scores[:, 0]
|
|
449
|
+
rows.append({
|
|
450
|
+
'treatment': treatment_names[j],
|
|
451
|
+
'dropped_covariates': [covariate_names[k] for k in sorted(dropped)],
|
|
452
|
+
'retained_covariates': [covariate_names[k] for k in retained],
|
|
453
|
+
'penalty_factors': {
|
|
454
|
+
covariate_names[k]: treatment_penalties[k]
|
|
455
|
+
for k in retained if treatment_penalties.get(k, 1.0) != 1.0
|
|
456
|
+
},
|
|
457
|
+
'n_retained': len(retained),
|
|
458
|
+
'degenerate_design': degenerate,
|
|
459
|
+
'score_std': float(np.std(scores[eligible, 0])),
|
|
460
|
+
})
|
|
461
|
+
|
|
462
|
+
if clip is not None:
|
|
463
|
+
lower, upper = _validate_clip(clip)
|
|
464
|
+
pi_updated = np.clip(pi_updated, lower, upper)
|
|
465
|
+
report = pd.DataFrame(rows, columns=[
|
|
466
|
+
'treatment', 'dropped_covariates', 'retained_covariates',
|
|
467
|
+
'penalty_factors', 'n_retained', 'degenerate_design', 'score_std',
|
|
468
|
+
])
|
|
469
|
+
return pi_updated, report
|
|
470
|
+
|
|
471
|
+
|
|
181
472
|
def cross_fitting(
|
|
182
473
|
Y, A, X, X_A, family='poisson', K=1, glm_alpha=1e-4,
|
|
183
474
|
ps_model='logistic', ps_class_weight='balanced',
|
causarray/DR_learner.py
CHANGED
|
@@ -10,6 +10,24 @@ from causarray.DR_inference import fdx_control, bh_correction
|
|
|
10
10
|
from causarray.utils import reset_random_seeds, pprint, tqdm, comp_size_factor, _filter_params
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
def _add_log2fc_columns(df_res):
|
|
14
|
+
"""Add base-2 LFC aliases while retaining natural-log result columns."""
|
|
15
|
+
log2 = np.log(2.0)
|
|
16
|
+
log2fc = df_res['tau'].to_numpy() / log2
|
|
17
|
+
log2fc_se = df_res['std'].to_numpy() / log2
|
|
18
|
+
|
|
19
|
+
# Recompute existing aliases as well as missing ones. This normalizes
|
|
20
|
+
# mixed old/new frames loaded from resumable batch caches and guarantees
|
|
21
|
+
# that the aliases remain exact transformations of tau and std.
|
|
22
|
+
for name in ('log2fc', 'log2fc_se'):
|
|
23
|
+
if name in df_res.columns:
|
|
24
|
+
df_res.pop(name)
|
|
25
|
+
insert_at = df_res.columns.get_loc('std') + 1
|
|
26
|
+
df_res.insert(insert_at, 'log2fc', log2fc)
|
|
27
|
+
df_res.insert(insert_at + 1, 'log2fc_se', log2fc_se)
|
|
28
|
+
return df_res
|
|
29
|
+
|
|
30
|
+
|
|
13
31
|
|
|
14
32
|
def compute_causal_estimand(
|
|
15
33
|
estimand,
|
|
@@ -301,19 +319,29 @@ def LFC(
|
|
|
301
319
|
|
|
302
320
|
* ``'unequal'`` (default, v0.0.6+): Welch variance
|
|
303
321
|
``s₀²/n₀ + s₁²/n₁`` with Welch-Satterthwaite degrees of
|
|
304
|
-
freedom; p-values use the t-distribution.
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
322
|
+
freedom; p-values use the t-distribution. Prefer this estimator when
|
|
323
|
+
treatment and control sample sizes or effective sample sizes are
|
|
324
|
+
meaningfully unbalanced, when arm-specific pseudo-outcome variances
|
|
325
|
+
may differ, and for case-control, bulk, and donor-level pseudo-bulk
|
|
326
|
+
analyses. Independence of the rows does not imply equal
|
|
327
|
+
treatment-arm variances, and biological heterogeneity or imbalance
|
|
328
|
+
can make pooled inference anti-conservative.
|
|
310
329
|
* ``'pooled'``: pooled-variance estimator ``(s² + eps_var) / n``.
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
330
|
+
For a small, approximately balanced perturbation comparison, this
|
|
331
|
+
estimator may provide better power when the independent sampling
|
|
332
|
+
units and arm-specific pseudo-outcome variances are reasonably
|
|
333
|
+
comparable. Treat it as an opt-in, empirically justified analysis,
|
|
334
|
+
not as an automatic choice for every small study. Balanced sample
|
|
335
|
+
counts alone are insufficient for case-control data, where biological
|
|
336
|
+
heterogeneity commonly favors ``'unequal'``. Pooled inference can
|
|
337
|
+
produce substantially smaller standard errors and many more
|
|
338
|
+
discoveries.
|
|
339
|
+
|
|
340
|
+
There is no universal arm-size ratio at which the choice should switch.
|
|
341
|
+
Inspect nominal and propensity-weighted effective sample sizes,
|
|
342
|
+
arm-specific pseudo-outcome variability, and sensitivity of the
|
|
343
|
+
discoveries. When those diagnostics are uncertain, retain
|
|
344
|
+
``'unequal'``.
|
|
317
345
|
|
|
318
346
|
``'unequal'`` accommodates arm-specific variance but does not model
|
|
319
347
|
within-donor or within-subject correlation. Repeated cells from the
|
|
@@ -359,9 +387,17 @@ def LFC(
|
|
|
359
387
|
Returns
|
|
360
388
|
-------
|
|
361
389
|
df_res : DataFrame
|
|
362
|
-
Test results with effect
|
|
363
|
-
``
|
|
364
|
-
``
|
|
390
|
+
Test results with natural-log effect estimate ``tau`` and standard
|
|
391
|
+
error ``std``, together with their base-2 equivalents ``log2fc`` and
|
|
392
|
+
``log2fc_se``. The fold change is treatment relative to control, and
|
|
393
|
+
``log2fc_se`` is a standard error (not a sample standard deviation).
|
|
394
|
+
The result also contains inference columns, raw ``mean_control`` and
|
|
395
|
+
``mean_treated`` counterfactual means, and an ``estimable`` flag (plus
|
|
396
|
+
``trt`` for multiple treatments).
|
|
397
|
+
|
|
398
|
+
.. versionadded:: 0.0.8
|
|
399
|
+
Added the ``log2fc`` and ``log2fc_se`` convenience columns. The
|
|
400
|
+
original natural-log ``tau`` and ``std`` columns remain unchanged.
|
|
365
401
|
"""
|
|
366
402
|
|
|
367
403
|
def estimand(etas, A, **kwargs):
|
|
@@ -448,12 +484,13 @@ def LFC(
|
|
|
448
484
|
if K is None:
|
|
449
485
|
K = 2 if cross_est else 1
|
|
450
486
|
|
|
451
|
-
|
|
487
|
+
df_res, estimation = compute_causal_estimand(
|
|
452
488
|
estimand, Y, W, A, W_A, family, offset,
|
|
453
489
|
Y_hat=Y_hat, pi_hat=pi_hat, mask=mask,
|
|
454
490
|
fdx=fdx, fdx_alpha=fdx_alpha, fdx_c=fdx_c,
|
|
455
491
|
verbose=verbose, backend=backend, K=K, ps_clip=ps_clip,
|
|
456
492
|
ps_class_weight=ps_class_weight, **kwargs)
|
|
493
|
+
return _add_log2fc_columns(df_res), estimation
|
|
457
494
|
|
|
458
495
|
|
|
459
496
|
|
|
@@ -634,10 +671,13 @@ def gcate_lfc_batch(
|
|
|
634
671
|
lfc_kwargs : dict or None
|
|
635
672
|
Extra keyword arguments forwarded to :func:`LFC`
|
|
636
673
|
(e.g. ``usevar``, ``fdx``, ``thres_min``). Retain the default
|
|
637
|
-
``usevar='unequal'``
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
674
|
+
``usevar='unequal'`` when arm sizes or effective sample sizes are
|
|
675
|
+
meaningfully unbalanced, when arm-specific variability may differ, and
|
|
676
|
+
for case-control, bulk, or pseudo-bulk analyses. For a small,
|
|
677
|
+
approximately balanced perturbation comparison with comparable
|
|
678
|
+
pseudo-outcome variability, ``lfc_kwargs=dict(usevar='pooled')`` may
|
|
679
|
+
improve power. There is no universal balance threshold; compare the
|
|
680
|
+
relevant diagnostics and retain ``'unequal'`` when uncertain.
|
|
641
681
|
**kwargs
|
|
642
682
|
Additional arguments forwarded to both :func:`fit_gcate_batch` and
|
|
643
683
|
:func:`LFC`. When a key collides with ``gcate_kwargs`` /
|
|
@@ -649,8 +689,10 @@ def gcate_lfc_batch(
|
|
|
649
689
|
Returns
|
|
650
690
|
-------
|
|
651
691
|
df_res : DataFrame
|
|
652
|
-
Concatenated result from all batches.
|
|
653
|
-
|
|
692
|
+
Concatenated result from all batches. Includes natural-log ``tau`` and
|
|
693
|
+
``std`` columns, base-2 ``log2fc`` and ``log2fc_se`` columns, and a
|
|
694
|
+
``'batch'`` column with the 0-based batch index. Older compatible
|
|
695
|
+
caches containing only ``tau`` and ``std`` are upgraded in memory.
|
|
654
696
|
"""
|
|
655
697
|
import gc
|
|
656
698
|
from causarray.gcate import fit_gcate_batch
|
|
@@ -821,7 +863,7 @@ def gcate_lfc_batch(
|
|
|
821
863
|
result = pd.concat(
|
|
822
864
|
[new_dfs[i] for i in all_indices], axis=0
|
|
823
865
|
).reset_index(drop=True)
|
|
824
|
-
return result
|
|
866
|
+
return _add_log2fc_columns(result)
|
|
825
867
|
|
|
826
868
|
|
|
827
869
|
def LFC_batch(*args, **kwargs):
|
causarray/__about__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.0.
|
|
1
|
+
__version__ = "0.0.9"
|
causarray/__init__.py
CHANGED
|
@@ -11,13 +11,19 @@ __all__ = [
|
|
|
11
11
|
'fit_glm', 'fit_glm_fast', 'fit_glm_ondisk',
|
|
12
12
|
'reset_random_seeds', 'fit_gcate', 'fit_gcate_batch',
|
|
13
13
|
'estimate_propensity_scores', 'summarize_propensity_scores',
|
|
14
|
-
'plot_propensity_scores',
|
|
14
|
+
'plot_propensity_scores', 'refit_propensity_scores',
|
|
15
|
+
'summarize_treatment_associations', 'plot_treatment_associations',
|
|
16
|
+
'align_test_mask',
|
|
15
17
|
]
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
from causarray.DR_learner import LFC, gcate_lfc_batch, LFC_batch # ATE, SATE, FC
|
|
19
|
-
from causarray.DR_estimation import estimate_propensity_scores
|
|
20
|
-
from causarray.diagnostics import
|
|
21
|
+
from causarray.DR_estimation import estimate_propensity_scores, refit_propensity_scores
|
|
22
|
+
from causarray.diagnostics import (
|
|
23
|
+
summarize_propensity_scores, plot_propensity_scores,
|
|
24
|
+
summarize_treatment_associations, plot_treatment_associations,
|
|
25
|
+
align_test_mask,
|
|
26
|
+
)
|
|
21
27
|
from causarray.gcate_glm import fit_glm
|
|
22
28
|
from causarray.nb_glm_fast import fit_glm_fast, fit_glm_ondisk
|
|
23
29
|
from causarray.utils import prep_causarray_data, reset_random_seeds, comp_size_factor
|
causarray/diagnostics.py
CHANGED
|
@@ -1,11 +1,441 @@
|
|
|
1
|
-
"""Diagnostics for treatment overlap and
|
|
1
|
+
"""Diagnostics for treatment overlap, propensity scores, and result masks."""
|
|
2
2
|
|
|
3
3
|
import math
|
|
4
|
+
from typing import Hashable, Optional, Sequence, Union
|
|
4
5
|
|
|
5
6
|
import matplotlib.pyplot as plt
|
|
6
7
|
import numpy as np
|
|
7
8
|
import pandas as pd
|
|
9
|
+
from scipy import stats
|
|
8
10
|
from sklearn.metrics import brier_score_loss, roc_auc_score
|
|
11
|
+
from statsmodels.stats.multitest import multipletests
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
_TestMask = Union[np.ndarray, pd.DataFrame, pd.Series]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _normalize_test_labels(values, name):
|
|
18
|
+
raw_labels = pd.Index(values)
|
|
19
|
+
if raw_labels.hasnans:
|
|
20
|
+
raise ValueError(f'{name} names must not be missing')
|
|
21
|
+
labels = pd.Index([str(value) for value in raw_labels], dtype=object)
|
|
22
|
+
if labels.has_duplicates:
|
|
23
|
+
duplicates = labels[labels.duplicated()].unique().tolist()
|
|
24
|
+
raise ValueError(f'{name} names must be unique; duplicates: {duplicates}')
|
|
25
|
+
return labels
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _require_boolean_mask(values):
|
|
29
|
+
values = np.asarray(values)
|
|
30
|
+
if values.dtype.kind != 'b':
|
|
31
|
+
raise ValueError('test_mask must contain only Boolean values')
|
|
32
|
+
return values
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def align_test_mask(
|
|
36
|
+
results: pd.DataFrame,
|
|
37
|
+
test_mask: _TestMask,
|
|
38
|
+
treatment_names: Optional[Sequence[Hashable]] = None,
|
|
39
|
+
gene_names: Optional[Sequence[Hashable]] = None,
|
|
40
|
+
treatment_col: str = 'trt',
|
|
41
|
+
gene_col: str = 'gene_names',
|
|
42
|
+
) -> np.ndarray:
|
|
43
|
+
"""Align a treatment-by-gene diagnostic mask to an LFC result table.
|
|
44
|
+
|
|
45
|
+
Alignment uses treatment and gene labels rather than row position. This
|
|
46
|
+
makes it possible to annotate or subset existing causarray results with a
|
|
47
|
+
post-hoc diagnostic rule without refitting effects or changing their
|
|
48
|
+
standard errors and p-values. Labels are compared after conversion to
|
|
49
|
+
strings, matching the labeling convention of batch LFC results.
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
results : DataFrame
|
|
54
|
+
Result table containing one row per treatment-gene test.
|
|
55
|
+
test_mask : ndarray, DataFrame, or Series
|
|
56
|
+
Boolean diagnostic mask. A two-dimensional array requires
|
|
57
|
+
``treatment_names`` and ``gene_names``. A DataFrame uses its index as
|
|
58
|
+
treatment names and columns as gene names unless names are supplied
|
|
59
|
+
explicitly. A Series must have a two-level ``(treatment, gene)``
|
|
60
|
+
MultiIndex.
|
|
61
|
+
treatment_names, gene_names : sequence, optional
|
|
62
|
+
Labels for the rows and columns of a two-dimensional mask.
|
|
63
|
+
treatment_col, gene_col : str, optional
|
|
64
|
+
Columns containing treatment and gene labels in ``results``.
|
|
65
|
+
|
|
66
|
+
Returns
|
|
67
|
+
-------
|
|
68
|
+
aligned : ndarray of bool, shape (n_tests,)
|
|
69
|
+
Mask values in the original row order of ``results``.
|
|
70
|
+
|
|
71
|
+
Raises
|
|
72
|
+
------
|
|
73
|
+
ValueError
|
|
74
|
+
If labels are missing or duplicated, the mask is malformed or
|
|
75
|
+
non-Boolean, or a result test is absent from the mask.
|
|
76
|
+
|
|
77
|
+
Notes
|
|
78
|
+
-----
|
|
79
|
+
This function only aligns a diagnostic mask. It does not modify the
|
|
80
|
+
multiple-testing family or recompute adjusted p-values.
|
|
81
|
+
|
|
82
|
+
.. versionadded:: 0.0.9
|
|
83
|
+
"""
|
|
84
|
+
if not isinstance(results, pd.DataFrame):
|
|
85
|
+
raise ValueError('results must be a pandas DataFrame')
|
|
86
|
+
missing_columns = {treatment_col, gene_col}.difference(results.columns)
|
|
87
|
+
if missing_columns:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
f'results is missing required columns: {sorted(missing_columns)}'
|
|
90
|
+
)
|
|
91
|
+
if results[[treatment_col, gene_col]].isna().any().any():
|
|
92
|
+
raise ValueError('result treatment and gene labels must not be missing')
|
|
93
|
+
|
|
94
|
+
result_keys = pd.MultiIndex.from_arrays([
|
|
95
|
+
results[treatment_col].astype(str),
|
|
96
|
+
results[gene_col].astype(str),
|
|
97
|
+
])
|
|
98
|
+
if result_keys.has_duplicates:
|
|
99
|
+
duplicates = result_keys[result_keys.duplicated()].unique().tolist()
|
|
100
|
+
raise ValueError(
|
|
101
|
+
'results contains duplicate treatment-gene tests: '
|
|
102
|
+
f'{duplicates[:5]}'
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
if isinstance(test_mask, pd.Series):
|
|
106
|
+
if treatment_names is not None or gene_names is not None:
|
|
107
|
+
raise ValueError(
|
|
108
|
+
'treatment_names and gene_names must be omitted for a Series mask'
|
|
109
|
+
)
|
|
110
|
+
if (
|
|
111
|
+
not isinstance(test_mask.index, pd.MultiIndex)
|
|
112
|
+
or test_mask.index.nlevels != 2
|
|
113
|
+
):
|
|
114
|
+
raise ValueError(
|
|
115
|
+
'a Series test_mask must have a two-level treatment-gene MultiIndex'
|
|
116
|
+
)
|
|
117
|
+
values = _require_boolean_mask(test_mask.to_numpy())
|
|
118
|
+
raw_treatments = pd.Index(test_mask.index.get_level_values(0))
|
|
119
|
+
raw_genes = pd.Index(test_mask.index.get_level_values(1))
|
|
120
|
+
if raw_treatments.hasnans or raw_genes.hasnans:
|
|
121
|
+
raise ValueError(
|
|
122
|
+
'test_mask treatment and gene labels must not be missing'
|
|
123
|
+
)
|
|
124
|
+
treatment_index = pd.Index(
|
|
125
|
+
[str(value) for value in raw_treatments], dtype=object,
|
|
126
|
+
)
|
|
127
|
+
gene_index = pd.Index(
|
|
128
|
+
[str(value) for value in raw_genes], dtype=object,
|
|
129
|
+
)
|
|
130
|
+
mask_index = pd.MultiIndex.from_arrays([treatment_index, gene_index])
|
|
131
|
+
if mask_index.has_duplicates:
|
|
132
|
+
duplicates = mask_index[mask_index.duplicated()].unique().tolist()
|
|
133
|
+
raise ValueError(
|
|
134
|
+
'test_mask contains duplicate treatment-gene tests: '
|
|
135
|
+
f'{duplicates[:5]}'
|
|
136
|
+
)
|
|
137
|
+
lookup = pd.Series(values, index=mask_index)
|
|
138
|
+
else:
|
|
139
|
+
if isinstance(test_mask, pd.DataFrame):
|
|
140
|
+
values = _require_boolean_mask(test_mask.to_numpy())
|
|
141
|
+
if treatment_names is None:
|
|
142
|
+
treatment_names = list(test_mask.index)
|
|
143
|
+
if gene_names is None:
|
|
144
|
+
gene_names = list(test_mask.columns)
|
|
145
|
+
else:
|
|
146
|
+
values = _require_boolean_mask(test_mask)
|
|
147
|
+
if values.ndim != 2:
|
|
148
|
+
raise ValueError('test_mask must be two-dimensional')
|
|
149
|
+
if treatment_names is None or gene_names is None:
|
|
150
|
+
raise ValueError(
|
|
151
|
+
'treatment_names and gene_names are required for an array mask'
|
|
152
|
+
)
|
|
153
|
+
treatments = _normalize_test_labels(treatment_names, 'treatment')
|
|
154
|
+
genes = _normalize_test_labels(gene_names, 'gene')
|
|
155
|
+
expected_shape = (len(treatments), len(genes))
|
|
156
|
+
if values.shape != expected_shape:
|
|
157
|
+
raise ValueError(
|
|
158
|
+
f'test_mask has shape {values.shape}; expected {expected_shape}'
|
|
159
|
+
)
|
|
160
|
+
mask_index = pd.MultiIndex.from_product([treatments, genes])
|
|
161
|
+
lookup = pd.Series(values.ravel(), index=mask_index)
|
|
162
|
+
|
|
163
|
+
aligned = lookup.reindex(result_keys)
|
|
164
|
+
if aligned.isna().any():
|
|
165
|
+
missing = result_keys[aligned.isna().to_numpy()].unique().tolist()
|
|
166
|
+
raise ValueError(
|
|
167
|
+
'test_mask is missing treatment-gene tests required by results: '
|
|
168
|
+
f'{missing[:5]}'
|
|
169
|
+
)
|
|
170
|
+
return aligned.to_numpy(dtype=bool)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _coerce_named_matrix(values, names, name, generated_prefix):
|
|
174
|
+
if isinstance(values, pd.DataFrame):
|
|
175
|
+
if names is None:
|
|
176
|
+
names = list(values.columns)
|
|
177
|
+
values = values.to_numpy()
|
|
178
|
+
else:
|
|
179
|
+
values = np.asarray(values)
|
|
180
|
+
if values.ndim == 1:
|
|
181
|
+
values = values[:, None]
|
|
182
|
+
if values.ndim != 2:
|
|
183
|
+
raise ValueError(f'{name} must be one- or two-dimensional')
|
|
184
|
+
if names is None:
|
|
185
|
+
names = [f'{generated_prefix}{j + 1}' for j in range(values.shape[1])]
|
|
186
|
+
names = list(names)
|
|
187
|
+
if len(names) != values.shape[1]:
|
|
188
|
+
raise ValueError(f'{name} names must match the number of columns')
|
|
189
|
+
if len(set(names)) != len(names):
|
|
190
|
+
raise ValueError(f'{name} names must be unique')
|
|
191
|
+
return values, names
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def summarize_treatment_associations(
|
|
195
|
+
A, Z, treatment_names=None, covariate_names=None, covariate_types=None,
|
|
196
|
+
bh_scope='global',
|
|
197
|
+
):
|
|
198
|
+
"""Summarize covariate association with each treatment.
|
|
199
|
+
|
|
200
|
+
Each treatment is compared with shared all-zero controls; rows assigned to
|
|
201
|
+
other treatments are excluded. Spearman p-values are adjusted with the
|
|
202
|
+
Benjamini-Hochberg procedure over the family selected by ``bh_scope``. The
|
|
203
|
+
returned statistics are diagnostics and do not automatically identify
|
|
204
|
+
variables that should be removed from an adjustment set.
|
|
205
|
+
|
|
206
|
+
Parameters
|
|
207
|
+
----------
|
|
208
|
+
A : array-like, shape (n, a)
|
|
209
|
+
Binary treatment indicator matrix. DataFrame column names are retained.
|
|
210
|
+
Z : array-like, shape (n, q)
|
|
211
|
+
Observed covariates and/or estimated latent factors to diagnose.
|
|
212
|
+
treatment_names : sequence, optional
|
|
213
|
+
Treatment labels. Inferred from a DataFrame or generated when omitted.
|
|
214
|
+
covariate_names : sequence, optional
|
|
215
|
+
Covariate labels. Inferred from a DataFrame or generated when omitted.
|
|
216
|
+
covariate_types : sequence, optional
|
|
217
|
+
Labels such as ``'observed'`` and ``'latent'``. Defaults to
|
|
218
|
+
``'observed'`` for every column.
|
|
219
|
+
bh_scope : {'global', 'per_treatment'}, optional
|
|
220
|
+
Multiple-testing family. ``'global'`` (the default) adjusts across every
|
|
221
|
+
finite treatment-by-covariate test at once, which with many treatments
|
|
222
|
+
is a large and correspondingly conservative family.
|
|
223
|
+
``'per_treatment'`` adjusts within each treatment's own block, which is
|
|
224
|
+
the right choice when each treatment is screened on its own.
|
|
225
|
+
|
|
226
|
+
Returns
|
|
227
|
+
-------
|
|
228
|
+
summary : DataFrame
|
|
229
|
+
Long-form table containing ``spearman_rho``, ``pvalue``, BH-adjusted
|
|
230
|
+
``padj``, ``n_tests_in_family``, and ``standardized_mean_difference``
|
|
231
|
+
for every treatment-by-covariate pair. ``n_tests_in_family`` records how
|
|
232
|
+
many tests entered that row's adjustment, so the correction is
|
|
233
|
+
self-describing.
|
|
234
|
+
|
|
235
|
+
.. versionadded:: 0.0.9
|
|
236
|
+
"""
|
|
237
|
+
if bh_scope not in ('global', 'per_treatment'):
|
|
238
|
+
raise ValueError("bh_scope must be 'global' or 'per_treatment'")
|
|
239
|
+
A, treatment_names = _coerce_named_matrix(
|
|
240
|
+
A, treatment_names, 'treatment', 'treatment_',
|
|
241
|
+
)
|
|
242
|
+
Z, covariate_names = _coerce_named_matrix(
|
|
243
|
+
Z, covariate_names, 'covariate', 'covariate_',
|
|
244
|
+
)
|
|
245
|
+
if A.shape[0] != Z.shape[0]:
|
|
246
|
+
raise ValueError('A and Z must have the same number of rows')
|
|
247
|
+
if not np.all(np.isin(A, (0, 1))):
|
|
248
|
+
raise ValueError('A must contain only binary treatment indicators')
|
|
249
|
+
try:
|
|
250
|
+
Z = np.asarray(Z, dtype=float)
|
|
251
|
+
except (TypeError, ValueError) as exc:
|
|
252
|
+
raise ValueError('Z must contain numeric covariates') from exc
|
|
253
|
+
if not np.all(np.isfinite(Z)):
|
|
254
|
+
raise ValueError('Z must contain only finite values')
|
|
255
|
+
|
|
256
|
+
if covariate_types is None:
|
|
257
|
+
covariate_types = ['observed'] * Z.shape[1]
|
|
258
|
+
covariate_types = list(covariate_types)
|
|
259
|
+
if len(covariate_types) != Z.shape[1]:
|
|
260
|
+
raise ValueError('covariate_types must match the number of covariates')
|
|
261
|
+
|
|
262
|
+
ctrl = np.sum(A, axis=1) == 0
|
|
263
|
+
if not np.any(ctrl):
|
|
264
|
+
raise ValueError('At least one all-zero control row is required')
|
|
265
|
+
|
|
266
|
+
rows = []
|
|
267
|
+
for j, treatment in enumerate(treatment_names):
|
|
268
|
+
case = A[:, j] == 1
|
|
269
|
+
if not np.any(case):
|
|
270
|
+
raise ValueError(f'Treatment {treatment} has no treated rows')
|
|
271
|
+
eligible = ctrl | case
|
|
272
|
+
y = A[eligible, j]
|
|
273
|
+
for k, covariate in enumerate(covariate_names):
|
|
274
|
+
values = Z[eligible, k]
|
|
275
|
+
is_constant = bool(np.ptp(values) == 0)
|
|
276
|
+
if is_constant:
|
|
277
|
+
rho, pvalue = np.nan, np.nan
|
|
278
|
+
else:
|
|
279
|
+
rho, pvalue = stats.spearmanr(values, y)
|
|
280
|
+
|
|
281
|
+
control_values = Z[ctrl, k]
|
|
282
|
+
treated_values = Z[case, k]
|
|
283
|
+
if len(control_values) < 2 or len(treated_values) < 2:
|
|
284
|
+
smd = np.nan
|
|
285
|
+
else:
|
|
286
|
+
pooled_sd = np.sqrt(
|
|
287
|
+
(np.var(control_values, ddof=1)
|
|
288
|
+
+ np.var(treated_values, ddof=1)) / 2
|
|
289
|
+
)
|
|
290
|
+
if pooled_sd == 0:
|
|
291
|
+
smd = np.nan
|
|
292
|
+
else:
|
|
293
|
+
smd = (
|
|
294
|
+
np.mean(treated_values) - np.mean(control_values)
|
|
295
|
+
) / pooled_sd
|
|
296
|
+
rows.append({
|
|
297
|
+
'treatment': treatment,
|
|
298
|
+
'covariate': covariate,
|
|
299
|
+
'covariate_type': covariate_types[k],
|
|
300
|
+
'n_control': int(np.sum(ctrl)),
|
|
301
|
+
'n_treated': int(np.sum(case)),
|
|
302
|
+
'spearman_rho': float(rho),
|
|
303
|
+
'pvalue': float(pvalue),
|
|
304
|
+
'padj': np.nan,
|
|
305
|
+
'standardized_mean_difference': float(smd),
|
|
306
|
+
'constant': is_constant,
|
|
307
|
+
})
|
|
308
|
+
|
|
309
|
+
summary = pd.DataFrame(rows)
|
|
310
|
+
summary['n_tests_in_family'] = 0
|
|
311
|
+
finite = np.isfinite(summary['pvalue'].to_numpy())
|
|
312
|
+
if bh_scope == 'global':
|
|
313
|
+
families = [np.ones(len(summary), dtype=bool)]
|
|
314
|
+
else:
|
|
315
|
+
families = [
|
|
316
|
+
(summary['treatment'] == treatment).to_numpy()
|
|
317
|
+
for treatment in treatment_names
|
|
318
|
+
]
|
|
319
|
+
for family in families:
|
|
320
|
+
adjust = family & finite
|
|
321
|
+
summary.loc[family, 'n_tests_in_family'] = int(adjust.sum())
|
|
322
|
+
if np.any(adjust):
|
|
323
|
+
summary.loc[adjust, 'padj'] = multipletests(
|
|
324
|
+
summary.loc[adjust, 'pvalue'], method='fdr_bh',
|
|
325
|
+
)[1]
|
|
326
|
+
return summary
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def plot_treatment_associations(
|
|
330
|
+
summary, value='spearman_rho', treatments=None, covariates=None,
|
|
331
|
+
alpha=None, vmax=None, ax=None,
|
|
332
|
+
):
|
|
333
|
+
"""Plot a heatmap of treatment-by-covariate associations.
|
|
334
|
+
|
|
335
|
+
Parameters
|
|
336
|
+
----------
|
|
337
|
+
summary : DataFrame
|
|
338
|
+
Output from :func:`summarize_treatment_associations`.
|
|
339
|
+
value : {'spearman_rho', 'standardized_mean_difference'}, optional
|
|
340
|
+
Statistic displayed in the heatmap.
|
|
341
|
+
treatments, covariates : sequence, optional
|
|
342
|
+
Ordered subsets to display.
|
|
343
|
+
alpha : float or None, optional
|
|
344
|
+
If provided, mark cells whose adjusted p-value is at most ``alpha``. No
|
|
345
|
+
significance threshold is applied by default.
|
|
346
|
+
vmax : float or None, optional
|
|
347
|
+
Symmetric colour limit, so the heatmap spans ``(-vmax, vmax)``. When
|
|
348
|
+
omitted, ``'spearman_rho'`` uses the fixed range ``(-1, 1)`` given by
|
|
349
|
+
its natural bounds, which keeps panels comparable across subsets, while
|
|
350
|
+
the unbounded ``'standardized_mean_difference'`` scales to the largest
|
|
351
|
+
finite magnitude on display.
|
|
352
|
+
ax : matplotlib Axes, optional
|
|
353
|
+
Axes on which to draw the heatmap.
|
|
354
|
+
|
|
355
|
+
Returns
|
|
356
|
+
-------
|
|
357
|
+
fig, ax : matplotlib Figure and Axes
|
|
358
|
+
The populated figure and axes.
|
|
359
|
+
|
|
360
|
+
.. versionadded:: 0.0.9
|
|
361
|
+
"""
|
|
362
|
+
if value not in ('spearman_rho', 'standardized_mean_difference'):
|
|
363
|
+
raise ValueError(
|
|
364
|
+
"value must be 'spearman_rho' or 'standardized_mean_difference'"
|
|
365
|
+
)
|
|
366
|
+
if not isinstance(summary, pd.DataFrame):
|
|
367
|
+
raise ValueError('summary must be a pandas DataFrame')
|
|
368
|
+
required = {'treatment', 'covariate', value, 'padj'}
|
|
369
|
+
missing = required.difference(summary.columns)
|
|
370
|
+
if missing:
|
|
371
|
+
raise ValueError(f'summary is missing required columns: {sorted(missing)}')
|
|
372
|
+
if alpha is not None and not 0 < alpha <= 1:
|
|
373
|
+
raise ValueError('alpha must be None or lie in (0, 1]')
|
|
374
|
+
if vmax is not None and not (np.isfinite(vmax) and vmax > 0):
|
|
375
|
+
raise ValueError('vmax must be None or a finite positive number')
|
|
376
|
+
|
|
377
|
+
all_treatments = list(dict.fromkeys(summary['treatment']))
|
|
378
|
+
all_covariates = list(dict.fromkeys(summary['covariate']))
|
|
379
|
+
treatments = all_treatments if treatments is None else list(treatments)
|
|
380
|
+
covariates = all_covariates if covariates is None else list(covariates)
|
|
381
|
+
unknown_treatments = set(treatments).difference(all_treatments)
|
|
382
|
+
unknown_covariates = set(covariates).difference(all_covariates)
|
|
383
|
+
if unknown_treatments:
|
|
384
|
+
raise ValueError(f'Unknown treatments: {sorted(unknown_treatments)}')
|
|
385
|
+
if unknown_covariates:
|
|
386
|
+
raise ValueError(f'Unknown covariates: {sorted(unknown_covariates)}')
|
|
387
|
+
if not treatments or not covariates:
|
|
388
|
+
raise ValueError('At least one treatment and covariate must be selected')
|
|
389
|
+
|
|
390
|
+
subset = summary[
|
|
391
|
+
summary['treatment'].isin(treatments)
|
|
392
|
+
& summary['covariate'].isin(covariates)
|
|
393
|
+
]
|
|
394
|
+
if subset.duplicated(['treatment', 'covariate']).any():
|
|
395
|
+
raise ValueError('summary contains duplicate treatment-covariate pairs')
|
|
396
|
+
matrix = subset.pivot(
|
|
397
|
+
index='treatment', columns='covariate', values=value,
|
|
398
|
+
).reindex(index=treatments, columns=covariates)
|
|
399
|
+
adjusted = subset.pivot(
|
|
400
|
+
index='treatment', columns='covariate', values='padj',
|
|
401
|
+
).reindex(index=treatments, columns=covariates)
|
|
402
|
+
|
|
403
|
+
if ax is None:
|
|
404
|
+
fig, ax = plt.subplots(
|
|
405
|
+
figsize=(max(6, 0.7 * len(covariates)),
|
|
406
|
+
max(3, 0.3 * len(treatments))),
|
|
407
|
+
)
|
|
408
|
+
else:
|
|
409
|
+
fig = ax.figure
|
|
410
|
+
values = matrix.to_numpy(dtype=float)
|
|
411
|
+
if vmax is not None:
|
|
412
|
+
limit = float(vmax)
|
|
413
|
+
elif value == 'spearman_rho':
|
|
414
|
+
limit = 1.0
|
|
415
|
+
else:
|
|
416
|
+
finite_values = np.abs(values[np.isfinite(values)])
|
|
417
|
+
limit = float(np.max(finite_values)) if finite_values.size else 1.0
|
|
418
|
+
if limit == 0:
|
|
419
|
+
limit = 1.0
|
|
420
|
+
image = ax.imshow(values, aspect='auto', cmap='coolwarm',
|
|
421
|
+
vmin=-limit, vmax=limit)
|
|
422
|
+
ax.set_xticks(np.arange(len(covariates)))
|
|
423
|
+
ax.set_xticklabels(covariates, rotation=45, ha='right')
|
|
424
|
+
ax.set_yticks(np.arange(len(treatments)))
|
|
425
|
+
ax.set_yticklabels(treatments)
|
|
426
|
+
ax.set_xlabel('Covariate or latent factor')
|
|
427
|
+
ax.set_ylabel('Treatment')
|
|
428
|
+
colorbar = fig.colorbar(image, ax=ax)
|
|
429
|
+
colorbar.set_label(
|
|
430
|
+
'Spearman correlation' if value == 'spearman_rho'
|
|
431
|
+
else 'Standardized mean difference'
|
|
432
|
+
)
|
|
433
|
+
if alpha is not None:
|
|
434
|
+
padj = adjusted.to_numpy(dtype=float)
|
|
435
|
+
for row, col in np.argwhere(np.isfinite(padj) & (padj <= alpha)):
|
|
436
|
+
ax.text(col, row, '*', ha='center', va='center', color='black')
|
|
437
|
+
fig.tight_layout()
|
|
438
|
+
return fig, ax
|
|
9
439
|
|
|
10
440
|
|
|
11
441
|
def _coerce_treatment_inputs(A, pi_hat, treatment_names=None):
|
|
@@ -49,6 +479,21 @@ def summarize_propensity_scores(
|
|
|
49
479
|
|
|
50
480
|
Other perturbations are excluded from a treatment's diagnostic comparison;
|
|
51
481
|
each row compares that treatment with shared all-zero controls.
|
|
482
|
+
|
|
483
|
+
Parameters
|
|
484
|
+
----------
|
|
485
|
+
A, pi_hat, treatment_names, overlap_bounds, bins
|
|
486
|
+
Treatments, scores, labels, the reference overlap window, and the
|
|
487
|
+
histogram resolution used for ``overlap_ratio``.
|
|
488
|
+
clip_bounds : tuple(float, float) or None, optional
|
|
489
|
+
The bounds the caller **already applied** to ``pi_hat``, defaulting to
|
|
490
|
+
the ``LFC(ps_clip=(0.01, 0.99))`` default. ``clipped_fraction`` counts
|
|
491
|
+
scores sitting on those bounds. Pass ``None`` for raw, unclipped scores;
|
|
492
|
+
``clipped_fraction`` is then ``NaN`` rather than a misleading ``0.0``,
|
|
493
|
+
since clipping cannot be inferred from the values alone.
|
|
494
|
+
|
|
495
|
+
.. versionchanged:: 0.0.9
|
|
496
|
+
``clip_bounds=None`` reports ``NaN`` instead of ``0.0``.
|
|
52
497
|
"""
|
|
53
498
|
A, pi_hat, treatment_names = _coerce_treatment_inputs(
|
|
54
499
|
A, pi_hat, treatment_names)
|
|
@@ -80,9 +525,13 @@ def summarize_propensity_scores(
|
|
|
80
525
|
eps = np.finfo(float).eps
|
|
81
526
|
ess_ctrl = _effective_sample_size(1 / np.clip(1 - p_ctrl, eps, None))
|
|
82
527
|
ess_case = _effective_sample_size(1 / np.clip(p_case, eps, None))
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
528
|
+
if clip_bounds is None:
|
|
529
|
+
clipped_fraction = np.nan
|
|
530
|
+
else:
|
|
531
|
+
clipped = (
|
|
532
|
+
np.isclose(p, clip_bounds[0]) | np.isclose(p, clip_bounds[1])
|
|
533
|
+
)
|
|
534
|
+
clipped_fraction = float(clipped.mean())
|
|
86
535
|
|
|
87
536
|
rows.append({
|
|
88
537
|
'treatment': name,
|
|
@@ -93,7 +542,7 @@ def summarize_propensity_scores(
|
|
|
93
542
|
'auc': float(roc_auc_score(y, p)),
|
|
94
543
|
'brier_score': float(brier_score_loss(y, p)),
|
|
95
544
|
'outside_overlap_fraction': float(np.mean((p < lo) | (p > hi))),
|
|
96
|
-
'clipped_fraction':
|
|
545
|
+
'clipped_fraction': clipped_fraction,
|
|
97
546
|
'ess_control': ess_ctrl,
|
|
98
547
|
'ess_treated': ess_case,
|
|
99
548
|
'ess_control_fraction': ess_ctrl / max(len(p_ctrl), 1),
|
|
@@ -107,14 +556,21 @@ def summarize_propensity_scores(
|
|
|
107
556
|
|
|
108
557
|
def plot_propensity_scores(
|
|
109
558
|
A, pi_hat, treatments=None, treatment_names=None, overlap_bounds=(0.05, 0.95),
|
|
110
|
-
bins=40, max_panels=4, axes=None,
|
|
559
|
+
bins=40, max_panels=4, axes=None, clip_bounds=(0.01, 0.99),
|
|
111
560
|
):
|
|
112
|
-
"""Plot propensity distributions for treatment and control cells.
|
|
561
|
+
"""Plot propensity distributions for treatment and control cells.
|
|
562
|
+
|
|
563
|
+
``clip_bounds`` is forwarded to :func:`summarize_propensity_scores` for the
|
|
564
|
+
returned summary; pass ``None`` when ``pi_hat`` holds raw, unclipped scores.
|
|
565
|
+
|
|
566
|
+
.. versionchanged:: 0.0.9
|
|
567
|
+
``clip_bounds`` is no longer fixed at its default.
|
|
568
|
+
"""
|
|
113
569
|
A, pi_hat, treatment_names = _coerce_treatment_inputs(
|
|
114
570
|
A, pi_hat, treatment_names)
|
|
115
571
|
summary = summarize_propensity_scores(
|
|
116
572
|
A, pi_hat, treatment_names=treatment_names,
|
|
117
|
-
overlap_bounds=overlap_bounds, bins=bins,
|
|
573
|
+
overlap_bounds=overlap_bounds, clip_bounds=clip_bounds, bins=bins,
|
|
118
574
|
)
|
|
119
575
|
|
|
120
576
|
if treatments is None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: causarray
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.9
|
|
4
4
|
Summary: causarray is a Python module for simultaneous causal inference with an array of outcomes.
|
|
5
5
|
Author-email: Jin-Hong Du <jinhongd@hku.hk>, Maya Shen <myshen@andrew.cmu.edu>, Hansruedi Mathys <mathysh@pitt.edu>, Kathryn Roeder <jinhongd@hku.hk>
|
|
6
6
|
Maintainer-email: Jin-Hong Du <jinhongd@hku.hk>
|
|
@@ -54,7 +54,16 @@ For optimal parallel performance, we recommend installing `llvm-openmp` if using
|
|
|
54
54
|
conda install -c conda-forge llvm-openmp
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
-
For `R` users, `reticulate` can be used to call `causarray` from
|
|
57
|
+
For `R` users, `reticulate` can be used to call `causarray` from R while
|
|
58
|
+
keeping NumPy >2 in the Python environment. Create the separate R environment
|
|
59
|
+
with a current NumPy-2-compatible reticulate build:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
conda env create -f environment-r.yaml
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The R tutorial runs from `causarray-r` and connects to the Python package in
|
|
66
|
+
the `causarray` environment.
|
|
58
67
|
The documentation and tutorials using both `Python` and `R` are available at [causarray.readthedocs.io](https://causarray.readthedocs.io/en/latest/).
|
|
59
68
|
|
|
60
69
|
|
|
@@ -89,6 +98,30 @@ df_res = gcate_lfc_batch(
|
|
|
89
98
|
See the [Replogle-E-K562 tutorial](https://causarray.readthedocs.io/en/latest/tutorial/replogle/replogle-py.html)
|
|
90
99
|
for a demonstration on 200 perturbations from a genome-wide CRISPRi screen.
|
|
91
100
|
|
|
101
|
+
### Diagnostic masks without refitting effects
|
|
102
|
+
|
|
103
|
+
Treatment-by-gene support or quality-control rules can be aligned to an
|
|
104
|
+
existing causarray result table by label, even when its rows are reordered:
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from causarray import align_test_mask
|
|
108
|
+
|
|
109
|
+
keep = align_test_mask(
|
|
110
|
+
df_res,
|
|
111
|
+
support_mask, # treatments × genes, Boolean
|
|
112
|
+
treatment_names=perturbations,
|
|
113
|
+
gene_names=genes,
|
|
114
|
+
)
|
|
115
|
+
df_res_flagged = df_res.assign(support_keep=keep)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
This operation only annotates or subsets existing results. It does not refit
|
|
119
|
+
the causarray LFC or change standard errors and p-values. If a diagnostic rule
|
|
120
|
+
was selected after inspecting the outcomes, retain the original adjusted
|
|
121
|
+
p-values rather than redefining the multiple-testing family post hoc. The
|
|
122
|
+
[Replogle tutorial](https://causarray.readthedocs.io/en/latest/tutorial/replogle/replogle-py.html)
|
|
123
|
+
compares several expression-support rules with marginal Wilcoxon results.
|
|
124
|
+
|
|
92
125
|
## Changelog
|
|
93
126
|
|
|
94
127
|
See [CHANGELOG](https://causarray.readthedocs.io/en/latest/changelog.html) for a full version history.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
causarray/DR_estimation.py,sha256=qsfgK1mL8x7XJV4UIezD8XmQ5hHSWn9c50LAdLM_DBs,31778
|
|
2
|
+
causarray/DR_inference.py,sha256=a49LdlRRgIclceQmNnjXrLWGwtljjUjAAXVp8NjCcHk,5468
|
|
3
|
+
causarray/DR_learner.py,sha256=wL-qgh1w0U3uP9D0zYHdOm2aY6EBSZS2HG0zdSKHZcI,36682
|
|
4
|
+
causarray/__about__.py,sha256=46Yjk3fz9o8aTN8E95McnzpJcjGzVJmHmQqUZ5mXzfc,22
|
|
5
|
+
causarray/__init__.py,sha256=nySXtAigGS0Pu7H26twhwsZaHBJxRcqD5or3YOEfv_I,1692
|
|
6
|
+
causarray/diagnostics.py,sha256=h7dTZg_05-g4wZEOax8o4L5R89RSPcpEi6ppYPnioFA,25437
|
|
7
|
+
causarray/gcate.py,sha256=kH_1L25W74mAsNgzT0ok9tEo_XxpQcncDXKFfKJWojI,25738
|
|
8
|
+
causarray/gcate_glm.py,sha256=N66BNCRezRfkWP1jdsgtpUSFwQTIxqnyeaPvRozx0oI,18287
|
|
9
|
+
causarray/gcate_likelihood.py,sha256=srs2ql2uEhgn386w1lES41E_Gb0EKHT5YDIW-25YDec,4797
|
|
10
|
+
causarray/gcate_opt.py,sha256=tGbYNv_Q0-Pd4sWUMRMEn4Ms_fCnQ07oDLXe-XKcUSA,18630
|
|
11
|
+
causarray/nb_glm_fast.py,sha256=uCuKnx2Wu6-z6eLGjmdjbKh9g8vx1AnSmJX2LrbE1Bg,29702
|
|
12
|
+
causarray/utils.py,sha256=pmnrocTiyK0gS-1mo3vxDbTUptiAGUC9LYh6aGNCaqc,10440
|
|
13
|
+
causarray-0.0.9.dist-info/METADATA,sha256=_IjU7VdMpoLU6XaVhqAq6_B_X6720RopV1c4EIcOvWA,6893
|
|
14
|
+
causarray-0.0.9.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
15
|
+
causarray-0.0.9.dist-info/licenses/LICENSE,sha256=4zFElAgYMtL4dKsu0A1p70cEo2SXRsGEWmJFdFb2hNg,1067
|
|
16
|
+
causarray-0.0.9.dist-info/RECORD,,
|
causarray-0.0.7.dist-info/RECORD
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
causarray/DR_estimation.py,sha256=xuQXApVfEuNvOG4sne5hnn3hCiW5_H-wkiqX8bwTW08,18943
|
|
2
|
-
causarray/DR_inference.py,sha256=a49LdlRRgIclceQmNnjXrLWGwtljjUjAAXVp8NjCcHk,5468
|
|
3
|
-
causarray/DR_learner.py,sha256=6VlzdZSrUhg8HFhIa22f9-vrpx3VESONaBaM3Qid7Zc,34341
|
|
4
|
-
causarray/__about__.py,sha256=R9xOYoYrWKcfO5zvTeGC3m_eDNOvxMd8CocQs2tLufo,22
|
|
5
|
-
causarray/__init__.py,sha256=tTcGCcWjjmkP95c6Y30HaBpwDpTVoUv32qD_ju8DmBo,1449
|
|
6
|
-
causarray/diagnostics.py,sha256=V4l870QbX0XXH6cLJjy57tW0fhv9-wqEu28dcRdowsI,6741
|
|
7
|
-
causarray/gcate.py,sha256=kH_1L25W74mAsNgzT0ok9tEo_XxpQcncDXKFfKJWojI,25738
|
|
8
|
-
causarray/gcate_glm.py,sha256=N66BNCRezRfkWP1jdsgtpUSFwQTIxqnyeaPvRozx0oI,18287
|
|
9
|
-
causarray/gcate_likelihood.py,sha256=srs2ql2uEhgn386w1lES41E_Gb0EKHT5YDIW-25YDec,4797
|
|
10
|
-
causarray/gcate_opt.py,sha256=tGbYNv_Q0-Pd4sWUMRMEn4Ms_fCnQ07oDLXe-XKcUSA,18630
|
|
11
|
-
causarray/nb_glm_fast.py,sha256=uCuKnx2Wu6-z6eLGjmdjbKh9g8vx1AnSmJX2LrbE1Bg,29702
|
|
12
|
-
causarray/utils.py,sha256=pmnrocTiyK0gS-1mo3vxDbTUptiAGUC9LYh6aGNCaqc,10440
|
|
13
|
-
causarray-0.0.7.dist-info/METADATA,sha256=rUtapIuaqM-Tl21WTaFGdvh3vEIonSTO1FRumFYbLvc,5663
|
|
14
|
-
causarray-0.0.7.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
15
|
-
causarray-0.0.7.dist-info/licenses/LICENSE,sha256=4zFElAgYMtL4dKsu0A1p70cEo2SXRsGEWmJFdFb2hNg,1067
|
|
16
|
-
causarray-0.0.7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|