ebin 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ebin/__init__.py +32 -0
- ebin/__main__.py +76 -0
- ebin/activity.py +158 -0
- ebin/data.py +93 -0
- ebin/effects.py +161 -0
- ebin/fit.py +512 -0
- ebin/ftzkiller.py +70 -0
- ebin/gauss_rules.py +297 -0
- ebin/initialize.py +186 -0
- ebin/model.py +186 -0
- ebin/pipeline.py +173 -0
- ebin/plots.py +251 -0
- ebin/truncation.py +59 -0
- ebin-1.0.dist-info/METADATA +12 -0
- ebin-1.0.dist-info/RECORD +18 -0
- ebin-1.0.dist-info/WHEEL +5 -0
- ebin-1.0.dist-info/entry_points.txt +2 -0
- ebin-1.0.dist-info/top_level.txt +1 -0
ebin/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""EBin -- posterior bin-expectation normalization for sorting-based MPRA.
|
|
2
|
+
|
|
3
|
+
Reads raw per-bin read counts and returns, for every sequence in every cell
|
|
4
|
+
line, the posterior-mean expected bin E[bin] (the activity) together with its
|
|
5
|
+
posterior SD. The model is a compound NB-Poisson likelihood over a latent
|
|
6
|
+
Gaussian effect law per sequence; see model.py, effects.py and activity.py.
|
|
7
|
+
|
|
8
|
+
from ebin import activity_table
|
|
9
|
+
table, _ = activity_table("counts.csv", out="activity.csv")
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
|
|
14
|
+
# jax grabs 75% of VRAM otherwise; float64 throughout (the likelihood sums
|
|
15
|
+
# thousands of log terms per object)
|
|
16
|
+
os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false")
|
|
17
|
+
|
|
18
|
+
import jax # noqa: E402
|
|
19
|
+
|
|
20
|
+
jax.config.update("jax_enable_x64", True)
|
|
21
|
+
|
|
22
|
+
from .data import GroupData, read_counts, load_groups, prep # noqa: E402,F401
|
|
23
|
+
from .initialize import initialize, init_from_fit, InitResult # noqa: E402,F401
|
|
24
|
+
from .fit import fit_effects, FitResult # noqa: E402,F401
|
|
25
|
+
from .activity import posterior_activity # noqa: E402,F401
|
|
26
|
+
from .pipeline import (activity_table, readout_table, save_fits, # noqa: E402,F401
|
|
27
|
+
load_fits, write_netcdf)
|
|
28
|
+
from .plots import (plot_marginal_distribution, plot_activity_vs_raw, # noqa: E402,F401
|
|
29
|
+
plot_gc_vs_abundance, plot_all, raw_mass_center,
|
|
30
|
+
gc_content, marginal_density)
|
|
31
|
+
|
|
32
|
+
__version__ = "1.0"
|
ebin/__main__.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Command line: python -m ebin counts.csv -o activity.csv"""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
|
|
5
|
+
from .pipeline import activity_table, write_netcdf, TABLE_FIELDS
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _parse_args(argv=None):
|
|
9
|
+
p = argparse.ArgumentParser(
|
|
10
|
+
prog="python -m ebin",
|
|
11
|
+
description="Fit the EBin model to a bin-count table and write the "
|
|
12
|
+
"per-cell-line activity table. Defaults are the shipped "
|
|
13
|
+
"configuration.")
|
|
14
|
+
p.add_argument("data", help="count table, 3-level header "
|
|
15
|
+
"(cell line, replicate, bin)")
|
|
16
|
+
p.add_argument("-o", "--out", default="activity.csv", help="output CSV")
|
|
17
|
+
p.add_argument("--groups", help="comma-separated cell lines to run, in "
|
|
18
|
+
"output order (default: all)")
|
|
19
|
+
p.add_argument("--zero-truncation", action="store_true",
|
|
20
|
+
help="drop all-zero rows instead of fitting them as "
|
|
21
|
+
"structural zeros")
|
|
22
|
+
p.add_argument("--abundance", choices=("free", "gamma"), default="free",
|
|
23
|
+
help="per-object abundance: a free a_n, or integrated out "
|
|
24
|
+
"under a Gamma prior")
|
|
25
|
+
p.add_argument("--nodes", type=int, default=24,
|
|
26
|
+
help="quadrature nodes for the Gamma abundance integral")
|
|
27
|
+
p.add_argument("--a-max", type=float, default=15.0)
|
|
28
|
+
p.add_argument("--lambda-fix", type=float, default=50.0)
|
|
29
|
+
p.add_argument("--mu-prior", type=float, default=1.3,
|
|
30
|
+
help="sd of the prior on the effect location (0 disables)")
|
|
31
|
+
p.add_argument("--sigma-prior", default="0,0.5",
|
|
32
|
+
help="mean,sd of the prior on log sigma ('none' disables)")
|
|
33
|
+
p.add_argument("--grid", default="121x33",
|
|
34
|
+
help="readout grid, n_mu x n_log_sigma")
|
|
35
|
+
p.add_argument("--fields", help="comma-separated subset of the output "
|
|
36
|
+
"fields (default: all of them)")
|
|
37
|
+
p.add_argument("--save-fits", help="pickle the fitted parameters here")
|
|
38
|
+
p.add_argument("--warm-from", help="warm-start from a fits pickle")
|
|
39
|
+
p.add_argument("--netcdf", help="also write the full posterior to netCDF")
|
|
40
|
+
p.add_argument("--plots", metavar="DIR",
|
|
41
|
+
help="also write the diagnostic plots into DIR")
|
|
42
|
+
p.add_argument("-q", "--quiet", action="store_true")
|
|
43
|
+
return p.parse_args(argv)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def main(argv=None):
|
|
47
|
+
a = _parse_args(argv)
|
|
48
|
+
n_mu, n_ls = (int(v) for v in a.grid.lower().split("x"))
|
|
49
|
+
sigma_prior = None if a.sigma_prior.lower() == "none" else \
|
|
50
|
+
tuple(float(v) for v in a.sigma_prior.split(","))
|
|
51
|
+
fit_kw = dict(conditional=a.zero_truncation, a_max=a.a_max,
|
|
52
|
+
lambda_fix=a.lambda_fix, sigma_prior=sigma_prior,
|
|
53
|
+
mu_prior=a.mu_prior or None)
|
|
54
|
+
if a.abundance == "gamma":
|
|
55
|
+
fit_kw.update(abundance=False, abundance_prior="gamma",
|
|
56
|
+
n_abund_nodes=a.nodes)
|
|
57
|
+
|
|
58
|
+
table, results = activity_table(
|
|
59
|
+
a.data, groups=a.groups.split(",") if a.groups else None,
|
|
60
|
+
out=a.out, fits_out=a.save_fits, warm_from=a.warm_from,
|
|
61
|
+
readout=dict(n_mu=n_mu, n_ls=n_ls),
|
|
62
|
+
fields=tuple(a.fields.split(",")) if a.fields else TABLE_FIELDS,
|
|
63
|
+
verbose=not a.quiet, **fit_kw)
|
|
64
|
+
print(f"[saved] {a.out} {table.shape[0]} sequences x "
|
|
65
|
+
f"{len(results)} cell lines")
|
|
66
|
+
if a.netcdf:
|
|
67
|
+
write_netcdf(a.netcdf, results, table.index, order=list(results),
|
|
68
|
+
attrs=dict(model="EBin", readout=f"{n_mu}x{n_ls} grid"))
|
|
69
|
+
print(f"[saved] {a.netcdf}")
|
|
70
|
+
if a.plots:
|
|
71
|
+
from .plots import plot_all
|
|
72
|
+
plot_all(table, a.data, a.plots, groups=list(results))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == "__main__":
|
|
76
|
+
main()
|
ebin/activity.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Posterior-mean activity E[bin] and its uncertainty.
|
|
2
|
+
|
|
3
|
+
The plug-in readout E[bin] = sum_b b * Pi[n,b] at the fitted (mu_n, sigma_n) is
|
|
4
|
+
a MODE. For a low-information object the posterior over (mu_n, sigma_n) is
|
|
5
|
+
diffuse and skewed, so its mode sits near a simplex vertex (activity 1 or B,
|
|
6
|
+
regardless of depth) while the posterior MEAN is a mild tilt with a large SD --
|
|
7
|
+
the honest summary, and the one that ships.
|
|
8
|
+
|
|
9
|
+
E[bin] is a functional of theta = (mu_n, sigma_n), so its posterior mean is
|
|
10
|
+
|
|
11
|
+
activity_n = sum_j w_{n,j} ebin(theta_j), w_{n,j} ~ p(X_n | theta_j) p(theta_j)
|
|
12
|
+
|
|
13
|
+
on a shared (mu, log sigma) grid. The abundance is PROFILED per grid point (a
|
|
14
|
+
size factor a = total_n / E[total | a = 1, theta_j]), so a deep object's
|
|
15
|
+
likelihood is sharply peaked and a 3-read object's is flat: depth enters
|
|
16
|
+
honestly instead of being frozen at the joint mode. The prior is the same
|
|
17
|
+
penalty the fit used, so the posterior mean is coherent with the fitted model.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import jax
|
|
22
|
+
import jax.numpy as jnp
|
|
23
|
+
|
|
24
|
+
from .model import LoglikBuilder
|
|
25
|
+
from .effects import normal_bin_probs
|
|
26
|
+
from .truncation import log_zero_prob, log1mexp
|
|
27
|
+
|
|
28
|
+
jax.config.update("jax_enable_x64", True)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def posterior_activity(res, X, mask, *, mu_center=0.0, tau_within="auto",
|
|
32
|
+
sigma_prior="auto", conditional="auto",
|
|
33
|
+
n_mu=121, n_ls=33,
|
|
34
|
+
mu_lim=12.0, ls_lim=(0.2, 5.0)):
|
|
35
|
+
"""Posterior summary of every object of one cell line.
|
|
36
|
+
|
|
37
|
+
res : a fitted ``FitResult`` (supplies R, P, rates, cuts, config).
|
|
38
|
+
X, mask : the same (N, S, B) arrays the fit saw.
|
|
39
|
+
mu_center : scalar or (N,) shrinkage target of the mu prior (0 = neutral).
|
|
40
|
+
tau_within : sd of the mu prior; "auto" takes the fit's ``mu_prior``,
|
|
41
|
+
None gives a flat effect location.
|
|
42
|
+
sigma_prior : (m, s) log-sigma prior; "auto" takes the fit's.
|
|
43
|
+
conditional : whether the per-object likelihood is zero-truncated; "auto"
|
|
44
|
+
takes the fit's. Under zero inflation phi is a global
|
|
45
|
+
constant and drops out of the per-object weights.
|
|
46
|
+
|
|
47
|
+
Everything comes from the SAME posterior weights, so the summaries are
|
|
48
|
+
mutually consistent (activity == bin_prob @ [1..B], q2 == mu). Returns a
|
|
49
|
+
dict of (N,) arrays, NaN off ``res.observed``:
|
|
50
|
+
|
|
51
|
+
activity, activity_sd posterior mean and SD of E[bin]
|
|
52
|
+
activity_map the plug-in mode, for reference
|
|
53
|
+
mu, sigma posterior-mean effect location and scale
|
|
54
|
+
mu_sd, sigma_sd their posterior SDs (standard errors)
|
|
55
|
+
q1, q2, q3 quartiles of the posterior-mean effect law
|
|
56
|
+
abundance posterior-mean profiled size factor a_n
|
|
57
|
+
n_eff posterior grid support (larger = more diffuse)
|
|
58
|
+
tot observed reads
|
|
59
|
+
observed (N,) bool
|
|
60
|
+
|
|
61
|
+
plus bin_prob, the (N, B) posterior-mean distribution over bins.
|
|
62
|
+
"""
|
|
63
|
+
cfg = res.config
|
|
64
|
+
X = np.asarray(X)
|
|
65
|
+
N, S, B = X.shape
|
|
66
|
+
bvec = np.arange(1, B + 1, dtype=float)
|
|
67
|
+
lambda_fix = cfg["lambda_fix"]
|
|
68
|
+
# profile the abundance up to a_max whenever the fit modelled one, whether
|
|
69
|
+
# as a free a_n or integrated out under the Gamma prior
|
|
70
|
+
a_max = (cfg["a_max"]
|
|
71
|
+
if (cfg["abundance"] or cfg.get("abundance_prior")) else 1.0)
|
|
72
|
+
rate_max = lambda_fix * a_max
|
|
73
|
+
if tau_within == "auto":
|
|
74
|
+
tau_within = cfg.get("mu_prior")
|
|
75
|
+
if isinstance(sigma_prior, str):
|
|
76
|
+
sigma_prior = cfg.get("sigma_prior")
|
|
77
|
+
if conditional == "auto":
|
|
78
|
+
conditional = cfg.get("conditional", True)
|
|
79
|
+
|
|
80
|
+
cuts = jnp.asarray(res.cuts)
|
|
81
|
+
R, P = jnp.asarray(res.R), jnp.asarray(res.P)
|
|
82
|
+
rates, log_w = jnp.asarray(res.rates), jnp.asarray(res.log_w)
|
|
83
|
+
lam = np.asarray(res.rates)[:, 0] # (S,) K = 1
|
|
84
|
+
c = np.asarray(res.R) * (1 - np.asarray(res.P)) / np.asarray(res.P)
|
|
85
|
+
|
|
86
|
+
builder = LoglikBuilder(X, mask=mask, rate_max=rate_max)
|
|
87
|
+
kept = np.asarray(res.observed)
|
|
88
|
+
Xz = np.where(np.asarray(builder.mask), np.nan_to_num(X, nan=0.0), 0.0)
|
|
89
|
+
total = Xz.sum((1, 2)) # observed reads
|
|
90
|
+
|
|
91
|
+
# shared (mu, log sigma) grid, its activity readout and expected total
|
|
92
|
+
mu_g = np.linspace(-mu_lim, mu_lim, n_mu)
|
|
93
|
+
ls_g = np.linspace(np.log(ls_lim[0]), np.log(ls_lim[1]), n_ls)
|
|
94
|
+
MU, LS = np.meshgrid(mu_g, ls_g, indexing="ij")
|
|
95
|
+
MU, LS = MU.ravel(), LS.ravel()
|
|
96
|
+
G = MU.size
|
|
97
|
+
Pi_g = np.asarray(normal_bin_probs(jnp.asarray(MU), jnp.exp(jnp.asarray(LS)),
|
|
98
|
+
cuts))
|
|
99
|
+
ebin_g = Pi_g @ bvec # (G,)
|
|
100
|
+
denom_g = (c[None] * Pi_g[:, None, :]).sum(2) @ lam # E[tot | a = 1]
|
|
101
|
+
|
|
102
|
+
@jax.jit
|
|
103
|
+
def ll_grid(pi_row, a_col):
|
|
104
|
+
M = a_col[:, None] * pi_row[None, :]
|
|
105
|
+
obj = builder.object_loglik(R, P, M, rates, log_w, 0.0)
|
|
106
|
+
if not conditional:
|
|
107
|
+
return obj
|
|
108
|
+
lz = jnp.minimum(log_zero_prob(R, P, M, rates, log_w, builder.mask),
|
|
109
|
+
-1e-12)
|
|
110
|
+
return obj - log1mexp(lz)
|
|
111
|
+
|
|
112
|
+
ll = np.empty((N, G))
|
|
113
|
+
a_grid = np.empty((N, G))
|
|
114
|
+
for j in range(G):
|
|
115
|
+
a_col = np.clip(total / max(denom_g[j], 1e-12), 1e-4, a_max)
|
|
116
|
+
a_grid[:, j] = a_col
|
|
117
|
+
ll[:, j] = np.asarray(ll_grid(jnp.asarray(Pi_g[j]), jnp.asarray(a_col)))
|
|
118
|
+
|
|
119
|
+
# per-object prior over the grid: N(mu; center_n, tau^2) * N(ls; m_s, s_s)
|
|
120
|
+
logprior = np.zeros((N, G))
|
|
121
|
+
if tau_within is not None:
|
|
122
|
+
center = np.broadcast_to(np.asarray(mu_center, float), (N,))
|
|
123
|
+
logprior = logprior - (MU[None, :] - center[:, None]) ** 2 \
|
|
124
|
+
/ (2.0 * tau_within ** 2)
|
|
125
|
+
if sigma_prior is not None:
|
|
126
|
+
m_s, s_s = sigma_prior
|
|
127
|
+
logprior = logprior - ((LS - m_s) ** 2 / (2.0 * s_s ** 2))[None, :]
|
|
128
|
+
|
|
129
|
+
logpost = ll + logprior
|
|
130
|
+
logpost -= logpost.max(1, keepdims=True)
|
|
131
|
+
w = np.exp(logpost)
|
|
132
|
+
w /= w.sum(1, keepdims=True)
|
|
133
|
+
ebin_pm = w @ ebin_g
|
|
134
|
+
ebin_sd = np.sqrt(np.clip(w @ (ebin_g ** 2) - ebin_pm ** 2, 0, None))
|
|
135
|
+
|
|
136
|
+
sig_g = np.exp(LS)
|
|
137
|
+
mu_pm = w @ MU
|
|
138
|
+
sigma_pm = w @ sig_g
|
|
139
|
+
z25 = 0.6744897501960817 # N(0,1) third quartile
|
|
140
|
+
return dict(
|
|
141
|
+
activity=np.where(kept, ebin_pm, np.nan),
|
|
142
|
+
activity_sd=np.where(kept, ebin_sd, np.nan),
|
|
143
|
+
activity_map=np.where(kept, np.asarray(res.Pi) @ bvec, np.nan),
|
|
144
|
+
mu=np.where(kept, mu_pm, np.nan),
|
|
145
|
+
mu_sd=np.where(kept, np.sqrt(np.clip(w @ (MU ** 2) - mu_pm ** 2,
|
|
146
|
+
0, None)), np.nan),
|
|
147
|
+
sigma=np.where(kept, sigma_pm, np.nan),
|
|
148
|
+
sigma_sd=np.where(kept, np.sqrt(np.clip(w @ (sig_g ** 2)
|
|
149
|
+
- sigma_pm ** 2, 0, None)),
|
|
150
|
+
np.nan),
|
|
151
|
+
q1=np.where(kept, w @ (MU - z25 * sig_g), np.nan),
|
|
152
|
+
q2=np.where(kept, mu_pm, np.nan),
|
|
153
|
+
q3=np.where(kept, w @ (MU + z25 * sig_g), np.nan),
|
|
154
|
+
abundance=np.where(kept, (w * a_grid).sum(1), np.nan),
|
|
155
|
+
n_eff=np.where(kept, 1.0 / (w ** 2).sum(1), np.nan),
|
|
156
|
+
tot=np.where(kept, total, np.nan),
|
|
157
|
+
bin_prob=np.where(kept[:, None], w @ Pi_g, np.nan),
|
|
158
|
+
observed=kept)
|
ebin/data.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Reading the sorting-bin count table.
|
|
2
|
+
|
|
3
|
+
The expected layout is a table with one row per sequence and a 3-level column
|
|
4
|
+
header ``(cell_line, replicate, bin)``:
|
|
5
|
+
|
|
6
|
+
seq, A549,A549,A549,A549,A549,... <- cell line
|
|
7
|
+
rep1,rep1,rep1,rep1,rep2,... <- replicate
|
|
8
|
+
bin1,bin2,bin3,bin4,bin1,... <- expression bin
|
|
9
|
+
|
|
10
|
+
Each cell line is loaded as an independent (N, S, B) tensor of counts.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class GroupData:
|
|
21
|
+
name: str
|
|
22
|
+
X: np.ndarray # (N, S, B) counts
|
|
23
|
+
mask: np.ndarray # (N, S, B) bool, True = observed
|
|
24
|
+
index: pd.Index
|
|
25
|
+
s_names: list
|
|
26
|
+
b_names: list
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def read_counts(path, sep=None):
|
|
30
|
+
"""Read the count table; ``sep`` defaults to tab for .tsv, comma otherwise."""
|
|
31
|
+
if sep is None:
|
|
32
|
+
sep = "\t" if str(path).endswith((".tsv", ".txt")) else ","
|
|
33
|
+
return pd.read_csv(path, sep=sep, index_col=0, header=[0, 1, 2])
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load_groups(path_or_df, groups=None, verbose=False):
|
|
37
|
+
"""Split the count table into per-cell-line (N, S, B) tensors.
|
|
38
|
+
|
|
39
|
+
Empty fields are read as counts of ZERO, not as missing cells. Some
|
|
40
|
+
export formats write every zero as a blank; treating those as unobserved
|
|
41
|
+
deletes exactly the informative zeros of the low-abundance sequences (an
|
|
42
|
+
object recorded as (0,0,0,14) would reach the likelihood as (-,-,-,14),
|
|
43
|
+
which carries no information about its bin profile at all). Only rows that
|
|
44
|
+
are zero everywhere are meant to drop out, and the likelihood handles those.
|
|
45
|
+
|
|
46
|
+
Rows keep file order, so the returned index aligns with the fit output.
|
|
47
|
+
"""
|
|
48
|
+
df = path_or_df if isinstance(path_or_df, pd.DataFrame) \
|
|
49
|
+
else read_counts(path_or_df)
|
|
50
|
+
n_blank = int(df.isna().to_numpy().sum())
|
|
51
|
+
if n_blank and verbose:
|
|
52
|
+
print(f"[data] {n_blank} empty fields read as zero counts")
|
|
53
|
+
df = df.fillna(0.0)
|
|
54
|
+
|
|
55
|
+
out = {}
|
|
56
|
+
all_groups = list(dict.fromkeys(df.columns.get_level_values(0)))
|
|
57
|
+
unknown = [g for g in (groups or ()) if g not in all_groups]
|
|
58
|
+
if unknown:
|
|
59
|
+
raise ValueError(f"unknown cell line(s) {unknown}; "
|
|
60
|
+
f"the table has {all_groups}")
|
|
61
|
+
for g in (groups or all_groups):
|
|
62
|
+
sub = df[g]
|
|
63
|
+
s_names = list(dict.fromkeys(sub.columns.get_level_values(0)))
|
|
64
|
+
b_names = list(dict.fromkeys(sub.columns.get_level_values(1)))
|
|
65
|
+
cols = pd.MultiIndex.from_product([s_names, b_names])
|
|
66
|
+
missing = [c for c in cols if c not in sub.columns]
|
|
67
|
+
if missing:
|
|
68
|
+
raise ValueError(f"group {g}: non-rectangular columns {missing}")
|
|
69
|
+
sub = sub[cols]
|
|
70
|
+
X = sub.to_numpy(dtype=np.float64).reshape(
|
|
71
|
+
len(sub), len(s_names), len(b_names))
|
|
72
|
+
out[g] = GroupData(name=g, X=X, mask=~np.isnan(X), index=sub.index,
|
|
73
|
+
s_names=s_names, b_names=b_names)
|
|
74
|
+
return out
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def bin_order(b_names):
|
|
78
|
+
"""Column order that sorts bins by the number in their name (bin1 < bin2)."""
|
|
79
|
+
key = lambda b: int("".join(c for c in str(b) if c.isdigit()) or 0)
|
|
80
|
+
return sorted(range(len(b_names)), key=lambda i: key(b_names[i]))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def prep(gd):
|
|
84
|
+
"""(X, mask, measured) with bins in ascending order.
|
|
85
|
+
|
|
86
|
+
``measured`` marks the objects with at least one read; the all-zero rows
|
|
87
|
+
carry no activity signal (they are either silent or unsampled).
|
|
88
|
+
"""
|
|
89
|
+
bo = bin_order(gd.b_names)
|
|
90
|
+
X, mask = gd.X[:, :, bo], gd.mask[:, :, bo]
|
|
91
|
+
measured = mask.any((1, 2)) & (np.where(mask, np.nan_to_num(X), 0)
|
|
92
|
+
.sum((1, 2)) > 0)
|
|
93
|
+
return X, mask, measured
|
ebin/effects.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Normal-effects parameterization of the bin profile Pi.
|
|
2
|
+
|
|
3
|
+
Each object carries a latent Gaussian effect law E_n ~ Normal(mu_n, sigma_n^2)
|
|
4
|
+
-- how the cells of that sequence spread over the measured phenotype. The
|
|
5
|
+
library's marginal effect law is then the N-component mixture of those per-
|
|
6
|
+
sequence normals, and the B sorting bins are its quantile cells:
|
|
7
|
+
|
|
8
|
+
G(x) = sum_n w_n * Phi((x - mu_n) / sigma_n), w_n = 1 / N_observed,
|
|
9
|
+
|
|
10
|
+
with cut points q_1 < ... < q_{B-1} solving G(q_j) = j / B, so that
|
|
11
|
+
|
|
12
|
+
Pi[n, b] = Phi((q_b - mu_n)/sigma_n) - Phi((q_{b-1} - mu_n)/sigma_n).
|
|
13
|
+
|
|
14
|
+
That spends 2 parameters per object instead of B-1, and ties every object to
|
|
15
|
+
the global marginal through the shared cuts (which move when any object moves).
|
|
16
|
+
The cuts are an implicit function of all (mu, sigma); gradients flow through the
|
|
17
|
+
root solve by the implicit function theorem.
|
|
18
|
+
|
|
19
|
+
Gauge freedom: the likelihood sees the parameters only through the z-scores
|
|
20
|
+
(q_j - mu_n)/sigma_n, so it is exactly invariant under
|
|
21
|
+
(mu, sigma, q) -> (a + k*mu, k*sigma, a + k*q), k > 0. Location and scale of
|
|
22
|
+
the effect axis are conventions; ``gauge_normalize`` pins them (median cut = 0,
|
|
23
|
+
first cut = -1). B >= 3 is required for per-object sigma to mean anything.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from functools import partial
|
|
27
|
+
|
|
28
|
+
import numpy as np
|
|
29
|
+
import jax
|
|
30
|
+
import jax.numpy as jnp
|
|
31
|
+
from jax.scipy.stats import norm as _jnorm
|
|
32
|
+
from scipy.special import ndtri as _ndtri
|
|
33
|
+
|
|
34
|
+
jax.config.update("jax_enable_x64", True)
|
|
35
|
+
|
|
36
|
+
# Solver constants in standardized (mixture sd = 1) coordinates. The step clamp
|
|
37
|
+
# tames Newton in flat CDF regions; the bracket contains any p-quantile with
|
|
38
|
+
# p in [1/2500, 1 - 1/2500] by Chebyshev, far wider than any 1/B.
|
|
39
|
+
_STEP_CLAMP = 3.0
|
|
40
|
+
_BRACKET = 50.0
|
|
41
|
+
_XATOL = 1e-13
|
|
42
|
+
_MAX_ITER = 200
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@partial(jax.custom_jvp, nondiff_argnums=(0, 1))
|
|
46
|
+
def _newton_root(fun, dfun, args, x0):
|
|
47
|
+
"""Solve fun(x, args) = 0 by bracketed Newton from x0.
|
|
48
|
+
|
|
49
|
+
``fun`` must be increasing in x with derivative ``dfun``, and the root must
|
|
50
|
+
lie in [x0 - _BRACKET, x0 + _BRACKET]. Every iterate tightens the sign
|
|
51
|
+
bracket; a clamped Newton candidate that leaves the bracket is replaced by
|
|
52
|
+
bisection, so isolated modes of a spiky mixture cannot be jumped over.
|
|
53
|
+
"""
|
|
54
|
+
def cond(state):
|
|
55
|
+
x_prev, i, x, lo, hi = state
|
|
56
|
+
return (jnp.abs(x - x_prev) > _XATOL) & (i < _MAX_ITER)
|
|
57
|
+
|
|
58
|
+
def body(state):
|
|
59
|
+
_, i, x, lo, hi = state
|
|
60
|
+
f = fun(x, args)
|
|
61
|
+
lo = jnp.where(f < 0, jnp.maximum(lo, x), lo)
|
|
62
|
+
hi = jnp.where(f >= 0, jnp.minimum(hi, x), hi)
|
|
63
|
+
step = jnp.clip(f / dfun(x, args), -_STEP_CLAMP, _STEP_CLAMP)
|
|
64
|
+
x_newton = x - step
|
|
65
|
+
inside = (x_newton > lo) & (x_newton < hi)
|
|
66
|
+
x_next = jnp.where(inside, x_newton, 0.5 * (lo + hi))
|
|
67
|
+
return (x, i + 1, x_next, lo, hi)
|
|
68
|
+
|
|
69
|
+
state = (x0 + 1.0, 0, x0, x0 - _BRACKET, x0 + _BRACKET)
|
|
70
|
+
return jax.lax.while_loop(cond, body, state)[2]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@_newton_root.defjvp
|
|
74
|
+
def _newton_root_jvp(fun, dfun, primals, tangents):
|
|
75
|
+
args, x0 = primals
|
|
76
|
+
t_args, _ = tangents # the root is independent of x0
|
|
77
|
+
x_star = _newton_root(fun, dfun, args, x0)
|
|
78
|
+
# implicit function theorem: dx*/dargs = -(dfun/dargs) / (dfun/dx)
|
|
79
|
+
_, j_args = jax.jvp(lambda a: fun(x_star, a), (args,), (t_args,))
|
|
80
|
+
return x_star, -j_args / dfun(x_star, args)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _mix_resid(t, args, p):
|
|
84
|
+
"""G(m0 + s0*t) - p for the weighted normal mixture."""
|
|
85
|
+
mu, sigma, w, m0, s0 = args
|
|
86
|
+
x = m0 + s0 * t
|
|
87
|
+
return jnp.sum(w * _jnorm.cdf((x - mu) / sigma)) - p
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _mix_slope(t, args):
|
|
91
|
+
"""d/dt of _mix_resid (floored: a quantile in a density gap is genuinely
|
|
92
|
+
ill-conditioned; the floor only prevents NaNs)."""
|
|
93
|
+
mu, sigma, w, m0, s0 = args
|
|
94
|
+
x = m0 + s0 * t
|
|
95
|
+
dens = jnp.sum(w * _jnorm.pdf((x - mu) / sigma) / sigma)
|
|
96
|
+
return jnp.maximum(dens * s0, 1e-300)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def mixture_quantiles(mu, sigma, weights, target_probs):
|
|
100
|
+
"""Cut points q_j with sum_n w_n Phi((q_j - mu_n)/sigma_n) = p_j.
|
|
101
|
+
|
|
102
|
+
Differentiable in (mu, sigma) and in ``weights`` through the implicit
|
|
103
|
+
function theorem; ``target_probs`` are constants. The solve runs in
|
|
104
|
+
standardized coordinates, so Newton's start and step clamp are scale-free;
|
|
105
|
+
freezing the standardization costs nothing since the root is exact either
|
|
106
|
+
way and all derivatives flow through the root.
|
|
107
|
+
"""
|
|
108
|
+
target_probs = np.asarray(target_probs, dtype=float)
|
|
109
|
+
m = jnp.sum(weights * mu)
|
|
110
|
+
v = jnp.sum(weights * (sigma ** 2 + mu ** 2)) - m ** 2
|
|
111
|
+
m0 = jax.lax.stop_gradient(m)
|
|
112
|
+
s0 = jax.lax.stop_gradient(jnp.sqrt(jnp.clip(v, 1e-30, None)))
|
|
113
|
+
args = (mu, sigma, weights, m0, s0)
|
|
114
|
+
cuts = []
|
|
115
|
+
for p in target_probs:
|
|
116
|
+
fun = partial(_mix_resid, p=float(p))
|
|
117
|
+
t0 = jnp.asarray(float(_ndtri(p)), dtype=jnp.float64)
|
|
118
|
+
t_star = _newton_root(fun, _mix_slope, args, t0)
|
|
119
|
+
cuts.append(m0 + s0 * t_star)
|
|
120
|
+
return jnp.stack(cuts)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def normal_bin_probs(mu, sigma, cuts, floor=1e-12):
|
|
124
|
+
"""(N, B) bin probabilities Pi[n,b] = Phi(z_{n,b}) - Phi(z_{n,b-1}).
|
|
125
|
+
|
|
126
|
+
Rows sum to 1 exactly; ``floor`` keeps every cell strictly positive (an
|
|
127
|
+
object far outside the cuts would otherwise underflow whole bins to 0 and
|
|
128
|
+
give -inf loglik for nonzero counts there).
|
|
129
|
+
"""
|
|
130
|
+
z = (cuts[None, :] - mu[:, None]) / sigma[:, None] # (N, B-1)
|
|
131
|
+
cdf = _jnorm.cdf(z)
|
|
132
|
+
zero = jnp.zeros((cdf.shape[0], 1), dtype=cdf.dtype)
|
|
133
|
+
one = jnp.ones((cdf.shape[0], 1), dtype=cdf.dtype)
|
|
134
|
+
raw = jnp.maximum(jnp.concatenate([cdf, one], axis=1)
|
|
135
|
+
- jnp.concatenate([zero, cdf], axis=1), 0.0)
|
|
136
|
+
B = raw.shape[1]
|
|
137
|
+
return (raw + floor) / (1.0 + B * floor)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def gauge_normalize(mu, sigma, cuts, pin_median=0.0, pin_left=-1.0):
|
|
141
|
+
"""Exact affine normalization of the (unidentified) effect-axis gauge.
|
|
142
|
+
|
|
143
|
+
Maps (mu, sigma, cuts) -> (a + k*mu, k*sigma, a + k*cuts) so the middle cut
|
|
144
|
+
equals ``pin_median`` and (if ``pin_left`` is given and B >= 3) the first cut
|
|
145
|
+
equals ``pin_left``. The likelihood is exactly invariant under this map.
|
|
146
|
+
"""
|
|
147
|
+
mu = np.asarray(mu, dtype=float)
|
|
148
|
+
sigma = np.asarray(sigma, dtype=float)
|
|
149
|
+
cuts = np.asarray(cuts, dtype=float)
|
|
150
|
+
mid = len(cuts) // 2
|
|
151
|
+
if pin_left is not None and mid > 0:
|
|
152
|
+
if pin_median <= pin_left:
|
|
153
|
+
raise ValueError("pin_median must exceed pin_left")
|
|
154
|
+
k = (cuts[mid] - cuts[0]) / (pin_median - pin_left)
|
|
155
|
+
if k <= 0:
|
|
156
|
+
raise ValueError("cuts must be increasing")
|
|
157
|
+
else:
|
|
158
|
+
k = 1.0
|
|
159
|
+
mu2 = pin_median + (mu - cuts[mid]) / k
|
|
160
|
+
cuts2 = pin_median + (cuts - cuts[mid]) / k
|
|
161
|
+
return mu2, sigma / k, cuts2
|