ebin 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ebin/__init__.py +32 -0
- ebin/__main__.py +76 -0
- ebin/activity.py +158 -0
- ebin/data.py +93 -0
- ebin/effects.py +161 -0
- ebin/fit.py +512 -0
- ebin/ftzkiller.py +70 -0
- ebin/gauss_rules.py +297 -0
- ebin/initialize.py +186 -0
- ebin/model.py +186 -0
- ebin/pipeline.py +173 -0
- ebin/plots.py +251 -0
- ebin/truncation.py +59 -0
- ebin-1.0.dist-info/METADATA +12 -0
- ebin-1.0.dist-info/RECORD +18 -0
- ebin-1.0.dist-info/WHEEL +5 -0
- ebin-1.0.dist-info/entry_points.txt +2 -0
- ebin-1.0.dist-info/top_level.txt +1 -0
ebin/pipeline.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""End-to-end driver: fit every cell line, read out the activity, write tables."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import pickle
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from .data import load_groups, prep
|
|
11
|
+
from .fit import fit_effects
|
|
12
|
+
from .initialize import init_from_fit
|
|
13
|
+
from .activity import posterior_activity
|
|
14
|
+
|
|
15
|
+
# every per-object scalar the readout produces; all of it goes to the table
|
|
16
|
+
TABLE_FIELDS = ("activity", "activity_sd", "activity_map",
|
|
17
|
+
"mu", "mu_sd", "sigma", "sigma_sd", "q1", "q2", "q3",
|
|
18
|
+
"abundance", "n_eff", "tot")
|
|
19
|
+
|
|
20
|
+
FIELD_DESC = {
|
|
21
|
+
"activity": "posterior-mean E[bin] (the activity target)",
|
|
22
|
+
"activity_sd": "posterior SD of E[bin] (per-object confidence)",
|
|
23
|
+
"activity_map": "plug-in MAP E[bin] (point estimate, reference)",
|
|
24
|
+
"mu": "posterior-mean effect location (gauge units)",
|
|
25
|
+
"mu_sd": "posterior SD of mu (standard error)",
|
|
26
|
+
"sigma": "posterior-mean effect scale (gauge units)",
|
|
27
|
+
"sigma_sd": "posterior SD of sigma (standard error)",
|
|
28
|
+
"q1": "1st quartile of the posterior-mean effect distribution",
|
|
29
|
+
"q2": "2nd quartile (median = mu) of the effect distribution",
|
|
30
|
+
"q3": "3rd quartile of the posterior-mean effect distribution",
|
|
31
|
+
"abundance": "posterior-mean profiled abundance / size factor a_n",
|
|
32
|
+
"n_eff": "posterior effective grid support (larger = more uncertain)",
|
|
33
|
+
"tot": "total observed reads for this object",
|
|
34
|
+
"bin_prob": "posterior-mean probability of falling in each bin",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class StoredFit:
|
|
39
|
+
"""The part of a fit the readout needs, as loaded from disk."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, d):
|
|
42
|
+
self.__dict__.update(d)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def light(res):
|
|
46
|
+
"""A ``FitResult`` without the optimizer history: everything the readout
|
|
47
|
+
reads, plus the fitted per-object effect law and abundance."""
|
|
48
|
+
return dict(R=res.R, P=res.P, rates=res.rates, log_w=res.log_w,
|
|
49
|
+
cuts=res.cuts, Pi=res.Pi, mu=res.mu, sigma=res.sigma, a=res.a,
|
|
50
|
+
observed=res.observed, phi=res.phi, loglik=res.loglik,
|
|
51
|
+
converged=res.converged, extras=res.extras, config=res.config)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def save_fits(path, fits):
|
|
55
|
+
with open(path, "wb") as f:
|
|
56
|
+
pickle.dump(fits, f)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def load_fits(path):
|
|
60
|
+
with open(path, "rb") as f:
|
|
61
|
+
return {g: StoredFit(d) for g, d in pickle.load(f).items()}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def activity_table(data, groups=None, *, out=None, fits_out=None,
|
|
65
|
+
warm_from=None, readout=None, fields=TABLE_FIELDS,
|
|
66
|
+
verbose=True, **fit_kw):
|
|
67
|
+
"""Fit each cell line and assemble the activity table.
|
|
68
|
+
|
|
69
|
+
data : path to the count table, or a DataFrame / dict of GroupData.
|
|
70
|
+
groups : cell lines to run, in output order (default: all, file order).
|
|
71
|
+
out : CSV path; rewritten after every line so a long run can be
|
|
72
|
+
inspected while it goes.
|
|
73
|
+
fits_out : pickle path for the fitted parameters (reusable readouts).
|
|
74
|
+
warm_from : pickle of earlier fits to warm-start from (same data, e.g.
|
|
75
|
+
refitting under the Gamma abundance prior).
|
|
76
|
+
readout : kwargs for ``posterior_activity`` (defaults reproduce the
|
|
77
|
+
shipped readout: the fit's own priors on a 121 x 33 grid).
|
|
78
|
+
fields : which per-object scalars to write; all of them by default.
|
|
79
|
+
fit_kw : passed to ``fit_effects``.
|
|
80
|
+
|
|
81
|
+
Returns (table, results) where ``table`` has (cell line, field) columns and
|
|
82
|
+
``results`` maps cell line -> the full readout dict (including the (N, B)
|
|
83
|
+
``bin_prob``, which is too wide for the table).
|
|
84
|
+
"""
|
|
85
|
+
gdata = data if isinstance(data, dict) else load_groups(data, groups=groups,
|
|
86
|
+
verbose=verbose)
|
|
87
|
+
order = [g for g in (groups or gdata) if g in gdata]
|
|
88
|
+
if not order:
|
|
89
|
+
raise ValueError(f"no cell lines to fit (have {sorted(gdata)})")
|
|
90
|
+
warm = load_fits(warm_from) if warm_from else {}
|
|
91
|
+
index = gdata[order[0]].index
|
|
92
|
+
|
|
93
|
+
fits, results, cols = {}, {}, {}
|
|
94
|
+
for i, g in enumerate(order):
|
|
95
|
+
t0 = time.time()
|
|
96
|
+
X, mask, _ = prep(gdata[g])
|
|
97
|
+
init = init_from_fit(warm[g].__dict__) if g in warm else None
|
|
98
|
+
res = fit_effects(X, mask=mask, init=init, verbose=False, **fit_kw)
|
|
99
|
+
act = posterior_activity(res, X, mask, **(readout or {}))
|
|
100
|
+
fits[g], results[g] = light(res), act
|
|
101
|
+
for f in fields:
|
|
102
|
+
cols[(g, f)] = pd.Series(act[f], index=index)
|
|
103
|
+
if out:
|
|
104
|
+
_write_table(cols, index, out)
|
|
105
|
+
if fits_out:
|
|
106
|
+
save_fits(fits_out, fits)
|
|
107
|
+
if verbose:
|
|
108
|
+
a = act["activity"]
|
|
109
|
+
print(f"[{i+1}/{len(order)}] {g}: n={int(np.isfinite(a).sum())} "
|
|
110
|
+
f"loglik={res.loglik:.1f} conv={res.converged} "
|
|
111
|
+
f"phi={res.phi:.4f} activity mean={np.nanmean(a):.3f} "
|
|
112
|
+
f"({time.time()-t0:.0f}s)", flush=True)
|
|
113
|
+
return _write_table(cols, index, out), results
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def readout_table(data, fits_path, groups=None, *, out=None, readout=None,
|
|
117
|
+
fields=TABLE_FIELDS, verbose=True):
|
|
118
|
+
"""Redo the readout from saved fits -- no refit.
|
|
119
|
+
|
|
120
|
+
Useful for trying a different readout grid or prior, or for rebuilding the
|
|
121
|
+
table (or the netCDF) without paying for the fits again.
|
|
122
|
+
"""
|
|
123
|
+
gdata = data if isinstance(data, dict) else load_groups(data, groups=groups,
|
|
124
|
+
verbose=verbose)
|
|
125
|
+
fits = load_fits(fits_path)
|
|
126
|
+
order = [g for g in (groups or fits) if g in fits and g in gdata]
|
|
127
|
+
if not order:
|
|
128
|
+
raise ValueError(f"no cell lines in both the data and {fits_path}")
|
|
129
|
+
index = gdata[order[0]].index
|
|
130
|
+
|
|
131
|
+
results, cols = {}, {}
|
|
132
|
+
for g in order:
|
|
133
|
+
X, mask, _ = prep(gdata[g])
|
|
134
|
+
act = posterior_activity(fits[g], X, mask, **(readout or {}))
|
|
135
|
+
results[g] = act
|
|
136
|
+
for f in fields:
|
|
137
|
+
cols[(g, f)] = pd.Series(act[f], index=index)
|
|
138
|
+
if verbose:
|
|
139
|
+
print(f" {g}: activity median "
|
|
140
|
+
f"{np.nanmedian(act['activity']):.3f}", flush=True)
|
|
141
|
+
return _write_table(cols, index, out), results
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _write_table(cols, index, out=None):
|
|
145
|
+
tab = pd.DataFrame(cols, index=index)
|
|
146
|
+
tab.columns = pd.MultiIndex.from_tuples(tab.columns,
|
|
147
|
+
names=["group", "score"])
|
|
148
|
+
if out:
|
|
149
|
+
os.makedirs(os.path.dirname(os.path.abspath(out)), exist_ok=True)
|
|
150
|
+
tab.to_csv(out)
|
|
151
|
+
return tab
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def write_netcdf(path, results, index, order=None, attrs=None):
|
|
155
|
+
"""Write the readout to netCDF: every (cell_type, seq) field plus the
|
|
156
|
+
per-bin distribution bin_prob (cell_type, seq, bin)."""
|
|
157
|
+
import xarray as xr
|
|
158
|
+
|
|
159
|
+
order = list(order or results)
|
|
160
|
+
B = results[order[0]]["bin_prob"].shape[1]
|
|
161
|
+
binp = np.stack([results[g]["bin_prob"] for g in order]).astype(np.float32)
|
|
162
|
+
ds = xr.Dataset(
|
|
163
|
+
{"bin_prob": (("cell_type", "seq", "bin"), binp,
|
|
164
|
+
{"description": FIELD_DESC["bin_prob"]}),
|
|
165
|
+
**{f: (("cell_type", "seq"),
|
|
166
|
+
np.stack([results[g][f] for g in order]).astype(np.float32),
|
|
167
|
+
{"description": FIELD_DESC[f]})
|
|
168
|
+
for f in TABLE_FIELDS}},
|
|
169
|
+
coords={"cell_type": order, "seq": np.asarray(index),
|
|
170
|
+
"bin": np.arange(1, B + 1)},
|
|
171
|
+
attrs=attrs or {})
|
|
172
|
+
ds.to_netcdf(path)
|
|
173
|
+
return ds
|
ebin/plots.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""Diagnostic plots for a fitted activity table.
|
|
2
|
+
|
|
3
|
+
Each function takes the table returned by ``activity_table`` (or read back from
|
|
4
|
+
the CSV with ``pd.read_csv(path, index_col=0, header=[0, 1])``), draws one panel
|
|
5
|
+
per cell line, and returns the figure. matplotlib is imported lazily, so it is
|
|
6
|
+
only needed if you plot.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from .data import load_groups, prep
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _plt():
|
|
16
|
+
import matplotlib
|
|
17
|
+
import matplotlib.pyplot as plt
|
|
18
|
+
return matplotlib, plt
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _lines(table, groups=None):
|
|
22
|
+
have = list(dict.fromkeys(table.columns.get_level_values(0)))
|
|
23
|
+
return [g for g in (groups or have) if g in have]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _grid(n, ncols=5, size=(3.2, 2.7)):
|
|
27
|
+
_, plt = _plt()
|
|
28
|
+
ncols = max(1, min(ncols, n))
|
|
29
|
+
nrows = int(np.ceil(n / ncols))
|
|
30
|
+
fig, axes = plt.subplots(nrows, ncols, squeeze=False,
|
|
31
|
+
figsize=(size[0] * ncols, size[1] * nrows))
|
|
32
|
+
for ax in axes.ravel()[n:]:
|
|
33
|
+
ax.axis("off")
|
|
34
|
+
return fig, list(axes.ravel()[:n])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _finish(fig, out, layout=True):
|
|
38
|
+
if layout:
|
|
39
|
+
fig.tight_layout()
|
|
40
|
+
if out:
|
|
41
|
+
fig.savefig(out, dpi=150, bbox_inches="tight")
|
|
42
|
+
print(f"[saved] {out}")
|
|
43
|
+
return fig
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def raw_mass_center(counts, groups=None):
|
|
47
|
+
"""Depth-normalized empirical E[bin] straight from the counts.
|
|
48
|
+
|
|
49
|
+
Every (replicate, bin) column is divided by its own library size — the
|
|
50
|
+
bins are separately sequenced, so this is the normalization that has to
|
|
51
|
+
happen before the bins are comparable — then replicates are summed and each
|
|
52
|
+
sequence's profile is normalized to give ``sum_b b * p_b``. No model:
|
|
53
|
+
this is the yardstick to check the fitted activity against.
|
|
54
|
+
"""
|
|
55
|
+
gdata = counts if isinstance(counts, dict) else \
|
|
56
|
+
load_groups(counts, groups=groups, verbose=False)
|
|
57
|
+
out, index = {}, None
|
|
58
|
+
for g, gd in gdata.items():
|
|
59
|
+
X, mask, _ = prep(gd)
|
|
60
|
+
Xz = np.where(mask, np.nan_to_num(X), 0.0)
|
|
61
|
+
p = Xz / np.maximum(Xz.sum(0)[None], 1e-9) # per-channel depth
|
|
62
|
+
y = p.sum(1) # over replicates, (N,B)
|
|
63
|
+
s = y.sum(1)
|
|
64
|
+
b = np.arange(1, X.shape[2] + 1, dtype=float)
|
|
65
|
+
out[g] = np.where(s > 0, (y * b).sum(1) / np.maximum(s, 1e-30), np.nan)
|
|
66
|
+
index = gd.index
|
|
67
|
+
return pd.DataFrame(out, index=index)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def gc_content(sequences):
|
|
71
|
+
"""GC fraction of each sequence."""
|
|
72
|
+
s = pd.Index(sequences).astype(str).str.upper()
|
|
73
|
+
letters = s.str.count("[ACGTN]").to_numpy()
|
|
74
|
+
if not np.all(letters == s.str.len().to_numpy()):
|
|
75
|
+
raise ValueError("sequences must be DNA strings; pass sequences= "
|
|
76
|
+
"explicitly if the table index is not the sequence")
|
|
77
|
+
return ((s.str.count("G") + s.str.count("C")) / s.str.len()).to_numpy()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def marginal_density(mu, sigma, x, weights=None, chunk=4096):
|
|
81
|
+
"""(pdf, cdf) of the marginal effect law on the grid ``x``.
|
|
82
|
+
|
|
83
|
+
The marginal is the N-component normal mixture the model actually
|
|
84
|
+
parameterizes -- one component per sequence, not an averaged-out curve:
|
|
85
|
+
|
|
86
|
+
G(x) = sum_n w_n * Phi((x - mu_n) / sigma_n), sum_n w_n = 1.
|
|
87
|
+
|
|
88
|
+
``weights`` default to uniform, which is what the fit uses; pass e.g. the
|
|
89
|
+
fitted abundances to weight sequences by how many cells they contribute.
|
|
90
|
+
Accumulated in chunks, so the N x len(x) array never materializes.
|
|
91
|
+
"""
|
|
92
|
+
from scipy.stats import norm
|
|
93
|
+
mu = np.asarray(mu, float)
|
|
94
|
+
sigma = np.asarray(sigma, float)
|
|
95
|
+
ok = np.isfinite(mu) & np.isfinite(sigma) & (sigma > 0)
|
|
96
|
+
w = np.ones(mu.size) if weights is None else np.asarray(weights, float)
|
|
97
|
+
ok &= np.isfinite(w)
|
|
98
|
+
mu, sigma, w = mu[ok], sigma[ok], w[ok]
|
|
99
|
+
w = w / w.sum()
|
|
100
|
+
pdf = np.zeros_like(x)
|
|
101
|
+
cdf = np.zeros_like(x)
|
|
102
|
+
for i in range(0, mu.size, chunk):
|
|
103
|
+
s = slice(i, i + chunk)
|
|
104
|
+
z = (x[None, :] - mu[s, None]) / sigma[s, None]
|
|
105
|
+
pdf += (w[s, None] * norm.pdf(z) / sigma[s, None]).sum(0)
|
|
106
|
+
cdf += (w[s, None] * norm.cdf(z)).sum(0)
|
|
107
|
+
return pdf, cdf
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def plot_marginal_distribution(table, groups=None, n_bins=4, cuts=None,
|
|
111
|
+
weights=None, out=None, ncols=5, grid=1500,
|
|
112
|
+
pad=3.0):
|
|
113
|
+
"""The marginal effect law of each cell line, sliced into the sorting bins.
|
|
114
|
+
|
|
115
|
+
The curve is the N-component mixture the model estimates,
|
|
116
|
+
G(x) = sum_n w_n Phi((x - mu_n)/sigma_n) — one normal per sequence, drawn
|
|
117
|
+
as the density it defines — and the bins are its quantile cells. Each bin's
|
|
118
|
+
slice is filled in its own colour, and the x ticks are the cut points (the
|
|
119
|
+
quartiles of G for B = 4) labelled with their values. Every slice holds
|
|
120
|
+
1/B of the mass by construction, so unequal-looking areas mean the density
|
|
121
|
+
is concentrated there, not that the bins are unbalanced.
|
|
122
|
+
|
|
123
|
+
cuts : {cell line: array} to draw the fit's own cut points; by default
|
|
124
|
+
they are read off G itself, which is how the fit defines them.
|
|
125
|
+
weights : {cell line: array} of mixture weights; uniform by default, as in
|
|
126
|
+
the fit.
|
|
127
|
+
"""
|
|
128
|
+
matplotlib, plt = _plt()
|
|
129
|
+
lines = _lines(table, groups)
|
|
130
|
+
B = n_bins
|
|
131
|
+
colors = plt.get_cmap("viridis")(np.linspace(0.15, 0.9, B))
|
|
132
|
+
probs = np.arange(1, B) / B
|
|
133
|
+
|
|
134
|
+
fig, axes = _grid(len(lines), ncols, (3.4, 2.8))
|
|
135
|
+
for ax, g in zip(axes, lines):
|
|
136
|
+
mu = table[(g, "mu")].to_numpy(float)
|
|
137
|
+
sd = table[(g, "sigma")].to_numpy(float)
|
|
138
|
+
m = np.isfinite(mu) & np.isfinite(sd)
|
|
139
|
+
lo = np.percentile(mu[m], 0.2) - pad * np.median(sd[m])
|
|
140
|
+
hi = np.percentile(mu[m], 99.8) + pad * np.median(sd[m])
|
|
141
|
+
x = np.linspace(lo, hi, grid)
|
|
142
|
+
w = None if weights is None else np.asarray(weights[g], float)[m]
|
|
143
|
+
pdf, cdf = marginal_density(mu[m], sd[m], x, weights=w)
|
|
144
|
+
q = np.asarray(cuts[g]) if cuts is not None else np.interp(probs, cdf, x)
|
|
145
|
+
|
|
146
|
+
edges = np.concatenate([[x[0]], q, [x[-1]]])
|
|
147
|
+
for b in range(B):
|
|
148
|
+
sel = (x >= edges[b]) & (x <= edges[b + 1])
|
|
149
|
+
ax.fill_between(x[sel], pdf[sel], color=colors[b], linewidth=0)
|
|
150
|
+
ax.plot(x, pdf, color="0.15", lw=1.0)
|
|
151
|
+
for v in q:
|
|
152
|
+
ax.axvline(v, color="0.15", lw=0.8, ls="--")
|
|
153
|
+
ax.set_xticks(q)
|
|
154
|
+
ax.set_xticklabels([f"{v:.2f}" for v in q], fontsize=7, rotation=45,
|
|
155
|
+
ha="right", rotation_mode="anchor")
|
|
156
|
+
ax.set_xlim(lo, hi)
|
|
157
|
+
ax.set_ylim(bottom=0)
|
|
158
|
+
ax.set_yticks([])
|
|
159
|
+
ax.set_title(f"{g} (n={int(m.sum())})", fontsize=9)
|
|
160
|
+
# reserve a strip at the top for the bin legend (a fixed ~0.35 inch,
|
|
161
|
+
# whatever the number of rows)
|
|
162
|
+
top = 1.0 - 0.35 / fig.get_figheight()
|
|
163
|
+
fig.tight_layout(rect=(0, 0, 1, top))
|
|
164
|
+
fig.legend(handles=[matplotlib.patches.Patch(color=colors[b],
|
|
165
|
+
label=f"bin {b + 1}")
|
|
166
|
+
for b in range(B)],
|
|
167
|
+
loc="upper center", ncol=B, fontsize=8, frameon=False,
|
|
168
|
+
bbox_to_anchor=(0.5, 1.0))
|
|
169
|
+
return _finish(fig, out, layout=False)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def plot_activity_vs_raw(table, counts, groups=None, out=None, ncols=5):
|
|
173
|
+
"""Fitted activity against the depth-normalized raw mass centre, coloured
|
|
174
|
+
by the fitted abundance.
|
|
175
|
+
|
|
176
|
+
The two agree for well-measured sequences; the low-abundance ones are where
|
|
177
|
+
the model shrinks and the raw estimate is noise."""
|
|
178
|
+
from scipy.stats import spearmanr
|
|
179
|
+
_, plt = _plt()
|
|
180
|
+
lines = _lines(table, groups)
|
|
181
|
+
raw = raw_mass_center(counts, lines).reindex(table.index)
|
|
182
|
+
ab = np.concatenate([table[(g, "abundance")].to_numpy(float) for g in lines])
|
|
183
|
+
ab = np.log10(np.clip(ab[np.isfinite(ab)], 1e-4, None))
|
|
184
|
+
# a shared scale so panels are comparable; 5-95% because the a_min floor
|
|
185
|
+
# piles up a long left tail that would otherwise eat the whole colormap
|
|
186
|
+
vmin, vmax = np.percentile(ab, [5, 95])
|
|
187
|
+
|
|
188
|
+
fig, axes = _grid(len(lines), ncols)
|
|
189
|
+
sc = None
|
|
190
|
+
for ax, g in zip(axes, lines):
|
|
191
|
+
x = raw[g].to_numpy(float)
|
|
192
|
+
y = table[(g, "activity")].to_numpy(float)
|
|
193
|
+
c = np.log10(np.clip(table[(g, "abundance")].to_numpy(float), 1e-4, None))
|
|
194
|
+
m = np.isfinite(x) & np.isfinite(y) & np.isfinite(c)
|
|
195
|
+
sc = ax.scatter(x[m], y[m], c=c[m], s=2, alpha=0.4, linewidths=0,
|
|
196
|
+
cmap="viridis", vmin=vmin, vmax=vmax, rasterized=True)
|
|
197
|
+
lim = [min(np.nanmin(x[m]), np.nanmin(y[m])),
|
|
198
|
+
max(np.nanmax(x[m]), np.nanmax(y[m]))]
|
|
199
|
+
ax.plot(lim, lim, color="0.4", lw=0.8, ls="--")
|
|
200
|
+
rho = spearmanr(x[m], y[m]).statistic
|
|
201
|
+
ax.set_title(f"{g} rho={rho:.3f}", fontsize=9)
|
|
202
|
+
ax.set_xlabel("raw E[bin] (depth-normalized)", fontsize=7)
|
|
203
|
+
ax.set_ylabel("activity", fontsize=7)
|
|
204
|
+
ax.tick_params(labelsize=7)
|
|
205
|
+
if sc is not None:
|
|
206
|
+
fig.colorbar(sc, ax=axes, shrink=0.6, label="log10 abundance")
|
|
207
|
+
if out:
|
|
208
|
+
fig.savefig(out, dpi=150, bbox_inches="tight")
|
|
209
|
+
print(f"[saved] {out}")
|
|
210
|
+
return fig
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def plot_gc_vs_abundance(table, sequences=None, groups=None, out=None,
|
|
214
|
+
ncols=5, n_trend=25):
|
|
215
|
+
"""GC content against the fitted abundance, with the binned median trend.
|
|
216
|
+
|
|
217
|
+
A strong trend means the size factor is picking up a sequencing/cloning
|
|
218
|
+
bias rather than a biological one."""
|
|
219
|
+
from scipy.stats import spearmanr
|
|
220
|
+
_, plt = _plt()
|
|
221
|
+
lines = _lines(table, groups)
|
|
222
|
+
gc = gc_content(table.index if sequences is None else sequences)
|
|
223
|
+
|
|
224
|
+
fig, axes = _grid(len(lines), ncols)
|
|
225
|
+
for ax, g in zip(axes, lines):
|
|
226
|
+
y = table[(g, "abundance")].to_numpy(float)
|
|
227
|
+
m = np.isfinite(y) & np.isfinite(gc) & (y > 0)
|
|
228
|
+
ax.scatter(gc[m], y[m], s=2, alpha=0.3, linewidths=0, color="#3b6ea5",
|
|
229
|
+
rasterized=True)
|
|
230
|
+
edges = np.quantile(gc[m], np.linspace(0, 1, n_trend + 1))
|
|
231
|
+
idx = np.clip(np.digitize(gc[m], edges[1:-1]), 0, n_trend - 1)
|
|
232
|
+
med = np.array([np.median(y[m][idx == k]) if (idx == k).any() else np.nan
|
|
233
|
+
for k in range(n_trend)])
|
|
234
|
+
ax.plot(0.5 * (edges[:-1] + edges[1:]), med, color="#c0392b", lw=1.4)
|
|
235
|
+
ax.set_yscale("log")
|
|
236
|
+
ax.set_title(f"{g} rho={spearmanr(gc[m], y[m]).statistic:.3f}",
|
|
237
|
+
fontsize=9)
|
|
238
|
+
ax.set_xlabel("GC content", fontsize=7)
|
|
239
|
+
ax.set_ylabel("abundance", fontsize=7)
|
|
240
|
+
ax.tick_params(labelsize=7)
|
|
241
|
+
return _finish(fig, out)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def plot_all(table, counts, outdir=".", groups=None):
|
|
245
|
+
"""Write all three diagnostics into ``outdir``."""
|
|
246
|
+
import os
|
|
247
|
+
os.makedirs(outdir, exist_ok=True)
|
|
248
|
+
p = lambda n: os.path.join(outdir, n)
|
|
249
|
+
plot_marginal_distribution(table, groups, out=p("marginal_effect_law.png"))
|
|
250
|
+
plot_activity_vs_raw(table, counts, groups, out=p("activity_vs_raw.png"))
|
|
251
|
+
plot_gc_vs_abundance(table, groups=groups, out=p("gc_vs_abundance.png"))
|
ebin/truncation.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Zero handling: the probability that an object's observed cells are all zero.
|
|
2
|
+
|
|
3
|
+
Two ways to treat the all-zero rows:
|
|
4
|
+
|
|
5
|
+
* zero-TRUNCATION (``conditional=True``): drop them and condition every kept
|
|
6
|
+
object on Sum_{s,b} X > 0, i.e. subtract log(1 - P_n(0));
|
|
7
|
+
* zero-INFLATION (``conditional=False``): fit them, with a probability phi that
|
|
8
|
+
an object is a structural zero.
|
|
9
|
+
|
|
10
|
+
P_n(0) has a closed form -- for one cell it is the Poisson PGF evaluated at the
|
|
11
|
+
NB zero mass,
|
|
12
|
+
|
|
13
|
+
sum_tau Pois(tau | Pi_b r) NB(0 | R tau, P) = exp(Pi_b r (P^R - 1)),
|
|
14
|
+
|
|
15
|
+
so no tau summation is needed.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import jax.numpy as jnp
|
|
19
|
+
from jax.scipy.special import logsumexp
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def log1mexp(x):
|
|
23
|
+
"""log(1 - e^x) for x < 0, stable on both ends."""
|
|
24
|
+
return jnp.where(x > -0.6931471805599453, # -log 2
|
|
25
|
+
jnp.log(-jnp.expm1(x)),
|
|
26
|
+
jnp.log1p(-jnp.exp(x)))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def log_zero_prob(R, P, Pi, rates, log_wmix, mask):
|
|
30
|
+
"""(N,) log P(all observed cells of the object are zero), phi = 0.
|
|
31
|
+
|
|
32
|
+
Mirrors ``LoglikBuilder.object_loglik``: the rate mixture sits outside the
|
|
33
|
+
product over b, per (object, replicate).
|
|
34
|
+
"""
|
|
35
|
+
rates = jnp.atleast_2d(rates)
|
|
36
|
+
log_wmix = jnp.atleast_2d(log_wmix)
|
|
37
|
+
zeta = P ** R - 1.0 # (S, B) <= 0
|
|
38
|
+
mzeta = jnp.where(mask, zeta[None, :, :], 0.0) # (N, S, B)
|
|
39
|
+
a = jnp.einsum("nb,nsb->ns", Pi, mzeta) # (N, S)
|
|
40
|
+
t = a[:, :, None] * rates[None, :, :] + log_wmix[None, :, :]
|
|
41
|
+
return logsumexp(t, axis=2).sum(axis=1)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def log_zero_prob_abund(R, P, Pi, base_lambda, a_nodes, log_wa, mask):
|
|
45
|
+
"""(N,) log P(all observed cells zero) with the abundance integrated out.
|
|
46
|
+
|
|
47
|
+
Given a_n = a the cells are independent, so log P(all zero | a) = a * c_n
|
|
48
|
+
with c_n = sum_{observed (s,b)} Pi_nb * base_lambda[s] * (P^R - 1)_{s,b} <= 0,
|
|
49
|
+
and marginally log P_n(0) = logsumexp_k [log_wa_k + a_k * c_n] -- the
|
|
50
|
+
discretized-Gamma MGF at c_n. Exact: no tau truncation enters.
|
|
51
|
+
"""
|
|
52
|
+
base_lambda = jnp.atleast_1d(jnp.asarray(base_lambda))
|
|
53
|
+
a_nodes = jnp.asarray(a_nodes)
|
|
54
|
+
zeta = P ** R - 1.0 # (S, B) <= 0
|
|
55
|
+
c = jnp.zeros(mask.shape[0], dtype=jnp.float64)
|
|
56
|
+
for s in range(mask.shape[1]):
|
|
57
|
+
contrib = Pi * (zeta[s] * base_lambda[s])[None, :] # (N, B)
|
|
58
|
+
c = c + jnp.sum(jnp.where(mask[:, s, :], contrib, 0.0), axis=1)
|
|
59
|
+
return logsumexp(log_wa[None, :] + a_nodes[None, :] * c[:, None], axis=1)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ebin
|
|
3
|
+
Version: 1.0
|
|
4
|
+
Summary: Posterior bin-expectation normalization for sorting-based MPRA
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Requires-Dist: jax>=0.4
|
|
7
|
+
Requires-Dist: numpy
|
|
8
|
+
Requires-Dist: scipy
|
|
9
|
+
Requires-Dist: pandas
|
|
10
|
+
Provides-Extra: netcdf
|
|
11
|
+
Requires-Dist: xarray; extra == "netcdf"
|
|
12
|
+
Requires-Dist: h5netcdf; extra == "netcdf"
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
ebin/__init__.py,sha256=4WMcz10RxBu99SBaOXMw65P8HNFmTYQp2KKaYSERfhQ,1384
|
|
2
|
+
ebin/__main__.py,sha256=VKOx4fl0ngX3ygb3E60WoZQsf9DpxHTqKlXLC2ol_Fs,3691
|
|
3
|
+
ebin/activity.py,sha256=0efyOBp7kl3PbOINvQ8Hg3YK3EGuAckDOVvfVtNdoBA,7183
|
|
4
|
+
ebin/data.py,sha256=JU35xOSWbp0Cs-YGxMCrjRQZEgES_bfq5x3aX79fsCQ,3594
|
|
5
|
+
ebin/effects.py,sha256=SkqJka6ywb6k2DiSW__6v6p2ZKYApwcK8SIGuDm3_TE,6559
|
|
6
|
+
ebin/fit.py,sha256=_Wz0g_-006kaKkf2m6yV6JXMUEhUOJ8unD9OU6IUNac,23008
|
|
7
|
+
ebin/ftzkiller.py,sha256=esXm2i6MzWa6t9OEsA7-PYKiV9E3EFx6R2gfecU4f1M,2319
|
|
8
|
+
ebin/gauss_rules.py,sha256=I_r9uF_k0Q-4rW2qVaWXgeZMZWgWVvMc_SYCVqChJvI,9196
|
|
9
|
+
ebin/initialize.py,sha256=odJFSGmrQKhE0VRDtB_WJ8Ob04jqG5G5ViotZw02aWo,7317
|
|
10
|
+
ebin/model.py,sha256=mJ82VEwFfAc_h1EHEre8QCHxUdtXMHlteGv80VzTVeg,8270
|
|
11
|
+
ebin/pipeline.py,sha256=8pCiUc6D52JEtugMrHmPuMcjgCnbnESArJy7mfDiBs4,7282
|
|
12
|
+
ebin/plots.py,sha256=M1RjMPS-I7qLNLZhVFqCodhOltZe_lNXcRNTYPvEoRw,10783
|
|
13
|
+
ebin/truncation.py,sha256=z3_Qy0_rE9jacpVRlKcE6-cz9E9tdl8EHYgXjcqGl1Y,2459
|
|
14
|
+
ebin-1.0.dist-info/METADATA,sha256=W48F-2s0jvd9zkpshf1tO5Q6o_g0FZ0EvpJQcrMytYs,336
|
|
15
|
+
ebin-1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
16
|
+
ebin-1.0.dist-info/entry_points.txt,sha256=4flx-gzw4pc-7fdlcwV47IdTW0lZg0k5DPzmAEh3rpA,44
|
|
17
|
+
ebin-1.0.dist-info/top_level.txt,sha256=8YXMFv4GwJ62SJPK-tp-2XWbwetQ0HlE0CGp9nbPfrI,5
|
|
18
|
+
ebin-1.0.dist-info/RECORD,,
|
ebin-1.0.dist-info/WHEEL
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ebin
|