pertrank 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pertrank/__init__.py +11 -0
- pertrank/_version.py +3 -0
- pertrank/api.py +457 -0
- pertrank/core.py +255 -0
- pertrank/data/replogle_k562.npz +0 -0
- pertrank/data/replogle_k562.provenance.json +63 -0
- pertrank/data/tcell_d3_rest.npz +0 -0
- pertrank/data/tcell_d3_rest.provenance.json +64 -0
- pertrank/datasets.py +52 -0
- pertrank/evaluation.py +107 -0
- pertrank/py.typed +0 -0
- pertrank-0.2.0.dist-info/METADATA +177 -0
- pertrank-0.2.0.dist-info/RECORD +16 -0
- pertrank-0.2.0.dist-info/WHEEL +5 -0
- pertrank-0.2.0.dist-info/licenses/LICENSE +14 -0
- pertrank-0.2.0.dist-info/top_level.txt +1 -0
pertrank/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""PertRank's fixed paper method with an explicit AnnData input contract."""
|
|
2
|
+
|
|
3
|
+
from .api import PertRankResult, from_statistics, rank, rank_effects
|
|
4
|
+
from .core import fit_cells, nominate, normalize_counts, reliability_weights
|
|
5
|
+
from ._version import __version__
|
|
6
|
+
from .evaluation import evaluate_reference
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"PertRankResult", "rank", "rank_effects", "from_statistics", "evaluate_reference",
|
|
10
|
+
"fit_cells", "nominate", "normalize_counts", "reliability_weights",
|
|
11
|
+
]
|
pertrank/_version.py
ADDED
pertrank/api.py
ADDED
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
"""AnnData adapters and labelled outputs for the unchanged PertRank engine.
|
|
2
|
+
|
|
3
|
+
Input cells and matching strata are supplied by the caller. No reference,
|
|
4
|
+
query selection, cell subsampling, model tuning or performance evaluation is
|
|
5
|
+
performed here. The counts path follows the archived full-axis adapter.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from copy import deepcopy
|
|
11
|
+
import json
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from ._version import __version__
|
|
19
|
+
|
|
20
|
+
from .core import (
|
|
21
|
+
ARMS, BETA, K_DIRECTIONS, _controls, _directions, _strings,
|
|
22
|
+
nominate, reliability_weights,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _queries(values: Sequence[str] | str) -> np.ndarray:
|
|
27
|
+
values = [values] if isinstance(values, str) else values
|
|
28
|
+
out = _strings(values, len(values), "queries")
|
|
29
|
+
if not len(out) or len(set(out)) != len(out):
|
|
30
|
+
raise ValueError("queries must be nonempty and unique")
|
|
31
|
+
return out
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _options(top_k: int, seed: int) -> None:
|
|
35
|
+
if not isinstance(top_k, (int, np.integer)) or isinstance(top_k, bool) or top_k < 1:
|
|
36
|
+
raise ValueError("top_k must be a positive integer")
|
|
37
|
+
if not isinstance(seed, (int, np.integer)) or isinstance(seed, bool):
|
|
38
|
+
raise ValueError("seed must be an integer")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _read(matrix: Any, rows: Any, input_scale: str) -> np.ndarray:
|
|
42
|
+
x = matrix[rows, :]
|
|
43
|
+
if hasattr(x, "toarray"):
|
|
44
|
+
x = x.toarray()
|
|
45
|
+
x = np.asarray(x, dtype=np.float64)
|
|
46
|
+
if x.ndim != 2 or not np.isfinite(x).all():
|
|
47
|
+
raise ValueError("Expression values must form a finite cell-by-gene matrix")
|
|
48
|
+
if (x < 0).any():
|
|
49
|
+
raise ValueError("counts and log input require nonnegative values; scaled residuals are unsupported")
|
|
50
|
+
if input_scale == "counts" and (x != np.floor(x)).any():
|
|
51
|
+
raise ValueError("counts input must contain nonnegative integer raw counts")
|
|
52
|
+
return x
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _strata(adata: Any, keys: Sequence[str]) -> np.ndarray:
|
|
56
|
+
if isinstance(keys, str) or keys is None:
|
|
57
|
+
raise ValueError("strata_keys must be an explicit sequence of obs keys; use [] to declare one stratum")
|
|
58
|
+
keys = list(keys)
|
|
59
|
+
if len(set(keys)) != len(keys):
|
|
60
|
+
raise ValueError("strata_keys must be unique")
|
|
61
|
+
if not keys:
|
|
62
|
+
return np.repeat("__all__", adata.shape[0])
|
|
63
|
+
arrays = []
|
|
64
|
+
for key in keys:
|
|
65
|
+
if key not in adata.obs:
|
|
66
|
+
raise ValueError(f"Missing matching-stratum obs column {key!r}")
|
|
67
|
+
arrays.append(_strings(adata.obs[key].to_numpy(), adata.shape[0], key))
|
|
68
|
+
if len(arrays) == 1:
|
|
69
|
+
return arrays[0]
|
|
70
|
+
return np.array([json.dumps(list(parts), separators=(",", ":")) for parts in zip(*arrays)])
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _candidate_rows(stats: Mapping[str, Any], queries: np.ndarray,
|
|
74
|
+
probabilities: np.ndarray, top_k: int):
|
|
75
|
+
gmap = {gene: i for i, gene in enumerate(stats["gene_ids"])}
|
|
76
|
+
for qi, query in enumerate(queries):
|
|
77
|
+
eligible = np.flatnonzero(stats["target_gene_ids"] != query)
|
|
78
|
+
for di, sign in enumerate((1, -1)):
|
|
79
|
+
for ai, arm in enumerate(ARMS):
|
|
80
|
+
score = sign * stats["effects"][eligible, gmap[query]] * stats["weights"][arm][eligible]
|
|
81
|
+
ordered = np.sort(score)
|
|
82
|
+
for local, target in enumerate(eligible):
|
|
83
|
+
first = len(score) - int(np.searchsorted(ordered, score[local], side="right")) + 1
|
|
84
|
+
last = len(score) - int(np.searchsorted(ordered, score[local], side="left"))
|
|
85
|
+
yield {
|
|
86
|
+
"method": arm, "query_id": str(query),
|
|
87
|
+
"direction": "up" if sign > 0 else "down",
|
|
88
|
+
"target_id": str(stats["target_ids"][target]),
|
|
89
|
+
"target_gene_id": str(stats["target_gene_ids"][target]),
|
|
90
|
+
"original_effect": float(stats["effects"][target, gmap[query]]),
|
|
91
|
+
"weight": float(stats["weights"][arm][target]),
|
|
92
|
+
"signed_score": float(score[local]), "rank_first": first,
|
|
93
|
+
"rank_last": last,
|
|
94
|
+
"tie_inclusion_probability": float(probabilities[ai, qi, di, target]),
|
|
95
|
+
"top_k": top_k,
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass
|
|
100
|
+
class PertRankResult:
|
|
101
|
+
"""Labelled nominations, effects, reliability diagnostics and provenance.
|
|
102
|
+
|
|
103
|
+
``effects`` has target IDs as its index and gene IDs as columns. ``weights``
|
|
104
|
+
contains one row per target and the Unweighted, Euclidean and PertRank
|
|
105
|
+
columns. ``controls`` holds the fitted matched-control moments and leading
|
|
106
|
+
covariance directions. ``statistics`` retains the original reliability-
|
|
107
|
+
input statistics, including source effects, even after ``rank_effects``
|
|
108
|
+
supplies an external readout. Replaying those statistics therefore returns
|
|
109
|
+
the source readout; external estimates remain in ``effects``. Nothing is
|
|
110
|
+
written into AnnData.
|
|
111
|
+
|
|
112
|
+
The tie inclusion probabilities describe random boundary-tie selection;
|
|
113
|
+
they are not probabilities of biological validity.
|
|
114
|
+
"""
|
|
115
|
+
nominations: pd.DataFrame
|
|
116
|
+
all_candidate_scores: pd.DataFrame
|
|
117
|
+
weights: pd.DataFrame
|
|
118
|
+
effects: pd.DataFrame
|
|
119
|
+
controls: dict[str, Any]
|
|
120
|
+
statistics: dict[str, Any]
|
|
121
|
+
metadata: dict[str, Any]
|
|
122
|
+
|
|
123
|
+
def rank_effects(self, effects: pd.DataFrame, queries: Sequence[str] | str | None = None,
|
|
124
|
+
*, top_k: int | None = None, seed: int | None = None) -> PertRankResult:
|
|
125
|
+
"""Reuse these fixed weights on a labelled external effect estimate."""
|
|
126
|
+
return rank_effects(self, effects, queries, top_k=top_k, seed=seed)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _result(stats: dict[str, Any], queries: np.ndarray, top_k: int,
|
|
130
|
+
seed: int, metadata: dict[str, Any]) -> PertRankResult:
|
|
131
|
+
_options(top_k, seed)
|
|
132
|
+
records, probabilities = nominate(stats, queries, top_k=top_k, seed=seed)
|
|
133
|
+
weights = pd.DataFrame({
|
|
134
|
+
"target_id": stats["target_ids"], "target_gene_id": stats["target_gene_ids"],
|
|
135
|
+
"n_input_cells": stats["n_cells"], "stratum": stats["batch_ids"],
|
|
136
|
+
"energy": stats["energy"], "noise_trace": stats["noise_trace"],
|
|
137
|
+
**stats["weights"],
|
|
138
|
+
}).set_index("target_id", drop=False)
|
|
139
|
+
effects = pd.DataFrame(stats["effects"], index=stats["target_ids"], columns=stats["gene_ids"])
|
|
140
|
+
effects.index.name = "target_id"
|
|
141
|
+
effects.columns.name = "gene_id"
|
|
142
|
+
metadata = {
|
|
143
|
+
"weight_gene_count": int(len(stats["gene_ids"])),
|
|
144
|
+
"input_selection": "All supplied input cells; no sampling or reference access",
|
|
145
|
+
"effect_origin": "source", **metadata,
|
|
146
|
+
"pertrank_version": __version__,
|
|
147
|
+
"k": K_DIRECTIONS, "beta": BETA, "top_k": int(top_k),
|
|
148
|
+
"tie_seed": int(seed), "query_ids": queries.tolist(),
|
|
149
|
+
"targets": int(len(stats["target_ids"])),
|
|
150
|
+
"tie_probability_semantics": "Boundary-tie inclusion, not biological confidence",
|
|
151
|
+
}
|
|
152
|
+
return PertRankResult(
|
|
153
|
+
pd.DataFrame.from_records(records),
|
|
154
|
+
pd.DataFrame.from_records(_candidate_rows(stats, queries, probabilities, top_k)),
|
|
155
|
+
weights, effects, stats.get("control_models", {}), stats, metadata,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def rank(adata: Any, queries: Sequence[str] | str, *, input_scale: str,
|
|
160
|
+
strata_keys: Sequence[str], target_key: str = "target_ids",
|
|
161
|
+
target_gene_key: str = "target_gene_ids", control_key: str = "is_control",
|
|
162
|
+
layer: str | None = None, top_k: int = 25, seed: int = 2026,
|
|
163
|
+
batch_rows: int = 512, full_gene_axis: bool = False,
|
|
164
|
+
library_totals_key: str | None = None) -> PertRankResult:
|
|
165
|
+
"""Prioritize perturbations for supplied query genes using fixed k=5,beta=.9.
|
|
166
|
+
|
|
167
|
+
Parameters
|
|
168
|
+
----------
|
|
169
|
+
adata
|
|
170
|
+
AnnData with unique cell and gene IDs. Dense, sparse CSR/CSC and backed
|
|
171
|
+
matrices are read in row blocks; matched controls and one target at a
|
|
172
|
+
time are materialized for exact covariance and sample variance.
|
|
173
|
+
queries
|
|
174
|
+
Gene IDs present on the analysis axis. Own-target candidates are
|
|
175
|
+
excluded independently for every query.
|
|
176
|
+
input_scale
|
|
177
|
+
Required: ``"counts"`` or ``"log"``. Counts use the full-library
|
|
178
|
+
median-control normalization from the paper adapter. Log input must
|
|
179
|
+
already be nonnegative log-normalized expression on one supplied gene
|
|
180
|
+
axis; it is used unchanged and must not be scaled residuals.
|
|
181
|
+
strata_keys
|
|
182
|
+
Required obs keys defining matched controls. ``[]`` explicitly
|
|
183
|
+
declares a single stratum. Counts support one stratum in this release;
|
|
184
|
+
log input supports several, with each target confined to one stratum.
|
|
185
|
+
target_key, target_gene_key, control_key
|
|
186
|
+
Obs columns containing perturbation IDs, corresponding gene IDs, and
|
|
187
|
+
Boolean/0-or-1 control flags. Control rows require nonempty ID values.
|
|
188
|
+
layer
|
|
189
|
+
Expression layer, or ``None`` for X. The selected matrix alone is read.
|
|
190
|
+
full_gene_axis
|
|
191
|
+
Counts require ``True``: the caller certifies that row totals can be
|
|
192
|
+
calculated from the complete raw library axis, before gene filtering.
|
|
193
|
+
This declaration cannot detect a previously subsetted matrix. Optional
|
|
194
|
+
library_totals_key provides a check against retained full-library totals.
|
|
195
|
+
library_totals_key
|
|
196
|
+
Optional obs column of original full-library totals; when supplied for
|
|
197
|
+
counts it must exactly equal the selected matrix's row sums.
|
|
198
|
+
|
|
199
|
+
Returns
|
|
200
|
+
-------
|
|
201
|
+
PertRankResult
|
|
202
|
+
Three method arms, labelled effects, target weights and control models.
|
|
203
|
+
The input AnnData is not modified. ``top_k`` must fit every query's
|
|
204
|
+
eligible candidate pool; it is never silently reduced.
|
|
205
|
+
"""
|
|
206
|
+
if input_scale not in {"counts", "log"}:
|
|
207
|
+
raise ValueError("input_scale must explicitly be 'counts' or 'log'")
|
|
208
|
+
_options(top_k, seed)
|
|
209
|
+
if not isinstance(batch_rows, (int, np.integer)) or isinstance(batch_rows, bool) or batch_rows < 1:
|
|
210
|
+
raise ValueError("batch_rows must be a positive integer")
|
|
211
|
+
if not all(hasattr(adata, key) for key in ("shape", "obs", "var_names", "obs_names", "X")):
|
|
212
|
+
raise TypeError("adata must provide the AnnData matrix and axis interface")
|
|
213
|
+
n, g = adata.shape
|
|
214
|
+
if n == 0 or g == 0:
|
|
215
|
+
raise ValueError("AnnData must contain cells and genes")
|
|
216
|
+
genes = _strings(adata.var_names, g, "var_names")
|
|
217
|
+
cells = _strings(adata.obs_names, n, "obs_names")
|
|
218
|
+
if len(set(genes)) != g or len(set(cells)) != n:
|
|
219
|
+
raise ValueError("AnnData gene and cell IDs must be unique")
|
|
220
|
+
for key in (target_key, target_gene_key, control_key):
|
|
221
|
+
if key not in adata.obs:
|
|
222
|
+
raise ValueError(f"Missing obs column {key!r}")
|
|
223
|
+
targets = _strings(adata.obs[target_key].to_numpy(), n, target_key)
|
|
224
|
+
own = _strings(adata.obs[target_gene_key].to_numpy(), n, target_gene_key)
|
|
225
|
+
ctrl = _controls(adata.obs[control_key].to_numpy(), n)
|
|
226
|
+
batches = _strata(adata, strata_keys)
|
|
227
|
+
names = np.unique(targets[~ctrl])
|
|
228
|
+
if not len(names) or set(names) & set(targets[ctrl]):
|
|
229
|
+
raise ValueError("Distinct perturbation and control target IDs are required")
|
|
230
|
+
if layer is not None and layer not in adata.layers:
|
|
231
|
+
raise ValueError(f"Missing AnnData layer {layer!r}")
|
|
232
|
+
matrix = adata.X if layer is None else adata.layers[layer]
|
|
233
|
+
if matrix is None:
|
|
234
|
+
raise ValueError("The selected expression matrix is empty")
|
|
235
|
+
queries = _queries(queries)
|
|
236
|
+
if input_scale == "counts":
|
|
237
|
+
if full_gene_axis is not True:
|
|
238
|
+
raise ValueError("counts require full_gene_axis=True and the complete raw library gene axis")
|
|
239
|
+
if len(set(batches)) != 1:
|
|
240
|
+
raise ValueError("counts input supports one prespecified matching stratum; use separately normalized log input for an explicit multi-stratum design")
|
|
241
|
+
if ctrl.sum() < 6:
|
|
242
|
+
raise ValueError("At least six matched input controls are required")
|
|
243
|
+
totals = np.zeros(n)
|
|
244
|
+
csums = np.zeros(g)
|
|
245
|
+
for start in range(0, n, batch_rows):
|
|
246
|
+
stop = min(start + batch_rows, n)
|
|
247
|
+
x = _read(matrix, slice(start, stop), input_scale)
|
|
248
|
+
totals[start:stop] = x.sum(1)
|
|
249
|
+
csums += x[ctrl[start:stop]].sum(0)
|
|
250
|
+
if (totals <= 0).any():
|
|
251
|
+
raise ValueError("Zero-library cells must be QC-filtered before input")
|
|
252
|
+
if library_totals_key is not None:
|
|
253
|
+
if library_totals_key not in adata.obs:
|
|
254
|
+
raise ValueError(f"Missing full-library total column {library_totals_key!r}")
|
|
255
|
+
if not np.array_equal(np.asarray(adata.obs[library_totals_key], dtype=float), totals):
|
|
256
|
+
raise ValueError("Provided library totals differ from row sums; supply the full raw gene axis")
|
|
257
|
+
keep = np.flatnonzero(csums > 0)
|
|
258
|
+
scales = {str(batches[0]): float(np.median(totals[ctrl]))}
|
|
259
|
+
else:
|
|
260
|
+
if library_totals_key is not None:
|
|
261
|
+
raise ValueError("library_totals_key applies only to counts input")
|
|
262
|
+
keep = np.arange(g)
|
|
263
|
+
totals, scales = None, {}
|
|
264
|
+
if set(queries) - set(genes[keep]):
|
|
265
|
+
raise ValueError("Queries must be present on the input-control gene axis")
|
|
266
|
+
models = {}
|
|
267
|
+
|
|
268
|
+
def expression(rows: np.ndarray, control: bool = False) -> np.ndarray:
|
|
269
|
+
out = np.empty((len(rows), len(keep)), dtype=float)
|
|
270
|
+
for start in range(0, len(rows), batch_rows):
|
|
271
|
+
ix = rows[start:start + batch_rows]
|
|
272
|
+
x = _read(matrix, ix, input_scale)[:, keep]
|
|
273
|
+
if input_scale == "counts":
|
|
274
|
+
x = np.log1p(x * (scales[str(batches[ix[0]])] / totals[ix])[:, None])
|
|
275
|
+
if control:
|
|
276
|
+
x = x.astype(np.float32).astype(np.float64)
|
|
277
|
+
out[start:start + len(ix)] = x
|
|
278
|
+
return out
|
|
279
|
+
|
|
280
|
+
for batch in np.unique(batches[~ctrl]):
|
|
281
|
+
ci = np.flatnonzero(ctrl & (batches == batch))
|
|
282
|
+
if len(ci) < 6:
|
|
283
|
+
raise ValueError(f"Stratum {batch} needs at least six matched input controls")
|
|
284
|
+
xc = expression(ci, control=True)
|
|
285
|
+
mean = xc.mean(0)
|
|
286
|
+
directions, eigenvalues = _directions(xc - mean)
|
|
287
|
+
models[str(batch)] = dict(directions=directions, eigenvalues=eigenvalues,
|
|
288
|
+
mean=mean, variance=xc.var(0, ddof=1), n_control=len(ci))
|
|
289
|
+
stats = []
|
|
290
|
+
for name in names:
|
|
291
|
+
rows = np.flatnonzero((~ctrl) & (targets == name))
|
|
292
|
+
bs, gs = np.unique(batches[rows]), np.unique(own[rows])
|
|
293
|
+
if len(bs) != 1:
|
|
294
|
+
raise ValueError(f"Target {name} spans matching strata; supply a prespecified pooling design")
|
|
295
|
+
if len(gs) != 1 or len(rows) < 2:
|
|
296
|
+
raise ValueError(f"Target {name} needs >=2 cells and one target_gene_id")
|
|
297
|
+
model = models[str(bs[0])]
|
|
298
|
+
xt = expression(rows)
|
|
299
|
+
effect = xt.mean(0) - model["mean"]
|
|
300
|
+
variance = xt.var(0, ddof=1)
|
|
301
|
+
py = effect @ model["directions"]
|
|
302
|
+
pv = (xt @ model["directions"]).var(0, ddof=1)
|
|
303
|
+
depth = len(rows)
|
|
304
|
+
stats.append((effect, float(effect @ effect),
|
|
305
|
+
float(variance.sum() / depth + model["variance"].sum() / model["n_control"]),
|
|
306
|
+
py * py, pv / depth + model["eigenvalues"] / model["n_control"],
|
|
307
|
+
depth, str(bs[0]), str(gs[0])))
|
|
308
|
+
statistics = dict(
|
|
309
|
+
gene_ids=genes[keep], target_ids=names, effects=np.array([s[0] for s in stats]),
|
|
310
|
+
energy=np.array([s[1] for s in stats]), noise_trace=np.array([s[2] for s in stats]),
|
|
311
|
+
direction_energy=np.array([s[3] for s in stats]), direction_noise=np.array([s[4] for s in stats]),
|
|
312
|
+
n_cells=np.array([s[5] for s in stats]), batch_ids=np.array([s[6] for s in stats]),
|
|
313
|
+
target_gene_ids=np.array([s[7] for s in stats]), control_models=models,
|
|
314
|
+
)
|
|
315
|
+
statistics["weights"] = reliability_weights(statistics["energy"], statistics["noise_trace"],
|
|
316
|
+
statistics["direction_energy"], statistics["direction_noise"])
|
|
317
|
+
metadata = dict(
|
|
318
|
+
input_format="AnnData", input_scale=input_scale, layer=layer, strata_keys=list(strata_keys),
|
|
319
|
+
normalization_scales=scales, full_gene_count=g, weight_gene_positions=keep.tolist(),
|
|
320
|
+
n_input_controls=int(ctrl.sum()), n_input_target_cells=int((~ctrl).sum()),
|
|
321
|
+
target_depths=sorted(set(statistics["n_cells"].tolist())), batch_rows=int(batch_rows),
|
|
322
|
+
full_gene_axis_declared=bool(full_gene_axis), library_totals_key=library_totals_key,
|
|
323
|
+
normalization=("log1p(count * median full-library input-control total / full-library total)"
|
|
324
|
+
if input_scale == "counts" else "Caller-supplied log expression, unchanged"),
|
|
325
|
+
control_precision=("float32 per log value, then float64 moments" if input_scale == "counts" else "float64"),
|
|
326
|
+
target_precision="float64",
|
|
327
|
+
)
|
|
328
|
+
return _result(statistics, queries, top_k, seed, metadata)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def from_statistics(statistics: Mapping[str, Any], queries: Sequence[str] | str,
|
|
332
|
+
*, top_k: int = 25, seed: int = 2026,
|
|
333
|
+
effect_gene_subset: bool = False,
|
|
334
|
+
weight_gene_count: int | None = None) -> PertRankResult:
|
|
335
|
+
"""Rank archived sufficient statistics without reading or fitting cells.
|
|
336
|
+
|
|
337
|
+
Required keys are ``gene_ids``, ``target_ids``, ``target_gene_ids``,
|
|
338
|
+
``effects``, ``energy``, ``noise_trace``, ``direction_energy`` and
|
|
339
|
+
``direction_noise``. Direction arrays must contain the five original
|
|
340
|
+
matched-control directions. Optional ``n_cells``, ``batch_ids`` and
|
|
341
|
+
``control_models`` retain provenance. Weights are always recomputed using
|
|
342
|
+
the fixed paper equation; supplied weights are not trusted or reused.
|
|
343
|
+
By default, effects must contain the complete original weight gene axis
|
|
344
|
+
and their squared norm must equal ``energy``. For an explicitly declared
|
|
345
|
+
stored subset of effect columns, set ``effect_gene_subset=True`` and
|
|
346
|
+
provide the original ``weight_gene_count``. Whole-axis energy, noise and
|
|
347
|
+
projected statistics remain unchanged; they cannot be derived from the
|
|
348
|
+
subset. The subset squared norm must not exceed whole-axis energy.
|
|
349
|
+
A ``rank_effects`` result retains these original source statistics, so
|
|
350
|
+
passing its ``statistics`` here replays the source effects, not its external
|
|
351
|
+
readout. Use ``rank_effects`` again to combine saved external effects.
|
|
352
|
+
"""
|
|
353
|
+
required = {"gene_ids", "target_ids", "target_gene_ids", "effects", "energy",
|
|
354
|
+
"noise_trace", "direction_energy", "direction_noise"}
|
|
355
|
+
missing = required - statistics.keys()
|
|
356
|
+
if missing:
|
|
357
|
+
raise ValueError("Missing sufficient statistics: " + ", ".join(sorted(missing)))
|
|
358
|
+
_options(top_k, seed)
|
|
359
|
+
effect = np.asarray(statistics["effects"], dtype=float)
|
|
360
|
+
if effect.ndim != 2 or min(effect.shape) == 0 or not np.isfinite(effect).all():
|
|
361
|
+
raise ValueError("effects must be a nonempty finite target-by-gene matrix")
|
|
362
|
+
nt, ng = effect.shape
|
|
363
|
+
stats = {key: np.asarray(statistics[key]).copy() for key in required}
|
|
364
|
+
for key, length in (("gene_ids", ng), ("target_ids", nt), ("target_gene_ids", nt)):
|
|
365
|
+
stats[key] = _strings(statistics[key], length, key)
|
|
366
|
+
if len(set(stats["gene_ids"])) != ng or len(set(stats["target_ids"])) != nt:
|
|
367
|
+
raise ValueError("gene_ids and target_ids must be unique")
|
|
368
|
+
stats["effects"] = effect.copy()
|
|
369
|
+
for key in ("energy", "noise_trace", "direction_energy", "direction_noise"):
|
|
370
|
+
stats[key] = np.asarray(statistics[key], dtype=float).copy()
|
|
371
|
+
if stats["energy"].shape != (nt,) or stats["noise_trace"].shape != (nt,):
|
|
372
|
+
raise ValueError("energy and noise_trace must align with target IDs")
|
|
373
|
+
if stats["direction_energy"].shape != (nt, K_DIRECTIONS):
|
|
374
|
+
raise ValueError("direction_energy must have exactly five frozen directions per target")
|
|
375
|
+
if not isinstance(effect_gene_subset, (bool, np.bool_)):
|
|
376
|
+
raise ValueError("effect_gene_subset must be a Boolean declaration")
|
|
377
|
+
if weight_gene_count is not None and (
|
|
378
|
+
not isinstance(weight_gene_count, (int, np.integer))
|
|
379
|
+
or isinstance(weight_gene_count, bool) or weight_gene_count < 1):
|
|
380
|
+
raise ValueError("weight_gene_count must be a positive integer")
|
|
381
|
+
observed_energy = np.sum(effect * effect, axis=1)
|
|
382
|
+
if effect_gene_subset:
|
|
383
|
+
if (not isinstance(weight_gene_count, (int, np.integer)) or isinstance(weight_gene_count, bool)
|
|
384
|
+
or weight_gene_count < ng):
|
|
385
|
+
raise ValueError("A declared effect gene subset requires integer weight_gene_count >= supplied gene count")
|
|
386
|
+
if np.any(observed_energy > stats["energy"] + 1e-6 * np.maximum(1, stats["energy"])):
|
|
387
|
+
raise ValueError("Effect subset energy exceeds the saved full-axis energy")
|
|
388
|
+
else:
|
|
389
|
+
if weight_gene_count is not None and weight_gene_count != ng:
|
|
390
|
+
raise ValueError("Full-axis effects require weight_gene_count to equal the supplied gene count")
|
|
391
|
+
weight_gene_count = ng
|
|
392
|
+
if not np.allclose(stats["energy"], observed_energy, rtol=1e-6, atol=1e-10):
|
|
393
|
+
raise ValueError("energy does not match the full supplied effect gene axis")
|
|
394
|
+
stats["weights"] = reliability_weights(stats["energy"], stats["noise_trace"],
|
|
395
|
+
stats["direction_energy"], stats["direction_noise"])
|
|
396
|
+
depth = np.asarray(statistics.get("n_cells", np.full(nt, np.nan)), dtype=float)
|
|
397
|
+
if depth.shape != (nt,) or ("n_cells" in statistics and
|
|
398
|
+
(not np.isfinite(depth).all() or (depth < 2).any() or (depth != np.floor(depth)).any())):
|
|
399
|
+
raise ValueError("n_cells must align to targets and contain integer depths >=2")
|
|
400
|
+
stats["n_cells"] = depth
|
|
401
|
+
stats["batch_ids"] = _strings(statistics.get("batch_ids", np.repeat("not_recorded", nt)), nt, "batch_ids")
|
|
402
|
+
stats["control_models"] = statistics.get("control_models", {})
|
|
403
|
+
return _result(stats, _queries(queries), top_k, seed,
|
|
404
|
+
{"input_format": "Archived sufficient statistics", "input_scale": "archived",
|
|
405
|
+
"source": "Caller-provided frozen sufficient statistics; no model fit",
|
|
406
|
+
"effect_gene_subset": bool(effect_gene_subset),
|
|
407
|
+
"weight_gene_count": int(weight_gene_count), "effect_gene_count": ng,
|
|
408
|
+
"input_selection": "Archived selected inputs; no cells read or resampled"})
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def rank_effects(result: PertRankResult, effects: pd.DataFrame,
|
|
412
|
+
queries: Sequence[str] | str | None = None, *, top_k: int | None = None,
|
|
413
|
+
seed: int | None = None) -> PertRankResult:
|
|
414
|
+
"""Apply existing weights to labelled external target-by-gene effects.
|
|
415
|
+
|
|
416
|
+
Every fitted target must occur exactly once in the index. Columns may be
|
|
417
|
+
a subset of the fitted gene axis, but must contain every requested query.
|
|
418
|
+
Rows and columns are aligned by identifiers, never by their position.
|
|
419
|
+
Source weights and their diagnostics stay fixed; uncertainty is not
|
|
420
|
+
re-estimated for the external estimator. The returned ``statistics`` retain
|
|
421
|
+
the original source effects and axis; ``effects`` contain the external
|
|
422
|
+
readout. Source and returned objects do not share mutable statistics.
|
|
423
|
+
Repeated calls may use any gene retained in the original source statistics.
|
|
424
|
+
"""
|
|
425
|
+
if not isinstance(result, PertRankResult) or not isinstance(effects, pd.DataFrame):
|
|
426
|
+
raise TypeError("Supply a PertRankResult and a target-by-gene pandas DataFrame")
|
|
427
|
+
targets = _strings(effects.index, len(effects), "external target IDs")
|
|
428
|
+
genes = _strings(effects.columns, effects.shape[1], "external gene IDs")
|
|
429
|
+
if len(set(targets)) != len(targets) or len(set(genes)) != len(genes):
|
|
430
|
+
raise ValueError("External effect target and gene IDs must be unique")
|
|
431
|
+
if set(targets) != set(result.statistics["target_ids"]):
|
|
432
|
+
raise ValueError("External effects must contain exactly the fitted target IDs")
|
|
433
|
+
if not set(genes).issubset(set(result.statistics["gene_ids"])):
|
|
434
|
+
raise ValueError("External effect genes must belong to the fitted gene namespace and axis")
|
|
435
|
+
aligned = effects.copy()
|
|
436
|
+
aligned.index, aligned.columns = targets, genes
|
|
437
|
+
aligned = aligned.loc[result.statistics["target_ids"]]
|
|
438
|
+
values = aligned.to_numpy(dtype=float, copy=True)
|
|
439
|
+
if not np.isfinite(values).all():
|
|
440
|
+
raise ValueError("External effects must contain finite values")
|
|
441
|
+
queries = _queries(result.metadata["query_ids"] if queries is None else queries)
|
|
442
|
+
if set(queries) - set(genes):
|
|
443
|
+
raise ValueError("External effects must include every requested query")
|
|
444
|
+
source_statistics = deepcopy(result.statistics)
|
|
445
|
+
stats = {**source_statistics, "effects": values, "gene_ids": genes}
|
|
446
|
+
metadata = {**deepcopy(result.metadata), "effect_estimator": "external labelled effects",
|
|
447
|
+
"effect_origin": "external",
|
|
448
|
+
"weight_source": "Unchanged source-cell reliability statistics",
|
|
449
|
+
"weight_gene_count": result.metadata["weight_gene_count"],
|
|
450
|
+
"source_effect_gene_count": len(source_statistics["gene_ids"]),
|
|
451
|
+
"external_effect_gene_count": len(genes), "effect_gene_count": len(genes)}
|
|
452
|
+
answer = _result(stats, queries, result.metadata["top_k"] if top_k is None else top_k,
|
|
453
|
+
result.metadata["tie_seed"] if seed is None else seed, metadata)
|
|
454
|
+
# Reliability statistics and readout are distinct: preserve the source
|
|
455
|
+
# energy/effect relationship and its labelled control-model gene axis.
|
|
456
|
+
answer.statistics = source_statistics
|
|
457
|
+
return answer
|
pertrank/core.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Fixed k=5, beta=0.9 PertRank; no query selection or evaluation data input.
|
|
2
|
+
|
|
3
|
+
The equations follow stage3/read_e4_w1.py and evaluate_w1_e4.py in the
|
|
4
|
+
research archive. This module accepts only the cells already selected as
|
|
5
|
+
inputs. It does not infer experimental strata or recreate source cohorts.
|
|
6
|
+
"""
|
|
7
|
+
import hashlib
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from numbers import Real
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
ARMS = ("Unweighted", "Euclidean", "PertRank")
|
|
13
|
+
K_DIRECTIONS, BETA = 5, 0.9
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _strings(values, n, name):
|
|
17
|
+
a = np.asarray(values)
|
|
18
|
+
if a.ndim != 1 or len(a) != n:
|
|
19
|
+
raise ValueError(f"{name} must have length {n}")
|
|
20
|
+
a = a.astype(str)
|
|
21
|
+
if any(not s or s != s.strip() or s.lower() in ("none", "nan", "nat", "<na>") for s in a):
|
|
22
|
+
raise ValueError(f"{name} contains a missing or whitespace-padded identifier")
|
|
23
|
+
return a
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _matrix(values):
|
|
27
|
+
x = np.asarray(values, dtype=np.float64)
|
|
28
|
+
if x.ndim != 2 or min(x.shape) == 0 or not np.isfinite(x).all():
|
|
29
|
+
raise ValueError("X must be a nonempty finite dense cell-by-gene matrix")
|
|
30
|
+
return x
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _controls(values, n):
|
|
34
|
+
a = np.asarray(values)
|
|
35
|
+
if a.shape != (n,) or not np.isin(a, [False, True, 0, 1]).all():
|
|
36
|
+
raise ValueError("is_control must be a Boolean or 0/1 vector")
|
|
37
|
+
return a.astype(bool)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def normalize_counts(counts, library_totals, batch_ids, is_control):
|
|
41
|
+
"""log1p(count * median matched-input-control total / full-library total).
|
|
42
|
+
|
|
43
|
+
library_totals MUST be totals before weight-gene filtering. Control
|
|
44
|
+
strata must be the intended normalization strata; all input controls
|
|
45
|
+
are used and no 2,000-control subset is chosen here. Zero-total cells
|
|
46
|
+
are rejected instead of silently modifying their denominator.
|
|
47
|
+
"""
|
|
48
|
+
x = _matrix(counts)
|
|
49
|
+
totals = np.asarray(library_totals, dtype=np.float64)
|
|
50
|
+
batch = _strings(batch_ids, len(x), "batch_ids")
|
|
51
|
+
ctrl = _controls(is_control, len(x))
|
|
52
|
+
if (x < 0).any() or (x != np.floor(x)).any():
|
|
53
|
+
raise ValueError("Counts must be nonnegative integer raw counts")
|
|
54
|
+
if totals.shape != (len(x),) or not np.isfinite(totals).all() or (totals <= 0).any():
|
|
55
|
+
raise ValueError("Full-library totals must be finite and positive")
|
|
56
|
+
if np.any(x.sum(1) > totals + 1e-7 * np.maximum(1, totals)):
|
|
57
|
+
raise ValueError("Selected-gene counts exceed the supplied full-library total")
|
|
58
|
+
output = np.empty_like(x)
|
|
59
|
+
scales = {}
|
|
60
|
+
for b in np.unique(batch):
|
|
61
|
+
take = batch == b
|
|
62
|
+
if not np.any(take & ctrl):
|
|
63
|
+
raise ValueError(f"No input controls in normalization stratum {b}")
|
|
64
|
+
scale = float(np.median(totals[take & ctrl]))
|
|
65
|
+
output[take] = np.log1p(x[take] * (scale / totals[take])[:, None])
|
|
66
|
+
scales[str(b)] = scale
|
|
67
|
+
return output, scales
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def reliability_weights(energy, noise_trace, direction_energy, direction_noise,
|
|
71
|
+
beta=BETA, k=K_DIRECTIONS):
|
|
72
|
+
"""Evaluate the paper formula from sums, not means, over the gene axis.
|
|
73
|
+
|
|
74
|
+
Direction arrays are target by direction. Projected sample variances
|
|
75
|
+
include each target's own cell variance/B and matched controls/n_C.
|
|
76
|
+
beta/k are exposed for algebraic verification; fit_cells always uses
|
|
77
|
+
the fixed paper values and performs no parameter selection.
|
|
78
|
+
"""
|
|
79
|
+
e, v = np.asarray(energy, float), np.asarray(noise_trace, float)
|
|
80
|
+
pe, pv = np.asarray(direction_energy, float), np.asarray(direction_noise, float)
|
|
81
|
+
if e.ndim != 1 or v.shape != e.shape or pe.ndim != 2 or pv.shape != pe.shape or pe.shape[0] != len(e):
|
|
82
|
+
raise ValueError("Expected target energies and target-by-direction statistics")
|
|
83
|
+
if (not isinstance(k, (int, np.integer)) or isinstance(k, bool)
|
|
84
|
+
or not 0 <= k <= pe.shape[1] or not isinstance(beta, Real)
|
|
85
|
+
or isinstance(beta, (bool, np.bool_)) or not 0 <= beta < 1):
|
|
86
|
+
raise ValueError("Require an available integer k and 0 <= beta < 1")
|
|
87
|
+
if not all(np.isfinite(a).all() and (a >= 0).all() for a in (e, v, pe, pv)):
|
|
88
|
+
raise ValueError("All sufficient statistics must be finite and nonnegative")
|
|
89
|
+
se, sv = pe[:, :k].sum(1), pv[:, :k].sum(1)
|
|
90
|
+
if np.any(se > e + 1e-10*np.maximum(1, e)) or np.any(sv > v + 1e-10*np.maximum(1, v)):
|
|
91
|
+
raise ValueError("Projected contributions exceed total; check axes and orthogonality")
|
|
92
|
+
def weight(a, n):
|
|
93
|
+
ratio = np.zeros_like(a)
|
|
94
|
+
np.divide(n, a, out=ratio, where=a > 0)
|
|
95
|
+
return np.where(a > 0, np.clip(1-ratio, 0, 1), 0)
|
|
96
|
+
return {"Unweighted": np.ones_like(e), "Euclidean": weight(e, v),
|
|
97
|
+
"PertRank": weight(np.maximum(e-beta*se, 0), np.maximum(v-beta*sv, 0))}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _directions(centered):
|
|
101
|
+
"""Leading exact covariance eigenvectors via the smaller Gram matrix."""
|
|
102
|
+
n, g = centered.shape
|
|
103
|
+
if min(n-1, g) < K_DIRECTIONS:
|
|
104
|
+
raise ValueError("Fixed k=5 requires at least six controls and five weight genes per stratum")
|
|
105
|
+
gram = centered @ centered.T if n < g else centered.T @ centered
|
|
106
|
+
ev, vec = np.linalg.eigh(gram / (n-1))
|
|
107
|
+
order = np.argsort(ev)[::-1][:K_DIRECTIONS]
|
|
108
|
+
lam = ev[order]
|
|
109
|
+
if lam[-1] <= 1e-12 * max(float(lam[0]), np.finfo(float).tiny):
|
|
110
|
+
raise ValueError("Control covariance has fewer than five resolved positive directions")
|
|
111
|
+
u = centered.T @ vec[:, order] if n < g else vec[:, order]
|
|
112
|
+
u /= np.linalg.norm(u, axis=0)
|
|
113
|
+
# Signs do not affect the scalar; canonical signs simplify saved files.
|
|
114
|
+
signs = np.sign(u[np.argmax(np.abs(u), axis=0), np.arange(K_DIRECTIONS)])
|
|
115
|
+
u *= signs
|
|
116
|
+
if not np.allclose(u.T @ u, np.eye(K_DIRECTIONS), atol=1e-8, rtol=0):
|
|
117
|
+
raise ValueError("Computed control directions are not orthonormal")
|
|
118
|
+
lam = np.var(centered @ u, axis=0, ddof=1)
|
|
119
|
+
return u, lam
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def fit_cells(X, gene_ids, target_ids, batch_ids, is_control, target_gene_ids):
|
|
123
|
+
"""Compute weights and original-coordinate effects from input cells.
|
|
124
|
+
|
|
125
|
+
target_gene_ids is a per-cell identifier in the same namespace as
|
|
126
|
+
gene_ids. Control rows may use any nonempty placeholder. A target
|
|
127
|
+
must belong to one control-matching stratum; multi-stratum pooling
|
|
128
|
+
needs an explicit experimental design and is rejected here.
|
|
129
|
+
"""
|
|
130
|
+
x = _matrix(X)
|
|
131
|
+
n, g = x.shape
|
|
132
|
+
genes = _strings(gene_ids, g, "gene_ids")
|
|
133
|
+
targets = _strings(target_ids, n, "target_ids")
|
|
134
|
+
batch = _strings(batch_ids, n, "batch_ids")
|
|
135
|
+
own = _strings(target_gene_ids, n, "target_gene_ids")
|
|
136
|
+
ctrl = _controls(is_control, n)
|
|
137
|
+
if len(set(genes)) != len(genes):
|
|
138
|
+
raise ValueError("gene_ids must be unique and use one declared namespace")
|
|
139
|
+
names = np.unique(targets[~ctrl])
|
|
140
|
+
if len(names) == 0:
|
|
141
|
+
raise ValueError("At least one perturbation target is required")
|
|
142
|
+
if set(names) & set(targets[ctrl]):
|
|
143
|
+
raise ValueError("A target_id cannot label both controls and perturbations")
|
|
144
|
+
stats, models = [], {}
|
|
145
|
+
for b in np.unique(batch[~ctrl]):
|
|
146
|
+
xc = x[ctrl & (batch == b)]
|
|
147
|
+
if len(xc) < 6:
|
|
148
|
+
raise ValueError(f"Stratum {b} needs at least six matched input controls")
|
|
149
|
+
cm = xc.mean(0)
|
|
150
|
+
u, lam = _directions(xc-cm)
|
|
151
|
+
models[str(b)] = dict(directions=u, eigenvalues=lam, mean=cm,
|
|
152
|
+
variance=xc.var(0, ddof=1), n_control=len(xc))
|
|
153
|
+
for name in names:
|
|
154
|
+
rows = (~ctrl) & (targets == name)
|
|
155
|
+
bs, gs = np.unique(batch[rows]), np.unique(own[rows])
|
|
156
|
+
if len(bs) != 1:
|
|
157
|
+
raise ValueError(f"Target {name} spans strata; supply a prespecified pooling design")
|
|
158
|
+
if len(gs) != 1:
|
|
159
|
+
raise ValueError(f"Target {name} has inconsistent target_gene_ids")
|
|
160
|
+
if rows.sum() < 2:
|
|
161
|
+
raise ValueError(f"Target {name} needs at least two input cells for sample variance")
|
|
162
|
+
m = models[str(bs[0])]
|
|
163
|
+
xt = x[rows]
|
|
164
|
+
y = xt.mean(0)-m["mean"]
|
|
165
|
+
var = xt.var(0, ddof=1)
|
|
166
|
+
py = y @ m["directions"]
|
|
167
|
+
pv = (xt @ m["directions"]).var(0, ddof=1)
|
|
168
|
+
b = len(xt)
|
|
169
|
+
stats.append((y, float(y@y), float(var.sum()/b+m["variance"].sum()/m["n_control"]),
|
|
170
|
+
py*py, pv/b+m["eigenvalues"]/m["n_control"], b, str(bs[0]), str(gs[0])))
|
|
171
|
+
result = dict(gene_ids=genes, target_ids=names, effects=np.array([s[0] for s in stats]),
|
|
172
|
+
energy=np.array([s[1] for s in stats]), noise_trace=np.array([s[2] for s in stats]),
|
|
173
|
+
direction_energy=np.array([s[3] for s in stats]), direction_noise=np.array([s[4] for s in stats]),
|
|
174
|
+
n_cells=np.array([s[5] for s in stats]), batch_ids=np.array([s[6] for s in stats]),
|
|
175
|
+
target_gene_ids=np.array([s[7] for s in stats]), control_models=models)
|
|
176
|
+
result["weights"] = reliability_weights(result["energy"], result["noise_trace"],
|
|
177
|
+
result["direction_energy"], result["direction_noise"])
|
|
178
|
+
return result
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _boundary_probabilities(v, k):
|
|
182
|
+
threshold = np.partition(v, len(v)-k)[len(v)-k]
|
|
183
|
+
above, tied = v > threshold, v == threshold
|
|
184
|
+
p = above.astype(float)
|
|
185
|
+
p[tied] = (k-int(above.sum())) / int(tied.sum())
|
|
186
|
+
return p
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def nominate(result, query_ids, top_k=25, seed=2026):
|
|
190
|
+
"""Return lists and tie-expectation inclusion probabilities, no truth.
|
|
191
|
+
|
|
192
|
+
Stable SHA-256 keys provide reproducible delivered shortlists. The
|
|
193
|
+
probabilities separately implement independent boundary-tie selection;
|
|
194
|
+
they are not calibrated probabilities of biological correctness.
|
|
195
|
+
"""
|
|
196
|
+
if not isinstance(top_k, (int, np.integer)) or isinstance(top_k, bool) or top_k < 1:
|
|
197
|
+
raise ValueError("top_k must be a positive integer")
|
|
198
|
+
if not isinstance(seed, (int, np.integer)) or isinstance(seed, bool):
|
|
199
|
+
raise ValueError("seed must be an integer")
|
|
200
|
+
if not isinstance(result, Mapping):
|
|
201
|
+
raise TypeError("result must be a labelled statistics mapping")
|
|
202
|
+
required = {"gene_ids", "target_ids", "target_gene_ids", "effects", "weights"}
|
|
203
|
+
if required - result.keys():
|
|
204
|
+
raise ValueError("Missing nomination inputs: " + ", ".join(sorted(required - result.keys())))
|
|
205
|
+
effects = np.asarray(result["effects"], dtype=float)
|
|
206
|
+
if effects.ndim != 2 or min(effects.shape) == 0 or not np.isfinite(effects).all():
|
|
207
|
+
raise ValueError("effects must be a nonempty finite target-by-gene matrix")
|
|
208
|
+
nt, ng = effects.shape
|
|
209
|
+
genes = _strings(result["gene_ids"], ng, "gene_ids")
|
|
210
|
+
targets = _strings(result["target_ids"], nt, "target_ids")
|
|
211
|
+
own = _strings(result["target_gene_ids"], nt, "target_gene_ids")
|
|
212
|
+
if len(set(genes)) != ng or len(set(targets)) != nt:
|
|
213
|
+
raise ValueError("gene_ids and target_ids must be unique")
|
|
214
|
+
weights = result["weights"]
|
|
215
|
+
if not isinstance(weights, Mapping) or set(ARMS) - weights.keys():
|
|
216
|
+
raise ValueError("weights must contain Unweighted, Euclidean and PertRank")
|
|
217
|
+
weights = {arm: np.asarray(weights[arm], dtype=float) for arm in ARMS}
|
|
218
|
+
if any(w.shape != (nt,) or not np.isfinite(w).all()
|
|
219
|
+
or (w < 0).any() or (w > 1).any() for w in weights.values()):
|
|
220
|
+
raise ValueError("weights must align to targets and be finite values in [0, 1]")
|
|
221
|
+
result = {**result, "gene_ids": genes, "target_ids": targets,
|
|
222
|
+
"target_gene_ids": own, "effects": effects, "weights": weights}
|
|
223
|
+
queries = _strings(query_ids, len(query_ids), "query_ids")
|
|
224
|
+
if len(queries) == 0:
|
|
225
|
+
raise ValueError("At least one supplied query is required")
|
|
226
|
+
if len(set(queries)) != len(queries):
|
|
227
|
+
raise ValueError("query_ids must be unique")
|
|
228
|
+
gmap = {g: i for i, g in enumerate(result["gene_ids"])}
|
|
229
|
+
missing = [q for q in queries if q not in gmap]
|
|
230
|
+
if missing:
|
|
231
|
+
raise ValueError(f"Queries not present on the input gene axis: {missing[:5]}")
|
|
232
|
+
probabilities = np.zeros((len(ARMS), len(queries), 2, nt))
|
|
233
|
+
records = []
|
|
234
|
+
for qi, q in enumerate(queries):
|
|
235
|
+
keep = np.flatnonzero(result["target_gene_ids"] != q)
|
|
236
|
+
if len(keep) < top_k:
|
|
237
|
+
raise ValueError(f"Query {q}: fewer than top_k candidates after own-target exclusion")
|
|
238
|
+
for di, sign in enumerate((1, -1)):
|
|
239
|
+
direction = "up" if sign == 1 else "down"
|
|
240
|
+
keys = np.array([hashlib.sha256(f"{seed}|{q}|{direction}|{result['target_ids'][j]}".encode()).hexdigest()
|
|
241
|
+
for j in keep])
|
|
242
|
+
for ai, arm in enumerate(ARMS):
|
|
243
|
+
score = sign * result["effects"][keep, gmap[q]] * result["weights"][arm][keep]
|
|
244
|
+
p = _boundary_probabilities(score, top_k)
|
|
245
|
+
probabilities[ai, qi, di, keep] = p
|
|
246
|
+
selected = np.lexsort((keys, -score))[:top_k]
|
|
247
|
+
for rank, z in enumerate(selected, 1):
|
|
248
|
+
j = keep[z]
|
|
249
|
+
records.append(dict(method=arm, query_id=str(q), direction=direction, rank=rank,
|
|
250
|
+
target_id=str(result["target_ids"][j]),
|
|
251
|
+
target_gene_id=str(result["target_gene_ids"][j]),
|
|
252
|
+
effect=float(result["effects"][j, gmap[q]]),
|
|
253
|
+
weight=float(result["weights"][arm][j]), signed_score=float(score[z]),
|
|
254
|
+
tie_inclusion_probability=float(p[z])))
|
|
255
|
+
return records, probabilities
|
|
Binary file
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"dataset": "Replogle et al. 2022 K562 genome-wide Perturb-seq",
|
|
3
|
+
"fixture_kind": "Authentic archived sufficient statistics and query-effect matrix; not cell-level AnnData",
|
|
4
|
+
"candidate_selection": "Complete 2,000-candidate principal cohort from the archived input task; reference-screen targets excluded by the historical task rule",
|
|
5
|
+
"query_selection": "All 300 historical reference-informed queries retained. Tutorial displays the first three unique query IDs in lexicographic order, without consulting outcomes.",
|
|
6
|
+
"selected_query_ids": [
|
|
7
|
+
"ENSG00000004478",
|
|
8
|
+
"ENSG00000005249",
|
|
9
|
+
"ENSG00000016490"
|
|
10
|
+
],
|
|
11
|
+
"selected_query_symbols": [
|
|
12
|
+
"FKBP4",
|
|
13
|
+
"PRKAR2B",
|
|
14
|
+
"CLCA1"
|
|
15
|
+
],
|
|
16
|
+
"n_target_cells": 20,
|
|
17
|
+
"n_control_cells": 2000,
|
|
18
|
+
"weight_gene_count": 8202,
|
|
19
|
+
"effects_shape": [
|
|
20
|
+
2000,
|
|
21
|
+
300
|
|
22
|
+
],
|
|
23
|
+
"geometry_scope": "Energies and projected noise retain the full 8,202-gene input axis; query effects retain the original-coordinate 300-column panel.",
|
|
24
|
+
"normalization": "Archived input mean minus archived input-control mean, after the original library-total log1p normalization; no new normalization or fit performed.",
|
|
25
|
+
"units_conversion": {
|
|
26
|
+
"energy": "8202 * m2_20",
|
|
27
|
+
"noise_trace": "8202 * (vb20 / 20 + cvm / ncin)",
|
|
28
|
+
"direction_energy": "(proj_sum[0]/20 - proj_control)^2, first five directions",
|
|
29
|
+
"direction_noise": "(proj_sumsq[0] - 20*(proj_sum[0]/20)^2)/19/20 + eigenvalue/ncin, first five directions"
|
|
30
|
+
},
|
|
31
|
+
"source_study_doi": "10.1016/j.cell.2022.05.013",
|
|
32
|
+
"used_mirror": "https://zenodo.org/records/13350497",
|
|
33
|
+
"symbol_annotation_note": "Human Ensembl-to-symbol annotations reused from the archived T-cell feature table; unmatched IDs remain unchanged.",
|
|
34
|
+
"sources": {
|
|
35
|
+
"reduced_input": {
|
|
36
|
+
"path": "external_sources/reduced_input/e1_stage6.npz",
|
|
37
|
+
"sha256": "d79f3d5c18f6317fbe8d1ef62b86efdbfaeb7f1f6463af5267955ba7cd67beba"
|
|
38
|
+
},
|
|
39
|
+
"projection_statistics": {
|
|
40
|
+
"path": "external_sources/projection_statistics/e1_w1_stage3.npz",
|
|
41
|
+
"sha256": "9a47d008d68c35e9520fe7202dadf646e8234bad34f3cecaadd4e8a75270009b"
|
|
42
|
+
},
|
|
43
|
+
"symbol_annotation": {
|
|
44
|
+
"path": "external_sources/symbol_annotation/tcell_gene_symbols.json",
|
|
45
|
+
"sha256": "19775af812cff6486caf595868368e29f98666933d4f341da6a24c600e3b55a0"
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
"original_input_lock_sha256_verified": true,
|
|
49
|
+
"fixture": {
|
|
50
|
+
"file": "replogle_k562.npz",
|
|
51
|
+
"sha256": "0087e4d915105caac801069499428d58032ec64ab89e224230216404fda51dff",
|
|
52
|
+
"bytes": 2610808
|
|
53
|
+
},
|
|
54
|
+
"frozen_formula_weight_max_absolute_error": {
|
|
55
|
+
"Euclidean": 2.220446049250313e-16,
|
|
56
|
+
"PertRank": 4.440892098500626e-16
|
|
57
|
+
},
|
|
58
|
+
"new_model_fits": 0,
|
|
59
|
+
"scientific_performance_evaluations": 0,
|
|
60
|
+
"redistribution": "Derived numerical tutorial fixture from public-source data; retain source-study attribution. Source data are not relicensed by the software license.",
|
|
61
|
+
"source_dataset_doi": "10.25452/figshare.plus.20029387",
|
|
62
|
+
"source_path_note": "Logical source filenames replace preparation-workspace paths; SHA256 identifiers and numerical conversion definitions are preserved. These upstream files are not required for tutorial execution."
|
|
63
|
+
}
|
|
Binary file
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"dataset": "Primary CD4+ T-cell atlas, donor D3, resting state",
|
|
3
|
+
"fixture_kind": "Authentic archived sufficient statistics and query-effect matrix; not cell-level AnnData",
|
|
4
|
+
"candidate_selection": "All 52 candidates in the completed independent-knockout-RNA comparison; no additional candidate filtering",
|
|
5
|
+
"query_selection": "All 293 archived query columns retained. Tutorial displays first three unique query IDs in lexicographic order, without consulting outcomes.",
|
|
6
|
+
"selected_query_ids": [
|
|
7
|
+
"ENSG00000008517",
|
|
8
|
+
"ENSG00000010278",
|
|
9
|
+
"ENSG00000010610"
|
|
10
|
+
],
|
|
11
|
+
"selected_query_symbols": [
|
|
12
|
+
"IL32",
|
|
13
|
+
"CD9",
|
|
14
|
+
"CD4"
|
|
15
|
+
],
|
|
16
|
+
"n_target_cells": 20,
|
|
17
|
+
"n_control_cells": 2000,
|
|
18
|
+
"weight_gene_count": 13390,
|
|
19
|
+
"effects_shape": [
|
|
20
|
+
52,
|
|
21
|
+
293
|
|
22
|
+
],
|
|
23
|
+
"geometry_scope": "Energies and projected noise are from the full 13,390-gene archived input axis; displayed effect columns are the original 293 queries.",
|
|
24
|
+
"units_conversion": {
|
|
25
|
+
"energy": "G * m2",
|
|
26
|
+
"noise_trace": "G * vb / 20 + trC / n_control",
|
|
27
|
+
"direction_energy": "(Psum/20 - proj_control)^2, first 5 directions",
|
|
28
|
+
"direction_noise": "(Psumsq - Psum^2/20)/19/20 + eigenvalue/n_control, first 5 directions"
|
|
29
|
+
},
|
|
30
|
+
"source_study_doi": "10.1016/j.cell.2026.08.002",
|
|
31
|
+
"source_release": "genome-scale-tcell-perturb-seq/marson2025_data",
|
|
32
|
+
"sources": {
|
|
33
|
+
"moments": {
|
|
34
|
+
"path": "external_sources/moments/sufficient_state.npz",
|
|
35
|
+
"sha256": "b93e5fac94a2644709eb05c347932ebfcc5ceba2723f3e55e8d691e387cae464"
|
|
36
|
+
},
|
|
37
|
+
"predictions": {
|
|
38
|
+
"path": "external_sources/predictions/predictions_locked.npz",
|
|
39
|
+
"sha256": "14023f8044a499b5dd49885a41e81915bd061e2b1da041e09d1e142f9c42b50b"
|
|
40
|
+
},
|
|
41
|
+
"fr": {
|
|
42
|
+
"path": "external_sources/fr/effects.npz",
|
|
43
|
+
"sha256": "eb133f67647640e3e014429357913f7dac05df70adc6800394a1c42d9fd23285"
|
|
44
|
+
},
|
|
45
|
+
"symbols": {
|
|
46
|
+
"path": "external_sources/symbols/tcell_gene_symbols.json",
|
|
47
|
+
"sha256": "19775af812cff6486caf595868368e29f98666933d4f341da6a24c600e3b55a0"
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
"fixture": {
|
|
51
|
+
"file": "tcell_d3_rest.npz",
|
|
52
|
+
"sha256": "6e80c7819e659fe3b901ba2a6e5e6030ccd25a3cd4b65443afbb8f8c4edc7ca8",
|
|
53
|
+
"bytes": 246138
|
|
54
|
+
},
|
|
55
|
+
"archived_weight_max_absolute_error": {
|
|
56
|
+
"Euclidean": 2.220446049250313e-16,
|
|
57
|
+
"PertRank": 3.3306690738754696e-16
|
|
58
|
+
},
|
|
59
|
+
"new_model_fits": 0,
|
|
60
|
+
"reference_matrices_read": 0,
|
|
61
|
+
"scientific_performance_evaluations": 0,
|
|
62
|
+
"redistribution": "Derived numerical tutorial fixture from public-source data; retain source-study attribution. Source data are not relicensed by the software license.",
|
|
63
|
+
"source_path_note": "Logical source filenames replace preparation-workspace paths; SHA256 identifiers and numerical conversion definitions are preserved. These upstream files are not required for tutorial execution."
|
|
64
|
+
}
|
pertrank/datasets.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Small, offline tutorial fixtures with explicit source identities."""
|
|
2
|
+
from importlib.resources import files
|
|
3
|
+
import hashlib
|
|
4
|
+
import io
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def load_example(name):
|
|
11
|
+
"""Return (archived_statistics, provenance) for ``tcell`` or ``replogle``.
|
|
12
|
+
|
|
13
|
+
These are authentic saved sufficient statistics, not reconstructed cells.
|
|
14
|
+
Full-axis geometry is retained with a subset of gene-effect columns.
|
|
15
|
+
Every candidate in each archived task remains present. Loading performs
|
|
16
|
+
a SHA-256 check, reads no network data and opens no evaluation reference.
|
|
17
|
+
"""
|
|
18
|
+
names = {"tcell": "tcell_d3_rest", "replogle": "replogle_k562"}
|
|
19
|
+
if name not in names:
|
|
20
|
+
raise ValueError("name must be 'tcell' or 'replogle'")
|
|
21
|
+
stem = names[name]
|
|
22
|
+
root = files("pertrank").joinpath("data")
|
|
23
|
+
metadata = json.loads(root.joinpath(stem + ".provenance.json").read_text())
|
|
24
|
+
payload = root.joinpath(stem + ".npz").read_bytes()
|
|
25
|
+
if hashlib.sha256(payload).hexdigest() != metadata["fixture"]["sha256"]:
|
|
26
|
+
raise ValueError("Tutorial fixture integrity check failed")
|
|
27
|
+
with np.load(io.BytesIO(payload), allow_pickle=False) as archive:
|
|
28
|
+
arrays = {key: archive[key] for key in archive.files}
|
|
29
|
+
return arrays, metadata
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def software_example(seed=2026):
|
|
33
|
+
"""Build a tiny synthetic AnnData for installation and interface checks.
|
|
34
|
+
|
|
35
|
+
This example carries no biological performance or validation claim.
|
|
36
|
+
It has 120 controls, 8 targets with 20 cells each and 60 measured genes.
|
|
37
|
+
"""
|
|
38
|
+
import anndata as ad
|
|
39
|
+
import pandas as pd
|
|
40
|
+
rng = np.random.default_rng(seed)
|
|
41
|
+
counts = rng.poisson(3, (280, 60)).astype(np.int32)
|
|
42
|
+
for j in range(8):
|
|
43
|
+
counts[120 + 20*j:140 + 20*j, 20 + j] += 4
|
|
44
|
+
targets = np.array(["NTC"] * 120 + [f"T{j}" for j in range(8) for _ in range(20)])
|
|
45
|
+
own = np.array(["NTC"] * 120 + [f"G{j}" for j in range(8) for _ in range(20)])
|
|
46
|
+
obs = pd.DataFrame({"target_ids": targets, "target_gene_ids": own,
|
|
47
|
+
"is_control": targets == "NTC", "batch": "example"},
|
|
48
|
+
index=[f"cell{i}" for i in range(280)])
|
|
49
|
+
adata = ad.AnnData(counts, obs=obs,
|
|
50
|
+
var=pd.DataFrame(index=[f"G{i}" for i in range(60)]))
|
|
51
|
+
adata.uns["data_identity"] = "Synthetic software example; not experimental evidence"
|
|
52
|
+
return adata
|
pertrank/evaluation.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Evaluate frozen nominations against an independently supplied effect reference."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from .api import PertRankResult, _options, _queries
|
|
8
|
+
from .core import ARMS, _boundary_probabilities, _strings
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def evaluate_reference(result: PertRankResult, reference_effects: pd.DataFrame) -> pd.DataFrame:
|
|
12
|
+
"""Compute exact expected P@K on the result's fixed candidate pool.
|
|
13
|
+
|
|
14
|
+
``reference_effects`` has target identifiers as its index and response-gene
|
|
15
|
+
identifiers as columns. Every fitted target and requested query must be
|
|
16
|
+
present; extra reference genes are permitted. Both axes are matched by
|
|
17
|
+
identifier. The reference must use the same effect orientation as the
|
|
18
|
+
supplied predictions. Its construction and independence remain the caller's
|
|
19
|
+
responsibility.
|
|
20
|
+
|
|
21
|
+
The result's shortlist size, queries, candidate pool and own-target
|
|
22
|
+
exclusions are retained. At each boundary tie, marginal inclusion is the
|
|
23
|
+
number of remaining slots divided by tie size. Expected overlap is the dot
|
|
24
|
+
product of prediction and reference inclusion probabilities, integrating
|
|
25
|
+
independent uniform tie breaking. Deterministic display lists are not used.
|
|
26
|
+
|
|
27
|
+
Returns one row per weighting arm, query and direction with
|
|
28
|
+
``precision_at_k``, ``expected_overlap``, ``top_k`` and ``n_candidates``.
|
|
29
|
+
The arm names describe weighting of the supplied effect estimator: after
|
|
30
|
+
``rank_effects``, ``Unweighted`` means that external estimator without a
|
|
31
|
+
reliability weight. Scores measure reference-list agreement, not a
|
|
32
|
+
probability of biological validity. No model fitting, confidence intervals
|
|
33
|
+
or pooling across experiments occurs, and neither input is modified.
|
|
34
|
+
"""
|
|
35
|
+
if not isinstance(result, PertRankResult) or not isinstance(reference_effects, pd.DataFrame):
|
|
36
|
+
raise TypeError("Supply a PertRankResult and a target-by-gene reference DataFrame")
|
|
37
|
+
queries = _queries(result.metadata["query_ids"])
|
|
38
|
+
top_k = result.metadata["top_k"]
|
|
39
|
+
_options(top_k, result.metadata["tie_seed"])
|
|
40
|
+
targets = _strings(result.statistics["target_ids"], len(result.statistics["target_ids"]), "target IDs")
|
|
41
|
+
own = _strings(result.statistics["target_gene_ids"], len(targets), "target gene IDs")
|
|
42
|
+
if len(set(targets)) != len(targets):
|
|
43
|
+
raise ValueError("Fitted target IDs must be unique")
|
|
44
|
+
ref_targets = _strings(reference_effects.index, len(reference_effects), "reference target IDs")
|
|
45
|
+
ref_genes = _strings(reference_effects.columns, reference_effects.shape[1], "reference gene IDs")
|
|
46
|
+
if len(set(ref_targets)) != len(ref_targets) or len(set(ref_genes)) != len(ref_genes):
|
|
47
|
+
raise ValueError("Reference target and gene IDs must be unique")
|
|
48
|
+
if set(ref_targets) != set(targets):
|
|
49
|
+
raise ValueError("Reference effects must contain exactly the fitted target IDs")
|
|
50
|
+
if set(queries) - set(ref_genes):
|
|
51
|
+
raise ValueError("Reference effects must include every requested query")
|
|
52
|
+
reference = reference_effects.copy()
|
|
53
|
+
reference.index, reference.columns = ref_targets, ref_genes
|
|
54
|
+
if not np.isfinite(reference.to_numpy(dtype=float)).all():
|
|
55
|
+
raise ValueError("Reference effects must contain finite values")
|
|
56
|
+
reference = reference.loc[targets, queries]
|
|
57
|
+
|
|
58
|
+
scores = result.all_candidate_scores
|
|
59
|
+
required = {"method", "query_id", "direction", "target_id", "target_gene_id",
|
|
60
|
+
"signed_score", "tie_inclusion_probability", "top_k"}
|
|
61
|
+
if not isinstance(scores, pd.DataFrame) or not required.issubset(scores.columns):
|
|
62
|
+
raise ValueError("Result requires complete all_candidate_scores, including tie probabilities")
|
|
63
|
+
scores = scores.loc[:, sorted(required)].copy()
|
|
64
|
+
for key in ("method", "query_id", "direction", "target_id", "target_gene_id"):
|
|
65
|
+
scores[key] = _strings(scores[key], len(scores), key)
|
|
66
|
+
group_keys = ["method", "query_id", "direction"]
|
|
67
|
+
expected_groups = {(arm, query, direction) for arm in ARMS for query in queries
|
|
68
|
+
for direction in ("up", "down")}
|
|
69
|
+
groups = scores.groupby(group_keys, sort=False)
|
|
70
|
+
if set(groups.indices) != expected_groups:
|
|
71
|
+
raise ValueError("Candidate scores must contain exactly every method, query and direction")
|
|
72
|
+
if not np.all(scores["top_k"].to_numpy() == top_k):
|
|
73
|
+
raise ValueError("Candidate scores must use the result's fixed top_k")
|
|
74
|
+
rows = []
|
|
75
|
+
for arm in ARMS:
|
|
76
|
+
for query in queries:
|
|
77
|
+
eligible = targets[own != query]
|
|
78
|
+
eligible_own = own[own != query]
|
|
79
|
+
if len(eligible) < top_k:
|
|
80
|
+
raise ValueError("Fewer than top_k candidates remain after own-target exclusion")
|
|
81
|
+
truth = reference.loc[eligible, query].to_numpy(dtype=float)
|
|
82
|
+
for direction, sign in (("up", 1), ("down", -1)):
|
|
83
|
+
group = groups.get_group((arm, query, direction))
|
|
84
|
+
if (group["target_id"].duplicated().any()
|
|
85
|
+
or set(group["target_id"]) != set(eligible)):
|
|
86
|
+
raise ValueError("Candidate scores must contain the complete eligible pool exactly once")
|
|
87
|
+
group = group.set_index("target_id").loc[eligible]
|
|
88
|
+
if not np.array_equal(group["target_gene_id"].to_numpy(), eligible_own):
|
|
89
|
+
raise ValueError("Candidate own-target identifiers disagree with fitted target identities")
|
|
90
|
+
values = group["signed_score"].to_numpy(dtype=float)
|
|
91
|
+
saved_p = group["tie_inclusion_probability"].to_numpy(dtype=float)
|
|
92
|
+
if not np.isfinite(values).all() or not np.isfinite(saved_p).all():
|
|
93
|
+
raise ValueError("Candidate scores and tie probabilities must be finite")
|
|
94
|
+
predicted_p = _boundary_probabilities(values, top_k)
|
|
95
|
+
if not np.allclose(saved_p, predicted_p, rtol=0, atol=1e-12):
|
|
96
|
+
raise ValueError("Stored tie probabilities disagree with candidate scores")
|
|
97
|
+
reference_p = _boundary_probabilities(sign * truth, top_k)
|
|
98
|
+
overlap = float(predicted_p @ reference_p)
|
|
99
|
+
rows.append({"method": arm, "query_id": str(query), "direction": direction,
|
|
100
|
+
"precision_at_k": overlap / top_k, "expected_overlap": overlap,
|
|
101
|
+
"top_k": int(top_k), "n_candidates": len(eligible)})
|
|
102
|
+
answer = pd.DataFrame.from_records(rows)
|
|
103
|
+
answer.attrs["scope"] = (
|
|
104
|
+
"Fixed-query, fixed-pool reference-list agreement; independent uniform boundary ties. "
|
|
105
|
+
"No biological probability, uncertainty interval or across-experiment inference."
|
|
106
|
+
)
|
|
107
|
+
return answer
|
pertrank/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pertrank
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Reusable control-geometry reliability for gene-specific perturbation prioritization
|
|
5
|
+
License: Private review distribution; public license pending author selection
|
|
6
|
+
Project-URL: Repository, https://github.com/anonymous-rl-lab/PertRank/tree/v0.1.2
|
|
7
|
+
Project-URL: Documentation, https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/README.md
|
|
8
|
+
Project-URL: Scientific Design, https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/docs/scientific_design.md
|
|
9
|
+
Project-URL: API Reference, https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/docs/api.md
|
|
10
|
+
Project-URL: Replogle Tutorial, https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/tutorials/01_replogle.ipynb
|
|
11
|
+
Project-URL: T-cell Tutorial, https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/tutorials/02_tcell.ipynb
|
|
12
|
+
Project-URL: Code Releases, https://github.com/anonymous-rl-lab/PertRank/releases
|
|
13
|
+
Project-URL: Baseline Archive DOI, https://doi.org/10.5281/zenodo.24281072
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: numpy>=1.24
|
|
22
|
+
Requires-Dist: scipy<1.17,>=1.10
|
|
23
|
+
Requires-Dist: pandas>=1.5
|
|
24
|
+
Requires-Dist: anndata>=0.9
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
27
|
+
Requires-Dist: h5py>=3.8; extra == "test"
|
|
28
|
+
Provides-Extra: tutorial
|
|
29
|
+
Requires-Dist: jupyterlab>=4; extra == "tutorial"
|
|
30
|
+
Requires-Dist: matplotlib>=3.6; extra == "tutorial"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# PertRank
|
|
34
|
+
|
|
35
|
+
**Use control geometry to improve candidate selection while preserving the biological readout.** PertRank combines profile-wide effect and variance information with leading covariance directions from matched controls to calculate one reliability weight per perturbation. It applies that weight to the queried gene effect, producing separate candidate lists for increasing and decreasing the response.
|
|
36
|
+
|
|
37
|
+
The package provides an AnnData interface, reuse of fitted reliability with labeled external effect estimates, two real-data statistics tutorials, and numerical checks against the research implementation. The distribution and import name is `pertrank`.
|
|
38
|
+
|
|
39
|
+
Version **0.2.0** is a documentation and packaging maintenance release. It connects the reusable tool to the completed rvalues and transcriptional-program studies while preserving the numerical API and fixed parameters. The reproduction guide (`docs/reproducibility.md` in the source archive) explains which results belong to the Python tool and which are reproduced by the separate experiment bundle. Bundle releases are listed in the [repository](https://github.com/anonymous-rl-lab/PertRank/releases).
|
|
40
|
+
|
|
41
|
+
## What the software separates
|
|
42
|
+
|
|
43
|
+
| Step | Information used | Output |
|
|
44
|
+
|---|---|---|
|
|
45
|
+
| Estimate reliability | Matched controls and each target's own effect and sampling variation across the expression profile | One bounded weight per target |
|
|
46
|
+
| Read the response | Original gene effects, or a labeled upstream effect estimate such as FR-Perturb | A response-specific up/down score |
|
|
47
|
+
| Allocate shortlist positions | The same eligible candidates and a fixed list length | Candidate lists and exact boundary-tie inclusion probabilities |
|
|
48
|
+
|
|
49
|
+
The three returned methods make the contribution inspectable: **Unweighted** uses the effect directly, **Euclidean** uses target-level reliability without directional geometry, and **PertRank** adds the fixed control geometry. A complete-workflow comparison against Unweighted and a geometry comparison against Euclidean answer different questions. See [scientific design](https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/docs/scientific_design.md) for the equations, estimator reuse and connection to the manuscript evidence.
|
|
50
|
+
|
|
51
|
+
## Install
|
|
52
|
+
|
|
53
|
+
Use Python 3.10 or later. From the extracted source distribution, install the package in a separate environment:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
python -m pip install .
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
To create a wheel from the source tree:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m pip wheel --no-deps . --wheel-dir dist
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Dependencies are declared in `pyproject.toml`. For development and checks:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
python -m pip install '.[test]'
|
|
69
|
+
python -m pytest
|
|
70
|
+
python scripts/check_tutorials.py --output-dir ../pertrank-tutorial-check
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Rank candidates from AnnData
|
|
74
|
+
|
|
75
|
+
Start with quality-controlled input cells and their matched non-targeting controls. Keep planning cells and reference cells separate. Gene and cell identifiers must be unique, and perturbed genes must use the same identifier namespace as query genes.
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import anndata as ad
|
|
79
|
+
import pertrank as pr
|
|
80
|
+
|
|
81
|
+
adata = ad.read_h5ad("screen_inputs.h5ad", backed="r")
|
|
82
|
+
result = pr.rank(
|
|
83
|
+
adata,
|
|
84
|
+
queries=["CTLA4", "IL2"],
|
|
85
|
+
input_scale="counts",
|
|
86
|
+
layer=None, # raw counts in X
|
|
87
|
+
full_gene_axis=True, # assert the full measured axis is present
|
|
88
|
+
strata_keys=["donor", "condition", "batch"],
|
|
89
|
+
target_key="target_ids",
|
|
90
|
+
target_gene_key="target_gene_ids",
|
|
91
|
+
control_key="is_control",
|
|
92
|
+
top_k=25,
|
|
93
|
+
)
|
|
94
|
+
adata.file.close()
|
|
95
|
+
|
|
96
|
+
result.nominations.to_csv("nominations.csv", index=False)
|
|
97
|
+
result.weights.to_csv("reliability_weights.csv", index=False)
|
|
98
|
+
result.all_candidate_scores.to_csv("all_candidate_scores.csv", index=False)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
The count path accepts one prespecified matching stratum per call. The example's donor, condition and batch columns must therefore each have one value in the supplied object. Split the screen into its prespecified matching strata before calling this path. `strata_keys=[]` explicitly declares that the supplied cells belong to one stratum; use it only when the experimental design supports that declaration.
|
|
102
|
+
|
|
103
|
+
All supplied input cells contribute to the fit; define the query set and input-cell selection before calling the package. The paper's shallow examples use 20 cells per target and 2,000 controls; these experimental budgets differ from the numerical minimum. See input contracts (`docs/input_contracts.md` in the source archive) for scale, control matching and memory behavior.
|
|
104
|
+
|
|
105
|
+
## Read the outputs
|
|
106
|
+
|
|
107
|
+
| Output | Meaning |
|
|
108
|
+
|---|---|
|
|
109
|
+
| `nominations` | Deterministic top-k lists for up/down responses and the Unweighted, Euclidean and PertRank methods |
|
|
110
|
+
| `all_candidate_scores` | Eligible query–target scores, ranks and boundary-tie inclusion probabilities |
|
|
111
|
+
| `weights` | Target-level reliability weights and input counts |
|
|
112
|
+
| `effects` | Target-by-gene original-coordinate effects |
|
|
113
|
+
| `controls`, `statistics`, `metadata` | Control models, sufficient statistics and computation provenance |
|
|
114
|
+
|
|
115
|
+
The up score is `effect × weight`; the down score reverses its sign. The perturbed gene is excluded from its own query. A zero weight retains the candidate at a zero score. Tie inclusion probabilities describe random selection among equally scored candidates at the cutoff; they are not probabilities of biological correctness.
|
|
116
|
+
|
|
117
|
+
## Reuse reliability with an existing effect estimate
|
|
118
|
+
|
|
119
|
+
An external effect matrix must have unique target and gene labels, the same target population, and compatible effect orientation. Align it by labels rather than by row order.
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
import pandas as pd
|
|
123
|
+
|
|
124
|
+
external = pd.read_csv("external_effects.csv", index_col=0)
|
|
125
|
+
external_result = result.rank_effects(
|
|
126
|
+
external, queries=["CTLA4", "IL2"], top_k=25
|
|
127
|
+
)
|
|
128
|
+
external_result.nominations.to_csv("external_nominations.csv", index=False)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
This holds the fitted reliability calculation fixed while replacing the effect estimator. The returned `Unweighted`, `Euclidean` and `PertRank` arms then all use the supplied external effects, so the geometry comparison remains matched. The operation neither fits the upstream estimator nor re-estimates its uncertainty. Record that estimator's configuration and input design alongside the exported ranking.
|
|
132
|
+
|
|
133
|
+
## Evaluate a fixed shortlist against a separate reference
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
reference = pd.read_csv("reference_effects.csv", index_col=0)
|
|
137
|
+
scores = pr.evaluate_reference(result, reference)
|
|
138
|
+
scores.to_csv("reference_precision.csv", index=False)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Provide a target-by-gene reference effect table with the same target population and every queried gene. The evaluator returns precision and expected overlap for each method, query and direction, using the result's fixed list length and self-target exclusions. It accounts for boundary ties in both the predicted and reference lists. The reference is used only for scoring. Define its construction independently of the fitted ranking; biological interpretation depends on that design. See [API reference](https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/docs/api.md#evaluate_reference) for the exact scoring rule. Query-level uncertainty and manuscript-specific aggregation remain in the full scientific reproduction bundle.
|
|
142
|
+
|
|
143
|
+
## Try the interface immediately
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
python tutorials/00_anndata_quickstart.py
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
This quickstart creates a small synthetic AnnData solely for a software
|
|
150
|
+
check. The two atlas notebooks instead use authentic archived statistics.
|
|
151
|
+
|
|
152
|
+
## Learn and reproduce
|
|
153
|
+
|
|
154
|
+
- [API reference](https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/docs/api.md): public functions and result fields.
|
|
155
|
+
- [Scientific design](https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/docs/scientific_design.md): reliability, readout and the interpretation of matched comparisons.
|
|
156
|
+
- Input contracts (`docs/input_contracts.md` in the source archive): counts, normalized inputs and matching strata.
|
|
157
|
+
- [Replogle tutorial](https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/tutorials/01_replogle.ipynb) and [T-cell tutorial](https://github.com/anonymous-rl-lab/PertRank/blob/v0.1.2/modules/paper_analysis/software/tutorials/02_tcell.ipynb): worked examples using archived real-data sufficient statistics. The tutorials identify the source and processing level of each fixture. They replay candidate ranking from saved statistics; the AnnData example above is the cell-input route.
|
|
158
|
+
- Reproduction guide (`docs/reproducibility.md` in the source archive): package examples, the two completed experimental additions, and full-bundle execution.
|
|
159
|
+
- `tests/`: checks of the public interface and numerical agreement with the frozen implementation.
|
|
160
|
+
- Community integration proposal (`docs/community_integration.md` in the source archive): prepared scverse registration and pertpy contribution scope.
|
|
161
|
+
- Distribution and release process (`docs/release_process.md` in the source archive): installation, artifact verification and the remaining public-release metadata.
|
|
162
|
+
|
|
163
|
+
The scientific use is candidate prioritization for follow-up experiments. Weights describe estimated profile reliability, not calibrated probabilities of biological validation. The manuscript evaluates decision quality with fixed-length lists, matched exclusions and reference-supported precision; its independent RNA comparison supports the complete workflow, while matched Euclidean contrasts isolate the added geometry.
|
|
164
|
+
|
|
165
|
+
## Research resources and attribution
|
|
166
|
+
|
|
167
|
+
PertRank-Q is a separate, optional checkpoint-replay module in the full reproduction bundle (`modules/pertrank_q`); it is not part of the `pertrank` Python API. Its reference-supervised candidate-subpool evaluation must not be read as an unsupervised full-pool deployment result.
|
|
168
|
+
|
|
169
|
+
The computational equations and fixed parameters (`k=5`, `beta=0.9`) follow the manuscript implementation. The accompanying paper now also evaluates a fixed native rvalues comparator and predefined transcriptional-program nomination in a drug screen. Their experiment-specific inputs, reference resampling and diagnostics are distributed in the separate reproduction bundle. They do not add a program-ranking or PertRank-Q API to this wheel.
|
|
170
|
+
|
|
171
|
+
The source manifests identify the method and datasets, whose rights remain with their original owners. The inherited software-license decision remains pending; see release process (`docs/release_process.md` in the source archive) for the publication requirements.
|
|
172
|
+
|
|
173
|
+
## Where the documentation lives
|
|
174
|
+
|
|
175
|
+
The scientific design, API reference and both tutorial links above open the stable interface documentation at `v0.1.2` in the anonymous GitHub repository over HTTPS. These links describe the numerical interface retained by this maintenance release; they are not evidence that the newer experiment bundle is contained in that archived version. These documents and notebooks are also included in the source distribution for offline reading. PyPI hosts distribution archives, not separate web pages for files inside them. The wheel includes the runtime and tutorial data, while the notebooks and prose documentation remain in the source archive.
|
|
176
|
+
|
|
177
|
+
`PYPI_DESCRIPTION.md` is generated from this README by `python scripts/build_pypi_description.py`. It preserves HTTPS links, including the version-pinned documentation targets and the API `#evaluate_reference` anchor, and converts the remaining local links into explicit source-archive paths. `pyproject.toml` uses it as the PyPI description so local paths cannot resolve to nonexistent PyPI pages. Run `python scripts/check_distribution.py` against the newly built archives before a release. Package versions, reproduction-bundle versions and paper versions identify separate artifacts.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
pertrank/__init__.py,sha256=jxWWjSfkzdVQdxMfX6mnil3nhotoOQad1gWVhwobp08,475
|
|
2
|
+
pertrank/_version.py,sha256=mOk-roqsRZJM4H37idnSN75M5Ng_6njvNC16u5_vRxA,109
|
|
3
|
+
pertrank/api.py,sha256=5p24P-3ZM7Ed0uk0IGgAhRZZXtJzcIn0YZ5aCjEV0_M,25494
|
|
4
|
+
pertrank/core.py,sha256=jadyds7RWRdzsGDzTJnIolMfsuoo0kIVTPZZs5WB-28,13224
|
|
5
|
+
pertrank/datasets.py,sha256=-Gaji-c5huaY5dEcQsgo6EZw8hG9vWkonNiUJBfBt1E,2343
|
|
6
|
+
pertrank/evaluation.py,sha256=80L2frDJwiTIUs5BWLOfl-qo4zzetwYjPhChwv7KFj8,6661
|
|
7
|
+
pertrank/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
pertrank/data/replogle_k562.npz,sha256=AIfk2RUQXKrIAQaUmUKNWAMuxkq4niJCMCFkBP2lHf8,2610808
|
|
9
|
+
pertrank/data/replogle_k562.provenance.json,sha256=wwxDKDVtHIaTg9s1BbW6mWLNS8XE6d6a4x73dPfLhOM,3142
|
|
10
|
+
pertrank/data/tcell_d3_rest.npz,sha256=boDHgZ5ln-O5Aboqbl5gMMzSWjzUtlRDr7uPjE7cfKg,246138
|
|
11
|
+
pertrank/data/tcell_d3_rest.provenance.json,sha256=EKeUVd2HhHPj_x3NTGv3ujc35Nf1FvE9cyaQsDonQBI,2787
|
|
12
|
+
pertrank-0.2.0.dist-info/licenses/LICENSE,sha256=3ZFW6FOQ3J4e7SRsx3K8YQ8JjpGWw81E0J076DzpvQU,742
|
|
13
|
+
pertrank-0.2.0.dist-info/METADATA,sha256=qBt4DreqGQfmZiKIL_nSNa9yWK9dioMXKh6ckgYDBCU,13666
|
|
14
|
+
pertrank-0.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
15
|
+
pertrank-0.2.0.dist-info/top_level.txt,sha256=XFHsSqieNhzzva2bAfPOdFuD2W0aiWAjdA2nBTa5O9c,9
|
|
16
|
+
pertrank-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Private review distribution
|
|
2
|
+
|
|
3
|
+
This anonymous release is supplied for installation, examination and
|
|
4
|
+
reproduction of the accompanying method during private review. Copyright
|
|
5
|
+
and other rights remain with their respective owners. No new public
|
|
6
|
+
open-source license is granted by this packaging step.
|
|
7
|
+
|
|
8
|
+
Before a public PyPI release or scverse submission, the authors must select
|
|
9
|
+
and apply an OSI-approved license to their software. Third-party datasets
|
|
10
|
+
retain their original terms and attribution; the tutorial provenance files
|
|
11
|
+
identify the sources. Public dataset redistribution must respect those terms.
|
|
12
|
+
|
|
13
|
+
This notice describes the current distribution status and does not replace
|
|
14
|
+
any existing license or rights attached to a third-party component.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pertrank
|