pyplasmode 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyplasmode/__init__.py +178 -0
- pyplasmode/_validation.py +29 -0
- pyplasmode/data.py +221 -0
- pyplasmode/evaluate.py +392 -0
- pyplasmode/generate.py +349 -0
- pyplasmode/nominations.py +623 -0
- pyplasmode/outcomes.py +709 -0
- pyplasmode/py.typed +0 -0
- pyplasmode/results.py +172 -0
- pyplasmode/structures.py +123 -0
- pyplasmode/truth.py +510 -0
- pyplasmode-1.0.0.dist-info/METADATA +142 -0
- pyplasmode-1.0.0.dist-info/RECORD +15 -0
- pyplasmode-1.0.0.dist-info/WHEEL +4 -0
- pyplasmode-1.0.0.dist-info/licenses/LICENSE +29 -0
pyplasmode/truth.py
ADDED
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
"""Truth specifications and deterministic signal materialisation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Literal
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
from numpy.typing import NDArray
|
|
11
|
+
|
|
12
|
+
from pyplasmode._validation import identities, integer
|
|
13
|
+
from pyplasmode.data import Population
|
|
14
|
+
from pyplasmode.structures import FeatureSets
|
|
15
|
+
|
|
16
|
+
type TruthKind = Literal["null", "sparse", "correlated", "module", "interaction", "custom"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True, slots=True)
|
|
20
|
+
class RecoveryGroup:
|
|
21
|
+
"""A direct feature and the explicitly admissible substitutes for its group."""
|
|
22
|
+
|
|
23
|
+
name: str
|
|
24
|
+
direct_feature: str | None
|
|
25
|
+
features: tuple[str, ...]
|
|
26
|
+
|
|
27
|
+
def __post_init__(self) -> None:
|
|
28
|
+
identities((self.name,), "group name")
|
|
29
|
+
identities(self.features, "group features")
|
|
30
|
+
if self.direct_feature is not None and self.direct_feature not in self.features:
|
|
31
|
+
raise ValueError("group direct feature must belong to its members")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True, slots=True)
|
|
35
|
+
class FeatureWeight:
|
|
36
|
+
"""One feature's unstandardized contribution to a distributed signal."""
|
|
37
|
+
|
|
38
|
+
feature: str
|
|
39
|
+
weight: float
|
|
40
|
+
|
|
41
|
+
def __post_init__(self) -> None:
|
|
42
|
+
identities((self.feature,), "weight feature")
|
|
43
|
+
if not np.isfinite(self.weight) or self.weight == 0.0:
|
|
44
|
+
raise ValueError("truth weights must be finite and non-zero")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True, slots=True)
|
|
48
|
+
class MaterializedTruth:
|
|
49
|
+
"""The realised signal and mechanism-specific recovery targets."""
|
|
50
|
+
|
|
51
|
+
kind: TruthKind
|
|
52
|
+
signal: NDArray[np.float64]
|
|
53
|
+
feature_universe: tuple[str, ...]
|
|
54
|
+
direct_features: tuple[str, ...] = ()
|
|
55
|
+
admissible_groups: tuple[RecoveryGroup, ...] = ()
|
|
56
|
+
weights: tuple[FeatureWeight, ...] = ()
|
|
57
|
+
graph_region: tuple[str, ...] = ()
|
|
58
|
+
center: float = 0.0
|
|
59
|
+
scale: float = 1.0
|
|
60
|
+
description: str = ""
|
|
61
|
+
|
|
62
|
+
def __post_init__(self) -> None:
|
|
63
|
+
if self.kind not in {"null", "sparse", "correlated", "module", "interaction", "custom"}:
|
|
64
|
+
raise ValueError("unsupported truth kind")
|
|
65
|
+
identities(self.feature_universe, "feature universe")
|
|
66
|
+
identities(self.direct_features, "direct features", allow_empty=True)
|
|
67
|
+
identities(
|
|
68
|
+
tuple(item.feature for item in self.weights), "weighted features", allow_empty=True
|
|
69
|
+
)
|
|
70
|
+
identities(
|
|
71
|
+
tuple(group.name for group in self.admissible_groups), "group names", allow_empty=True
|
|
72
|
+
)
|
|
73
|
+
identities(self.graph_region, "graph region", allow_empty=True)
|
|
74
|
+
declared = (
|
|
75
|
+
self.direct_features
|
|
76
|
+
+ tuple(item.feature for item in self.weights)
|
|
77
|
+
+ tuple(feature for group in self.admissible_groups for feature in group.features)
|
|
78
|
+
+ self.graph_region
|
|
79
|
+
)
|
|
80
|
+
if not set(declared) <= set(self.feature_universe):
|
|
81
|
+
raise ValueError("truth targets must belong to the feature universe")
|
|
82
|
+
if not np.isfinite(self.center) or not np.isfinite(self.scale) or self.scale < 0.0:
|
|
83
|
+
raise ValueError("truth center and scale must be finite with non-negative scale")
|
|
84
|
+
signal = np.asarray(self.signal, dtype=np.float64)
|
|
85
|
+
if signal.ndim != 1 or not signal.size or not np.isfinite(signal).all():
|
|
86
|
+
raise ValueError("materialized truth signal must be finite and one-dimensional")
|
|
87
|
+
if self.kind == "null" and (np.any(signal != 0.0) or declared):
|
|
88
|
+
raise ValueError("null truth cannot contain signal or recovery targets")
|
|
89
|
+
copied = np.array(signal, copy=True)
|
|
90
|
+
copied.setflags(write=False)
|
|
91
|
+
object.__setattr__(self, "signal", copied)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@dataclass(frozen=True, slots=True)
|
|
95
|
+
class NullTruth:
|
|
96
|
+
"""A truth with no biomarker contribution."""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(frozen=True, slots=True)
|
|
100
|
+
class SparseTruth:
|
|
101
|
+
"""An exact sparse truth selected explicitly or by seeded weak-correlation search."""
|
|
102
|
+
|
|
103
|
+
count: int | None = None
|
|
104
|
+
features: tuple[str, ...] = ()
|
|
105
|
+
weights: tuple[float, ...] = ()
|
|
106
|
+
max_absolute_correlation: float = 0.7
|
|
107
|
+
|
|
108
|
+
def __post_init__(self) -> None:
|
|
109
|
+
if (self.count is None) == (not self.features):
|
|
110
|
+
raise ValueError("SparseTruth requires exactly one of count or features")
|
|
111
|
+
if self.count is not None:
|
|
112
|
+
integer(self.count, "sparse count", minimum=1)
|
|
113
|
+
identities(self.features, "sparse features", allow_empty=True)
|
|
114
|
+
if self.weights and len(self.weights) != len(self.features):
|
|
115
|
+
raise ValueError("sparse weights require one value per explicit feature")
|
|
116
|
+
_validate_weights(self.weights)
|
|
117
|
+
if not 0.0 <= self.max_absolute_correlation < 1.0:
|
|
118
|
+
raise ValueError("max_absolute_correlation must be in [0, 1)")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass(frozen=True, slots=True)
|
|
122
|
+
class CorrelatedTruth:
|
|
123
|
+
"""One direct sentinel and admissible substitutes from each declared group."""
|
|
124
|
+
|
|
125
|
+
group_names: tuple[str, ...]
|
|
126
|
+
sentinels: tuple[str, ...] = ()
|
|
127
|
+
weights: tuple[float, ...] = ()
|
|
128
|
+
signal_form: Literal["sentinels", "group_means"] = "sentinels"
|
|
129
|
+
|
|
130
|
+
def __post_init__(self) -> None:
|
|
131
|
+
identities(self.group_names, "correlated group_names")
|
|
132
|
+
identities(self.sentinels, "sentinels", allow_empty=True)
|
|
133
|
+
if self.signal_form not in {"sentinels", "group_means"}:
|
|
134
|
+
raise ValueError("signal_form must be sentinels or group_means")
|
|
135
|
+
if self.signal_form == "group_means" and self.sentinels:
|
|
136
|
+
raise ValueError("group_means has no direct sentinels")
|
|
137
|
+
if self.sentinels and len(self.sentinels) != len(self.group_names):
|
|
138
|
+
raise ValueError("sentinels must contain one feature per group")
|
|
139
|
+
if self.weights and len(self.weights) != len(self.group_names):
|
|
140
|
+
raise ValueError("correlated weights require one value per group")
|
|
141
|
+
_validate_weights(self.weights)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@dataclass(frozen=True, slots=True)
|
|
145
|
+
class ModuleTruth:
|
|
146
|
+
"""A weighted state distributed across one declared feature set."""
|
|
147
|
+
|
|
148
|
+
set_name: str
|
|
149
|
+
weights: tuple[float, ...] = ()
|
|
150
|
+
|
|
151
|
+
def __post_init__(self) -> None:
|
|
152
|
+
identities((self.set_name,), "module set_name")
|
|
153
|
+
_validate_weights(self.weights)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@dataclass(frozen=True, slots=True)
|
|
157
|
+
class InteractionTruth:
|
|
158
|
+
"""A bounded pairwise product or threshold interaction sensitivity."""
|
|
159
|
+
|
|
160
|
+
features: tuple[str, str]
|
|
161
|
+
operation: Literal["product", "threshold"] = "product"
|
|
162
|
+
threshold: float = 0.0
|
|
163
|
+
|
|
164
|
+
def __post_init__(self) -> None:
|
|
165
|
+
identities(self.features, "interaction features")
|
|
166
|
+
if len(self.features) != 2:
|
|
167
|
+
raise ValueError("interaction truth requires two distinct features")
|
|
168
|
+
if self.operation not in {"product", "threshold"}:
|
|
169
|
+
raise ValueError("operation must be product or threshold")
|
|
170
|
+
if not np.isfinite(self.threshold):
|
|
171
|
+
raise ValueError("interaction threshold must be finite")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@dataclass(frozen=True, slots=True)
|
|
175
|
+
class CustomSignal:
|
|
176
|
+
"""A custom signal and its explicit recovery semantics."""
|
|
177
|
+
|
|
178
|
+
signal: NDArray[np.float64]
|
|
179
|
+
direct_features: tuple[str, ...] = ()
|
|
180
|
+
groups: tuple[RecoveryGroup, ...] = ()
|
|
181
|
+
weights: tuple[FeatureWeight, ...] = ()
|
|
182
|
+
graph_region: tuple[str, ...] = ()
|
|
183
|
+
description: str = ""
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
type CustomTruthFunction = Callable[
|
|
187
|
+
[NDArray[np.float64], tuple[str, ...], np.random.Generator], CustomSignal
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@dataclass(frozen=True, slots=True)
|
|
192
|
+
class CustomTruth:
|
|
193
|
+
"""A conforming callable used to materialise an extension truth."""
|
|
194
|
+
|
|
195
|
+
function: CustomTruthFunction
|
|
196
|
+
|
|
197
|
+
def __post_init__(self) -> None:
|
|
198
|
+
if not callable(self.function):
|
|
199
|
+
raise ValueError("custom truth function must be callable")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
type TruthSpec = (
|
|
203
|
+
NullTruth | SparseTruth | CorrelatedTruth | ModuleTruth | InteractionTruth | CustomTruth
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _validate_weights(weights: tuple[float, ...]) -> None:
|
|
208
|
+
if any(not np.isfinite(weight) or weight == 0.0 for weight in weights):
|
|
209
|
+
raise ValueError("truth weights must be finite and non-zero")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _completed_standardized(
|
|
213
|
+
population: Population, *, required_features: tuple[str, ...] = ()
|
|
214
|
+
) -> NDArray[np.float64]:
|
|
215
|
+
"""Median-complete observed columns and retain missing columns as constants."""
|
|
216
|
+
required = _indices(population, required_features)
|
|
217
|
+
matrix = np.array(population.X[:, required] if required_features else population.X, copy=True)
|
|
218
|
+
all_missing = np.isnan(matrix).all(axis=0)
|
|
219
|
+
unavailable = tuple(
|
|
220
|
+
feature for feature, missing in zip(required_features, all_missing, strict=False) if missing
|
|
221
|
+
)
|
|
222
|
+
if unavailable:
|
|
223
|
+
raise ValueError(f"truth features are entirely missing: {unavailable}")
|
|
224
|
+
medians = np.zeros(matrix.shape[1], dtype=np.float64)
|
|
225
|
+
observed = ~all_missing
|
|
226
|
+
medians[observed] = np.nanmedian(matrix[:, observed], axis=0)
|
|
227
|
+
missing_rows, missing_columns = np.where(np.isnan(matrix))
|
|
228
|
+
matrix[missing_rows, missing_columns] = medians[missing_columns]
|
|
229
|
+
scales = np.std(matrix, axis=0)
|
|
230
|
+
constants = tuple(
|
|
231
|
+
feature for feature, scale in zip(required_features, scales, strict=False) if scale == 0.0
|
|
232
|
+
)
|
|
233
|
+
if constants:
|
|
234
|
+
raise ValueError(f"truth features are constant: {constants}")
|
|
235
|
+
scales[scales == 0.0] = 1.0
|
|
236
|
+
return (matrix - np.mean(matrix, axis=0)) / scales
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _indices(population: Population, features: tuple[str, ...]) -> NDArray[np.int64]:
|
|
240
|
+
positions = {feature: index for index, feature in enumerate(population.feature_ids)}
|
|
241
|
+
unknown = tuple(feature for feature in features if feature not in positions)
|
|
242
|
+
if unknown:
|
|
243
|
+
raise ValueError(f"truth contains unknown feature identities: {unknown}")
|
|
244
|
+
return np.asarray([positions[feature] for feature in features], dtype=np.int64)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _standardize_signal(signal: NDArray[np.float64]) -> tuple[NDArray[np.float64], float, float]:
|
|
248
|
+
if signal.ndim != 1 or not np.isfinite(signal).all():
|
|
249
|
+
raise ValueError("truth signal must be finite and one-dimensional")
|
|
250
|
+
center = float(np.mean(signal))
|
|
251
|
+
scale = float(np.std(signal))
|
|
252
|
+
if not np.isfinite(scale) or scale <= 0.0:
|
|
253
|
+
raise ValueError("a non-null truth signal must be nonconstant")
|
|
254
|
+
return (signal - center) / scale, center, scale
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _weighted_signal(
|
|
258
|
+
standardized: NDArray[np.float64], indices: NDArray[np.int64], weights: NDArray[np.float64]
|
|
259
|
+
) -> NDArray[np.float64]:
|
|
260
|
+
if not np.isfinite(weights).all() or np.all(weights == 0.0):
|
|
261
|
+
raise ValueError("truth weights must be finite and not all zero")
|
|
262
|
+
return np.asarray(standardized[:, indices] @ weights, dtype=np.float64)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _select_sparse(
|
|
266
|
+
standardized: NDArray[np.float64], count: int, maximum: float, rng: np.random.Generator
|
|
267
|
+
) -> NDArray[np.int64]:
|
|
268
|
+
variable = np.flatnonzero(np.std(standardized, axis=0) > 0.0)
|
|
269
|
+
selected: list[int] = []
|
|
270
|
+
for candidate in rng.permutation(variable):
|
|
271
|
+
if all(
|
|
272
|
+
abs(float(np.corrcoef(standardized[:, candidate], standardized[:, prior])[0, 1]))
|
|
273
|
+
<= maximum
|
|
274
|
+
for prior in selected
|
|
275
|
+
):
|
|
276
|
+
selected.append(int(candidate))
|
|
277
|
+
if len(selected) == count:
|
|
278
|
+
return np.asarray(selected, dtype=np.int64)
|
|
279
|
+
raise ValueError(f"could not select {count} features under max_absolute_correlation={maximum}")
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _materialize_custom(truth: CustomTruth, population: Population, seed: int) -> MaterializedTruth:
|
|
283
|
+
readonly = np.array(population.X, copy=True)
|
|
284
|
+
readonly.setflags(write=False)
|
|
285
|
+
first = truth.function(readonly, population.feature_ids, np.random.default_rng(seed))
|
|
286
|
+
second = truth.function(readonly, population.feature_ids, np.random.default_rng(seed))
|
|
287
|
+
first_signal = np.asarray(first.signal, dtype=np.float64)
|
|
288
|
+
second_signal = np.asarray(second.signal, dtype=np.float64)
|
|
289
|
+
if first_signal.shape != (population.sample_count,):
|
|
290
|
+
raise ValueError("custom truth signal shape must equal the population row count")
|
|
291
|
+
same_metadata = (
|
|
292
|
+
first.direct_features == second.direct_features
|
|
293
|
+
and first.groups == second.groups
|
|
294
|
+
and first.weights == second.weights
|
|
295
|
+
and first.graph_region == second.graph_region
|
|
296
|
+
and first.description == second.description
|
|
297
|
+
)
|
|
298
|
+
if not np.array_equal(first_signal, second_signal, equal_nan=True) or not same_metadata:
|
|
299
|
+
raise ValueError("custom truth must be deterministic for identical inputs and seed")
|
|
300
|
+
declared = (
|
|
301
|
+
first.direct_features
|
|
302
|
+
+ tuple(feature for group in first.groups for feature in group.features)
|
|
303
|
+
+ tuple(item.feature for item in first.weights)
|
|
304
|
+
+ first.graph_region
|
|
305
|
+
)
|
|
306
|
+
_indices(population, declared)
|
|
307
|
+
signal, center, scale = _standardize_signal(first_signal)
|
|
308
|
+
return MaterializedTruth(
|
|
309
|
+
kind="custom",
|
|
310
|
+
signal=signal,
|
|
311
|
+
feature_universe=population.feature_ids,
|
|
312
|
+
direct_features=first.direct_features,
|
|
313
|
+
admissible_groups=first.groups,
|
|
314
|
+
weights=first.weights,
|
|
315
|
+
graph_region=first.graph_region,
|
|
316
|
+
center=center,
|
|
317
|
+
scale=scale,
|
|
318
|
+
description=first.description,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def materialize_truth(
|
|
323
|
+
truth: TruthSpec,
|
|
324
|
+
population: Population,
|
|
325
|
+
*,
|
|
326
|
+
seed: int,
|
|
327
|
+
feature_sets: FeatureSets | None = None,
|
|
328
|
+
) -> MaterializedTruth:
|
|
329
|
+
"""Materialise a declared mechanism on an empirical matrix.
|
|
330
|
+
|
|
331
|
+
Args:
|
|
332
|
+
truth: Built-in mechanism or deterministic custom signal function.
|
|
333
|
+
population: Numeric feature matrix and ordered identifiers.
|
|
334
|
+
seed: Non-negative integer controlling data-derived truth selection.
|
|
335
|
+
feature_sets: Named memberships required by correlated and module truths.
|
|
336
|
+
|
|
337
|
+
Returns:
|
|
338
|
+
Standardised latent signal and explicit recovery targets. The original
|
|
339
|
+
matrix is not modified. Null truth has a zero signal and no targets.
|
|
340
|
+
|
|
341
|
+
Raises:
|
|
342
|
+
ValueError: Invalid identities, weights or settings, unavailable or
|
|
343
|
+
constant required features, or an infeasible sparse selection.
|
|
344
|
+
TypeError: Unsupported specification type.
|
|
345
|
+
"""
|
|
346
|
+
integer(seed, "seed")
|
|
347
|
+
if isinstance(truth, NullTruth):
|
|
348
|
+
return MaterializedTruth(
|
|
349
|
+
kind="null",
|
|
350
|
+
signal=np.zeros(population.sample_count),
|
|
351
|
+
feature_universe=population.feature_ids,
|
|
352
|
+
center=0.0,
|
|
353
|
+
scale=0.0,
|
|
354
|
+
description="No biomarker signal",
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
rng = np.random.default_rng(seed)
|
|
358
|
+
if isinstance(truth, SparseTruth):
|
|
359
|
+
if truth.features:
|
|
360
|
+
features = truth.features
|
|
361
|
+
standardized = _completed_standardized(population, required_features=features)
|
|
362
|
+
selected = np.arange(len(features), dtype=np.int64)
|
|
363
|
+
else:
|
|
364
|
+
if truth.count is None: # Narrowing guard for static type checkers.
|
|
365
|
+
raise RuntimeError("sparse truth count is missing")
|
|
366
|
+
standardized = _completed_standardized(population)
|
|
367
|
+
selected = _select_sparse(
|
|
368
|
+
standardized, truth.count, truth.max_absolute_correlation, rng
|
|
369
|
+
)
|
|
370
|
+
features = tuple(population.feature_ids[index] for index in selected)
|
|
371
|
+
raw_weights = truth.weights or tuple(1.0 for _ in features)
|
|
372
|
+
weights = np.asarray(raw_weights, dtype=np.float64)
|
|
373
|
+
raw = _weighted_signal(standardized, selected, weights)
|
|
374
|
+
signal, center, scale = _standardize_signal(raw)
|
|
375
|
+
return MaterializedTruth(
|
|
376
|
+
kind="sparse",
|
|
377
|
+
signal=signal,
|
|
378
|
+
feature_universe=population.feature_ids,
|
|
379
|
+
direct_features=features,
|
|
380
|
+
weights=tuple(
|
|
381
|
+
FeatureWeight(feature, float(weight))
|
|
382
|
+
for feature, weight in zip(features, weights, strict=True)
|
|
383
|
+
),
|
|
384
|
+
center=center,
|
|
385
|
+
scale=scale,
|
|
386
|
+
description=f"Sparse exact truth over {len(features)} features",
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
if isinstance(truth, CorrelatedTruth):
|
|
390
|
+
if feature_sets is None:
|
|
391
|
+
raise ValueError("CorrelatedTruth requires FeatureSets")
|
|
392
|
+
groups = tuple(feature_sets.get(name) for name in truth.group_names)
|
|
393
|
+
all_members = tuple(feature for group in groups for feature in group.members)
|
|
394
|
+
if len(set(all_members)) != len(all_members):
|
|
395
|
+
raise ValueError("correlated truth requires disjoint groups")
|
|
396
|
+
for group in groups:
|
|
397
|
+
_indices(population, group.members)
|
|
398
|
+
sentinels = (
|
|
399
|
+
(
|
|
400
|
+
truth.sentinels
|
|
401
|
+
or tuple(
|
|
402
|
+
group.members[int(rng.integers(0, len(group.members)))] for group in groups
|
|
403
|
+
)
|
|
404
|
+
)
|
|
405
|
+
if truth.signal_form == "sentinels"
|
|
406
|
+
else ()
|
|
407
|
+
)
|
|
408
|
+
if any(
|
|
409
|
+
sentinel not in group.members
|
|
410
|
+
for sentinel, group in zip(sentinels, groups, strict=False)
|
|
411
|
+
):
|
|
412
|
+
raise ValueError("every correlated sentinel must belong to its declared group")
|
|
413
|
+
required = (
|
|
414
|
+
sentinels
|
|
415
|
+
if truth.signal_form == "sentinels"
|
|
416
|
+
else tuple(dict.fromkeys(feature for group in groups for feature in group.members))
|
|
417
|
+
)
|
|
418
|
+
standardized = _completed_standardized(population, required_features=required)
|
|
419
|
+
positions = {feature: index for index, feature in enumerate(required)}
|
|
420
|
+
weights = np.asarray(truth.weights or tuple(1.0 for _ in groups), dtype=np.float64)
|
|
421
|
+
if truth.signal_form == "sentinels":
|
|
422
|
+
selected = np.arange(len(sentinels), dtype=np.int64)
|
|
423
|
+
raw = _weighted_signal(standardized, selected, weights)
|
|
424
|
+
feature_weights = tuple(
|
|
425
|
+
FeatureWeight(feature, float(weight))
|
|
426
|
+
for feature, weight in zip(sentinels, weights, strict=True)
|
|
427
|
+
)
|
|
428
|
+
else:
|
|
429
|
+
group_states = np.column_stack(
|
|
430
|
+
[
|
|
431
|
+
np.mean(
|
|
432
|
+
standardized[:, [positions[feature] for feature in group.members]], axis=1
|
|
433
|
+
)
|
|
434
|
+
for group in groups
|
|
435
|
+
]
|
|
436
|
+
)
|
|
437
|
+
raw = np.asarray(group_states @ weights, dtype=np.float64)
|
|
438
|
+
contributions: dict[str, float] = dict.fromkeys(required, 0.0)
|
|
439
|
+
for group, weight in zip(groups, weights, strict=True):
|
|
440
|
+
for feature in group.members:
|
|
441
|
+
contributions[feature] += float(weight) / len(group.members)
|
|
442
|
+
feature_weights = tuple(
|
|
443
|
+
FeatureWeight(feature, weight)
|
|
444
|
+
for feature, weight in contributions.items()
|
|
445
|
+
if weight != 0.0
|
|
446
|
+
)
|
|
447
|
+
signal, center, scale = _standardize_signal(raw)
|
|
448
|
+
recovery_groups = tuple(
|
|
449
|
+
RecoveryGroup(group.name, sentinels[index] if sentinels else None, group.members)
|
|
450
|
+
for index, group in enumerate(groups)
|
|
451
|
+
)
|
|
452
|
+
return MaterializedTruth(
|
|
453
|
+
kind="correlated",
|
|
454
|
+
signal=signal,
|
|
455
|
+
feature_universe=population.feature_ids,
|
|
456
|
+
direct_features=sentinels,
|
|
457
|
+
admissible_groups=recovery_groups,
|
|
458
|
+
weights=feature_weights,
|
|
459
|
+
center=center,
|
|
460
|
+
scale=scale,
|
|
461
|
+
description=f"Correlated {truth.signal_form} truth over {len(groups)} groups",
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
if isinstance(truth, ModuleTruth):
|
|
465
|
+
if feature_sets is None:
|
|
466
|
+
raise ValueError("ModuleTruth requires FeatureSets")
|
|
467
|
+
feature_set = feature_sets.get(truth.set_name)
|
|
468
|
+
standardized = _completed_standardized(population, required_features=feature_set.members)
|
|
469
|
+
selected = np.arange(len(feature_set.members), dtype=np.int64)
|
|
470
|
+
raw_weights = truth.weights or tuple(1.0 for _ in feature_set.members)
|
|
471
|
+
if len(raw_weights) != len(feature_set.members):
|
|
472
|
+
raise ValueError("module weights require one value per set member")
|
|
473
|
+
weights = np.asarray(raw_weights, dtype=np.float64)
|
|
474
|
+
raw = _weighted_signal(standardized, selected, weights)
|
|
475
|
+
signal, center, scale = _standardize_signal(raw)
|
|
476
|
+
return MaterializedTruth(
|
|
477
|
+
kind="module",
|
|
478
|
+
signal=signal,
|
|
479
|
+
feature_universe=population.feature_ids,
|
|
480
|
+
weights=tuple(
|
|
481
|
+
FeatureWeight(feature, float(weight))
|
|
482
|
+
for feature, weight in zip(feature_set.members, weights, strict=True)
|
|
483
|
+
),
|
|
484
|
+
center=center,
|
|
485
|
+
scale=scale,
|
|
486
|
+
description=f"Weighted module truth: {feature_set.name}",
|
|
487
|
+
)
|
|
488
|
+
|
|
489
|
+
if isinstance(truth, InteractionTruth):
|
|
490
|
+
standardized = _completed_standardized(population, required_features=truth.features)
|
|
491
|
+
left = standardized[:, 0]
|
|
492
|
+
right = standardized[:, 1]
|
|
493
|
+
if truth.operation == "product":
|
|
494
|
+
raw = left * right
|
|
495
|
+
else:
|
|
496
|
+
raw = ((left > truth.threshold) & (right > truth.threshold)).astype(float)
|
|
497
|
+
signal, center, scale = _standardize_signal(raw)
|
|
498
|
+
return MaterializedTruth(
|
|
499
|
+
kind="interaction",
|
|
500
|
+
signal=signal,
|
|
501
|
+
feature_universe=population.feature_ids,
|
|
502
|
+
direct_features=truth.features,
|
|
503
|
+
center=center,
|
|
504
|
+
scale=scale,
|
|
505
|
+
description=f"Pairwise {truth.operation} interaction",
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
if isinstance(truth, CustomTruth):
|
|
509
|
+
return _materialize_custom(truth, population, seed)
|
|
510
|
+
raise TypeError(f"unsupported truth specification: {type(truth).__name__}")
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyplasmode
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Simulate biomarker signals and evaluate prediction and feature recovery.
|
|
5
|
+
Project-URL: Repository, https://github.com/BradSegal/PyPlasmode
|
|
6
|
+
Project-URL: Documentation, https://github.com/BradSegal/PyPlasmode/tree/main/docs
|
|
7
|
+
Project-URL: Issues, https://github.com/BradSegal/PyPlasmode/issues
|
|
8
|
+
License-Expression: BSD-3-Clause
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
18
|
+
Requires-Python: <3.15,>=3.12
|
|
19
|
+
Requires-Dist: numpy<3,>=2.0
|
|
20
|
+
Requires-Dist: scikit-learn<2,>=1.6
|
|
21
|
+
Requires-Dist: scipy<2,>=1.14
|
|
22
|
+
Provides-Extra: validation
|
|
23
|
+
Requires-Dist: matplotlib<4,>=3.10; extra == 'validation'
|
|
24
|
+
Requires-Dist: statsmodels<0.16,>=0.14; extra == 'validation'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# PyPlasmode
|
|
28
|
+
|
|
29
|
+
PyPlasmode benchmarks the reliability of biomarker discovery. A model can predict disease
|
|
30
|
+
accurately while selecting proteins that only track the underlying signal. To evaluate its
|
|
31
|
+
explanations, you need a comparison in which the relevant features are known. PyPlasmode
|
|
32
|
+
creates that comparison by generating outcomes from specified effects within your measurements.
|
|
33
|
+
|
|
34
|
+
Use it with your existing modelling workflow: generate a dataset, fit and tune your models,
|
|
35
|
+
then evaluate their predictions, feature rankings or selected panels.
|
|
36
|
+
|
|
37
|
+
[](docs/methods.md)
|
|
38
|
+
|
|
39
|
+
The same generated dataset supports two comparisons: predictions against outcomes, and
|
|
40
|
+
selected features against the signal that generated them.
|
|
41
|
+
|
|
42
|
+
Explore the [visual gallery](docs/gallery.md) for signal patterns, correlated-feature recovery
|
|
43
|
+
and the five outcome families.
|
|
44
|
+
|
|
45
|
+
The [biomarker tutorial](docs/tutorial.md) follows a complete example from correlated
|
|
46
|
+
measurements through model fitting, attribution and recovery above chance.
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
Requires Python 3.12-3.14. Install from GitHub:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
python -m pip install "git+https://github.com/BradSegal/PyPlasmode.git@v1.0.0"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The runtime dependencies are NumPy, SciPy and scikit-learn. To include the reference
|
|
57
|
+
validation and survival examples, install the validation extra:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
python -m pip install "PyPlasmode[validation] @ git+https://github.com/BradSegal/PyPlasmode.git@v1.0.0"
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Ten-minute example
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
import numpy as np
|
|
67
|
+
from sklearn.linear_model import LogisticRegression
|
|
68
|
+
import pyplasmode as ppm
|
|
69
|
+
|
|
70
|
+
rng = np.random.default_rng(1)
|
|
71
|
+
population = ppm.Population(
|
|
72
|
+
rng.normal(size=(2_000, 100)),
|
|
73
|
+
tuple(f"protein_{index}" for index in range(100)),
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
partition = ppm.partition_population(
|
|
77
|
+
population,
|
|
78
|
+
training_fraction=0.6,
|
|
79
|
+
validation_fraction=0.2,
|
|
80
|
+
seed=7,
|
|
81
|
+
)
|
|
82
|
+
samples = ppm.generate_partitioned(
|
|
83
|
+
partition,
|
|
84
|
+
truth=ppm.SparseTruth(features=("protein_3", "protein_17", "protein_42")),
|
|
85
|
+
outcome=ppm.BinaryOutcome(probability=0.15, odds_ratio=2.0),
|
|
86
|
+
sample_sizes=ppm.PartitionSampleSizes(1_200, 400, 400),
|
|
87
|
+
sampling_method="without_replacement",
|
|
88
|
+
seed=42,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
train, test = samples.training, samples.evaluation
|
|
92
|
+
model = LogisticRegression(max_iter=1_000).fit(train.X, train.outcome.values)
|
|
93
|
+
ranking = tuple(train.feature_ids[index] for index in np.argsort(-np.abs(model.coef_[0])))
|
|
94
|
+
|
|
95
|
+
recovery = ppm.evaluate_ranking(ranking, train.truth, depths=(10, 20, 50))
|
|
96
|
+
prediction = ppm.evaluate_prediction(
|
|
97
|
+
test.outcome.values,
|
|
98
|
+
model.predict_proba(test.X)[:, 1],
|
|
99
|
+
metric="roc_auc",
|
|
100
|
+
)
|
|
101
|
+
print(recovery.at_depth[0].recall)
|
|
102
|
+
print(prediction.value)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Recall measures how many of the three generating features appear in the model's top ten;
|
|
106
|
+
AUC measures prediction on held-out participants. Repeat the comparison with different seeds
|
|
107
|
+
to estimate average performance and its Monte Carlo uncertainty.
|
|
108
|
+
|
|
109
|
+
Source participants are split before resampling, keeping training, validation and test
|
|
110
|
+
rows separate. This example uses a fixed model; the validation sample is available for
|
|
111
|
+
hyperparameter tuning. See [Methods](docs/methods.md) for the generation process.
|
|
112
|
+
|
|
113
|
+
## What can be varied
|
|
114
|
+
|
|
115
|
+
- **Signal:** null, sparse features, correlated groups, distributed modules, pairwise
|
|
116
|
+
interactions or a custom function.
|
|
117
|
+
- **Outcome:** binary probability, continuous mean and spread, count rate, ordinal category
|
|
118
|
+
probabilities or cumulative event risks at native-time horizons.
|
|
119
|
+
- **Evaluation:** feature and group recovery, weighted-module and graph-region coverage,
|
|
120
|
+
signal reconstruction, prediction accuracy, panel performance and ranking stability.
|
|
121
|
+
Additional functions handle tied scores, incomplete selections and matched chance references.
|
|
122
|
+
|
|
123
|
+
Start with [Methods](docs/methods.md), then choose a [signal](docs/truths.md) and
|
|
124
|
+
[outcome](docs/outcomes.md). The [evaluation guide](docs/evaluation.md) defines each result.
|
|
125
|
+
The [validation example](docs/validation.md) checks the generators and recovery metrics against
|
|
126
|
+
known values and reference estimators.
|
|
127
|
+
|
|
128
|
+
The executable [quickstart](https://github.com/BradSegal/PyPlasmode/blob/main/examples/quickstart.py),
|
|
129
|
+
[survival example](https://github.com/BradSegal/PyPlasmode/blob/main/examples/survival.py)
|
|
130
|
+
and [custom signal](https://github.com/BradSegal/PyPlasmode/blob/main/examples/custom_signal.py)
|
|
131
|
+
use synthetic inputs. The [documentation](https://github.com/BradSegal/PyPlasmode/tree/main/docs)
|
|
132
|
+
includes the generation methods, evaluation definitions and API reference.
|
|
133
|
+
|
|
134
|
+
## Working with empirical data
|
|
135
|
+
|
|
136
|
+
Resampling preserves the source data's feature relationships and missingness patterns.
|
|
137
|
+
It can reproduce individual records, so generated datasets require the same privacy protections
|
|
138
|
+
as their source. See [Data handling](docs/privacy-and-limitations.md).
|
|
139
|
+
|
|
140
|
+
## Licence
|
|
141
|
+
|
|
142
|
+
PyPlasmode uses the [BSD 3-Clause License](LICENSE).
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
pyplasmode/__init__.py,sha256=rja5bwpPzlCbiy4LFqxW20Ae80GbhscZeDqVm9BQFSM,4050
|
|
2
|
+
pyplasmode/_validation.py,sha256=Cpz2cPvIbNE4huu8SWbDPyhnniAlxV_D5NCqqK8KTRI,1239
|
|
3
|
+
pyplasmode/data.py,sha256=t8lGLhjKTp_R52eTDE8KG_NTfyiXUBZXpmSXN4Bnw44,9053
|
|
4
|
+
pyplasmode/evaluate.py,sha256=r1V-3EOg6SGLQSFTfa0n5qveFoXobUAYAIVnOPCH49Q,15504
|
|
5
|
+
pyplasmode/generate.py,sha256=S7qMj_uq06b1ah9Ctu3ad-YA6XjmGPR4m2XgNIwt9gw,12166
|
|
6
|
+
pyplasmode/nominations.py,sha256=P0kZ3ihp80Ja6fvDHqqki-tilbYgbsCE83dKDkfmZQA,23338
|
|
7
|
+
pyplasmode/outcomes.py,sha256=6Rv594rEnzLFgsFQwYysd4C_lQ1Sk53yWMdm6tN-i68,27890
|
|
8
|
+
pyplasmode/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
pyplasmode/results.py,sha256=yBh41GeFKKcb5Fa6Gxo-0Pi8X4OpvZg4KhcHDpK70yM,4288
|
|
10
|
+
pyplasmode/structures.py,sha256=skBF1lOx-wl9fkMiAYleEFHyOZGliYY_Fp3rx7rQ66s,4545
|
|
11
|
+
pyplasmode/truth.py,sha256=GVCbYjxdGjJouv6VRjhjCCTsQXB_mBN04DkmDEhAuJs,21237
|
|
12
|
+
pyplasmode-1.0.0.dist-info/METADATA,sha256=IU5rU9SFmqo4N0j960Jnf6cRbsxdFMtxrRZ_kiFAsX4,6119
|
|
13
|
+
pyplasmode-1.0.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
14
|
+
pyplasmode-1.0.0.dist-info/licenses/LICENSE,sha256=e2PYeltfQUlLtcYi9YJ91fEgDiEhCEp0BKvf-swZzq8,1531
|
|
15
|
+
pyplasmode-1.0.0.dist-info/RECORD,,
|