shellde 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shellde/__init__.py +36 -0
- shellde/acquisition.py +135 -0
- shellde/advisor.py +158 -0
- shellde/bench/__init__.py +17 -0
- shellde/bench/falsification.py +95 -0
- shellde/bench/harness.py +134 -0
- shellde/bench/stats.py +46 -0
- shellde/campaign.py +257 -0
- shellde/candidates.py +341 -0
- shellde/cli.py +1517 -0
- shellde/colab.py +257 -0
- shellde/conformal.py +160 -0
- shellde/consensus.py +52 -0
- shellde/design_space.py +141 -0
- shellde/embeddings/__init__.py +6 -0
- shellde/embeddings/esm2.py +76 -0
- shellde/embeddings/esmc.py +94 -0
- shellde/embeddings/provider.py +105 -0
- shellde/features/__init__.py +22 -0
- shellde/features/base.py +49 -0
- shellde/features/defaults.py +11 -0
- shellde/features/embedding.py +80 -0
- shellde/features/inverse_folding.py +66 -0
- shellde/features/matrix.py +50 -0
- shellde/features/naturalness.py +65 -0
- shellde/features/onehot.py +55 -0
- shellde/features/pairwise.py +64 -0
- shellde/funclib.py +331 -0
- shellde/gating.py +181 -0
- shellde/holo.py +175 -0
- shellde/hotspots.py +108 -0
- shellde/loop.py +161 -0
- shellde/msa.py +186 -0
- shellde/naturalness.py +142 -0
- shellde/oracle.py +94 -0
- shellde/plm.py +145 -0
- shellde/prereg.py +41 -0
- shellde/protocols.py +87 -0
- shellde/rank.py +103 -0
- shellde/report.py +132 -0
- shellde/selector.py +63 -0
- shellde/sitefinder.py +465 -0
- shellde/structure.py +356 -0
- shellde/surrogate.py +323 -0
- shellde/types.py +66 -0
- shellde/zero_shot.py +160 -0
- shellde-0.2.0.dist-info/METADATA +285 -0
- shellde-0.2.0.dist-info/RECORD +51 -0
- shellde-0.2.0.dist-info/WHEEL +5 -0
- shellde-0.2.0.dist-info/entry_points.txt +2 -0
- shellde-0.2.0.dist-info/top_level.txt +1 -0
shellde/holo.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Find a ligand-bound (holo) structural homolog when the target is apo.
|
|
2
|
+
|
|
3
|
+
ShellDE's resolve-site needs a ligand pose to define the active-site shell; it abstains on apo
|
|
4
|
+
structures. This module fills that gap: submit a query structure to the public Foldseek search server
|
|
5
|
+
(same ticket-based API family as the ColabFold MMseqs2 MSA API already used here), take the PDB hits,
|
|
6
|
+
and keep the ones that actually have a bound ligand (RCSB check). It returns candidate holo PDB ids for
|
|
7
|
+
the user to feed to ``resolve-site --pdb <id>`` (the existing shell machinery aligns it onto WT).
|
|
8
|
+
|
|
9
|
+
Network + polling; every failure returns an empty list so callers ABSTAIN, never fabricate. Advisory
|
|
10
|
+
only: it proposes candidates, it does not auto-superpose. stdlib urllib (no requests dependency).
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import io
|
|
16
|
+
import re
|
|
17
|
+
import tarfile
|
|
18
|
+
import time
|
|
19
|
+
import urllib.request
|
|
20
|
+
import uuid
|
|
21
|
+
|
|
22
|
+
from shellde.structure import _NON_LIGAND
|
|
23
|
+
|
|
24
|
+
_UA = {"User-Agent": "shellde (https://pypi.org/project/shellde/)"}
|
|
25
|
+
_PDB_ID = re.compile(r"(?<![A-Za-z0-9])([1-9][A-Za-z0-9]{3})(?![A-Za-z0-9])")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _encode_multipart(fields: list[tuple[str, str]], filename: str, filedata: bytes) -> tuple[bytes, str]:
|
|
29
|
+
"""Build a multipart/form-data body (repeated keys allowed, e.g. ``database[]``) for the ``q`` file."""
|
|
30
|
+
boundary = "----shelldeFoldseek" + uuid.uuid4().hex
|
|
31
|
+
crlf = "\r\n"
|
|
32
|
+
parts = []
|
|
33
|
+
for key, val in fields:
|
|
34
|
+
parts.append(
|
|
35
|
+
f'--{boundary}{crlf}Content-Disposition: form-data; name="{key}"{crlf}{crlf}{val}{crlf}'
|
|
36
|
+
)
|
|
37
|
+
parts.append(
|
|
38
|
+
f'--{boundary}{crlf}Content-Disposition: form-data; name="q"; filename="{filename}"{crlf}'
|
|
39
|
+
f"Content-Type: application/octet-stream{crlf}{crlf}"
|
|
40
|
+
)
|
|
41
|
+
body = "".join(parts).encode() + filedata + f"{crlf}--{boundary}--{crlf}".encode()
|
|
42
|
+
return body, f"multipart/form-data; boundary={boundary}"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def foldseek_search(
|
|
46
|
+
pdb_text: str,
|
|
47
|
+
*,
|
|
48
|
+
host: str = "https://search.foldseek.com/api",
|
|
49
|
+
databases: tuple[str, ...] = ("pdb100",),
|
|
50
|
+
mode: str = "3diaa",
|
|
51
|
+
timeout: float = 30.0,
|
|
52
|
+
max_wait: float = 600.0,
|
|
53
|
+
poll: float = 5.0,
|
|
54
|
+
max_evalue: float = 1e-3,
|
|
55
|
+
min_coverage: float = 0.5,
|
|
56
|
+
) -> list[dict]:
|
|
57
|
+
"""Foldseek-search a query structure, gate on structural significance, return ranked hits.
|
|
58
|
+
|
|
59
|
+
Submits the structure (POST /ticket), polls, downloads the .m8 result, and keeps only hits that are
|
|
60
|
+
a SIGNIFICANT, well-covering structural match -- E-value <= ``max_evalue`` AND alignment coverage
|
|
61
|
+
(alnlen/qlen) >= ``min_coverage`` -- so a distant/partial homolog whose ligand pocket may not
|
|
62
|
+
correspond to the target is NOT returned. Each hit is ``{"pdb_id","evalue","coverage","bits"}``,
|
|
63
|
+
ranked by bits (best first). [] on any failure or if nothing passes the gate.
|
|
64
|
+
"""
|
|
65
|
+
data = pdb_text.encode() if isinstance(pdb_text, str) else pdb_text
|
|
66
|
+
if not data:
|
|
67
|
+
return []
|
|
68
|
+
fields = [("mode", mode)] + [("database[]", db) for db in databases]
|
|
69
|
+
body, ctype = _encode_multipart(fields, "query.pdb", data)
|
|
70
|
+
|
|
71
|
+
def _get_json(url: str, payload: bytes | None = None, ctype_hdr: str | None = None) -> dict | None:
|
|
72
|
+
try:
|
|
73
|
+
headers = dict(_UA)
|
|
74
|
+
if ctype_hdr:
|
|
75
|
+
headers["Content-Type"] = ctype_hdr
|
|
76
|
+
req = urllib.request.Request(url, data=payload, headers=headers)
|
|
77
|
+
with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310 (trusted host)
|
|
78
|
+
return json.loads(r.read().decode())
|
|
79
|
+
except Exception: # noqa: BLE001 (network/HTTP/JSON -> abstain)
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
job = _get_json(f"{host}/ticket", body, ctype)
|
|
83
|
+
if not job or "id" not in job or job.get("status") in ("ERROR", "MAINTENANCE", None):
|
|
84
|
+
return []
|
|
85
|
+
jid, st, waited = job["id"], job.get("status"), 0.0
|
|
86
|
+
while st in ("PENDING", "RUNNING", "UNKNOWN", "RATELIMIT"):
|
|
87
|
+
time.sleep(poll)
|
|
88
|
+
waited += poll
|
|
89
|
+
if waited > max_wait:
|
|
90
|
+
return []
|
|
91
|
+
s = _get_json(f"{host}/ticket/{jid}")
|
|
92
|
+
st = s.get("status") if s else None
|
|
93
|
+
if st != "COMPLETE":
|
|
94
|
+
return []
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
req = urllib.request.Request(f"{host}/result/download/{jid}", headers=_UA)
|
|
98
|
+
with urllib.request.urlopen(req, timeout=max(timeout, 120.0)) as r: # noqa: S310
|
|
99
|
+
blob = r.read()
|
|
100
|
+
except Exception: # noqa: BLE001
|
|
101
|
+
return []
|
|
102
|
+
|
|
103
|
+
rows: dict[str, dict] = {}
|
|
104
|
+
try:
|
|
105
|
+
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
|
|
106
|
+
for m in tar.getmembers():
|
|
107
|
+
if not m.name.endswith(".m8"):
|
|
108
|
+
continue
|
|
109
|
+
fh = tar.extractfile(m)
|
|
110
|
+
if fh is None:
|
|
111
|
+
continue
|
|
112
|
+
for line in fh.read().decode("utf-8", "ignore").splitlines():
|
|
113
|
+
cols = line.split("\t")
|
|
114
|
+
if len(cols) < 14: # need through qlen (col 13)
|
|
115
|
+
continue
|
|
116
|
+
hit = _PDB_ID.search(cols[1])
|
|
117
|
+
if not hit:
|
|
118
|
+
continue
|
|
119
|
+
try:
|
|
120
|
+
alnlen = int(cols[3])
|
|
121
|
+
evalue = float(cols[11])
|
|
122
|
+
bits = float(cols[12])
|
|
123
|
+
qlen = int(cols[13])
|
|
124
|
+
except (ValueError, IndexError):
|
|
125
|
+
continue
|
|
126
|
+
cov = alnlen / qlen if qlen else 0.0
|
|
127
|
+
if evalue > max_evalue or cov < min_coverage: # structural-significance gate
|
|
128
|
+
continue
|
|
129
|
+
pid = hit.group(1).upper()
|
|
130
|
+
rec = {"pdb_id": pid, "evalue": evalue, "coverage": round(cov, 3), "bits": bits}
|
|
131
|
+
if pid not in rows or bits > rows[pid]["bits"]:
|
|
132
|
+
rows[pid] = rec
|
|
133
|
+
except Exception: # noqa: BLE001
|
|
134
|
+
return []
|
|
135
|
+
return sorted(rows.values(), key=lambda r: -r["bits"])
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def rcsb_ligands(pdb_id: str, *, timeout: float = 15.0) -> list[str]:
|
|
139
|
+
"""Bound non-polymer ligand comp-ids for a PDB entry, minus crystallization additives. [] on failure."""
|
|
140
|
+
try:
|
|
141
|
+
req = urllib.request.Request(
|
|
142
|
+
f"https://data.rcsb.org/rest/v1/core/entry/{pdb_id.upper()}", headers=_UA
|
|
143
|
+
)
|
|
144
|
+
with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310
|
|
145
|
+
info = json.loads(r.read().decode())
|
|
146
|
+
except Exception: # noqa: BLE001
|
|
147
|
+
return []
|
|
148
|
+
comps = info.get("rcsb_entry_info", {}).get("nonpolymer_bound_components") or []
|
|
149
|
+
return [c for c in comps if c.upper() not in _NON_LIGAND]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def find_holo_homolog(
|
|
153
|
+
pdb_text: str,
|
|
154
|
+
*,
|
|
155
|
+
top_n: int = 10,
|
|
156
|
+
max_candidates: int = 3,
|
|
157
|
+
timeout: float = 30.0,
|
|
158
|
+
max_wait: float = 600.0,
|
|
159
|
+
) -> list[dict]:
|
|
160
|
+
"""Advisory: rank ligand-bound (holo) PDB homologs of a query structure.
|
|
161
|
+
|
|
162
|
+
Runs :func:`foldseek_search` (pdb100), then keeps the top hits that RCSB reports as having a bound
|
|
163
|
+
(non-additive) ligand. Returns up to ``max_candidates`` dicts ``{"pdb_id", "ligands", "rank"}`` for
|
|
164
|
+
the user to pass to ``resolve-site --pdb``. Empty on any failure (ABSTAIN).
|
|
165
|
+
"""
|
|
166
|
+
hits = foldseek_search(pdb_text, timeout=timeout, max_wait=max_wait)
|
|
167
|
+
out: list[dict] = []
|
|
168
|
+
for rank, h in enumerate(hits[:top_n], 1):
|
|
169
|
+
ligs = rcsb_ligands(h["pdb_id"], timeout=min(timeout, 15.0))
|
|
170
|
+
if ligs:
|
|
171
|
+
out.append({"pdb_id": h["pdb_id"], "ligands": ligs, "evalue": h["evalue"],
|
|
172
|
+
"coverage": h["coverage"], "rank": rank})
|
|
173
|
+
if len(out) >= max_candidates:
|
|
174
|
+
break
|
|
175
|
+
return out
|
shellde/hotspots.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Bridge (niche C): a whole-protein single-mutant SCAN -> hotspot positions for combinatorial AL.
|
|
2
|
+
|
|
3
|
+
The scan->focus handoff that neither EVOLVEpro (whole-protein singles + additive stacking) nor
|
|
4
|
+
FuncLib/ALDE (you must already know the positions) provides:
|
|
5
|
+
|
|
6
|
+
1. SCAN -- a whole-protein single-mutant table (measured or PLM-predicted, EVOLVEpro-style,
|
|
7
|
+
mutation-notation like ``V100F``) ranks how impactful each POSITION is.
|
|
8
|
+
2. BRIDGE -- pick the top-k hotspot positions here.
|
|
9
|
+
3. FOCUS -- feed those positions to ``campaign``/``round`` (combinatorial AL: the candidate pool is
|
|
10
|
+
multi-mutant, the fitted model is additive by default), which MEASURES real
|
|
11
|
+
combinations instead of assuming additive stacking.
|
|
12
|
+
|
|
13
|
+
This module is the BRIDGE. It is pure/offline (parsing + per-position aggregation); the scan itself
|
|
14
|
+
(producing the single-mutant scores) and the combinatorial phase live elsewhere.
|
|
15
|
+
|
|
16
|
+
Honest scope: ranking positions from single-mutant scores is a heuristic to CHOOSE where to look; it
|
|
17
|
+
does not predict the combinatorial winner (that needs the measured combinatorial AL). If the scores
|
|
18
|
+
are predictions (e.g. EVOLVEpro y_pred), the choice inherits that model's ceiling.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import re
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
|
|
25
|
+
_MUT = re.compile(r"^([A-Za-z])?(\d+)([A-Za-z*])$")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Hotspot:
|
|
30
|
+
"""One design position summarised from the single-mutant scan."""
|
|
31
|
+
|
|
32
|
+
position: int
|
|
33
|
+
wt: str
|
|
34
|
+
best_effect: float # score of the best single substitution at this position
|
|
35
|
+
best_sub: str # e.g. "V100F"
|
|
36
|
+
n_beneficial: int # substitutions scoring above the baseline
|
|
37
|
+
subs: dict[str, float] = field(default_factory=dict) # aa -> score
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def parse_mutation(token: str) -> tuple[str, int, str] | None:
|
|
41
|
+
"""``'V100F'`` -> ``('V', 100, 'F')``; ``'100F'``/``'100 F'`` -> ``('', 100, 'F')``.
|
|
42
|
+
|
|
43
|
+
The wild-type letter is optional so position+AA scan exports (e.g. ``29I``, common when the WT is
|
|
44
|
+
implied) parse too; ranking only needs the position and the substituted residue. None if the token
|
|
45
|
+
is not a single substitution.
|
|
46
|
+
"""
|
|
47
|
+
m = _MUT.match(token.strip())
|
|
48
|
+
if not m:
|
|
49
|
+
return None
|
|
50
|
+
return (m.group(1) or "").upper(), int(m.group(2)), m.group(3).upper()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def rank_hotspots(
|
|
54
|
+
singles,
|
|
55
|
+
*,
|
|
56
|
+
baseline: float = 0.0,
|
|
57
|
+
top_k: int = 6,
|
|
58
|
+
metric: str = "max",
|
|
59
|
+
min_effect: float | None = None,
|
|
60
|
+
) -> list[Hotspot]:
|
|
61
|
+
"""Aggregate single-mutant scores per position and return the top-``top_k`` hotspots.
|
|
62
|
+
|
|
63
|
+
``singles``: iterable of ``(mutation_token, score)`` where higher score == more desirable
|
|
64
|
+
(activity / predicted benefit). ``metric`` ranks positions by ``'max'`` (best single here, the
|
|
65
|
+
default), ``'mean'`` (average over its substitutions), or ``'count'`` (number scoring above
|
|
66
|
+
``baseline``). ``min_effect`` drops positions whose best single is below it. Pure.
|
|
67
|
+
"""
|
|
68
|
+
by_pos: dict[int, dict] = {}
|
|
69
|
+
for token, score in singles:
|
|
70
|
+
parsed = parse_mutation(token)
|
|
71
|
+
if parsed is None:
|
|
72
|
+
continue
|
|
73
|
+
wt, pos, mut = parsed
|
|
74
|
+
try:
|
|
75
|
+
val = float(score)
|
|
76
|
+
except (TypeError, ValueError):
|
|
77
|
+
continue
|
|
78
|
+
slot = by_pos.setdefault(pos, {"wt": wt, "subs": {}})
|
|
79
|
+
slot["subs"][mut] = val
|
|
80
|
+
|
|
81
|
+
hotspots: list[Hotspot] = []
|
|
82
|
+
for pos, slot in by_pos.items():
|
|
83
|
+
subs = slot["subs"]
|
|
84
|
+
if not subs:
|
|
85
|
+
continue
|
|
86
|
+
best_aa = max(subs, key=subs.get)
|
|
87
|
+
hotspots.append(
|
|
88
|
+
Hotspot(
|
|
89
|
+
position=pos,
|
|
90
|
+
wt=slot["wt"],
|
|
91
|
+
best_effect=subs[best_aa],
|
|
92
|
+
best_sub=f"{slot['wt']}{pos}{best_aa}",
|
|
93
|
+
n_beneficial=sum(1 for v in subs.values() if v > baseline),
|
|
94
|
+
subs=dict(subs),
|
|
95
|
+
)
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
key = {
|
|
99
|
+
"max": lambda h: h.best_effect,
|
|
100
|
+
"mean": lambda h: sum(h.subs.values()) / len(h.subs),
|
|
101
|
+
"count": lambda h: h.n_beneficial,
|
|
102
|
+
}
|
|
103
|
+
if metric not in key:
|
|
104
|
+
raise ValueError(f"metric must be one of {sorted(key)}, got {metric!r}")
|
|
105
|
+
hotspots.sort(key=key[metric], reverse=True)
|
|
106
|
+
if min_effect is not None:
|
|
107
|
+
hotspots = [h for h in hotspots if h.best_effect >= min_effect]
|
|
108
|
+
return hotspots[:top_k]
|
shellde/loop.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""The active-learning / experimental-design loop: fit -> acquire -> measure -> repeat.
|
|
2
|
+
|
|
3
|
+
This is the organizing axis of the tool. Each round re-fits a fresh surrogate on ALL
|
|
4
|
+
accumulated measurements, predicts (calibrated mean+std) over the unmeasured pool,
|
|
5
|
+
the pluggable acquisition selects the next batch, the oracle measures it, and the
|
|
6
|
+
data accumulates. The candidate pool is the oracle's enumerable universe, so a
|
|
7
|
+
campaign reports best-found over genuinely measurable variants and identical budget
|
|
8
|
+
across methods is the harness's responsibility.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
from shellde.design_space import DesignSpace
|
|
18
|
+
from shellde.features.matrix import FeatureMatrix
|
|
19
|
+
from shellde.protocols import Acquisition, Oracle, Surrogate
|
|
20
|
+
from shellde.types import CampaignResult, RoundLog
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _spearman(a: np.ndarray, b: np.ndarray) -> float:
|
|
24
|
+
"""Spearman rank correlation via numpy (avoids a scipy typing dependency here)."""
|
|
25
|
+
ra = np.argsort(np.argsort(a)).astype(float)
|
|
26
|
+
rb = np.argsort(np.argsort(b)).astype(float)
|
|
27
|
+
if np.std(ra) < 1e-12 or np.std(rb) < 1e-12:
|
|
28
|
+
return 0.0
|
|
29
|
+
return float(np.corrcoef(ra, rb)[0, 1])
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def run_campaign(
|
|
33
|
+
oracle: Oracle,
|
|
34
|
+
space: DesignSpace,
|
|
35
|
+
*,
|
|
36
|
+
make_matrix: Callable[[], FeatureMatrix],
|
|
37
|
+
regate: Callable[[list[str], np.ndarray], FeatureMatrix] | None = None,
|
|
38
|
+
make_surrogate: Callable[[], Surrogate],
|
|
39
|
+
acquisition: Acquisition,
|
|
40
|
+
n_init: int = 95,
|
|
41
|
+
batch_size: int = 95,
|
|
42
|
+
n_rounds: int = 3,
|
|
43
|
+
seed: int = 0,
|
|
44
|
+
init_variants: Sequence[str] | None = None,
|
|
45
|
+
anchors: Mapping[str, float] | None = None,
|
|
46
|
+
) -> CampaignResult:
|
|
47
|
+
"""Run an active-learning campaign against an oracle with an enumerable universe."""
|
|
48
|
+
universe = oracle.universe
|
|
49
|
+
if universe is None:
|
|
50
|
+
raise ValueError("run_campaign requires an oracle exposing an enumerable universe")
|
|
51
|
+
rng = np.random.default_rng(seed)
|
|
52
|
+
if init_variants is not None:
|
|
53
|
+
# Explicit seed plate (e.g. a funclib R0 library): use it as the initial measured set
|
|
54
|
+
# instead of a random draw. The AL pool below is still the FULL universe, so the loop
|
|
55
|
+
# explores beyond the seed (never hard-restricted to it). Variants outside the universe
|
|
56
|
+
# are dropped; if the seed is short, top up with a random draw to keep n_init honest.
|
|
57
|
+
uni_set = set(universe)
|
|
58
|
+
init = list(dict.fromkeys(v for v in init_variants if v in uni_set))[:n_init]
|
|
59
|
+
if len(init) < n_init:
|
|
60
|
+
extra = [universe[int(i)] for i in rng.permutation(len(universe))
|
|
61
|
+
if universe[int(i)] not in set(init)]
|
|
62
|
+
init.extend(extra[: n_init - len(init)])
|
|
63
|
+
else:
|
|
64
|
+
init_idx = rng.choice(len(universe), min(n_init, len(universe)), replace=False)
|
|
65
|
+
init = [universe[int(i)] for i in init_idx]
|
|
66
|
+
measured: dict[str, float] = dict(zip(init, (float(x) for x in oracle.evaluate(init)), strict=True))
|
|
67
|
+
|
|
68
|
+
static_matrix = make_matrix() if regate is None else None # fixed blocks unless re-gated
|
|
69
|
+
# Encode-once: with fixed blocks (no per-round regate) the design matrix never changes, so
|
|
70
|
+
# encode the whole universe a SINGLE time and index rows by a measured-mask instead of
|
|
71
|
+
# re-encoding the ~|universe| pool every round. Byte-identical: encode is row-independent, so
|
|
72
|
+
# universe_enc[i] == static_matrix.encode([universe[i]])[0], and the masked pool keeps universe
|
|
73
|
+
# order (== the old `[v for v in universe if v not in measured]`).
|
|
74
|
+
universe_enc = static_matrix.encode(universe) if static_matrix is not None else None
|
|
75
|
+
idx_of = {v: i for i, v in enumerate(universe)} if static_matrix is not None else {}
|
|
76
|
+
measured_mask = np.zeros(len(universe), dtype=bool)
|
|
77
|
+
if static_matrix is not None:
|
|
78
|
+
for v in measured:
|
|
79
|
+
measured_mask[idx_of[v]] = True
|
|
80
|
+
# Anchors are FIT-only free rows; the invariant is they never become pool candidates
|
|
81
|
+
# (so they cannot be acquired into measured/best/trajectory). This mask is a no-op when
|
|
82
|
+
# `anchors` is None -- the pool-construction path below is then byte-identical.
|
|
83
|
+
anchor_keys = set(anchors) if anchors else set()
|
|
84
|
+
anchor_mask = np.zeros(len(universe), dtype=bool)
|
|
85
|
+
if anchors and static_matrix is not None:
|
|
86
|
+
for v in anchor_keys:
|
|
87
|
+
if v in idx_of:
|
|
88
|
+
anchor_mask[idx_of[v]] = True
|
|
89
|
+
trajectory = [max(measured.values())]
|
|
90
|
+
rounds: list[RoundLog] = []
|
|
91
|
+
heldout_spearman: list[float] = []
|
|
92
|
+
|
|
93
|
+
for r in range(n_rounds):
|
|
94
|
+
vs = list(measured)
|
|
95
|
+
if static_matrix is not None:
|
|
96
|
+
pool_mask = ~measured_mask
|
|
97
|
+
if anchors:
|
|
98
|
+
pool_mask = pool_mask & ~anchor_mask
|
|
99
|
+
pool_idx = np.where(pool_mask)[0]
|
|
100
|
+
pool = [universe[int(i)] for i in pool_idx]
|
|
101
|
+
else:
|
|
102
|
+
pool_idx = None
|
|
103
|
+
if anchors:
|
|
104
|
+
pool = [v for v in universe if v not in measured and v not in anchor_keys]
|
|
105
|
+
else:
|
|
106
|
+
pool = [v for v in universe if v not in measured]
|
|
107
|
+
if len(vs) < 2 or not pool:
|
|
108
|
+
break
|
|
109
|
+
ys = np.asarray([measured[v] for v in vs], dtype=float)
|
|
110
|
+
if static_matrix is not None:
|
|
111
|
+
matrix = static_matrix
|
|
112
|
+
x_vs = universe_enc[[idx_of[v] for v in vs]]
|
|
113
|
+
x_pool = universe_enc[pool_idx]
|
|
114
|
+
else:
|
|
115
|
+
matrix = regate(vs, ys)
|
|
116
|
+
assert matrix is not None
|
|
117
|
+
x_vs = matrix.encode(vs)
|
|
118
|
+
x_pool = matrix.encode(pool)
|
|
119
|
+
# Anchor: extra free labeled rows into the FIT ONLY (kept out of `measured`,
|
|
120
|
+
# the pool mask, and best-found). Budget parity: anchors add on top of the draw.
|
|
121
|
+
if anchors:
|
|
122
|
+
extra = [(v, fv) for v, fv in anchors.items() if v not in measured]
|
|
123
|
+
if extra:
|
|
124
|
+
ev = [v for v, _ in extra]
|
|
125
|
+
x_extra = static_matrix.encode(ev) if static_matrix is not None else matrix.encode(ev)
|
|
126
|
+
x_vs = np.vstack([x_vs, x_extra])
|
|
127
|
+
ys = np.concatenate([ys, np.asarray([fv for _, fv in extra], dtype=float)])
|
|
128
|
+
sur = make_surrogate().fit(x_vs, ys)
|
|
129
|
+
pred = sur.predict(x_pool)
|
|
130
|
+
k = min(batch_size, len(pool))
|
|
131
|
+
sel = acquisition.select(pred, k, candidates=pool, space=space)
|
|
132
|
+
batch = [pool[i] for i in sel]
|
|
133
|
+
pred_batch = np.asarray(pred.mean, dtype=float)[sel]
|
|
134
|
+
batch_y = np.asarray(oracle.evaluate(batch), dtype=float)
|
|
135
|
+
for v, fv in zip(batch, batch_y, strict=True):
|
|
136
|
+
measured[v] = float(fv)
|
|
137
|
+
if static_matrix is not None:
|
|
138
|
+
measured_mask[idx_of[v]] = True
|
|
139
|
+
ho_rmse = float(np.sqrt(np.mean((pred_batch - batch_y) ** 2))) if batch_y.size else None
|
|
140
|
+
ho_rho: float | None = None
|
|
141
|
+
if batch_y.size >= 3 and float(np.std(batch_y)) > 1e-12 and float(np.std(pred_batch)) > 1e-12:
|
|
142
|
+
ho_rho = _spearman(pred_batch, batch_y)
|
|
143
|
+
heldout_spearman.append(ho_rho)
|
|
144
|
+
best = max(measured.values())
|
|
145
|
+
trajectory.append(best)
|
|
146
|
+
rounds.append(RoundLog(
|
|
147
|
+
round_index=r, n_measured=len(measured), best_fitness=best, batch=batch,
|
|
148
|
+
heldout_rmse=ho_rmse, heldout_spearman=ho_rho,
|
|
149
|
+
notes={"active_blocks": matrix.active_blocks()},
|
|
150
|
+
))
|
|
151
|
+
|
|
152
|
+
best_v = max(measured, key=lambda kk: measured[kk])
|
|
153
|
+
return CampaignResult(
|
|
154
|
+
trajectory=trajectory,
|
|
155
|
+
measured=measured,
|
|
156
|
+
best_variant=best_v,
|
|
157
|
+
best_fitness=measured[best_v],
|
|
158
|
+
n_rounds=len(rounds),
|
|
159
|
+
rounds=rounds,
|
|
160
|
+
heldout_spearman=heldout_spearman,
|
|
161
|
+
)
|
shellde/msa.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""MSA-derived evolutionary tolerance: Henikoff-weighted PSSM log-probabilities.
|
|
2
|
+
|
|
3
|
+
A per-position amino-acid log-probability table built from a multiple sequence alignment (a3m),
|
|
4
|
+
usable as a funclib tolerance signal exactly like an inverse-folding or ddG table. This is the
|
|
5
|
+
classic evolutionary-conservation signal (the "independent" MSA model that funclib's design names
|
|
6
|
+
as a canonical tolerance). On a small 4-protein gate test it matched/beat ESM-2, BUT on the
|
|
7
|
+
217-DMS ProteinGym benchmark MSA and ESM are statistically INDISTINGUISHABLE (per-DMS paired, ns
|
|
8
|
+
on both Spearman and top-recall). So this is a GPU-free MSA ALTERNATIVE to PLM tolerance, NOT a
|
|
9
|
+
proven improvement; the 4-protein advantage (esp. GB1's shallow-MSA anti-enrichment) did not
|
|
10
|
+
generalize. See ``scripts/proteingym_revalidate.py`` and decision log D11/D13.
|
|
11
|
+
|
|
12
|
+
Honest scope: this is the independent (per-site) conservation model, not the epistasis-aware
|
|
13
|
+
GEMME / EVmutation (Potts) signal. Same recall ceiling as all tolerance signals (it predicts
|
|
14
|
+
"tolerated/functional", not "improved").
|
|
15
|
+
|
|
16
|
+
Dependency-light: numpy + stdlib. Produce the a3m once with any MSA tool (jackhmmer, hhblits, or
|
|
17
|
+
the ColabFold MMseqs2 API); this module turns it into the log-prob table.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import io
|
|
22
|
+
import json
|
|
23
|
+
import tarfile
|
|
24
|
+
import time
|
|
25
|
+
import urllib.parse
|
|
26
|
+
import urllib.request
|
|
27
|
+
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
import numpy as np
|
|
31
|
+
|
|
32
|
+
AA20 = "ACDEFGHIKLMNPQRSTVWY"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def read_a3m(path: str | Path) -> list[str]:
|
|
36
|
+
"""Parse an a3m into match-state-only aligned rows (all of equal length = query length).
|
|
37
|
+
|
|
38
|
+
a3m insertion columns are lowercase; dropping every lowercase character collapses each record
|
|
39
|
+
to the query's match columns, so all returned rows share the query length. The first record is
|
|
40
|
+
the query.
|
|
41
|
+
"""
|
|
42
|
+
seqs: list[str] = []
|
|
43
|
+
cur: str | None = None
|
|
44
|
+
for line in Path(path).read_text().splitlines():
|
|
45
|
+
if line.startswith(">"):
|
|
46
|
+
if cur is not None:
|
|
47
|
+
seqs.append(cur)
|
|
48
|
+
cur = ""
|
|
49
|
+
elif cur is not None:
|
|
50
|
+
cur += line.strip()
|
|
51
|
+
if cur:
|
|
52
|
+
seqs.append(cur)
|
|
53
|
+
aligned = ["".join(c for c in s if not c.islower()) for s in seqs]
|
|
54
|
+
if not aligned:
|
|
55
|
+
raise ValueError(f"{path}: no sequences parsed from a3m")
|
|
56
|
+
length = len(aligned[0])
|
|
57
|
+
return [a for a in aligned if len(a) == length]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _henikoff_weights(matrix: np.ndarray) -> np.ndarray:
|
|
61
|
+
"""Henikoff position-based sequence weights over an (N, L) residue-char matrix.
|
|
62
|
+
|
|
63
|
+
Each column contributes 1/(r_j * count_{j,a}) to the weight of a sequence with residue a in
|
|
64
|
+
column j, where r_j is the number of distinct amino acids in column j (gaps ignored). Down-
|
|
65
|
+
weights redundant sequences without the O(N^2) cost of identity clustering.
|
|
66
|
+
"""
|
|
67
|
+
n, length = matrix.shape
|
|
68
|
+
aaset = set(AA20)
|
|
69
|
+
w = np.zeros(n, dtype=float)
|
|
70
|
+
for j in range(length):
|
|
71
|
+
col = matrix[:, j]
|
|
72
|
+
uniq, counts = np.unique(col, return_counts=True)
|
|
73
|
+
cmap = {u: c for u, c in zip(uniq, counts)}
|
|
74
|
+
r = sum(1 for u in uniq if u in aaset)
|
|
75
|
+
if r == 0:
|
|
76
|
+
continue
|
|
77
|
+
for i in range(n):
|
|
78
|
+
a = col[i]
|
|
79
|
+
if a in aaset:
|
|
80
|
+
w[i] += 1.0 / (r * cmap[a])
|
|
81
|
+
total = float(w.sum())
|
|
82
|
+
return w / total if total > 0 else np.full(n, 1.0 / n)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def load_msa_logprobs(
|
|
86
|
+
path: str | Path, *, pseudocount: float | None = None, reweight: bool = True
|
|
87
|
+
) -> dict[int, dict[str, float]]:
|
|
88
|
+
"""Henikoff-weighted PSSM as a {position: {aa: log_probability}} table from an a3m.
|
|
89
|
+
|
|
90
|
+
Positions are 1-based query (first-record) columns. Frequencies use a uniform pseudocount
|
|
91
|
+
(default ``1/20``) so unobserved residues get a finite log-prob. With ``reweight`` (default),
|
|
92
|
+
sequences are Henikoff-weighted to reduce alignment redundancy bias; set ``reweight=False`` for
|
|
93
|
+
raw counts. Feed the result to ``funclib.design_library`` / ``funclib.tolerance_from_logprob_table``
|
|
94
|
+
as a tolerance signal (top-K is invariant to the WT baseline, so it gates on conservation rank).
|
|
95
|
+
"""
|
|
96
|
+
rows = read_a3m(path)
|
|
97
|
+
matrix = np.array([list(r) for r in rows])
|
|
98
|
+
n, length = matrix.shape
|
|
99
|
+
pc = (1.0 / len(AA20)) if pseudocount is None else float(pseudocount)
|
|
100
|
+
w = _henikoff_weights(matrix) if reweight else np.full(n, 1.0 / max(n, 1))
|
|
101
|
+
aaidx = {a: i for i, a in enumerate(AA20)}
|
|
102
|
+
out: dict[int, dict[str, float]] = {}
|
|
103
|
+
for j in range(length):
|
|
104
|
+
f = np.zeros(len(AA20), dtype=float)
|
|
105
|
+
col = matrix[:, j]
|
|
106
|
+
for i in range(n):
|
|
107
|
+
a = col[i]
|
|
108
|
+
if a in aaidx:
|
|
109
|
+
f[aaidx[a]] += w[i]
|
|
110
|
+
f = (f + pc) / (f.sum() + pc * len(AA20))
|
|
111
|
+
lp = np.log(f)
|
|
112
|
+
out[j + 1] = {a: float(lp[aaidx[a]]) for a in AA20}
|
|
113
|
+
return out
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
_MMSEQS_UA = {"User-Agent": "shellde (https://pypi.org/project/shellde/)"}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def build_msa_mmseqs2(
|
|
120
|
+
seq: str,
|
|
121
|
+
*,
|
|
122
|
+
host: str = "https://api.colabfold.com",
|
|
123
|
+
timeout: float = 30.0,
|
|
124
|
+
max_wait: float = 600.0,
|
|
125
|
+
poll: float = 5.0,
|
|
126
|
+
) -> str | None:
|
|
127
|
+
"""Build an a3m MSA for a single sequence via the public ColabFold MMseqs2 API (no local DBs).
|
|
128
|
+
|
|
129
|
+
Submit -> poll -> download the result tar.gz, then concatenate its a3m members. Returns the a3m
|
|
130
|
+
text, or None on any error/timeout so callers ABSTAIN (never fabricate an alignment). This removes
|
|
131
|
+
the "R0 needs a tolerance signal" friction; it is a GPU-free evolutionary signal, NOT a proven
|
|
132
|
+
accuracy improvement (see module docstring). Network + polling. Fair-use: one job at a time
|
|
133
|
+
(https://colabfold.mmseqs.com).
|
|
134
|
+
"""
|
|
135
|
+
clean = "".join(seq.split()).upper()
|
|
136
|
+
if not clean:
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
def _call(url: str, data: bytes | None) -> dict | None:
|
|
140
|
+
try:
|
|
141
|
+
req = urllib.request.Request(url, data=data, headers=_MMSEQS_UA)
|
|
142
|
+
with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310 (trusted host)
|
|
143
|
+
return json.loads(r.read().decode())
|
|
144
|
+
except Exception: # noqa: BLE001 (network/HTTP/JSON -> abstain)
|
|
145
|
+
return None
|
|
146
|
+
|
|
147
|
+
body = urllib.parse.urlencode({"q": f">101\n{clean}\n", "mode": "env"}).encode()
|
|
148
|
+
job = _call(f"{host}/ticket/msa", body)
|
|
149
|
+
tries = 0
|
|
150
|
+
while job and job.get("status") in ("UNKNOWN", "RATELIMIT") and tries < 5:
|
|
151
|
+
time.sleep(max(poll, 8.0))
|
|
152
|
+
job = _call(f"{host}/ticket/msa", body)
|
|
153
|
+
tries += 1
|
|
154
|
+
if not job or "id" not in job or job.get("status") in ("ERROR", "MAINTENANCE", None):
|
|
155
|
+
return None
|
|
156
|
+
|
|
157
|
+
jid, st, waited = job["id"], job.get("status"), 0.0
|
|
158
|
+
while st in ("PENDING", "RUNNING", "UNKNOWN"):
|
|
159
|
+
time.sleep(poll)
|
|
160
|
+
waited += poll
|
|
161
|
+
if waited > max_wait:
|
|
162
|
+
return None
|
|
163
|
+
s = _call(f"{host}/ticket/{jid}", None)
|
|
164
|
+
st = s.get("status") if s else None
|
|
165
|
+
if st != "COMPLETE":
|
|
166
|
+
return None
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
req = urllib.request.Request(f"{host}/result/download/{jid}", headers=_MMSEQS_UA)
|
|
170
|
+
with urllib.request.urlopen(req, timeout=max(timeout, 120.0)) as r: # noqa: S310
|
|
171
|
+
blob = r.read()
|
|
172
|
+
except Exception: # noqa: BLE001
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
parts: list[str] = []
|
|
176
|
+
try:
|
|
177
|
+
with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
|
|
178
|
+
for m in tar.getmembers():
|
|
179
|
+
if m.name.endswith(".a3m"):
|
|
180
|
+
fh = tar.extractfile(m)
|
|
181
|
+
if fh is not None:
|
|
182
|
+
parts.append(fh.read().decode("utf-8", "ignore"))
|
|
183
|
+
except Exception: # noqa: BLE001
|
|
184
|
+
return None
|
|
185
|
+
a3m = "\n".join(p.strip() for p in parts if p.strip()).strip()
|
|
186
|
+
return a3m or None
|