shellde 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. shellde/__init__.py +36 -0
  2. shellde/acquisition.py +135 -0
  3. shellde/advisor.py +158 -0
  4. shellde/bench/__init__.py +17 -0
  5. shellde/bench/falsification.py +95 -0
  6. shellde/bench/harness.py +134 -0
  7. shellde/bench/stats.py +46 -0
  8. shellde/campaign.py +257 -0
  9. shellde/candidates.py +341 -0
  10. shellde/cli.py +1517 -0
  11. shellde/colab.py +257 -0
  12. shellde/conformal.py +160 -0
  13. shellde/consensus.py +52 -0
  14. shellde/design_space.py +141 -0
  15. shellde/embeddings/__init__.py +6 -0
  16. shellde/embeddings/esm2.py +76 -0
  17. shellde/embeddings/esmc.py +94 -0
  18. shellde/embeddings/provider.py +105 -0
  19. shellde/features/__init__.py +22 -0
  20. shellde/features/base.py +49 -0
  21. shellde/features/defaults.py +11 -0
  22. shellde/features/embedding.py +80 -0
  23. shellde/features/inverse_folding.py +66 -0
  24. shellde/features/matrix.py +50 -0
  25. shellde/features/naturalness.py +65 -0
  26. shellde/features/onehot.py +55 -0
  27. shellde/features/pairwise.py +64 -0
  28. shellde/funclib.py +331 -0
  29. shellde/gating.py +181 -0
  30. shellde/holo.py +175 -0
  31. shellde/hotspots.py +108 -0
  32. shellde/loop.py +161 -0
  33. shellde/msa.py +186 -0
  34. shellde/naturalness.py +142 -0
  35. shellde/oracle.py +94 -0
  36. shellde/plm.py +145 -0
  37. shellde/prereg.py +41 -0
  38. shellde/protocols.py +87 -0
  39. shellde/rank.py +103 -0
  40. shellde/report.py +132 -0
  41. shellde/selector.py +63 -0
  42. shellde/sitefinder.py +465 -0
  43. shellde/structure.py +356 -0
  44. shellde/surrogate.py +323 -0
  45. shellde/types.py +66 -0
  46. shellde/zero_shot.py +160 -0
  47. shellde-0.2.0.dist-info/METADATA +285 -0
  48. shellde-0.2.0.dist-info/RECORD +51 -0
  49. shellde-0.2.0.dist-info/WHEEL +5 -0
  50. shellde-0.2.0.dist-info/entry_points.txt +2 -0
  51. shellde-0.2.0.dist-info/top_level.txt +1 -0
shellde/holo.py ADDED
@@ -0,0 +1,175 @@
1
+ """Find a ligand-bound (holo) structural homolog when the target is apo.
2
+
3
+ ShellDE's resolve-site needs a ligand pose to define the active-site shell; it abstains on apo
4
+ structures. This module fills that gap: submit a query structure to the public Foldseek search server
5
+ (same ticket-based API family as the ColabFold MMseqs2 MSA API already used here), take the PDB hits,
6
+ and keep the ones that actually have a bound ligand (RCSB check). It returns candidate holo PDB ids for
7
+ the user to feed to ``resolve-site --pdb <id>`` (the existing shell machinery aligns it onto WT).
8
+
9
+ Network + polling; every failure returns an empty list so callers ABSTAIN, never fabricate. Advisory
10
+ only: it proposes candidates, it does not auto-superpose. stdlib urllib (no requests dependency).
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import io
16
+ import re
17
+ import tarfile
18
+ import time
19
+ import urllib.request
20
+ import uuid
21
+
22
+ from shellde.structure import _NON_LIGAND
23
+
24
+ _UA = {"User-Agent": "shellde (https://pypi.org/project/shellde/)"}
25
+ _PDB_ID = re.compile(r"(?<![A-Za-z0-9])([1-9][A-Za-z0-9]{3})(?![A-Za-z0-9])")
26
+
27
+
28
+ def _encode_multipart(fields: list[tuple[str, str]], filename: str, filedata: bytes) -> tuple[bytes, str]:
29
+ """Build a multipart/form-data body (repeated keys allowed, e.g. ``database[]``) for the ``q`` file."""
30
+ boundary = "----shelldeFoldseek" + uuid.uuid4().hex
31
+ crlf = "\r\n"
32
+ parts = []
33
+ for key, val in fields:
34
+ parts.append(
35
+ f'--{boundary}{crlf}Content-Disposition: form-data; name="{key}"{crlf}{crlf}{val}{crlf}'
36
+ )
37
+ parts.append(
38
+ f'--{boundary}{crlf}Content-Disposition: form-data; name="q"; filename="{filename}"{crlf}'
39
+ f"Content-Type: application/octet-stream{crlf}{crlf}"
40
+ )
41
+ body = "".join(parts).encode() + filedata + f"{crlf}--{boundary}--{crlf}".encode()
42
+ return body, f"multipart/form-data; boundary={boundary}"
43
+
44
+
45
+ def foldseek_search(
46
+ pdb_text: str,
47
+ *,
48
+ host: str = "https://search.foldseek.com/api",
49
+ databases: tuple[str, ...] = ("pdb100",),
50
+ mode: str = "3diaa",
51
+ timeout: float = 30.0,
52
+ max_wait: float = 600.0,
53
+ poll: float = 5.0,
54
+ max_evalue: float = 1e-3,
55
+ min_coverage: float = 0.5,
56
+ ) -> list[dict]:
57
+ """Foldseek-search a query structure, gate on structural significance, return ranked hits.
58
+
59
+ Submits the structure (POST /ticket), polls, downloads the .m8 result, and keeps only hits that are
60
+ a SIGNIFICANT, well-covering structural match -- E-value <= ``max_evalue`` AND alignment coverage
61
+ (alnlen/qlen) >= ``min_coverage`` -- so a distant/partial homolog whose ligand pocket may not
62
+ correspond to the target is NOT returned. Each hit is ``{"pdb_id","evalue","coverage","bits"}``,
63
+ ranked by bits (best first). [] on any failure or if nothing passes the gate.
64
+ """
65
+ data = pdb_text.encode() if isinstance(pdb_text, str) else pdb_text
66
+ if not data:
67
+ return []
68
+ fields = [("mode", mode)] + [("database[]", db) for db in databases]
69
+ body, ctype = _encode_multipart(fields, "query.pdb", data)
70
+
71
+ def _get_json(url: str, payload: bytes | None = None, ctype_hdr: str | None = None) -> dict | None:
72
+ try:
73
+ headers = dict(_UA)
74
+ if ctype_hdr:
75
+ headers["Content-Type"] = ctype_hdr
76
+ req = urllib.request.Request(url, data=payload, headers=headers)
77
+ with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310 (trusted host)
78
+ return json.loads(r.read().decode())
79
+ except Exception: # noqa: BLE001 (network/HTTP/JSON -> abstain)
80
+ return None
81
+
82
+ job = _get_json(f"{host}/ticket", body, ctype)
83
+ if not job or "id" not in job or job.get("status") in ("ERROR", "MAINTENANCE", None):
84
+ return []
85
+ jid, st, waited = job["id"], job.get("status"), 0.0
86
+ while st in ("PENDING", "RUNNING", "UNKNOWN", "RATELIMIT"):
87
+ time.sleep(poll)
88
+ waited += poll
89
+ if waited > max_wait:
90
+ return []
91
+ s = _get_json(f"{host}/ticket/{jid}")
92
+ st = s.get("status") if s else None
93
+ if st != "COMPLETE":
94
+ return []
95
+
96
+ try:
97
+ req = urllib.request.Request(f"{host}/result/download/{jid}", headers=_UA)
98
+ with urllib.request.urlopen(req, timeout=max(timeout, 120.0)) as r: # noqa: S310
99
+ blob = r.read()
100
+ except Exception: # noqa: BLE001
101
+ return []
102
+
103
+ rows: dict[str, dict] = {}
104
+ try:
105
+ with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
106
+ for m in tar.getmembers():
107
+ if not m.name.endswith(".m8"):
108
+ continue
109
+ fh = tar.extractfile(m)
110
+ if fh is None:
111
+ continue
112
+ for line in fh.read().decode("utf-8", "ignore").splitlines():
113
+ cols = line.split("\t")
114
+ if len(cols) < 14: # need through qlen (col 13)
115
+ continue
116
+ hit = _PDB_ID.search(cols[1])
117
+ if not hit:
118
+ continue
119
+ try:
120
+ alnlen = int(cols[3])
121
+ evalue = float(cols[11])
122
+ bits = float(cols[12])
123
+ qlen = int(cols[13])
124
+ except (ValueError, IndexError):
125
+ continue
126
+ cov = alnlen / qlen if qlen else 0.0
127
+ if evalue > max_evalue or cov < min_coverage: # structural-significance gate
128
+ continue
129
+ pid = hit.group(1).upper()
130
+ rec = {"pdb_id": pid, "evalue": evalue, "coverage": round(cov, 3), "bits": bits}
131
+ if pid not in rows or bits > rows[pid]["bits"]:
132
+ rows[pid] = rec
133
+ except Exception: # noqa: BLE001
134
+ return []
135
+ return sorted(rows.values(), key=lambda r: -r["bits"])
136
+
137
+
138
+ def rcsb_ligands(pdb_id: str, *, timeout: float = 15.0) -> list[str]:
139
+ """Bound non-polymer ligand comp-ids for a PDB entry, minus crystallization additives. [] on failure."""
140
+ try:
141
+ req = urllib.request.Request(
142
+ f"https://data.rcsb.org/rest/v1/core/entry/{pdb_id.upper()}", headers=_UA
143
+ )
144
+ with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310
145
+ info = json.loads(r.read().decode())
146
+ except Exception: # noqa: BLE001
147
+ return []
148
+ comps = info.get("rcsb_entry_info", {}).get("nonpolymer_bound_components") or []
149
+ return [c for c in comps if c.upper() not in _NON_LIGAND]
150
+
151
+
152
+ def find_holo_homolog(
153
+ pdb_text: str,
154
+ *,
155
+ top_n: int = 10,
156
+ max_candidates: int = 3,
157
+ timeout: float = 30.0,
158
+ max_wait: float = 600.0,
159
+ ) -> list[dict]:
160
+ """Advisory: rank ligand-bound (holo) PDB homologs of a query structure.
161
+
162
+ Runs :func:`foldseek_search` (pdb100), then keeps the top hits that RCSB reports as having a bound
163
+ (non-additive) ligand. Returns up to ``max_candidates`` dicts ``{"pdb_id", "ligands", "rank"}`` for
164
+ the user to pass to ``resolve-site --pdb``. Empty on any failure (ABSTAIN).
165
+ """
166
+ hits = foldseek_search(pdb_text, timeout=timeout, max_wait=max_wait)
167
+ out: list[dict] = []
168
+ for rank, h in enumerate(hits[:top_n], 1):
169
+ ligs = rcsb_ligands(h["pdb_id"], timeout=min(timeout, 15.0))
170
+ if ligs:
171
+ out.append({"pdb_id": h["pdb_id"], "ligands": ligs, "evalue": h["evalue"],
172
+ "coverage": h["coverage"], "rank": rank})
173
+ if len(out) >= max_candidates:
174
+ break
175
+ return out
shellde/hotspots.py ADDED
@@ -0,0 +1,108 @@
1
+ """Bridge (niche C): a whole-protein single-mutant SCAN -> hotspot positions for combinatorial AL.
2
+
3
+ The scan->focus handoff that neither EVOLVEpro (whole-protein singles + additive stacking) nor
4
+ FuncLib/ALDE (you must already know the positions) provides:
5
+
6
+ 1. SCAN -- a whole-protein single-mutant table (measured or PLM-predicted, EVOLVEpro-style,
7
+ mutation-notation like ``V100F``) ranks how impactful each POSITION is.
8
+ 2. BRIDGE -- pick the top-k hotspot positions here.
9
+ 3. FOCUS -- feed those positions to ``campaign``/``round`` (combinatorial AL: the candidate pool is
10
+ multi-mutant, the fitted model is additive by default), which MEASURES real
11
+ combinations instead of assuming additive stacking.
12
+
13
+ This module is the BRIDGE. It is pure/offline (parsing + per-position aggregation); the scan itself
14
+ (producing the single-mutant scores) and the combinatorial phase live elsewhere.
15
+
16
+ Honest scope: ranking positions from single-mutant scores is a heuristic to CHOOSE where to look; it
17
+ does not predict the combinatorial winner (that needs the measured combinatorial AL). If the scores
18
+ are predictions (e.g. EVOLVEpro y_pred), the choice inherits that model's ceiling.
19
+ """
20
+ from __future__ import annotations
21
+
22
+ import re
23
+ from dataclasses import dataclass, field
24
+
25
+ _MUT = re.compile(r"^([A-Za-z])?(\d+)([A-Za-z*])$")
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class Hotspot:
30
+ """One design position summarised from the single-mutant scan."""
31
+
32
+ position: int
33
+ wt: str
34
+ best_effect: float # score of the best single substitution at this position
35
+ best_sub: str # e.g. "V100F"
36
+ n_beneficial: int # substitutions scoring above the baseline
37
+ subs: dict[str, float] = field(default_factory=dict) # aa -> score
38
+
39
+
40
+ def parse_mutation(token: str) -> tuple[str, int, str] | None:
41
+ """``'V100F'`` -> ``('V', 100, 'F')``; ``'100F'``/``'100 F'`` -> ``('', 100, 'F')``.
42
+
43
+ The wild-type letter is optional so position+AA scan exports (e.g. ``29I``, common when the WT is
44
+ implied) parse too; ranking only needs the position and the substituted residue. None if the token
45
+ is not a single substitution.
46
+ """
47
+ m = _MUT.match(token.strip())
48
+ if not m:
49
+ return None
50
+ return (m.group(1) or "").upper(), int(m.group(2)), m.group(3).upper()
51
+
52
+
53
+ def rank_hotspots(
54
+ singles,
55
+ *,
56
+ baseline: float = 0.0,
57
+ top_k: int = 6,
58
+ metric: str = "max",
59
+ min_effect: float | None = None,
60
+ ) -> list[Hotspot]:
61
+ """Aggregate single-mutant scores per position and return the top-``top_k`` hotspots.
62
+
63
+ ``singles``: iterable of ``(mutation_token, score)`` where higher score == more desirable
64
+ (activity / predicted benefit). ``metric`` ranks positions by ``'max'`` (best single here, the
65
+ default), ``'mean'`` (average over its substitutions), or ``'count'`` (number scoring above
66
+ ``baseline``). ``min_effect`` drops positions whose best single is below it. Pure.
67
+ """
68
+ by_pos: dict[int, dict] = {}
69
+ for token, score in singles:
70
+ parsed = parse_mutation(token)
71
+ if parsed is None:
72
+ continue
73
+ wt, pos, mut = parsed
74
+ try:
75
+ val = float(score)
76
+ except (TypeError, ValueError):
77
+ continue
78
+ slot = by_pos.setdefault(pos, {"wt": wt, "subs": {}})
79
+ slot["subs"][mut] = val
80
+
81
+ hotspots: list[Hotspot] = []
82
+ for pos, slot in by_pos.items():
83
+ subs = slot["subs"]
84
+ if not subs:
85
+ continue
86
+ best_aa = max(subs, key=subs.get)
87
+ hotspots.append(
88
+ Hotspot(
89
+ position=pos,
90
+ wt=slot["wt"],
91
+ best_effect=subs[best_aa],
92
+ best_sub=f"{slot['wt']}{pos}{best_aa}",
93
+ n_beneficial=sum(1 for v in subs.values() if v > baseline),
94
+ subs=dict(subs),
95
+ )
96
+ )
97
+
98
+ key = {
99
+ "max": lambda h: h.best_effect,
100
+ "mean": lambda h: sum(h.subs.values()) / len(h.subs),
101
+ "count": lambda h: h.n_beneficial,
102
+ }
103
+ if metric not in key:
104
+ raise ValueError(f"metric must be one of {sorted(key)}, got {metric!r}")
105
+ hotspots.sort(key=key[metric], reverse=True)
106
+ if min_effect is not None:
107
+ hotspots = [h for h in hotspots if h.best_effect >= min_effect]
108
+ return hotspots[:top_k]
shellde/loop.py ADDED
@@ -0,0 +1,161 @@
1
+ """The active-learning / experimental-design loop: fit -> acquire -> measure -> repeat.
2
+
3
+ This is the organizing axis of the tool. Each round re-fits a fresh surrogate on ALL
4
+ accumulated measurements, predicts (calibrated mean+std) over the unmeasured pool,
5
+ the pluggable acquisition selects the next batch, the oracle measures it, and the
6
+ data accumulates. The candidate pool is the oracle's enumerable universe, so a
7
+ campaign reports best-found over genuinely measurable variants and identical budget
8
+ across methods is the harness's responsibility.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ from collections.abc import Callable, Mapping, Sequence
13
+
14
+ import numpy as np
15
+
16
+
17
+ from shellde.design_space import DesignSpace
18
+ from shellde.features.matrix import FeatureMatrix
19
+ from shellde.protocols import Acquisition, Oracle, Surrogate
20
+ from shellde.types import CampaignResult, RoundLog
21
+
22
+
23
+ def _spearman(a: np.ndarray, b: np.ndarray) -> float:
24
+ """Spearman rank correlation via numpy (avoids a scipy typing dependency here)."""
25
+ ra = np.argsort(np.argsort(a)).astype(float)
26
+ rb = np.argsort(np.argsort(b)).astype(float)
27
+ if np.std(ra) < 1e-12 or np.std(rb) < 1e-12:
28
+ return 0.0
29
+ return float(np.corrcoef(ra, rb)[0, 1])
30
+
31
+
32
+ def run_campaign(
33
+ oracle: Oracle,
34
+ space: DesignSpace,
35
+ *,
36
+ make_matrix: Callable[[], FeatureMatrix],
37
+ regate: Callable[[list[str], np.ndarray], FeatureMatrix] | None = None,
38
+ make_surrogate: Callable[[], Surrogate],
39
+ acquisition: Acquisition,
40
+ n_init: int = 95,
41
+ batch_size: int = 95,
42
+ n_rounds: int = 3,
43
+ seed: int = 0,
44
+ init_variants: Sequence[str] | None = None,
45
+ anchors: Mapping[str, float] | None = None,
46
+ ) -> CampaignResult:
47
+ """Run an active-learning campaign against an oracle with an enumerable universe."""
48
+ universe = oracle.universe
49
+ if universe is None:
50
+ raise ValueError("run_campaign requires an oracle exposing an enumerable universe")
51
+ rng = np.random.default_rng(seed)
52
+ if init_variants is not None:
53
+ # Explicit seed plate (e.g. a funclib R0 library): use it as the initial measured set
54
+ # instead of a random draw. The AL pool below is still the FULL universe, so the loop
55
+ # explores beyond the seed (never hard-restricted to it). Variants outside the universe
56
+ # are dropped; if the seed is short, top up with a random draw to keep n_init honest.
57
+ uni_set = set(universe)
58
+ init = list(dict.fromkeys(v for v in init_variants if v in uni_set))[:n_init]
59
+ if len(init) < n_init:
60
+ extra = [universe[int(i)] for i in rng.permutation(len(universe))
61
+ if universe[int(i)] not in set(init)]
62
+ init.extend(extra[: n_init - len(init)])
63
+ else:
64
+ init_idx = rng.choice(len(universe), min(n_init, len(universe)), replace=False)
65
+ init = [universe[int(i)] for i in init_idx]
66
+ measured: dict[str, float] = dict(zip(init, (float(x) for x in oracle.evaluate(init)), strict=True))
67
+
68
+ static_matrix = make_matrix() if regate is None else None # fixed blocks unless re-gated
69
+ # Encode-once: with fixed blocks (no per-round regate) the design matrix never changes, so
70
+ # encode the whole universe a SINGLE time and index rows by a measured-mask instead of
71
+ # re-encoding the ~|universe| pool every round. Byte-identical: encode is row-independent, so
72
+ # universe_enc[i] == static_matrix.encode([universe[i]])[0], and the masked pool keeps universe
73
+ # order (== the old `[v for v in universe if v not in measured]`).
74
+ universe_enc = static_matrix.encode(universe) if static_matrix is not None else None
75
+ idx_of = {v: i for i, v in enumerate(universe)} if static_matrix is not None else {}
76
+ measured_mask = np.zeros(len(universe), dtype=bool)
77
+ if static_matrix is not None:
78
+ for v in measured:
79
+ measured_mask[idx_of[v]] = True
80
+ # Anchors are FIT-only free rows; the invariant is they never become pool candidates
81
+ # (so they cannot be acquired into measured/best/trajectory). This mask is a no-op when
82
+ # `anchors` is None -- the pool-construction path below is then byte-identical.
83
+ anchor_keys = set(anchors) if anchors else set()
84
+ anchor_mask = np.zeros(len(universe), dtype=bool)
85
+ if anchors and static_matrix is not None:
86
+ for v in anchor_keys:
87
+ if v in idx_of:
88
+ anchor_mask[idx_of[v]] = True
89
+ trajectory = [max(measured.values())]
90
+ rounds: list[RoundLog] = []
91
+ heldout_spearman: list[float] = []
92
+
93
+ for r in range(n_rounds):
94
+ vs = list(measured)
95
+ if static_matrix is not None:
96
+ pool_mask = ~measured_mask
97
+ if anchors:
98
+ pool_mask = pool_mask & ~anchor_mask
99
+ pool_idx = np.where(pool_mask)[0]
100
+ pool = [universe[int(i)] for i in pool_idx]
101
+ else:
102
+ pool_idx = None
103
+ if anchors:
104
+ pool = [v for v in universe if v not in measured and v not in anchor_keys]
105
+ else:
106
+ pool = [v for v in universe if v not in measured]
107
+ if len(vs) < 2 or not pool:
108
+ break
109
+ ys = np.asarray([measured[v] for v in vs], dtype=float)
110
+ if static_matrix is not None:
111
+ matrix = static_matrix
112
+ x_vs = universe_enc[[idx_of[v] for v in vs]]
113
+ x_pool = universe_enc[pool_idx]
114
+ else:
115
+ matrix = regate(vs, ys)
116
+ assert matrix is not None
117
+ x_vs = matrix.encode(vs)
118
+ x_pool = matrix.encode(pool)
119
+ # Anchor: extra free labeled rows into the FIT ONLY (kept out of `measured`,
120
+ # the pool mask, and best-found). Budget parity: anchors add on top of the draw.
121
+ if anchors:
122
+ extra = [(v, fv) for v, fv in anchors.items() if v not in measured]
123
+ if extra:
124
+ ev = [v for v, _ in extra]
125
+ x_extra = static_matrix.encode(ev) if static_matrix is not None else matrix.encode(ev)
126
+ x_vs = np.vstack([x_vs, x_extra])
127
+ ys = np.concatenate([ys, np.asarray([fv for _, fv in extra], dtype=float)])
128
+ sur = make_surrogate().fit(x_vs, ys)
129
+ pred = sur.predict(x_pool)
130
+ k = min(batch_size, len(pool))
131
+ sel = acquisition.select(pred, k, candidates=pool, space=space)
132
+ batch = [pool[i] for i in sel]
133
+ pred_batch = np.asarray(pred.mean, dtype=float)[sel]
134
+ batch_y = np.asarray(oracle.evaluate(batch), dtype=float)
135
+ for v, fv in zip(batch, batch_y, strict=True):
136
+ measured[v] = float(fv)
137
+ if static_matrix is not None:
138
+ measured_mask[idx_of[v]] = True
139
+ ho_rmse = float(np.sqrt(np.mean((pred_batch - batch_y) ** 2))) if batch_y.size else None
140
+ ho_rho: float | None = None
141
+ if batch_y.size >= 3 and float(np.std(batch_y)) > 1e-12 and float(np.std(pred_batch)) > 1e-12:
142
+ ho_rho = _spearman(pred_batch, batch_y)
143
+ heldout_spearman.append(ho_rho)
144
+ best = max(measured.values())
145
+ trajectory.append(best)
146
+ rounds.append(RoundLog(
147
+ round_index=r, n_measured=len(measured), best_fitness=best, batch=batch,
148
+ heldout_rmse=ho_rmse, heldout_spearman=ho_rho,
149
+ notes={"active_blocks": matrix.active_blocks()},
150
+ ))
151
+
152
+ best_v = max(measured, key=lambda kk: measured[kk])
153
+ return CampaignResult(
154
+ trajectory=trajectory,
155
+ measured=measured,
156
+ best_variant=best_v,
157
+ best_fitness=measured[best_v],
158
+ n_rounds=len(rounds),
159
+ rounds=rounds,
160
+ heldout_spearman=heldout_spearman,
161
+ )
shellde/msa.py ADDED
@@ -0,0 +1,186 @@
1
+ """MSA-derived evolutionary tolerance: Henikoff-weighted PSSM log-probabilities.
2
+
3
+ A per-position amino-acid log-probability table built from a multiple sequence alignment (a3m),
4
+ usable as a funclib tolerance signal exactly like an inverse-folding or ddG table. This is the
5
+ classic evolutionary-conservation signal (the "independent" MSA model that funclib's design names
6
+ as a canonical tolerance). On a small 4-protein gate test it matched/beat ESM-2, BUT on the
7
+ 217-DMS ProteinGym benchmark MSA and ESM are statistically INDISTINGUISHABLE (per-DMS paired, ns
8
+ on both Spearman and top-recall). So this is a GPU-free MSA ALTERNATIVE to PLM tolerance, NOT a
9
+ proven improvement; the 4-protein advantage (esp. GB1's shallow-MSA anti-enrichment) did not
10
+ generalize. See ``scripts/proteingym_revalidate.py`` and decision log D11/D13.
11
+
12
+ Honest scope: this is the independent (per-site) conservation model, not the epistasis-aware
13
+ GEMME / EVmutation (Potts) signal. Same recall ceiling as all tolerance signals (it predicts
14
+ "tolerated/functional", not "improved").
15
+
16
+ Dependency-light: numpy + stdlib. Produce the a3m once with any MSA tool (jackhmmer, hhblits, or
17
+ the ColabFold MMseqs2 API); this module turns it into the log-prob table.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import io
22
+ import json
23
+ import tarfile
24
+ import time
25
+ import urllib.parse
26
+ import urllib.request
27
+
28
+ from pathlib import Path
29
+
30
+ import numpy as np
31
+
32
+ AA20 = "ACDEFGHIKLMNPQRSTVWY"
33
+
34
+
35
+ def read_a3m(path: str | Path) -> list[str]:
36
+ """Parse an a3m into match-state-only aligned rows (all of equal length = query length).
37
+
38
+ a3m insertion columns are lowercase; dropping every lowercase character collapses each record
39
+ to the query's match columns, so all returned rows share the query length. The first record is
40
+ the query.
41
+ """
42
+ seqs: list[str] = []
43
+ cur: str | None = None
44
+ for line in Path(path).read_text().splitlines():
45
+ if line.startswith(">"):
46
+ if cur is not None:
47
+ seqs.append(cur)
48
+ cur = ""
49
+ elif cur is not None:
50
+ cur += line.strip()
51
+ if cur:
52
+ seqs.append(cur)
53
+ aligned = ["".join(c for c in s if not c.islower()) for s in seqs]
54
+ if not aligned:
55
+ raise ValueError(f"{path}: no sequences parsed from a3m")
56
+ length = len(aligned[0])
57
+ return [a for a in aligned if len(a) == length]
58
+
59
+
60
+ def _henikoff_weights(matrix: np.ndarray) -> np.ndarray:
61
+ """Henikoff position-based sequence weights over an (N, L) residue-char matrix.
62
+
63
+ Each column contributes 1/(r_j * count_{j,a}) to the weight of a sequence with residue a in
64
+ column j, where r_j is the number of distinct amino acids in column j (gaps ignored). Down-
65
+ weights redundant sequences without the O(N^2) cost of identity clustering.
66
+ """
67
+ n, length = matrix.shape
68
+ aaset = set(AA20)
69
+ w = np.zeros(n, dtype=float)
70
+ for j in range(length):
71
+ col = matrix[:, j]
72
+ uniq, counts = np.unique(col, return_counts=True)
73
+ cmap = {u: c for u, c in zip(uniq, counts)}
74
+ r = sum(1 for u in uniq if u in aaset)
75
+ if r == 0:
76
+ continue
77
+ for i in range(n):
78
+ a = col[i]
79
+ if a in aaset:
80
+ w[i] += 1.0 / (r * cmap[a])
81
+ total = float(w.sum())
82
+ return w / total if total > 0 else np.full(n, 1.0 / n)
83
+
84
+
85
+ def load_msa_logprobs(
86
+ path: str | Path, *, pseudocount: float | None = None, reweight: bool = True
87
+ ) -> dict[int, dict[str, float]]:
88
+ """Henikoff-weighted PSSM as a {position: {aa: log_probability}} table from an a3m.
89
+
90
+ Positions are 1-based query (first-record) columns. Frequencies use a uniform pseudocount
91
+ (default ``1/20``) so unobserved residues get a finite log-prob. With ``reweight`` (default),
92
+ sequences are Henikoff-weighted to reduce alignment redundancy bias; set ``reweight=False`` for
93
+ raw counts. Feed the result to ``funclib.design_library`` / ``funclib.tolerance_from_logprob_table``
94
+ as a tolerance signal (top-K is invariant to the WT baseline, so it gates on conservation rank).
95
+ """
96
+ rows = read_a3m(path)
97
+ matrix = np.array([list(r) for r in rows])
98
+ n, length = matrix.shape
99
+ pc = (1.0 / len(AA20)) if pseudocount is None else float(pseudocount)
100
+ w = _henikoff_weights(matrix) if reweight else np.full(n, 1.0 / max(n, 1))
101
+ aaidx = {a: i for i, a in enumerate(AA20)}
102
+ out: dict[int, dict[str, float]] = {}
103
+ for j in range(length):
104
+ f = np.zeros(len(AA20), dtype=float)
105
+ col = matrix[:, j]
106
+ for i in range(n):
107
+ a = col[i]
108
+ if a in aaidx:
109
+ f[aaidx[a]] += w[i]
110
+ f = (f + pc) / (f.sum() + pc * len(AA20))
111
+ lp = np.log(f)
112
+ out[j + 1] = {a: float(lp[aaidx[a]]) for a in AA20}
113
+ return out
114
+
115
+
116
+ _MMSEQS_UA = {"User-Agent": "shellde (https://pypi.org/project/shellde/)"}
117
+
118
+
119
+ def build_msa_mmseqs2(
120
+ seq: str,
121
+ *,
122
+ host: str = "https://api.colabfold.com",
123
+ timeout: float = 30.0,
124
+ max_wait: float = 600.0,
125
+ poll: float = 5.0,
126
+ ) -> str | None:
127
+ """Build an a3m MSA for a single sequence via the public ColabFold MMseqs2 API (no local DBs).
128
+
129
+ Submit -> poll -> download the result tar.gz, then concatenate its a3m members. Returns the a3m
130
+ text, or None on any error/timeout so callers ABSTAIN (never fabricate an alignment). This removes
131
+ the "R0 needs a tolerance signal" friction; it is a GPU-free evolutionary signal, NOT a proven
132
+ accuracy improvement (see module docstring). Network + polling. Fair-use: one job at a time
133
+ (https://colabfold.mmseqs.com).
134
+ """
135
+ clean = "".join(seq.split()).upper()
136
+ if not clean:
137
+ return None
138
+
139
+ def _call(url: str, data: bytes | None) -> dict | None:
140
+ try:
141
+ req = urllib.request.Request(url, data=data, headers=_MMSEQS_UA)
142
+ with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310 (trusted host)
143
+ return json.loads(r.read().decode())
144
+ except Exception: # noqa: BLE001 (network/HTTP/JSON -> abstain)
145
+ return None
146
+
147
+ body = urllib.parse.urlencode({"q": f">101\n{clean}\n", "mode": "env"}).encode()
148
+ job = _call(f"{host}/ticket/msa", body)
149
+ tries = 0
150
+ while job and job.get("status") in ("UNKNOWN", "RATELIMIT") and tries < 5:
151
+ time.sleep(max(poll, 8.0))
152
+ job = _call(f"{host}/ticket/msa", body)
153
+ tries += 1
154
+ if not job or "id" not in job or job.get("status") in ("ERROR", "MAINTENANCE", None):
155
+ return None
156
+
157
+ jid, st, waited = job["id"], job.get("status"), 0.0
158
+ while st in ("PENDING", "RUNNING", "UNKNOWN"):
159
+ time.sleep(poll)
160
+ waited += poll
161
+ if waited > max_wait:
162
+ return None
163
+ s = _call(f"{host}/ticket/{jid}", None)
164
+ st = s.get("status") if s else None
165
+ if st != "COMPLETE":
166
+ return None
167
+
168
+ try:
169
+ req = urllib.request.Request(f"{host}/result/download/{jid}", headers=_MMSEQS_UA)
170
+ with urllib.request.urlopen(req, timeout=max(timeout, 120.0)) as r: # noqa: S310
171
+ blob = r.read()
172
+ except Exception: # noqa: BLE001
173
+ return None
174
+
175
+ parts: list[str] = []
176
+ try:
177
+ with tarfile.open(fileobj=io.BytesIO(blob), mode="r:gz") as tar:
178
+ for m in tar.getmembers():
179
+ if m.name.endswith(".a3m"):
180
+ fh = tar.extractfile(m)
181
+ if fh is not None:
182
+ parts.append(fh.read().decode("utf-8", "ignore"))
183
+ except Exception: # noqa: BLE001
184
+ return None
185
+ a3m = "\n".join(p.strip() for p in parts if p.strip()).strip()
186
+ return a3m or None