esm3di 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- esm3di/ESM3di_model.py +4 -0
- esm3di/__init__.py +40 -0
- esm3di/_core/__init__.py +1 -0
- esm3di/_core/checkpoints.py +529 -0
- esm3di/_core/model.py +369 -0
- esm3di/checkpoints.py +4 -0
- esm3di/cli.py +35 -0
- esm3di/cutting_cli.py +9 -0
- esm3di/fastas2foldseekdb.py +90 -0
- esm3di/foldseek_db.py +6 -0
- esm3di/inference/__init__.py +15 -0
- esm3di/inference/__main__.py +3 -0
- esm3di/inference/checkpoint_predictor.py +335 -0
- esm3di/inference/cli.py +437 -0
- esm3di/inference/cutting.py +460 -0
- esm3di/inference/cutting_bundle.py +169 -0
- esm3di/inference/foldseek_db.py +66 -0
- esm3di/inference/io.py +237 -0
- esm3di/inference/model.py +18 -0
- esm3di/inference/phylogeny.py +87 -0
- esm3di/inference/predictor.py +372 -0
- esm3di/inference/preprocessing.py +42 -0
- esm3di/inference/windowing.py +81 -0
- esm3di/io.py +6 -0
- esm3di/model.py +4 -0
- esm3di/predict.py +9 -0
- esm3di/preprocessing.py +6 -0
- esm3di/workflow/Snakefile +48 -0
- esm3di/workflow/__init__.py +0 -0
- esm3di/workflow/config.yaml +14 -0
- esm3di/workflow/envs/foldtree.yaml +30 -0
- esm3di/workflow/envs/prostt5_env.yaml +7 -0
- esm3di/workflow/madroot/mad.py +678 -0
- esm3di/workflow/rules/database_ops.smk +77 -0
- esm3di/workflow/rules/homolog_grouping.smk +2 -0
- esm3di/workflow/rules/prostt5_db.smk +72 -0
- esm3di/workflow/rules/tree_building.smk +87 -0
- esm3di/workflow/scripts/3di2foldseekdb.py +139 -0
- esm3di/workflow/scripts/foldseek2tree.py +294 -0
- esm3di/workflow/scripts/foldseekres2distmat_simple.py +53 -0
- esm3di/workflow/scripts/postprocess.py +19 -0
- esm3di/workflow/scripts/process_madroot.py +17 -0
- esm3di/workflow/scripts/run_madroot.py +80 -0
- esm3di/workflow/scripts/tipnames.py +38 -0
- esm3di-0.2.0.dist-info/METADATA +385 -0
- esm3di-0.2.0.dist-info/RECORD +50 -0
- esm3di-0.2.0.dist-info/WHEEL +5 -0
- esm3di-0.2.0.dist-info/entry_points.txt +6 -0
- esm3di-0.2.0.dist-info/licenses/LICENSE +21 -0
- esm3di-0.2.0.dist-info/top_level.txt +1 -0
esm3di/ESM3di_model.py
ADDED
esm3di/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""ESM3Di inference and shared ESM++ checkpoint runtime."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
from pkgutil import extend_path
|
|
5
|
+
|
|
6
|
+
__path__ = extend_path(__path__, __name__)
|
|
7
|
+
__version__ = "0.2.0"
|
|
8
|
+
|
|
9
|
+
_LAZY = {
|
|
10
|
+
"ESM3DiModel": "_core.model",
|
|
11
|
+
"ESMWithCNNHead": "_core.model",
|
|
12
|
+
"CNNClassificationHead": "_core.model",
|
|
13
|
+
"read_fasta": "_core.model",
|
|
14
|
+
"discover_lora_target_modules": "_core.model",
|
|
15
|
+
"freeze_all_but_lora_and_classifier": "_core.model",
|
|
16
|
+
"load_3di_checkpoint": "_core.checkpoints",
|
|
17
|
+
"load_mlm_model": "_core.checkpoints",
|
|
18
|
+
"Predictor": "predict",
|
|
19
|
+
"predict_records": "predict",
|
|
20
|
+
"write_foldseek_db": "foldseek_db",
|
|
21
|
+
"fasta2foldseek": "foldseek_db",
|
|
22
|
+
}
|
|
23
|
+
_TRAINING = {
|
|
24
|
+
"FocalLoss": "losses", "CyclicalFocalLoss": "losses",
|
|
25
|
+
"PLDDTWeightedFocalLoss": "losses", "PLDDTWeightedCyclicalFocalLoss": "losses",
|
|
26
|
+
"GammaSchedulerOnPlateau": "losses", "DEFAULT_PLDDT_BIN_WEIGHTS": "losses",
|
|
27
|
+
"prepare": "prepare_data",
|
|
28
|
+
}
|
|
29
|
+
__all__ = sorted(_LAZY)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def __getattr__(name):
|
|
33
|
+
if name in _LAZY:
|
|
34
|
+
value = getattr(importlib.import_module(f".{_LAZY[name]}", __name__), name)
|
|
35
|
+
elif name in _TRAINING:
|
|
36
|
+
value = getattr(importlib.import_module(f"esm3di_training.{_TRAINING[name]}"), name)
|
|
37
|
+
else:
|
|
38
|
+
raise AttributeError(f"module 'esm3di' has no attribute {name!r}")
|
|
39
|
+
globals()[name] = value
|
|
40
|
+
return value
|
esm3di/_core/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Shared model architectures and checkpoint reconstruction."""
|
|
@@ -0,0 +1,529 @@
|
|
|
1
|
+
"""Checkpoint utilities for the ESM++ MLM -> 3Di pipeline.
|
|
2
|
+
|
|
3
|
+
This module is the single place that knows how to
|
|
4
|
+
|
|
5
|
+
* pin a HuggingFace remote-code revision that works with the installed
|
|
6
|
+
``transformers`` version (:func:`resolve_hf_revision`),
|
|
7
|
+
* read backbone weights from an MLM fine-tuning output directory and copy them
|
|
8
|
+
into a freshly loaded HF model *before* LoRA is applied
|
|
9
|
+
(:func:`load_mlm_state_dict`, :func:`transfer_backbone_weights`),
|
|
10
|
+
* classify an ``--init-from`` argument (:func:`detect_init_source`), and
|
|
11
|
+
* rebuild a trained 3Di tagger from a ``.pt`` checkpoint
|
|
12
|
+
(:func:`load_3di_checkpoint`).
|
|
13
|
+
|
|
14
|
+
Checkpoint formats
|
|
15
|
+
------------------
|
|
16
|
+
MLM checkpoint (output of ``esm3di mlm``)
|
|
17
|
+
A HuggingFace model directory: ``config.json``, ``model.safetensors`` (or
|
|
18
|
+
``pytorch_model.bin``) and tokenizer files. Keys look like
|
|
19
|
+
``embed.weight``, ``transformer.blocks.N...``, ``sequence_head...``.
|
|
20
|
+
Directories containing only a PEFT adapter (``adapter_config.json``) are
|
|
21
|
+
also accepted; the adapter is merged into its base model on load.
|
|
22
|
+
|
|
23
|
+
3Di checkpoint (output of ``esm3di train``)
|
|
24
|
+
A ``torch.save`` dict with at least ``model_state_dict``, ``label_vocab``,
|
|
25
|
+
``mask_label_chars``, ``args``, ``epoch`` and ``optimizer_state_dict``.
|
|
26
|
+
``model_state_dict`` holds the *complete* model (backbone, LoRA adapters
|
|
27
|
+
and head), so only the architecture (``args['hf_model']``) is needed to
|
|
28
|
+
rebuild it.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import json
|
|
34
|
+
import os
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from typing import Dict, List, Optional, Tuple
|
|
37
|
+
|
|
38
|
+
import torch
|
|
39
|
+
|
|
40
|
+
# Remote-code revisions of the Synthyra ESM++ repos known to load with
|
|
41
|
+
# transformers 4.48.x. Newer revisions on the hub import ``fastplms`` and
|
|
42
|
+
# ``transformers.AttentionInterface`` which do not exist in 4.48.
|
|
43
|
+
KNOWN_HF_REVISIONS: Dict[str, str] = {
|
|
44
|
+
"Synthyra/ESMplusplus_small": "056ceb518bbcf1a69f18d2504e9701513684b846",
|
|
45
|
+
"Synthyra/ESMplusplus_large": "36ef85d43b09001f8006774ff35ae1fa62f06a66",
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
DEFAULT_HF_MODEL = "Synthyra/ESMplusplus_small"
|
|
49
|
+
|
|
50
|
+
# FoldSeek's 3Di alphabet uses the 20 amino-acid letters.
|
|
51
|
+
THREEDI_VOCAB: List[str] = list("ACDEFGHIKLMNPQRSTVWY")
|
|
52
|
+
|
|
53
|
+
_HEAD_PREFIXES = ("sequence_head", "lm_head", "classifier", "score")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def resolve_hf_revision(hf_model: str, revision: Optional[str] = None) -> Optional[str]:
|
|
57
|
+
"""Return the HF revision to use for ``hf_model``.
|
|
58
|
+
|
|
59
|
+
Priority: explicit ``revision`` > ``ESM3DI_HF_REVISION`` env var > pinned
|
|
60
|
+
known-good revision > ``None`` (hub default). Pass ``revision="main"`` to
|
|
61
|
+
deliberately use the latest hub code.
|
|
62
|
+
"""
|
|
63
|
+
if revision:
|
|
64
|
+
return revision
|
|
65
|
+
env_rev = os.environ.get("ESM3DI_HF_REVISION")
|
|
66
|
+
if env_rev:
|
|
67
|
+
return env_rev
|
|
68
|
+
if os.path.isdir(hf_model):
|
|
69
|
+
return None
|
|
70
|
+
return KNOWN_HF_REVISIONS.get(hf_model)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
# init-source detection
|
|
75
|
+
# ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
def is_3di_checkpoint(path: str) -> bool:
|
|
78
|
+
return os.path.isfile(path) and path.endswith((".pt", ".pth"))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def is_mlm_checkpoint_dir(path: str) -> bool:
|
|
82
|
+
if not os.path.isdir(path):
|
|
83
|
+
return False
|
|
84
|
+
names = set(os.listdir(path))
|
|
85
|
+
return "config.json" in names or "adapter_config.json" in names
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def detect_init_source(init_from: Optional[str]) -> str:
|
|
89
|
+
"""Classify ``--init-from``: ``'base'``, ``'mlm'`` or ``'3di'``."""
|
|
90
|
+
if not init_from:
|
|
91
|
+
return "base"
|
|
92
|
+
if is_3di_checkpoint(init_from):
|
|
93
|
+
return "3di"
|
|
94
|
+
if os.path.isdir(init_from):
|
|
95
|
+
if is_mlm_checkpoint_dir(init_from):
|
|
96
|
+
return "mlm"
|
|
97
|
+
raise ValueError(
|
|
98
|
+
f"--init-from directory {init_from!r} does not look like an MLM checkpoint "
|
|
99
|
+
"(expected config.json + model.safetensors, or a PEFT adapter). If this is an "
|
|
100
|
+
"MLM output directory, point at one of its sub-directories, e.g. "
|
|
101
|
+
f"{os.path.join(init_from, 'best')} or {os.path.join(init_from, 'final')}."
|
|
102
|
+
)
|
|
103
|
+
if os.path.exists(init_from):
|
|
104
|
+
raise ValueError(f"Unrecognised --init-from path: {init_from!r}")
|
|
105
|
+
# Anything else is treated as a HuggingFace hub id.
|
|
106
|
+
return "base"
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def mlm_base_model_name(mlm_dir: str) -> Optional[str]:
|
|
110
|
+
"""Best-effort lookup of the HF base model an MLM directory was trained from."""
|
|
111
|
+
meta_path = os.path.join(mlm_dir, "esm3di_mlm.json")
|
|
112
|
+
if os.path.isfile(meta_path):
|
|
113
|
+
with open(meta_path) as fh:
|
|
114
|
+
meta = json.load(fh)
|
|
115
|
+
if meta.get("base_model"):
|
|
116
|
+
return meta["base_model"]
|
|
117
|
+
for fname in ("adapter_config.json", "config.json"):
|
|
118
|
+
fpath = os.path.join(mlm_dir, fname)
|
|
119
|
+
if os.path.isfile(fpath):
|
|
120
|
+
with open(fpath) as fh:
|
|
121
|
+
cfg = json.load(fh)
|
|
122
|
+
name = cfg.get("base_model_name_or_path") or cfg.get("_name_or_path")
|
|
123
|
+
if name and not os.path.isdir(name):
|
|
124
|
+
return name
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# ---------------------------------------------------------------------------
|
|
129
|
+
# MLM -> backbone weight transfer
|
|
130
|
+
# ---------------------------------------------------------------------------
|
|
131
|
+
|
|
132
|
+
def _read_weight_file(path: str) -> Dict[str, torch.Tensor]:
|
|
133
|
+
if path.endswith(".safetensors"):
|
|
134
|
+
from safetensors.torch import load_file
|
|
135
|
+
|
|
136
|
+
return load_file(path, device="cpu")
|
|
137
|
+
obj = torch.load(path, map_location="cpu")
|
|
138
|
+
if isinstance(obj, dict) and "model_state_dict" in obj:
|
|
139
|
+
obj = obj["model_state_dict"]
|
|
140
|
+
return obj
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _find_weight_file(directory: str) -> Optional[str]:
|
|
144
|
+
for fname in ("model.safetensors", "pytorch_model.bin", "model.bin"):
|
|
145
|
+
fpath = os.path.join(directory, fname)
|
|
146
|
+
if os.path.isfile(fpath):
|
|
147
|
+
return fpath
|
|
148
|
+
return None
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _strip_known_prefixes(key: str) -> str:
|
|
152
|
+
for prefix in ("module.", "base_model.model.", "model."):
|
|
153
|
+
if key.startswith(prefix):
|
|
154
|
+
key = key[len(prefix):]
|
|
155
|
+
return key.replace(".base_layer.", ".")
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def load_mlm_state_dict(mlm_dir: str, revision: Optional[str] = None) -> Dict[str, torch.Tensor]:
|
|
159
|
+
"""Return the full (merged) backbone state dict stored in an MLM directory.
|
|
160
|
+
|
|
161
|
+
Handles full-weight HF directories and PEFT adapter directories.
|
|
162
|
+
"""
|
|
163
|
+
if os.path.isfile(os.path.join(mlm_dir, "adapter_config.json")) and _find_weight_file(mlm_dir) is None:
|
|
164
|
+
from peft import PeftModel
|
|
165
|
+
from transformers import AutoModelForMaskedLM
|
|
166
|
+
|
|
167
|
+
with open(os.path.join(mlm_dir, "adapter_config.json")) as fh:
|
|
168
|
+
base_name = json.load(fh)["base_model_name_or_path"]
|
|
169
|
+
base = AutoModelForMaskedLM.from_pretrained(
|
|
170
|
+
base_name, trust_remote_code=True, revision=resolve_hf_revision(base_name, revision)
|
|
171
|
+
)
|
|
172
|
+
merged = PeftModel.from_pretrained(base, mlm_dir).merge_and_unload()
|
|
173
|
+
return {k: v.detach().cpu() for k, v in merged.state_dict().items()}
|
|
174
|
+
|
|
175
|
+
weight_file = _find_weight_file(mlm_dir)
|
|
176
|
+
if weight_file is None:
|
|
177
|
+
raise FileNotFoundError(
|
|
178
|
+
f"No model weights (model.safetensors / pytorch_model.bin) found in {mlm_dir!r}. "
|
|
179
|
+
"Only config/tokenizer files are present - the weights may have been deleted."
|
|
180
|
+
)
|
|
181
|
+
raw = _read_weight_file(weight_file)
|
|
182
|
+
state = {}
|
|
183
|
+
for key, value in raw.items():
|
|
184
|
+
if "lora_" in key:
|
|
185
|
+
raise ValueError(
|
|
186
|
+
f"{weight_file} contains unmerged LoRA weights ({key}); export a merged model first."
|
|
187
|
+
)
|
|
188
|
+
state[_strip_known_prefixes(key)] = value
|
|
189
|
+
return state
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@dataclass
|
|
193
|
+
class TransferReport:
|
|
194
|
+
copied: int
|
|
195
|
+
partially_copied: List[str]
|
|
196
|
+
missing: List[str]
|
|
197
|
+
skipped_heads: int
|
|
198
|
+
|
|
199
|
+
def summary(self) -> str:
|
|
200
|
+
msg = f"copied {self.copied} backbone tensors"
|
|
201
|
+
if self.partially_copied:
|
|
202
|
+
msg += f", row-sliced {len(self.partially_copied)} ({', '.join(self.partially_copied)})"
|
|
203
|
+
if self.missing:
|
|
204
|
+
msg += f", {len(self.missing)} backbone tensors NOT found in MLM checkpoint"
|
|
205
|
+
return msg
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def transfer_backbone_weights(
|
|
209
|
+
target_model: torch.nn.Module,
|
|
210
|
+
source_state: Dict[str, torch.Tensor],
|
|
211
|
+
min_fraction: float = 0.95,
|
|
212
|
+
include_heads: bool = False,
|
|
213
|
+
) -> TransferReport:
|
|
214
|
+
"""Copy backbone weights from ``source_state`` into ``target_model`` in place.
|
|
215
|
+
|
|
216
|
+
``target_model`` must be the *plain* HF model (before LoRA wrapping). Head
|
|
217
|
+
weights (``sequence_head``/``classifier``) are skipped. Embedding matrices
|
|
218
|
+
whose vocabulary was resized during MLM training (e.g. 64 -> 33 rows) are
|
|
219
|
+
copied row-wise for the overlapping token ids.
|
|
220
|
+
|
|
221
|
+
Raises ``RuntimeError`` if fewer than ``min_fraction`` of the target's
|
|
222
|
+
backbone tensors could be matched - this guards against silently training
|
|
223
|
+
from the stock weights.
|
|
224
|
+
"""
|
|
225
|
+
target_state = target_model.state_dict()
|
|
226
|
+
backbone_keys = [k for k in target_state if include_heads or not k.startswith(_HEAD_PREFIXES)]
|
|
227
|
+
copied = 0
|
|
228
|
+
partial: List[str] = []
|
|
229
|
+
missing: List[str] = []
|
|
230
|
+
skipped_heads = 0 if include_heads else sum(1 for k in source_state if k.startswith(_HEAD_PREFIXES))
|
|
231
|
+
|
|
232
|
+
with torch.no_grad():
|
|
233
|
+
for key in backbone_keys:
|
|
234
|
+
src = source_state.get(key)
|
|
235
|
+
if src is None:
|
|
236
|
+
missing.append(key)
|
|
237
|
+
continue
|
|
238
|
+
dst = target_state[key]
|
|
239
|
+
if src.shape == dst.shape:
|
|
240
|
+
dst.copy_(src.to(dst.dtype))
|
|
241
|
+
copied += 1
|
|
242
|
+
elif (
|
|
243
|
+
src.dim() == dst.dim() >= 1
|
|
244
|
+
and src.shape[1:] == dst.shape[1:]
|
|
245
|
+
and src.shape[0] != dst.shape[0]
|
|
246
|
+
):
|
|
247
|
+
rows = min(src.shape[0], dst.shape[0])
|
|
248
|
+
dst[:rows].copy_(src[:rows].to(dst.dtype))
|
|
249
|
+
copied += 1
|
|
250
|
+
partial.append(key)
|
|
251
|
+
else:
|
|
252
|
+
missing.append(key)
|
|
253
|
+
|
|
254
|
+
if backbone_keys and copied / len(backbone_keys) < min_fraction:
|
|
255
|
+
raise RuntimeError(
|
|
256
|
+
f"Only {copied}/{len(backbone_keys)} backbone tensors matched the MLM checkpoint. "
|
|
257
|
+
f"Is the MLM checkpoint from the same base architecture? Missing e.g.: {missing[:5]}"
|
|
258
|
+
)
|
|
259
|
+
return TransferReport(copied=copied, partially_copied=partial, missing=missing, skipped_heads=skipped_heads)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def load_mlm_model(
|
|
263
|
+
source: str,
|
|
264
|
+
revision: Optional[str] = None,
|
|
265
|
+
base_model: Optional[str] = None,
|
|
266
|
+
):
|
|
267
|
+
"""Load an ESM++ masked-LM model from a hub id or an MLM checkpoint directory.
|
|
268
|
+
|
|
269
|
+
For directories the architecture/code is taken from the *pinned* base model
|
|
270
|
+
revision and the weights (including the LM head) are copied in. This avoids
|
|
271
|
+
the ``auto_map`` in saved ``config.json`` files pulling incompatible
|
|
272
|
+
remote code from the hub's latest revision.
|
|
273
|
+
"""
|
|
274
|
+
from transformers import AutoModelForMaskedLM
|
|
275
|
+
|
|
276
|
+
if os.path.isdir(source):
|
|
277
|
+
base = base_model or mlm_base_model_name(source)
|
|
278
|
+
if base is None:
|
|
279
|
+
raise ValueError(f"Cannot determine the base model of {source!r}; pass base_model=")
|
|
280
|
+
if not os.path.isdir(base) and not base.startswith("Synthyra/ESMplusplus_"):
|
|
281
|
+
raise ValueError(f"Only ESM++ MLM backbones are supported, not {base!r}.")
|
|
282
|
+
model = AutoModelForMaskedLM.from_pretrained(
|
|
283
|
+
base, trust_remote_code=True, revision=resolve_hf_revision(base, revision)
|
|
284
|
+
)
|
|
285
|
+
report = transfer_backbone_weights(model, load_mlm_state_dict(source, revision), include_heads=True)
|
|
286
|
+
return model, base, report
|
|
287
|
+
if not source.startswith("Synthyra/ESMplusplus_"):
|
|
288
|
+
raise ValueError(f"Only ESM++ MLM backbones are supported, not {source!r}.")
|
|
289
|
+
model = AutoModelForMaskedLM.from_pretrained(
|
|
290
|
+
source, trust_remote_code=True, revision=resolve_hf_revision(source, revision)
|
|
291
|
+
)
|
|
292
|
+
return model, source, None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# ---------------------------------------------------------------------------
|
|
296
|
+
# Tokenizer
|
|
297
|
+
# ---------------------------------------------------------------------------
|
|
298
|
+
|
|
299
|
+
def get_tokenizer(model=None, hf_model: Optional[str] = None, revision: Optional[str] = None):
|
|
300
|
+
"""Return a tokenizer for an ESM++ model.
|
|
301
|
+
|
|
302
|
+
ESM++ exposes its tokenizer as ``model.tokenizer`` (``AutoTokenizer`` does not
|
|
303
|
+
know ``EsmSequenceTokenizer``), so prefer that.
|
|
304
|
+
"""
|
|
305
|
+
seen = set()
|
|
306
|
+
current = model
|
|
307
|
+
while current is not None and id(current) not in seen:
|
|
308
|
+
seen.add(id(current))
|
|
309
|
+
tok = getattr(current, "tokenizer", None)
|
|
310
|
+
if tok is not None and not isinstance(tok, torch.nn.Module):
|
|
311
|
+
return tok
|
|
312
|
+
current = getattr(current, "base_model", None) or getattr(current, "model", None)
|
|
313
|
+
if hf_model is None:
|
|
314
|
+
raise ValueError("Cannot determine tokenizer: provide hf_model")
|
|
315
|
+
from transformers import AutoTokenizer, PreTrainedTokenizerFast
|
|
316
|
+
|
|
317
|
+
if os.path.isdir(hf_model) and os.path.isfile(os.path.join(hf_model, "tokenizer.json")):
|
|
318
|
+
try:
|
|
319
|
+
return AutoTokenizer.from_pretrained(hf_model, trust_remote_code=True)
|
|
320
|
+
except Exception:
|
|
321
|
+
return PreTrainedTokenizerFast.from_pretrained(hf_model)
|
|
322
|
+
return AutoTokenizer.from_pretrained(
|
|
323
|
+
hf_model, trust_remote_code=True, revision=resolve_hf_revision(hf_model, revision)
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
# ---------------------------------------------------------------------------
|
|
328
|
+
# 3Di checkpoint loading
|
|
329
|
+
# ---------------------------------------------------------------------------
|
|
330
|
+
|
|
331
|
+
def strip_module_prefix(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
|
|
332
|
+
"""Remove a ``module.`` prefix added by ``torch.nn.DataParallel``."""
|
|
333
|
+
if any(k.startswith("module.") for k in state_dict):
|
|
334
|
+
return {k[len("module."):] if k.startswith("module.") else k: v for k, v in state_dict.items()}
|
|
335
|
+
return state_dict
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def read_3di_checkpoint(path: str, map_location="cpu") -> dict:
|
|
339
|
+
ckpt = torch.load(path, map_location=map_location, weights_only=False)
|
|
340
|
+
if not isinstance(ckpt, dict) or "model_state_dict" not in ckpt:
|
|
341
|
+
raise ValueError(f"{path!r} is not an ESM3Di checkpoint (no 'model_state_dict').")
|
|
342
|
+
ckpt["model_state_dict"] = strip_module_prefix(ckpt["model_state_dict"])
|
|
343
|
+
args = ckpt.get("args") or {}
|
|
344
|
+
if not isinstance(args, dict):
|
|
345
|
+
args = vars(args)
|
|
346
|
+
ckpt["args"] = args
|
|
347
|
+
return ckpt
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
_UNSUPPORTED_LEGACY_FLAGS = (
|
|
351
|
+
"use_iterative_backbone_head", "use_iterative_transformer_head",
|
|
352
|
+
"use_transformer_head", "use_plddt_prediction_head",
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def model_kwargs_from_checkpoint(ckpt: dict) -> dict:
|
|
357
|
+
"""Translate a checkpoint's saved ``args`` into ``ESM3DiModel`` kwargs."""
|
|
358
|
+
args = ckpt["args"]
|
|
359
|
+
for flag in _UNSUPPORTED_LEGACY_FLAGS:
|
|
360
|
+
if args.get(flag):
|
|
361
|
+
raise ValueError(
|
|
362
|
+
f"Checkpoint was trained with {flag}=True, which is experimental. "
|
|
363
|
+
"Its implementation is archived under experiments/legacy_runtime."
|
|
364
|
+
)
|
|
365
|
+
hf_model = args.get("hf_model_name") or args.get("hf_model") or DEFAULT_HF_MODEL
|
|
366
|
+
if not os.path.isdir(hf_model) and not hf_model.startswith("Synthyra/ESMplusplus_"):
|
|
367
|
+
raise ValueError(f"Only ESM++ checkpoints are supported, not {hf_model!r}.")
|
|
368
|
+
if ckpt.get("aux_track_num_bins") or args.get("aux_track_num_bins"):
|
|
369
|
+
raise ValueError("Auxiliary-track checkpoints are archived and unsupported")
|
|
370
|
+
|
|
371
|
+
target_modules = ckpt.get("lora_target_modules") or args.get("lora_target_modules")
|
|
372
|
+
if isinstance(target_modules, str):
|
|
373
|
+
target_modules = [m.strip() for m in target_modules.split(",") if m.strip()]
|
|
374
|
+
|
|
375
|
+
kwargs = dict(
|
|
376
|
+
hf_model_name=hf_model,
|
|
377
|
+
num_labels=len(ckpt["label_vocab"]),
|
|
378
|
+
lora_r=args.get("lora_r", 16),
|
|
379
|
+
lora_alpha=args.get("lora_alpha", 32.0),
|
|
380
|
+
lora_dropout=args.get("lora_dropout", 0.01),
|
|
381
|
+
target_modules=target_modules or None,
|
|
382
|
+
use_cnn_head=args.get("use_cnn_head", False),
|
|
383
|
+
cnn_num_layers=args.get("cnn_num_layers", 2),
|
|
384
|
+
cnn_kernel_size=args.get("cnn_kernel_size", 3),
|
|
385
|
+
cnn_dropout=args.get("cnn_dropout", 0.1),
|
|
386
|
+
revision=args.get("hf_revision") or args.get("revision"),
|
|
387
|
+
)
|
|
388
|
+
return kwargs
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
# Parameters that some transformers versions create but never use (ESM-2 with
|
|
392
|
+
# rotary embeddings); older checkpoints legitimately lack them.
|
|
393
|
+
_IGNORABLE_MISSING_SUFFIXES = ("embeddings.position_embeddings.weight",)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _plain_key(key: str) -> str:
|
|
397
|
+
"""Map a PEFT-wrapped parameter name to the underlying dense parameter name."""
|
|
398
|
+
key = key.replace(".base_layer.", ".")
|
|
399
|
+
key = key.replace(".modules_to_save.default.", ".")
|
|
400
|
+
key = key.replace(".original_module.", ".")
|
|
401
|
+
return key
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def merge_lora_state_dict(state_dict: Dict[str, torch.Tensor], scaling: float) -> Dict[str, torch.Tensor]:
|
|
405
|
+
"""Fold every LoRA delta in ``state_dict`` into dense weights.
|
|
406
|
+
|
|
407
|
+
Returns a dict keyed by :func:`_plain_key` names without any ``lora_*``
|
|
408
|
+
entries. For PEFT ``modules_to_save`` wrappers the trained copy
|
|
409
|
+
(``modules_to_save.default``) wins over ``original_module``.
|
|
410
|
+
"""
|
|
411
|
+
sd = strip_module_prefix(state_dict)
|
|
412
|
+
merged: Dict[str, torch.Tensor] = {}
|
|
413
|
+
# original_module first so modules_to_save overwrites it
|
|
414
|
+
order = sorted(sd, key=lambda k: 0 if ".original_module." in k else 1)
|
|
415
|
+
for key in order:
|
|
416
|
+
if ".lora_" in key:
|
|
417
|
+
continue
|
|
418
|
+
tensor = sd[key].float() if sd[key].is_floating_point() else sd[key]
|
|
419
|
+
prefix = None
|
|
420
|
+
if key.endswith(".base_layer.weight"):
|
|
421
|
+
prefix = key[: -len(".base_layer.weight")]
|
|
422
|
+
if prefix is not None:
|
|
423
|
+
a, b = f"{prefix}.lora_A.default.weight", f"{prefix}.lora_B.default.weight"
|
|
424
|
+
ea, eb = f"{prefix}.lora_embedding_A.default", f"{prefix}.lora_embedding_B.default"
|
|
425
|
+
if a in sd and b in sd:
|
|
426
|
+
tensor = tensor + (sd[b].float() @ sd[a].float()) * scaling
|
|
427
|
+
elif ea in sd and eb in sd:
|
|
428
|
+
tensor = tensor + (sd[eb].float() @ sd[ea].float()).T * scaling
|
|
429
|
+
merged[_plain_key(key)] = tensor
|
|
430
|
+
return merged
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _load_merged(model: torch.nn.Module, state_dict: Dict[str, torch.Tensor], scaling: float) -> None:
|
|
434
|
+
"""Load a checkpoint with a foreign LoRA layout by merging it into base weights.
|
|
435
|
+
|
|
436
|
+
The checkpoint's LoRA deltas are folded into dense weights, which are copied
|
|
437
|
+
into the current model's base layers; the current model's ``lora_B``
|
|
438
|
+
matrices are zeroed so its adapters start as an exact no-op. Inference is
|
|
439
|
+
therefore identical to the original checkpoint.
|
|
440
|
+
"""
|
|
441
|
+
merged = merge_lora_state_dict(state_dict, scaling)
|
|
442
|
+
target = model.state_dict()
|
|
443
|
+
new_state, missing = {}, []
|
|
444
|
+
for key, value in target.items():
|
|
445
|
+
if ".lora_" in key:
|
|
446
|
+
if ".lora_B." in key or ".lora_embedding_B." in key:
|
|
447
|
+
new_state[key] = torch.zeros_like(value)
|
|
448
|
+
continue
|
|
449
|
+
src = merged.get(_plain_key(key))
|
|
450
|
+
if src is None:
|
|
451
|
+
if not key.endswith(_IGNORABLE_MISSING_SUFFIXES):
|
|
452
|
+
missing.append(key)
|
|
453
|
+
continue
|
|
454
|
+
if src.shape != value.shape:
|
|
455
|
+
raise RuntimeError(f"Shape mismatch for {key}: checkpoint {tuple(src.shape)} vs model {tuple(value.shape)}")
|
|
456
|
+
new_state[key] = src.to(value.dtype)
|
|
457
|
+
if missing:
|
|
458
|
+
raise RuntimeError(
|
|
459
|
+
"Checkpoint does not match the model architecture (even after merging LoRA).\n"
|
|
460
|
+
f" missing ({len(missing)}): {missing[:5]}\n"
|
|
461
|
+
"Check that hf_model / head settings match the checkpoint."
|
|
462
|
+
)
|
|
463
|
+
model.load_state_dict(new_state, strict=False)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def load_state_into_model(
|
|
467
|
+
model: torch.nn.Module,
|
|
468
|
+
state_dict: Dict[str, torch.Tensor],
|
|
469
|
+
lora_scaling: Optional[float] = None,
|
|
470
|
+
) -> str:
|
|
471
|
+
"""Load a 3Di state dict into ``model``.
|
|
472
|
+
|
|
473
|
+
Tries an exact load first. If the LoRA layout differs (checkpoints produced
|
|
474
|
+
by older PEFT versions sometimes wrapped extra modules) and ``lora_scaling``
|
|
475
|
+
is given, falls back to :func:`_load_merged`. Returns ``"exact"`` or
|
|
476
|
+
``"merged"``. Raises ``RuntimeError`` with a readable diff otherwise.
|
|
477
|
+
"""
|
|
478
|
+
result = model.load_state_dict(strip_module_prefix(state_dict), strict=False)
|
|
479
|
+
missing = [k for k in result.missing_keys if not k.endswith(_IGNORABLE_MISSING_SUFFIXES)]
|
|
480
|
+
if not missing and not result.unexpected_keys:
|
|
481
|
+
return "exact"
|
|
482
|
+
if lora_scaling is not None:
|
|
483
|
+
_load_merged(model, state_dict, lora_scaling)
|
|
484
|
+
print(" note: checkpoint LoRA layout differs from this PEFT version; "
|
|
485
|
+
"merged its adapters into the base weights (outputs are unchanged).")
|
|
486
|
+
return "merged"
|
|
487
|
+
raise RuntimeError(
|
|
488
|
+
"Checkpoint does not match the model architecture.\n"
|
|
489
|
+
f" missing ({len(missing)}): {missing[:5]}\n"
|
|
490
|
+
f" unexpected ({len(result.unexpected_keys)}): {result.unexpected_keys[:5]}\n"
|
|
491
|
+
"Check that hf_model / head / LoRA settings match the checkpoint."
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def lora_scaling_from_checkpoint(ckpt: dict) -> float:
|
|
496
|
+
args = ckpt["args"]
|
|
497
|
+
r = args.get("lora_r", 16)
|
|
498
|
+
alpha = args.get("lora_alpha", 32.0)
|
|
499
|
+
if args.get("use_rslora"):
|
|
500
|
+
return alpha / (r ** 0.5)
|
|
501
|
+
return alpha / r
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def load_3di_checkpoint(
|
|
505
|
+
path: str,
|
|
506
|
+
device: Optional[str] = None,
|
|
507
|
+
hf_revision: Optional[str] = None,
|
|
508
|
+
eval_mode: bool = True,
|
|
509
|
+
) -> Tuple["ESM3DiModel", dict]: # noqa: F821
|
|
510
|
+
"""Rebuild a trained 3Di tagger from a ``.pt`` checkpoint.
|
|
511
|
+
|
|
512
|
+
Returns ``(esm3di_model, checkpoint_dict)``. ``esm3di_model.model`` holds the
|
|
513
|
+
loaded ``nn.Module`` (already moved to ``device``) and
|
|
514
|
+
``esm3di_model.tokenizer`` the matching tokenizer.
|
|
515
|
+
"""
|
|
516
|
+
from .model import ESM3DiModel
|
|
517
|
+
|
|
518
|
+
ckpt = read_3di_checkpoint(path)
|
|
519
|
+
kwargs = model_kwargs_from_checkpoint(ckpt)
|
|
520
|
+
if hf_revision:
|
|
521
|
+
kwargs["revision"] = hf_revision
|
|
522
|
+
esm = ESM3DiModel(**kwargs)
|
|
523
|
+
load_state_into_model(esm.model, ckpt["model_state_dict"], lora_scaling=lora_scaling_from_checkpoint(ckpt))
|
|
524
|
+
if device is None:
|
|
525
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
526
|
+
esm.model.to(device)
|
|
527
|
+
if eval_mode:
|
|
528
|
+
esm.model.eval()
|
|
529
|
+
return esm, ckpt
|