esm3di 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. esm3di/ESM3di_model.py +4 -0
  2. esm3di/__init__.py +40 -0
  3. esm3di/_core/__init__.py +1 -0
  4. esm3di/_core/checkpoints.py +529 -0
  5. esm3di/_core/model.py +369 -0
  6. esm3di/checkpoints.py +4 -0
  7. esm3di/cli.py +35 -0
  8. esm3di/cutting_cli.py +9 -0
  9. esm3di/fastas2foldseekdb.py +90 -0
  10. esm3di/foldseek_db.py +6 -0
  11. esm3di/inference/__init__.py +15 -0
  12. esm3di/inference/__main__.py +3 -0
  13. esm3di/inference/checkpoint_predictor.py +335 -0
  14. esm3di/inference/cli.py +437 -0
  15. esm3di/inference/cutting.py +460 -0
  16. esm3di/inference/cutting_bundle.py +169 -0
  17. esm3di/inference/foldseek_db.py +66 -0
  18. esm3di/inference/io.py +237 -0
  19. esm3di/inference/model.py +18 -0
  20. esm3di/inference/phylogeny.py +87 -0
  21. esm3di/inference/predictor.py +372 -0
  22. esm3di/inference/preprocessing.py +42 -0
  23. esm3di/inference/windowing.py +81 -0
  24. esm3di/io.py +6 -0
  25. esm3di/model.py +4 -0
  26. esm3di/predict.py +9 -0
  27. esm3di/preprocessing.py +6 -0
  28. esm3di/workflow/Snakefile +48 -0
  29. esm3di/workflow/__init__.py +0 -0
  30. esm3di/workflow/config.yaml +14 -0
  31. esm3di/workflow/envs/foldtree.yaml +30 -0
  32. esm3di/workflow/envs/prostt5_env.yaml +7 -0
  33. esm3di/workflow/madroot/mad.py +678 -0
  34. esm3di/workflow/rules/database_ops.smk +77 -0
  35. esm3di/workflow/rules/homolog_grouping.smk +2 -0
  36. esm3di/workflow/rules/prostt5_db.smk +72 -0
  37. esm3di/workflow/rules/tree_building.smk +87 -0
  38. esm3di/workflow/scripts/3di2foldseekdb.py +139 -0
  39. esm3di/workflow/scripts/foldseek2tree.py +294 -0
  40. esm3di/workflow/scripts/foldseekres2distmat_simple.py +53 -0
  41. esm3di/workflow/scripts/postprocess.py +19 -0
  42. esm3di/workflow/scripts/process_madroot.py +17 -0
  43. esm3di/workflow/scripts/run_madroot.py +80 -0
  44. esm3di/workflow/scripts/tipnames.py +38 -0
  45. esm3di-0.2.0.dist-info/METADATA +385 -0
  46. esm3di-0.2.0.dist-info/RECORD +50 -0
  47. esm3di-0.2.0.dist-info/WHEEL +5 -0
  48. esm3di-0.2.0.dist-info/entry_points.txt +6 -0
  49. esm3di-0.2.0.dist-info/licenses/LICENSE +21 -0
  50. esm3di-0.2.0.dist-info/top_level.txt +1 -0
esm3di/ESM3di_model.py ADDED
@@ -0,0 +1,4 @@
1
+ """Compatibility alias for shared model architecture and training utilities."""
2
+ import sys
3
+ from ._core import model as _implementation
4
+ sys.modules[__name__] = _implementation
esm3di/__init__.py ADDED
@@ -0,0 +1,40 @@
1
+ """ESM3Di inference and shared ESM++ checkpoint runtime."""
2
+
3
+ import importlib
4
+ from pkgutil import extend_path
5
+
6
+ __path__ = extend_path(__path__, __name__)
7
+ __version__ = "0.2.0"
8
+
9
+ _LAZY = {
10
+ "ESM3DiModel": "_core.model",
11
+ "ESMWithCNNHead": "_core.model",
12
+ "CNNClassificationHead": "_core.model",
13
+ "read_fasta": "_core.model",
14
+ "discover_lora_target_modules": "_core.model",
15
+ "freeze_all_but_lora_and_classifier": "_core.model",
16
+ "load_3di_checkpoint": "_core.checkpoints",
17
+ "load_mlm_model": "_core.checkpoints",
18
+ "Predictor": "predict",
19
+ "predict_records": "predict",
20
+ "write_foldseek_db": "foldseek_db",
21
+ "fasta2foldseek": "foldseek_db",
22
+ }
23
+ _TRAINING = {
24
+ "FocalLoss": "losses", "CyclicalFocalLoss": "losses",
25
+ "PLDDTWeightedFocalLoss": "losses", "PLDDTWeightedCyclicalFocalLoss": "losses",
26
+ "GammaSchedulerOnPlateau": "losses", "DEFAULT_PLDDT_BIN_WEIGHTS": "losses",
27
+ "prepare": "prepare_data",
28
+ }
29
+ __all__ = sorted(_LAZY)
30
+
31
+
32
+ def __getattr__(name):
33
+ if name in _LAZY:
34
+ value = getattr(importlib.import_module(f".{_LAZY[name]}", __name__), name)
35
+ elif name in _TRAINING:
36
+ value = getattr(importlib.import_module(f"esm3di_training.{_TRAINING[name]}"), name)
37
+ else:
38
+ raise AttributeError(f"module 'esm3di' has no attribute {name!r}")
39
+ globals()[name] = value
40
+ return value
@@ -0,0 +1 @@
1
+ """Shared model architectures and checkpoint reconstruction."""
@@ -0,0 +1,529 @@
1
+ """Checkpoint utilities for the ESM++ MLM -> 3Di pipeline.
2
+
3
+ This module is the single place that knows how to
4
+
5
+ * pin a HuggingFace remote-code revision that works with the installed
6
+ ``transformers`` version (:func:`resolve_hf_revision`),
7
+ * read backbone weights from an MLM fine-tuning output directory and copy them
8
+ into a freshly loaded HF model *before* LoRA is applied
9
+ (:func:`load_mlm_state_dict`, :func:`transfer_backbone_weights`),
10
+ * classify an ``--init-from`` argument (:func:`detect_init_source`), and
11
+ * rebuild a trained 3Di tagger from a ``.pt`` checkpoint
12
+ (:func:`load_3di_checkpoint`).
13
+
14
+ Checkpoint formats
15
+ ------------------
16
+ MLM checkpoint (output of ``esm3di mlm``)
17
+ A HuggingFace model directory: ``config.json``, ``model.safetensors`` (or
18
+ ``pytorch_model.bin``) and tokenizer files. Keys look like
19
+ ``embed.weight``, ``transformer.blocks.N...``, ``sequence_head...``.
20
+ Directories containing only a PEFT adapter (``adapter_config.json``) are
21
+ also accepted; the adapter is merged into its base model on load.
22
+
23
+ 3Di checkpoint (output of ``esm3di train``)
24
+ A ``torch.save`` dict with at least ``model_state_dict``, ``label_vocab``,
25
+ ``mask_label_chars``, ``args``, ``epoch`` and ``optimizer_state_dict``.
26
+ ``model_state_dict`` holds the *complete* model (backbone, LoRA adapters
27
+ and head), so only the architecture (``args['hf_model']``) is needed to
28
+ rebuild it.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import json
34
+ import os
35
+ from dataclasses import dataclass
36
+ from typing import Dict, List, Optional, Tuple
37
+
38
+ import torch
39
+
40
+ # Remote-code revisions of the Synthyra ESM++ repos known to load with
41
+ # transformers 4.48.x. Newer revisions on the hub import ``fastplms`` and
42
+ # ``transformers.AttentionInterface`` which do not exist in 4.48.
43
+ KNOWN_HF_REVISIONS: Dict[str, str] = {
44
+ "Synthyra/ESMplusplus_small": "056ceb518bbcf1a69f18d2504e9701513684b846",
45
+ "Synthyra/ESMplusplus_large": "36ef85d43b09001f8006774ff35ae1fa62f06a66",
46
+ }
47
+
48
+ DEFAULT_HF_MODEL = "Synthyra/ESMplusplus_small"
49
+
50
+ # FoldSeek's 3Di alphabet uses the 20 amino-acid letters.
51
+ THREEDI_VOCAB: List[str] = list("ACDEFGHIKLMNPQRSTVWY")
52
+
53
+ _HEAD_PREFIXES = ("sequence_head", "lm_head", "classifier", "score")
54
+
55
+
56
+ def resolve_hf_revision(hf_model: str, revision: Optional[str] = None) -> Optional[str]:
57
+ """Return the HF revision to use for ``hf_model``.
58
+
59
+ Priority: explicit ``revision`` > ``ESM3DI_HF_REVISION`` env var > pinned
60
+ known-good revision > ``None`` (hub default). Pass ``revision="main"`` to
61
+ deliberately use the latest hub code.
62
+ """
63
+ if revision:
64
+ return revision
65
+ env_rev = os.environ.get("ESM3DI_HF_REVISION")
66
+ if env_rev:
67
+ return env_rev
68
+ if os.path.isdir(hf_model):
69
+ return None
70
+ return KNOWN_HF_REVISIONS.get(hf_model)
71
+
72
+
73
+ # ---------------------------------------------------------------------------
74
+ # init-source detection
75
+ # ---------------------------------------------------------------------------
76
+
77
+ def is_3di_checkpoint(path: str) -> bool:
78
+ return os.path.isfile(path) and path.endswith((".pt", ".pth"))
79
+
80
+
81
+ def is_mlm_checkpoint_dir(path: str) -> bool:
82
+ if not os.path.isdir(path):
83
+ return False
84
+ names = set(os.listdir(path))
85
+ return "config.json" in names or "adapter_config.json" in names
86
+
87
+
88
+ def detect_init_source(init_from: Optional[str]) -> str:
89
+ """Classify ``--init-from``: ``'base'``, ``'mlm'`` or ``'3di'``."""
90
+ if not init_from:
91
+ return "base"
92
+ if is_3di_checkpoint(init_from):
93
+ return "3di"
94
+ if os.path.isdir(init_from):
95
+ if is_mlm_checkpoint_dir(init_from):
96
+ return "mlm"
97
+ raise ValueError(
98
+ f"--init-from directory {init_from!r} does not look like an MLM checkpoint "
99
+ "(expected config.json + model.safetensors, or a PEFT adapter). If this is an "
100
+ "MLM output directory, point at one of its sub-directories, e.g. "
101
+ f"{os.path.join(init_from, 'best')} or {os.path.join(init_from, 'final')}."
102
+ )
103
+ if os.path.exists(init_from):
104
+ raise ValueError(f"Unrecognised --init-from path: {init_from!r}")
105
+ # Anything else is treated as a HuggingFace hub id.
106
+ return "base"
107
+
108
+
109
+ def mlm_base_model_name(mlm_dir: str) -> Optional[str]:
110
+ """Best-effort lookup of the HF base model an MLM directory was trained from."""
111
+ meta_path = os.path.join(mlm_dir, "esm3di_mlm.json")
112
+ if os.path.isfile(meta_path):
113
+ with open(meta_path) as fh:
114
+ meta = json.load(fh)
115
+ if meta.get("base_model"):
116
+ return meta["base_model"]
117
+ for fname in ("adapter_config.json", "config.json"):
118
+ fpath = os.path.join(mlm_dir, fname)
119
+ if os.path.isfile(fpath):
120
+ with open(fpath) as fh:
121
+ cfg = json.load(fh)
122
+ name = cfg.get("base_model_name_or_path") or cfg.get("_name_or_path")
123
+ if name and not os.path.isdir(name):
124
+ return name
125
+ return None
126
+
127
+
128
+ # ---------------------------------------------------------------------------
129
+ # MLM -> backbone weight transfer
130
+ # ---------------------------------------------------------------------------
131
+
132
+ def _read_weight_file(path: str) -> Dict[str, torch.Tensor]:
133
+ if path.endswith(".safetensors"):
134
+ from safetensors.torch import load_file
135
+
136
+ return load_file(path, device="cpu")
137
+ obj = torch.load(path, map_location="cpu")
138
+ if isinstance(obj, dict) and "model_state_dict" in obj:
139
+ obj = obj["model_state_dict"]
140
+ return obj
141
+
142
+
143
+ def _find_weight_file(directory: str) -> Optional[str]:
144
+ for fname in ("model.safetensors", "pytorch_model.bin", "model.bin"):
145
+ fpath = os.path.join(directory, fname)
146
+ if os.path.isfile(fpath):
147
+ return fpath
148
+ return None
149
+
150
+
151
+ def _strip_known_prefixes(key: str) -> str:
152
+ for prefix in ("module.", "base_model.model.", "model."):
153
+ if key.startswith(prefix):
154
+ key = key[len(prefix):]
155
+ return key.replace(".base_layer.", ".")
156
+
157
+
158
+ def load_mlm_state_dict(mlm_dir: str, revision: Optional[str] = None) -> Dict[str, torch.Tensor]:
159
+ """Return the full (merged) backbone state dict stored in an MLM directory.
160
+
161
+ Handles full-weight HF directories and PEFT adapter directories.
162
+ """
163
+ if os.path.isfile(os.path.join(mlm_dir, "adapter_config.json")) and _find_weight_file(mlm_dir) is None:
164
+ from peft import PeftModel
165
+ from transformers import AutoModelForMaskedLM
166
+
167
+ with open(os.path.join(mlm_dir, "adapter_config.json")) as fh:
168
+ base_name = json.load(fh)["base_model_name_or_path"]
169
+ base = AutoModelForMaskedLM.from_pretrained(
170
+ base_name, trust_remote_code=True, revision=resolve_hf_revision(base_name, revision)
171
+ )
172
+ merged = PeftModel.from_pretrained(base, mlm_dir).merge_and_unload()
173
+ return {k: v.detach().cpu() for k, v in merged.state_dict().items()}
174
+
175
+ weight_file = _find_weight_file(mlm_dir)
176
+ if weight_file is None:
177
+ raise FileNotFoundError(
178
+ f"No model weights (model.safetensors / pytorch_model.bin) found in {mlm_dir!r}. "
179
+ "Only config/tokenizer files are present - the weights may have been deleted."
180
+ )
181
+ raw = _read_weight_file(weight_file)
182
+ state = {}
183
+ for key, value in raw.items():
184
+ if "lora_" in key:
185
+ raise ValueError(
186
+ f"{weight_file} contains unmerged LoRA weights ({key}); export a merged model first."
187
+ )
188
+ state[_strip_known_prefixes(key)] = value
189
+ return state
190
+
191
+
192
+ @dataclass
193
+ class TransferReport:
194
+ copied: int
195
+ partially_copied: List[str]
196
+ missing: List[str]
197
+ skipped_heads: int
198
+
199
+ def summary(self) -> str:
200
+ msg = f"copied {self.copied} backbone tensors"
201
+ if self.partially_copied:
202
+ msg += f", row-sliced {len(self.partially_copied)} ({', '.join(self.partially_copied)})"
203
+ if self.missing:
204
+ msg += f", {len(self.missing)} backbone tensors NOT found in MLM checkpoint"
205
+ return msg
206
+
207
+
208
+ def transfer_backbone_weights(
209
+ target_model: torch.nn.Module,
210
+ source_state: Dict[str, torch.Tensor],
211
+ min_fraction: float = 0.95,
212
+ include_heads: bool = False,
213
+ ) -> TransferReport:
214
+ """Copy backbone weights from ``source_state`` into ``target_model`` in place.
215
+
216
+ ``target_model`` must be the *plain* HF model (before LoRA wrapping). Head
217
+ weights (``sequence_head``/``classifier``) are skipped. Embedding matrices
218
+ whose vocabulary was resized during MLM training (e.g. 64 -> 33 rows) are
219
+ copied row-wise for the overlapping token ids.
220
+
221
+ Raises ``RuntimeError`` if fewer than ``min_fraction`` of the target's
222
+ backbone tensors could be matched - this guards against silently training
223
+ from the stock weights.
224
+ """
225
+ target_state = target_model.state_dict()
226
+ backbone_keys = [k for k in target_state if include_heads or not k.startswith(_HEAD_PREFIXES)]
227
+ copied = 0
228
+ partial: List[str] = []
229
+ missing: List[str] = []
230
+ skipped_heads = 0 if include_heads else sum(1 for k in source_state if k.startswith(_HEAD_PREFIXES))
231
+
232
+ with torch.no_grad():
233
+ for key in backbone_keys:
234
+ src = source_state.get(key)
235
+ if src is None:
236
+ missing.append(key)
237
+ continue
238
+ dst = target_state[key]
239
+ if src.shape == dst.shape:
240
+ dst.copy_(src.to(dst.dtype))
241
+ copied += 1
242
+ elif (
243
+ src.dim() == dst.dim() >= 1
244
+ and src.shape[1:] == dst.shape[1:]
245
+ and src.shape[0] != dst.shape[0]
246
+ ):
247
+ rows = min(src.shape[0], dst.shape[0])
248
+ dst[:rows].copy_(src[:rows].to(dst.dtype))
249
+ copied += 1
250
+ partial.append(key)
251
+ else:
252
+ missing.append(key)
253
+
254
+ if backbone_keys and copied / len(backbone_keys) < min_fraction:
255
+ raise RuntimeError(
256
+ f"Only {copied}/{len(backbone_keys)} backbone tensors matched the MLM checkpoint. "
257
+ f"Is the MLM checkpoint from the same base architecture? Missing e.g.: {missing[:5]}"
258
+ )
259
+ return TransferReport(copied=copied, partially_copied=partial, missing=missing, skipped_heads=skipped_heads)
260
+
261
+
262
+ def load_mlm_model(
263
+ source: str,
264
+ revision: Optional[str] = None,
265
+ base_model: Optional[str] = None,
266
+ ):
267
+ """Load an ESM++ masked-LM model from a hub id or an MLM checkpoint directory.
268
+
269
+ For directories the architecture/code is taken from the *pinned* base model
270
+ revision and the weights (including the LM head) are copied in. This avoids
271
+ the ``auto_map`` in saved ``config.json`` files pulling incompatible
272
+ remote code from the hub's latest revision.
273
+ """
274
+ from transformers import AutoModelForMaskedLM
275
+
276
+ if os.path.isdir(source):
277
+ base = base_model or mlm_base_model_name(source)
278
+ if base is None:
279
+ raise ValueError(f"Cannot determine the base model of {source!r}; pass base_model=")
280
+ if not os.path.isdir(base) and not base.startswith("Synthyra/ESMplusplus_"):
281
+ raise ValueError(f"Only ESM++ MLM backbones are supported, not {base!r}.")
282
+ model = AutoModelForMaskedLM.from_pretrained(
283
+ base, trust_remote_code=True, revision=resolve_hf_revision(base, revision)
284
+ )
285
+ report = transfer_backbone_weights(model, load_mlm_state_dict(source, revision), include_heads=True)
286
+ return model, base, report
287
+ if not source.startswith("Synthyra/ESMplusplus_"):
288
+ raise ValueError(f"Only ESM++ MLM backbones are supported, not {source!r}.")
289
+ model = AutoModelForMaskedLM.from_pretrained(
290
+ source, trust_remote_code=True, revision=resolve_hf_revision(source, revision)
291
+ )
292
+ return model, source, None
293
+
294
+
295
+ # ---------------------------------------------------------------------------
296
+ # Tokenizer
297
+ # ---------------------------------------------------------------------------
298
+
299
+ def get_tokenizer(model=None, hf_model: Optional[str] = None, revision: Optional[str] = None):
300
+ """Return a tokenizer for an ESM++ model.
301
+
302
+ ESM++ exposes its tokenizer as ``model.tokenizer`` (``AutoTokenizer`` does not
303
+ know ``EsmSequenceTokenizer``), so prefer that.
304
+ """
305
+ seen = set()
306
+ current = model
307
+ while current is not None and id(current) not in seen:
308
+ seen.add(id(current))
309
+ tok = getattr(current, "tokenizer", None)
310
+ if tok is not None and not isinstance(tok, torch.nn.Module):
311
+ return tok
312
+ current = getattr(current, "base_model", None) or getattr(current, "model", None)
313
+ if hf_model is None:
314
+ raise ValueError("Cannot determine tokenizer: provide hf_model")
315
+ from transformers import AutoTokenizer, PreTrainedTokenizerFast
316
+
317
+ if os.path.isdir(hf_model) and os.path.isfile(os.path.join(hf_model, "tokenizer.json")):
318
+ try:
319
+ return AutoTokenizer.from_pretrained(hf_model, trust_remote_code=True)
320
+ except Exception:
321
+ return PreTrainedTokenizerFast.from_pretrained(hf_model)
322
+ return AutoTokenizer.from_pretrained(
323
+ hf_model, trust_remote_code=True, revision=resolve_hf_revision(hf_model, revision)
324
+ )
325
+
326
+
327
+ # ---------------------------------------------------------------------------
328
+ # 3Di checkpoint loading
329
+ # ---------------------------------------------------------------------------
330
+
331
+ def strip_module_prefix(state_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
332
+ """Remove a ``module.`` prefix added by ``torch.nn.DataParallel``."""
333
+ if any(k.startswith("module.") for k in state_dict):
334
+ return {k[len("module."):] if k.startswith("module.") else k: v for k, v in state_dict.items()}
335
+ return state_dict
336
+
337
+
338
+ def read_3di_checkpoint(path: str, map_location="cpu") -> dict:
339
+ ckpt = torch.load(path, map_location=map_location, weights_only=False)
340
+ if not isinstance(ckpt, dict) or "model_state_dict" not in ckpt:
341
+ raise ValueError(f"{path!r} is not an ESM3Di checkpoint (no 'model_state_dict').")
342
+ ckpt["model_state_dict"] = strip_module_prefix(ckpt["model_state_dict"])
343
+ args = ckpt.get("args") or {}
344
+ if not isinstance(args, dict):
345
+ args = vars(args)
346
+ ckpt["args"] = args
347
+ return ckpt
348
+
349
+
350
+ _UNSUPPORTED_LEGACY_FLAGS = (
351
+ "use_iterative_backbone_head", "use_iterative_transformer_head",
352
+ "use_transformer_head", "use_plddt_prediction_head",
353
+ )
354
+
355
+
356
+ def model_kwargs_from_checkpoint(ckpt: dict) -> dict:
357
+ """Translate a checkpoint's saved ``args`` into ``ESM3DiModel`` kwargs."""
358
+ args = ckpt["args"]
359
+ for flag in _UNSUPPORTED_LEGACY_FLAGS:
360
+ if args.get(flag):
361
+ raise ValueError(
362
+ f"Checkpoint was trained with {flag}=True, which is experimental. "
363
+ "Its implementation is archived under experiments/legacy_runtime."
364
+ )
365
+ hf_model = args.get("hf_model_name") or args.get("hf_model") or DEFAULT_HF_MODEL
366
+ if not os.path.isdir(hf_model) and not hf_model.startswith("Synthyra/ESMplusplus_"):
367
+ raise ValueError(f"Only ESM++ checkpoints are supported, not {hf_model!r}.")
368
+ if ckpt.get("aux_track_num_bins") or args.get("aux_track_num_bins"):
369
+ raise ValueError("Auxiliary-track checkpoints are archived and unsupported")
370
+
371
+ target_modules = ckpt.get("lora_target_modules") or args.get("lora_target_modules")
372
+ if isinstance(target_modules, str):
373
+ target_modules = [m.strip() for m in target_modules.split(",") if m.strip()]
374
+
375
+ kwargs = dict(
376
+ hf_model_name=hf_model,
377
+ num_labels=len(ckpt["label_vocab"]),
378
+ lora_r=args.get("lora_r", 16),
379
+ lora_alpha=args.get("lora_alpha", 32.0),
380
+ lora_dropout=args.get("lora_dropout", 0.01),
381
+ target_modules=target_modules or None,
382
+ use_cnn_head=args.get("use_cnn_head", False),
383
+ cnn_num_layers=args.get("cnn_num_layers", 2),
384
+ cnn_kernel_size=args.get("cnn_kernel_size", 3),
385
+ cnn_dropout=args.get("cnn_dropout", 0.1),
386
+ revision=args.get("hf_revision") or args.get("revision"),
387
+ )
388
+ return kwargs
389
+
390
+
391
+ # Parameters that some transformers versions create but never use (ESM-2 with
392
+ # rotary embeddings); older checkpoints legitimately lack them.
393
+ _IGNORABLE_MISSING_SUFFIXES = ("embeddings.position_embeddings.weight",)
394
+
395
+
396
+ def _plain_key(key: str) -> str:
397
+ """Map a PEFT-wrapped parameter name to the underlying dense parameter name."""
398
+ key = key.replace(".base_layer.", ".")
399
+ key = key.replace(".modules_to_save.default.", ".")
400
+ key = key.replace(".original_module.", ".")
401
+ return key
402
+
403
+
404
+ def merge_lora_state_dict(state_dict: Dict[str, torch.Tensor], scaling: float) -> Dict[str, torch.Tensor]:
405
+ """Fold every LoRA delta in ``state_dict`` into dense weights.
406
+
407
+ Returns a dict keyed by :func:`_plain_key` names without any ``lora_*``
408
+ entries. For PEFT ``modules_to_save`` wrappers the trained copy
409
+ (``modules_to_save.default``) wins over ``original_module``.
410
+ """
411
+ sd = strip_module_prefix(state_dict)
412
+ merged: Dict[str, torch.Tensor] = {}
413
+ # original_module first so modules_to_save overwrites it
414
+ order = sorted(sd, key=lambda k: 0 if ".original_module." in k else 1)
415
+ for key in order:
416
+ if ".lora_" in key:
417
+ continue
418
+ tensor = sd[key].float() if sd[key].is_floating_point() else sd[key]
419
+ prefix = None
420
+ if key.endswith(".base_layer.weight"):
421
+ prefix = key[: -len(".base_layer.weight")]
422
+ if prefix is not None:
423
+ a, b = f"{prefix}.lora_A.default.weight", f"{prefix}.lora_B.default.weight"
424
+ ea, eb = f"{prefix}.lora_embedding_A.default", f"{prefix}.lora_embedding_B.default"
425
+ if a in sd and b in sd:
426
+ tensor = tensor + (sd[b].float() @ sd[a].float()) * scaling
427
+ elif ea in sd and eb in sd:
428
+ tensor = tensor + (sd[eb].float() @ sd[ea].float()).T * scaling
429
+ merged[_plain_key(key)] = tensor
430
+ return merged
431
+
432
+
433
+ def _load_merged(model: torch.nn.Module, state_dict: Dict[str, torch.Tensor], scaling: float) -> None:
434
+ """Load a checkpoint with a foreign LoRA layout by merging it into base weights.
435
+
436
+ The checkpoint's LoRA deltas are folded into dense weights, which are copied
437
+ into the current model's base layers; the current model's ``lora_B``
438
+ matrices are zeroed so its adapters start as an exact no-op. Inference is
439
+ therefore identical to the original checkpoint.
440
+ """
441
+ merged = merge_lora_state_dict(state_dict, scaling)
442
+ target = model.state_dict()
443
+ new_state, missing = {}, []
444
+ for key, value in target.items():
445
+ if ".lora_" in key:
446
+ if ".lora_B." in key or ".lora_embedding_B." in key:
447
+ new_state[key] = torch.zeros_like(value)
448
+ continue
449
+ src = merged.get(_plain_key(key))
450
+ if src is None:
451
+ if not key.endswith(_IGNORABLE_MISSING_SUFFIXES):
452
+ missing.append(key)
453
+ continue
454
+ if src.shape != value.shape:
455
+ raise RuntimeError(f"Shape mismatch for {key}: checkpoint {tuple(src.shape)} vs model {tuple(value.shape)}")
456
+ new_state[key] = src.to(value.dtype)
457
+ if missing:
458
+ raise RuntimeError(
459
+ "Checkpoint does not match the model architecture (even after merging LoRA).\n"
460
+ f" missing ({len(missing)}): {missing[:5]}\n"
461
+ "Check that hf_model / head settings match the checkpoint."
462
+ )
463
+ model.load_state_dict(new_state, strict=False)
464
+
465
+
466
+ def load_state_into_model(
467
+ model: torch.nn.Module,
468
+ state_dict: Dict[str, torch.Tensor],
469
+ lora_scaling: Optional[float] = None,
470
+ ) -> str:
471
+ """Load a 3Di state dict into ``model``.
472
+
473
+ Tries an exact load first. If the LoRA layout differs (checkpoints produced
474
+ by older PEFT versions sometimes wrapped extra modules) and ``lora_scaling``
475
+ is given, falls back to :func:`_load_merged`. Returns ``"exact"`` or
476
+ ``"merged"``. Raises ``RuntimeError`` with a readable diff otherwise.
477
+ """
478
+ result = model.load_state_dict(strip_module_prefix(state_dict), strict=False)
479
+ missing = [k for k in result.missing_keys if not k.endswith(_IGNORABLE_MISSING_SUFFIXES)]
480
+ if not missing and not result.unexpected_keys:
481
+ return "exact"
482
+ if lora_scaling is not None:
483
+ _load_merged(model, state_dict, lora_scaling)
484
+ print(" note: checkpoint LoRA layout differs from this PEFT version; "
485
+ "merged its adapters into the base weights (outputs are unchanged).")
486
+ return "merged"
487
+ raise RuntimeError(
488
+ "Checkpoint does not match the model architecture.\n"
489
+ f" missing ({len(missing)}): {missing[:5]}\n"
490
+ f" unexpected ({len(result.unexpected_keys)}): {result.unexpected_keys[:5]}\n"
491
+ "Check that hf_model / head / LoRA settings match the checkpoint."
492
+ )
493
+
494
+
495
+ def lora_scaling_from_checkpoint(ckpt: dict) -> float:
496
+ args = ckpt["args"]
497
+ r = args.get("lora_r", 16)
498
+ alpha = args.get("lora_alpha", 32.0)
499
+ if args.get("use_rslora"):
500
+ return alpha / (r ** 0.5)
501
+ return alpha / r
502
+
503
+
504
+ def load_3di_checkpoint(
505
+ path: str,
506
+ device: Optional[str] = None,
507
+ hf_revision: Optional[str] = None,
508
+ eval_mode: bool = True,
509
+ ) -> Tuple["ESM3DiModel", dict]: # noqa: F821
510
+ """Rebuild a trained 3Di tagger from a ``.pt`` checkpoint.
511
+
512
+ Returns ``(esm3di_model, checkpoint_dict)``. ``esm3di_model.model`` holds the
513
+ loaded ``nn.Module`` (already moved to ``device``) and
514
+ ``esm3di_model.tokenizer`` the matching tokenizer.
515
+ """
516
+ from .model import ESM3DiModel
517
+
518
+ ckpt = read_3di_checkpoint(path)
519
+ kwargs = model_kwargs_from_checkpoint(ckpt)
520
+ if hf_revision:
521
+ kwargs["revision"] = hf_revision
522
+ esm = ESM3DiModel(**kwargs)
523
+ load_state_into_model(esm.model, ckpt["model_state_dict"], lora_scaling=lora_scaling_from_checkpoint(ckpt))
524
+ if device is None:
525
+ device = "cuda" if torch.cuda.is_available() else "cpu"
526
+ esm.model.to(device)
527
+ if eval_mode:
528
+ esm.model.eval()
529
+ return esm, ckpt