dualmapref 0.2.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. args_parse.py +593 -0
  2. candidate_corrector.py +161 -0
  3. candidate_scorer.py +359 -0
  4. conformend/__init__.py +36 -0
  5. conformend/refiner.py +25 -0
  6. dataset/__init__.py +1 -0
  7. dataset/drugdataset.py +644 -0
  8. dataset/manifold.py +237 -0
  9. dataset/sharded_drugdataset.py +124 -0
  10. dataset/smiles2graph.py +217 -0
  11. dualmapref/__init__.py +24 -0
  12. dualmapref/refiner.py +16 -0
  13. dualmapref-0.2.8.dist-info/LICENSE +21 -0
  14. dualmapref-0.2.8.dist-info/METADATA +167 -0
  15. dualmapref-0.2.8.dist-info/RECORD +186 -0
  16. dualmapref-0.2.8.dist-info/WHEEL +5 -0
  17. dualmapref-0.2.8.dist-info/entry_points.txt +10 -0
  18. dualmapref-0.2.8.dist-info/top_level.txt +29 -0
  19. exputils.py +289 -0
  20. infer.py +23 -0
  21. infer_base_conformers.py +5589 -0
  22. infer_mean_dispersion_torsion.py +1117 -0
  23. infer_torsion.py +752 -0
  24. infer_torsional_denoiser.py +1021 -0
  25. manimol/__init__.py +30 -0
  26. manimol/__main__.py +6 -0
  27. manimol/cli.py +36 -0
  28. manimol/full_cli.py +41 -0
  29. manimol/legacy_imports.py +43 -0
  30. manimol/manifest.py +56 -0
  31. manimol/postprocess.py +148 -0
  32. manimol/priors.py +60 -0
  33. manimol/protocols.py +119 -0
  34. manimol/refiner.py +645 -0
  35. manimol/runtime.py +194 -0
  36. manimol/selection.py +85 -0
  37. manimol/torsion.py +22 -0
  38. manimol/train_cli.py +14 -0
  39. mixture_flow/src/__init__.py +1 -0
  40. mixture_flow/src/_path.py +7 -0
  41. mixture_flow/src/infer28_mixture_torsion_flow.py +1093 -0
  42. mixture_flow/src/mixture_torsion_flow.py +291 -0
  43. mixture_flow/src/summarize_mixture_debug.py +80 -0
  44. mixture_flow/src/train28_mixture_torsion_flow.py +583 -0
  45. mixture_torsion_flow.py +7 -0
  46. models/ manifold_learning.py +86 -0
  47. models/__init__.py +8 -0
  48. models/dist.py +273 -0
  49. models/dist2coords.py +23 -0
  50. models/egnn.py +107 -0
  51. models/experimental_backbones.py +364 -0
  52. models/gnnconv.py +512 -0
  53. models/kernel_inversion.py +141 -0
  54. models/kernels.py +427 -0
  55. models/losses.py +154 -0
  56. models/mean_dispersion_bridge.py +119 -0
  57. models/mean_dispersion_torsion_context.py +155 -0
  58. models/model.py +73 -0
  59. models/model0.py +2375 -0
  60. models/vis.py +128 -0
  61. pgraph_controls.py +107 -0
  62. scripts/__init__.py +2 -0
  63. scripts/analyze_basin_alignment.py +63 -0
  64. scripts/analyze_hif2a_conformer_aggregation.py +267 -0
  65. scripts/analyze_hif2a_contact_pharmacophore.py +1111 -0
  66. scripts/analyze_hif2a_fixed_budget.py +202 -0
  67. scripts/analyze_hif2a_pharmacophore.py +405 -0
  68. scripts/analyze_hif2a_pharmacophore_match_metric.py +386 -0
  69. scripts/analyze_p_control_candidate_flow.py +154 -0
  70. scripts/analyze_scaffold_basin.py +83 -0
  71. scripts/audit_dispersion_proposals.py +208 -0
  72. scripts/audit_etflow_protocol.py +258 -0
  73. scripts/audit_external_overlap.py +150 -0
  74. scripts/audit_oldbest_fulltrainval_split.py +82 -0
  75. scripts/audit_pdbbind_contact_pharmacophores.py +146 -0
  76. scripts/audit_qm9_geomol_test_split.py +94 -0
  77. scripts/audit_stage1_stage2_cross.py +131 -0
  78. scripts/build_geom_drugs_lowref_diagnostic.py +84 -0
  79. scripts/build_qm9_backfill_manifest.py +99 -0
  80. scripts/check_etflow_runtime.py +56 -0
  81. scripts/compare_prior_sources.py +122 -0
  82. scripts/compose_binding_mode_figure.py +87 -0
  83. scripts/compose_hif2a_contact_pharmacophore_suppfig.py +255 -0
  84. scripts/compose_hif2a_functional_geometry_pipeline.py +266 -0
  85. scripts/compose_manimol_internal_architecture.py +291 -0
  86. scripts/diagnose_flow_basin_intervention.py +54 -0
  87. scripts/diagnose_hif2a_pharmacophore_budget.py +185 -0
  88. scripts/diagnose_route2_p_ranking.py +123 -0
  89. scripts/discover_binding_mode_inputs.py +30 -0
  90. scripts/evaluate_functional_prior_stage1.py +230 -0
  91. scripts/evaluate_geomanimol.py +74 -0
  92. scripts/evaluate_geomanimol_flow.py +23 -0
  93. scripts/evaluate_geomanimol_multiscaffold.py +46 -0
  94. scripts/evaluate_geomanimol_residual.py +45 -0
  95. scripts/evaluate_geomanimol_torsion.py +40 -0
  96. scripts/evaluate_hif2a_bioactive_enrichment.py +91 -0
  97. scripts/evaluate_hif2a_pms_suite.py +440 -0
  98. scripts/evaluate_pdbbind_contact_pms.py +155 -0
  99. scripts/evaluate_platinum_external.py +114 -0
  100. scripts/evaluate_platinum_pool.py +82 -0
  101. scripts/evaluate_route2_calibrated_p.py +104 -0
  102. scripts/evaluate_route2_library_components.py +596 -0
  103. scripts/evaluate_route2_p_basin_budget.py +288 -0
  104. scripts/evaluate_stage1_checkpoint_only.py +30 -0
  105. scripts/export_dispersion_bond_audit.py +136 -0
  106. scripts/export_top20_shards.py +207 -0
  107. scripts/external_validation_metrics.py +152 -0
  108. scripts/fit_topology_prior.py +117 -0
  109. scripts/frozen_dispersion_audit.py +281 -0
  110. scripts/geomanimol_debug.py +50 -0
  111. scripts/geomanimol_smoke.py +37 -0
  112. scripts/infer_etflow_matched.py +524 -0
  113. scripts/inspect_checkpoint.py +12 -0
  114. scripts/inspect_conformer_capacity.py +71 -0
  115. scripts/inspect_pool.py +12 -0
  116. scripts/make_figure3_selection_loss.py +474 -0
  117. scripts/make_figure3_selection_summary.py +275 -0
  118. scripts/make_geom_debug_subset.py +62 -0
  119. scripts/make_low_reference_test_subset.py +70 -0
  120. scripts/merge_stage1_into_stage2.py +82 -0
  121. scripts/place_hif2a_candidates_for_evaluation.py +66 -0
  122. scripts/plot_hif2a_low_budget_efficiency.py +100 -0
  123. scripts/plot_prior_validation.py +48 -0
  124. scripts/precompute_stage2_assignments.py +90 -0
  125. scripts/prepare_bace_activity.py +98 -0
  126. scripts/prepare_bace_etkdg_benchmark.py +90 -0
  127. scripts/prepare_figure5_4x4_cases.py +470 -0
  128. scripts/prepare_geom_drugs_geomol_top20.py +472 -0
  129. scripts/prepare_pdbbind_core_raw.py +90 -0
  130. scripts/prepare_platinum_raw.py +74 -0
  131. scripts/prepare_pubchem_etkdg.py +54 -0
  132. scripts/rebuild_hif2a_final_sdf_and_metrics.py +55 -0
  133. scripts/rebuild_hif2a_fixed_pool_sweep.py +175 -0
  134. scripts/rebuild_hif2a_strict.py +220 -0
  135. scripts/recompute_hif2a_bootstrap.py +63 -0
  136. scripts/reference_free_geometry.py +112 -0
  137. scripts/render_binding_modes.py +413 -0
  138. scripts/render_figure5_4x4.py +330 -0
  139. scripts/render_figure6_pharmacophore_pymol.py +162 -0
  140. scripts/render_hif2a_contact_pharmacophore_pymol.py +414 -0
  141. scripts/render_hif2a_figure11.py +397 -0
  142. scripts/run_bace_2d_baseline.py +73 -0
  143. scripts/run_drugs_validation_matrix.py +225 -0
  144. scripts/run_external_reference_free.py +53 -0
  145. scripts/run_manimol_finalk_sensitivity.py +106 -0
  146. scripts/run_manimol_oversampling_sensitivity.py +109 -0
  147. scripts/run_pdisp_matched_intervention.py +116 -0
  148. scripts/run_stage1_stage2_cross20.py +83 -0
  149. scripts/score_hif2a_candidates_with_pgraph.py +221 -0
  150. scripts/summarize_dispersion_intervention.py +88 -0
  151. scripts/test_mean_dispersion_bridge.py +50 -0
  152. scripts/train_dispersion_head_diagnostic.py +320 -0
  153. scripts/train_dispersion_head_streaming.py +519 -0
  154. scripts/train_geomanimol.py +100 -0
  155. scripts/train_geomanimol_flow.py +26 -0
  156. scripts/train_geomanimol_residual.py +70 -0
  157. scripts/train_geomanimol_torsion.py +50 -0
  158. scripts/train_quantile_representation_diagnostic.py +204 -0
  159. scripts/validate_mean_dispersion_prior.py +173 -0
  160. scripts/validate_packed_shard.py +41 -0
  161. scripts/validate_stage1_prior.py +221 -0
  162. scripts/validate_top20_shards.py +36 -0
  163. scripts/validation/__init__.py +1 -0
  164. scripts/validation/hif2a_downstream_baselines.py +105 -0
  165. scripts/validation/hif2a_pairwise_ranker.py +218 -0
  166. scripts/verify_binding_mode_figure.py +47 -0
  167. torsion_manifold.py +281 -0
  168. train.py +78 -0
  169. train_graph_encoder.py +206 -0
  170. train_local_torsion_corrector.py +511 -0
  171. train_mean_dispersion_torsion_prior.py +329 -0
  172. train_probability_basin.py +235 -0
  173. train_torsion_generator.py +1631 -0
  174. train_torsion_manifold_prior.py +258 -0
  175. train_torsional_denoiser.py +415 -0
  176. trainer.py +1038 -0
  177. utils/__init__.py +7 -0
  178. utils/checkpoint.py +40 -0
  179. utils/device.py +17 -0
  180. utils/helpers.py +98 -0
  181. utils/kabsch.py +29 -0
  182. utils/lookup_table.py +164 -0
  183. utils/metrics.py +50 -0
  184. utils/optuna.py +111 -0
  185. utils/save_mol.py +366 -0
  186. utils/util.py +36 -0
trainer.py ADDED
@@ -0,0 +1,1038 @@
1
+ # trainer.py
2
+ from __future__ import annotations
3
+ from datetime import datetime
4
+ import logging
5
+ import os
6
+ import random
7
+ import signal
8
+ from typing import Any, Optional, Tuple, Dict, List, Iterator
9
+
10
+ from rdkit import Chem
11
+ from rdkit.Geometry import Point3D
12
+ from torch import nn
13
+ from tqdm import tqdm
14
+ import torch
15
+ import torch.distributed as dist
16
+ import torch.nn.functional as F
17
+ from torch.optim.lr_scheduler import CosineAnnealingLR
18
+ from torch.utils.data import Sampler
19
+ from torch_geometric.loader import DataLoader
20
+ try:
21
+ from torch.utils.tensorboard import SummaryWriter
22
+ except ModuleNotFoundError:
23
+ class SummaryWriter: # type: ignore[no-redef]
24
+ """No-op fallback for inference environments without tensorboard."""
25
+
26
+ def __init__(self, *args, **kwargs):
27
+ pass
28
+
29
+ def add_scalar(self, *args, **kwargs):
30
+ pass
31
+
32
+ def add_histogram(self, *args, **kwargs):
33
+ pass
34
+
35
+ def flush(self):
36
+ pass
37
+
38
+ def close(self):
39
+ pass
40
+
41
+ from dataset import QM9Dataset
42
+ from dataset.sharded_drugdataset import ShardedDrugDataset
43
+ from models import GNNEncoder
44
+ from exputils import describe_model
45
+ from utils.device import get_device
46
+ from utils.checkpoint import CheckpointManager
47
+ from utils.helpers import to_runconfig_from_args, get_arg, safe_float, save_molecule_views
48
+ from utils.optuna import report_optuna_and_maybe_prune, write_final_metrics, handle_training_exception
49
+ from models.model0 import prob_low_dim, cross_entropy_loss, center_and_rescale, safe_kabsch_rmsd, heavy_atom_mask_from_x
50
+ from utils.kabsch import kabsch_rmsd
51
+
52
+ logger = logging.getLogger(__name__)
53
+
54
+
55
+ class MoleculeGroupedBatchSampler(Sampler[List[int]]):
56
+ def __init__(self, dataset, batch_size: int, shuffle: bool = True, drop_last: bool = False):
57
+ self.dataset = dataset
58
+ self.batch_size = int(batch_size)
59
+ self.shuffle = bool(shuffle)
60
+ self.drop_last = bool(drop_last)
61
+ self.groups_by_storage = getattr(dataset, "molecule_groups_by_shard", [])
62
+ self.groups = self._build_groups()
63
+
64
+ def _build_groups(self) -> List[List[int]]:
65
+ if self.groups_by_storage:
66
+ return [group for block in self.groups_by_storage for group in block]
67
+ mol_to_indices: Dict[int, List[int]] = {}
68
+ for local_idx in range(len(self.dataset)):
69
+ data = self.dataset[local_idx]
70
+ mol_id = int(data.mol_id.item()) if hasattr(data, "mol_id") else local_idx
71
+ if mol_id not in mol_to_indices:
72
+ mol_to_indices[mol_id] = []
73
+ mol_to_indices[mol_id].append(local_idx)
74
+ groups = list(mol_to_indices.values())
75
+ for group in groups:
76
+ group.sort(key=lambda idx: int(self.dataset[idx].conf_id.item()) if hasattr(self.dataset[idx], "conf_id") else idx)
77
+ return groups
78
+
79
+ def __iter__(self) -> Iterator[List[int]]:
80
+ if self.groups_by_storage:
81
+ blocks = [list(block) for block in self.groups_by_storage]
82
+ if self.shuffle:
83
+ random.shuffle(blocks)
84
+ for block in blocks:
85
+ random.shuffle(block)
86
+ groups = [group for block in blocks for group in block]
87
+ else:
88
+ groups = list(self.groups)
89
+ if self.shuffle:
90
+ random.shuffle(groups)
91
+ current_batch: List[int] = []
92
+ for group in groups:
93
+ if len(group) >= self.batch_size:
94
+ if len(current_batch) > 0:
95
+ yield current_batch
96
+ current_batch = []
97
+ yield list(group)
98
+ continue
99
+ if len(current_batch) + len(group) > self.batch_size:
100
+ if len(current_batch) > 0:
101
+ yield current_batch
102
+ current_batch = list(group)
103
+ else:
104
+ current_batch.extend(group)
105
+ if len(current_batch) > 0 and not self.drop_last:
106
+ yield current_batch
107
+
108
+ def __len__(self) -> int:
109
+ count = 0
110
+ current_size = 0
111
+ for group in self.groups:
112
+ gsize = len(group)
113
+ if gsize >= self.batch_size:
114
+ if current_size > 0:
115
+ count += 1
116
+ current_size = 0
117
+ count += 1
118
+ continue
119
+ if current_size + gsize > self.batch_size:
120
+ count += 1
121
+ current_size = gsize
122
+ else:
123
+ current_size += gsize
124
+ if current_size > 0 and not self.drop_last:
125
+ count += 1
126
+ return count
127
+
128
+
129
+ class DistributedMoleculeGroupedBatchSampler(MoleculeGroupedBatchSampler):
130
+ """Deterministically shard complete molecule batches across ranks."""
131
+
132
+ def __init__(self, dataset, batch_size: int, rank: int, world_size: int, seed: int = 0):
133
+ super().__init__(dataset, batch_size=batch_size, shuffle=False, drop_last=False)
134
+ self.rank = int(rank)
135
+ self.world_size = int(world_size)
136
+ self.seed = int(seed)
137
+ self.epoch = 0
138
+
139
+ def set_epoch(self, epoch: int) -> None:
140
+ self.epoch = int(epoch)
141
+
142
+ @staticmethod
143
+ def _pack_groups(groups: List[List[int]], batch_size: int) -> List[List[int]]:
144
+ batches: List[List[int]] = []
145
+ current: List[int] = []
146
+ for group in groups:
147
+ if len(group) >= batch_size:
148
+ if current:
149
+ batches.append(current)
150
+ current = []
151
+ batches.append(list(group))
152
+ elif len(current) + len(group) > batch_size:
153
+ if current:
154
+ batches.append(current)
155
+ current = list(group)
156
+ else:
157
+ current.extend(group)
158
+ if current:
159
+ batches.append(current)
160
+ return batches
161
+
162
+ def _batches(self) -> List[List[int]]:
163
+ rng = random.Random(self.seed + self.epoch)
164
+ if self.groups_by_storage:
165
+ # Keep batches shard-local. Globally shuffling the flattened groups
166
+ # defeats the tiny shard cache and makes every rank repeatedly
167
+ # deserialize unrelated files while its GPU waits for data.
168
+ blocks = [list(block) for block in self.groups_by_storage]
169
+ rng.shuffle(blocks)
170
+ batches: List[List[int]] = []
171
+ for block in blocks:
172
+ rng.shuffle(block)
173
+ batches.extend(self._pack_groups(block, self.batch_size))
174
+ else:
175
+ groups = list(self.groups)
176
+ rng.shuffle(groups)
177
+ batches = self._pack_groups(groups, self.batch_size)
178
+ if batches:
179
+ target = ((len(batches) + self.world_size - 1) // self.world_size) * self.world_size
180
+ batches.extend([list(batches[i % len(batches)]) for i in range(target - len(batches))])
181
+ return batches
182
+
183
+ def __iter__(self) -> Iterator[List[int]]:
184
+ return iter(self._batches()[self.rank::self.world_size])
185
+
186
+ def __len__(self) -> int:
187
+ return len(self._batches()) // self.world_size
188
+
189
+
190
+ class EarlyStopping:
191
+ def __init__(self, patience: int = 10, lower_better: bool = True):
192
+ self.patience = int(patience)
193
+ self.lower_better = bool(lower_better)
194
+ self.counter = 0
195
+ self.best_score = float("inf") if lower_better else -float("inf")
196
+ self.best_model_state = None
197
+ self.early_stop = False
198
+
199
+ def step(self, score: float, model: torch.nn.Module) -> None:
200
+ improved = score < self.best_score if self.lower_better else score > self.best_score
201
+ if improved:
202
+ self.best_score = score
203
+ self.best_model_state = {k: v.cpu().clone() if isinstance(v, torch.Tensor) else v for k, v in model.state_dict().items()}
204
+ self.counter = 0
205
+ else:
206
+ self.counter += 1
207
+ if self.counter >= self.patience:
208
+ self.early_stop = True
209
+
210
+
211
+ class Trainer:
212
+ def __init__(self, args: Any, writer: SummaryWriter, logger_path: str):
213
+ self.args = args
214
+ self.cfg = to_runconfig_from_args(args)
215
+ self.device = get_device(self.cfg.gpu)
216
+ self.writer = writer
217
+ self.logger_path = str(logger_path)
218
+ self.checkpoint_mgr = CheckpointManager(self.logger_path)
219
+ self.total_step = 0
220
+ self.did_optimizer_step = False
221
+ self.terminate_requested = False
222
+ self.checkpoint_interval_steps = max(
223
+ 0, int(getattr(args, "checkpoint_interval_steps", 1000))
224
+ )
225
+ self.max_confs = int(getattr(args, "max_confs", 5))
226
+
227
+ sharded_root = str(getattr(args, "sharded_data_root", "") or "")
228
+ if sharded_root:
229
+ logger.info("Using isolated tensor-only shard cache: %s", sharded_root)
230
+ dataset = None
231
+ assignment_root = str(getattr(args, "stage2_assignment_cache", "") or "")
232
+ self.train_set = ShardedDrugDataset(sharded_root, "train", getattr(args, "shard_cache_size", 2), assignment_root)
233
+ self.valid_set = ShardedDrugDataset(sharded_root, "val", getattr(args, "shard_cache_size", 2), assignment_root)
234
+ self.test_set = ShardedDrugDataset(sharded_root, "test", getattr(args, "shard_cache_size", 2), assignment_root)
235
+ self.max_confs = int(getattr(args, "max_confs", 20))
236
+ self.args.max_confs = self.max_confs
237
+ self.cfg.max_confs = self.max_confs
238
+ else:
239
+ dataset = QM9Dataset(
240
+ name=self.cfg.dataset,
241
+ root=self.cfg.data_root,
242
+ max_confs=self.max_confs,
243
+ raw_prefix=getattr(args, "raw_prefix", "converted"),
244
+ train_raw_filename=getattr(args, "train_raw_filename", None),
245
+ val_raw_filename=getattr(args, "val_raw_filename", None),
246
+ test_raw_filename=getattr(args, "test_raw_filename", None),
247
+ keep_full_test_conformers=bool(getattr(args, "full_test_conformers", False)),
248
+ )
249
+ # The dataset resolves auto mode before preprocessing. Propagate the
250
+ # resolved value into the encoder so its conformer embedding capacity
251
+ # exactly matches the cached dataset.
252
+ if dataset is not None:
253
+ self.max_confs = int(dataset.max_confs)
254
+ self.args.max_confs = self.max_confs
255
+ self.cfg.max_confs = self.max_confs
256
+ self.train_set = dataset[dataset.train_index]
257
+ self.valid_set = dataset[dataset.valid_index]
258
+ self.test_set = dataset[dataset.test_index]
259
+ self.train_set = self._limit_molecules(self.train_set, int(getattr(args, "train_mol_limit", 0)))
260
+ self.valid_set = self._limit_molecules(self.valid_set, int(getattr(args, "valid_mol_limit", 0)))
261
+ self.test_set = self._limit_molecules(self.test_set, int(getattr(args, "test_mol_limit", 0)))
262
+
263
+ loader_kwargs = dict(
264
+ num_workers=int(getattr(self.cfg, "num_workers", 0)),
265
+ pin_memory=bool(getattr(self.cfg, "pin_memory", False)) and self.device.type == "cuda",
266
+ )
267
+ if loader_kwargs["num_workers"] > 0:
268
+ loader_kwargs["persistent_workers"] = True
269
+ prefetch_factor = getattr(self.cfg, "prefetch_factor", None)
270
+ if prefetch_factor is not None:
271
+ loader_kwargs["prefetch_factor"] = int(prefetch_factor)
272
+
273
+ distributed_stage2 = bool(getattr(args, "distributed_stage2", False))
274
+ if distributed_stage2:
275
+ if not dist.is_available() or not dist.is_initialized():
276
+ raise RuntimeError("--distributed_stage2 requires torchrun with an initialized process group")
277
+ train_batch_sampler = DistributedMoleculeGroupedBatchSampler(
278
+ self.train_set,
279
+ batch_size=self.cfg.bs,
280
+ rank=dist.get_rank(),
281
+ world_size=dist.get_world_size(),
282
+ seed=int(getattr(args, "random_seed", 0)),
283
+ )
284
+ else:
285
+ train_batch_sampler = MoleculeGroupedBatchSampler(
286
+ self.train_set, batch_size=self.cfg.bs, shuffle=True, drop_last=False
287
+ )
288
+ self.train_loader = DataLoader(
289
+ self.train_set,
290
+ batch_sampler=train_batch_sampler,
291
+ **loader_kwargs,
292
+ )
293
+ self.valid_loader = DataLoader(
294
+ self.valid_set,
295
+ batch_sampler=MoleculeGroupedBatchSampler(self.valid_set, batch_size=self.cfg.bs, shuffle=False, drop_last=False),
296
+ **loader_kwargs,
297
+ )
298
+ self.test_loader = DataLoader(
299
+ self.test_set,
300
+ batch_sampler=MoleculeGroupedBatchSampler(self.test_set, batch_size=self.cfg.bs, shuffle=False, drop_last=False),
301
+ **loader_kwargs,
302
+ )
303
+
304
+ self.model = GNNEncoder(args=args, config={"model": {"model_level": "graph"}}).to(self.device)
305
+
306
+ self._load_checkpoint_if_needed()
307
+ self._apply_train_stage_freezing()
308
+
309
+ trainable_params = [p for p in self.model.parameters() if p.requires_grad]
310
+ if len(trainable_params) == 0:
311
+ raise RuntimeError("没有可训练参数,请检查 train_stage 和 requires_grad 设置。")
312
+
313
+ self.opt = self._build_optimizer()
314
+
315
+ trainable_count = sum(p.numel() for p in self.model.parameters() if p.requires_grad)
316
+ frozen_count = sum(p.numel() for p in self.model.parameters() if not p.requires_grad)
317
+ logger.info(f"Trainable parameters: {trainable_count}")
318
+ logger.info(f"Frozen parameters: {frozen_count}")
319
+
320
+ eta_min = float(self.cfg.eta_min)
321
+ if eta_min >= float(self.cfg.lr):
322
+ logger.warning(f"eta_min={eta_min} >= lr={self.cfg.lr},已自动改为 lr * 0.01 = {float(self.cfg.lr) * 0.01}")
323
+ eta_min = float(self.cfg.lr) * 0.01
324
+ self.scheduler = CosineAnnealingLR(self.opt, T_max=max(1, int(self.cfg.epoch)), eta_min=eta_min)
325
+ self.resume_epoch = 0
326
+ self.resume_batch_idx = -1
327
+ self.best_valid_score = float("inf")
328
+ self.best_test_score = float("inf")
329
+ self.best_epoch: Optional[int] = None
330
+ self.early_stopper = EarlyStopping(patience=self.cfg.patience, lower_better=True)
331
+ self._load_progress_checkpoint_if_needed()
332
+
333
+ self.last_found: Dict[str, Any] = dict(found=False, mol_pred=None, pos_gt_data=None, smiles_target=None)
334
+ try:
335
+ describe_model(self.model, path=self.logger_path)
336
+ except Exception:
337
+ logger.exception("describe_model 出错,可忽略")
338
+ logger.info(f"Trainer initialized on device {self.device}")
339
+ logger.info("Data unit = conformer; training batch is grouped by mol_id.")
340
+ signal.signal(signal.SIGTERM, self._request_termination)
341
+
342
+ def _request_termination(self, signum, frame):
343
+ self.terminate_requested = True
344
+ logger.warning(
345
+ "Received SIGTERM; saving an interruption checkpoint after the current batch."
346
+ )
347
+
348
+ def _save_progress_checkpoint(self, filename: str, epoch: int, batch_idx: int) -> None:
349
+ path = os.path.join(self.logger_path, filename)
350
+ tmp_path = f"{path}.tmp"
351
+ torch.save(
352
+ {
353
+ "model_state_dict": self.model.state_dict(),
354
+ "optimizer_state_dict": self.opt.state_dict(),
355
+ "epoch": int(epoch),
356
+ "batch_idx": int(batch_idx),
357
+ "total_step": int(self.total_step),
358
+ "scheduler_state_dict": self.scheduler.state_dict(),
359
+ "early_stopper_state": {
360
+ "counter": int(self.early_stopper.counter),
361
+ "best_score": float(self.early_stopper.best_score),
362
+ "best_model_state": self.early_stopper.best_model_state,
363
+ "early_stop": bool(self.early_stopper.early_stop),
364
+ },
365
+ "training_state": {
366
+ "best_valid_score": float(self.best_valid_score),
367
+ "best_test_score": float(self.best_test_score),
368
+ "best_epoch": self.best_epoch,
369
+ },
370
+ },
371
+ tmp_path,
372
+ )
373
+ os.replace(tmp_path, path)
374
+ logger.info("Saved progress checkpoint: %s", path)
375
+
376
+ def _limit_molecules(self, subset, mol_limit: int):
377
+ if mol_limit <= 0:
378
+ return subset
379
+ keep_indices = []
380
+ seen = set()
381
+ for idx in range(len(subset)):
382
+ data = subset[idx]
383
+ mol_id = int(data.mol_id.item()) if hasattr(data, "mol_id") else idx
384
+ if mol_id not in seen:
385
+ if len(seen) >= int(mol_limit):
386
+ break
387
+ seen.add(mol_id)
388
+ if mol_id in seen:
389
+ keep_indices.append(idx)
390
+ logger.info(f"Limited dataset split to {len(seen)} molecules / {len(keep_indices)} conformers.")
391
+ return subset[keep_indices]
392
+
393
+ def _build_optimizer(self):
394
+ if not bool(getattr(self.args, "use_end2end_pglobal", False)):
395
+ trainable_params = [p for p in self.model.parameters() if p.requires_grad]
396
+ return torch.optim.Adam(trainable_params, lr=self.cfg.lr)
397
+
398
+ pglobal_prefixes = ("gnn.", "global_edge_predictor.", "ab_head.", "conf_embedding.")
399
+ pglobal_params = []
400
+ other_params = []
401
+ for name, param in self.model.named_parameters():
402
+ if not param.requires_grad:
403
+ continue
404
+ if name.startswith(pglobal_prefixes):
405
+ pglobal_params.append(param)
406
+ else:
407
+ other_params.append(param)
408
+ groups = []
409
+ if other_params:
410
+ groups.append({"params": other_params, "lr": self.cfg.lr})
411
+ if pglobal_params:
412
+ groups.append({"params": pglobal_params, "lr": float(getattr(self.args, "pglobal_lr", 3e-5))})
413
+ if not groups:
414
+ raise RuntimeError("没有可训练参数,请检查 train_stage 和 requires_grad 设置。")
415
+ logger.info(
416
+ f"End-to-end optimizer groups: decoder/refiner lr={self.cfg.lr}, "
417
+ f"P_global lr={float(getattr(self.args, 'pglobal_lr', 3e-5))}"
418
+ )
419
+ return torch.optim.Adam(groups)
420
+
421
+ # --------------------------------------------------------
422
+ # Checkpoint loading
423
+ # --------------------------------------------------------
424
+ def _load_checkpoint_if_needed(self):
425
+ checkpoint_path = getattr(self.args, "checkpoint_path", None)
426
+ if not checkpoint_path:
427
+ return
428
+ try:
429
+ ckpt = torch.load(checkpoint_path, map_location=self.device)
430
+ if isinstance(ckpt, dict) and "model_state_dict" in ckpt:
431
+ state = ckpt["model_state_dict"]
432
+ epoch = ckpt.get("epoch", "unknown")
433
+ elif isinstance(ckpt, dict) and "state_dict" in ckpt:
434
+ state = ckpt["state_dict"]
435
+ epoch = ckpt.get("epoch", "unknown")
436
+ else:
437
+ state = ckpt
438
+ epoch = "unknown"
439
+ model_state = self.model.state_dict()
440
+ compatible_state = {}
441
+ skipped = []
442
+
443
+ stage = getattr(self.args, "train_stage", "gnn")
444
+ skip_prefixes = []
445
+ if stage == "multimode":
446
+ skip_prefixes = ["mode_generator", "coarse_decoder", "conditioned_decoder",
447
+ "refiner", "torsion_residual_head"]
448
+ if not bool(getattr(self.args, "use_end2end_pglobal", False)):
449
+ skip_prefixes.extend(["ab_head", "fusion_weight_head"])
450
+
451
+ for key, value in state.items():
452
+ should_skip = any(key.startswith(prefix + ".") or key == prefix for prefix in skip_prefixes)
453
+ if should_skip:
454
+ skipped.append(key)
455
+ continue
456
+ if key in model_state and model_state[key].shape == value.shape:
457
+ compatible_state[key] = value
458
+ else:
459
+ skipped.append(key)
460
+ model_state.update(compatible_state)
461
+ self.model.load_state_dict(model_state, strict=True)
462
+ logger.info(f"Loaded compatible checkpoint from {checkpoint_path} (epoch {epoch}), loaded={len(compatible_state)}, skipped={len(skipped)}")
463
+ if skipped:
464
+ logger.warning(f"Skipped incompatible keys: {skipped}")
465
+ except Exception:
466
+ logger.exception("加载 checkpoint 失败,继续从头开始")
467
+
468
+ def _load_progress_checkpoint_if_needed(self):
469
+ """Restore an interruption/latest checkpoint when explicitly requested.
470
+
471
+ This is separate from ``checkpoint_path`` so legacy runs keep their
472
+ historical weight-only loading behavior. Progress checkpoints also
473
+ restore optimizer state and resume at the first unprocessed batch.
474
+ """
475
+ checkpoint_path = getattr(self.args, "resume_progress_checkpoint", None)
476
+ if not checkpoint_path:
477
+ return
478
+ if not os.path.exists(checkpoint_path):
479
+ raise FileNotFoundError(f"Progress checkpoint not found: {checkpoint_path}")
480
+ ckpt = torch.load(checkpoint_path, map_location=self.device)
481
+ required = ("model_state_dict", "optimizer_state_dict", "epoch", "batch_idx", "total_step")
482
+ missing = [key for key in required if key not in ckpt]
483
+ if missing:
484
+ raise ValueError(f"Not a resumable progress checkpoint; missing keys: {missing}")
485
+ self.model.load_state_dict(ckpt["model_state_dict"], strict=True)
486
+ self.opt.load_state_dict(ckpt["optimizer_state_dict"])
487
+ self.resume_epoch = int(ckpt["epoch"])
488
+ self.resume_batch_idx = int(ckpt["batch_idx"])
489
+ self.total_step = int(ckpt["total_step"])
490
+ scheduler_state = ckpt.get("scheduler_state_dict")
491
+ if scheduler_state:
492
+ saved_t_max = int(scheduler_state.get("T_max", 0))
493
+ if int(self.cfg.epoch) > saved_t_max:
494
+ # The prior short run used a smaller cosine horizon. Preserve
495
+ # the current optimizer LR and decay it over the added epochs.
496
+ for group in self.opt.param_groups:
497
+ group["initial_lr"] = group["lr"]
498
+ remaining_epochs = max(1, int(self.cfg.epoch) - self.resume_epoch)
499
+ self.scheduler = CosineAnnealingLR(
500
+ self.opt,
501
+ T_max=remaining_epochs,
502
+ eta_min=float(self.cfg.eta_min),
503
+ )
504
+ logger.info(
505
+ "Extended scheduler horizon from T_max=%d to %d remaining epochs.",
506
+ saved_t_max,
507
+ remaining_epochs,
508
+ )
509
+ else:
510
+ self.scheduler.load_state_dict(scheduler_state)
511
+ elif int(self.cfg.epoch) > self.resume_epoch:
512
+ # Older progress checkpoints did not serialize the scheduler.
513
+ # Start an extended decay from the optimizer LR that was restored.
514
+ for group in self.opt.param_groups:
515
+ group["initial_lr"] = group["lr"]
516
+ remaining_epochs = max(1, int(self.cfg.epoch) - self.resume_epoch)
517
+ self.scheduler = CosineAnnealingLR(
518
+ self.opt,
519
+ T_max=remaining_epochs,
520
+ eta_min=float(self.cfg.eta_min),
521
+ )
522
+ logger.info(
523
+ "Rebased scheduler for legacy progress checkpoint over %d remaining epochs.",
524
+ remaining_epochs,
525
+ )
526
+
527
+ early_state = ckpt.get("early_stopper_state")
528
+ if early_state:
529
+ self.early_stopper.counter = int(early_state.get("counter", 0))
530
+ self.early_stopper.best_score = float(
531
+ early_state.get("best_score", self.early_stopper.best_score)
532
+ )
533
+ self.early_stopper.best_model_state = early_state.get("best_model_state")
534
+ self.early_stopper.early_stop = bool(early_state.get("early_stop", False))
535
+
536
+ training_state = ckpt.get("training_state", {})
537
+ self.best_valid_score = float(training_state.get("best_valid_score", self.best_valid_score))
538
+ self.best_test_score = float(training_state.get("best_test_score", self.best_test_score))
539
+ self.best_epoch = training_state.get("best_epoch", self.best_epoch)
540
+ if bool(getattr(self.args, "resume_next_epoch", False)) and self.resume_batch_idx >= 0:
541
+ logger.warning(
542
+ "Fast resume enabled: dropping the unfinished tail of epoch %d "
543
+ "and starting epoch %d to avoid replaying %d data-loader batches.",
544
+ self.resume_epoch,
545
+ self.resume_epoch + 1,
546
+ self.resume_batch_idx + 1,
547
+ )
548
+ self.resume_epoch += 1
549
+ self.resume_batch_idx = -1
550
+ logger.info(
551
+ "Resumed progress checkpoint %s: epoch=%d, last_batch=%d, "
552
+ "next_batch=%d, total_step=%d",
553
+ checkpoint_path,
554
+ self.resume_epoch,
555
+ self.resume_batch_idx,
556
+ self.resume_batch_idx + 1,
557
+ self.total_step,
558
+ )
559
+
560
+ # --------------------------------------------------------
561
+ # Freezing
562
+ # --------------------------------------------------------
563
+ def _set_requires_grad(self, module: torch.nn.Module, flag: bool):
564
+ for param in module.parameters():
565
+ param.requires_grad = flag
566
+
567
+ def _unfreeze_modules(self, names: List[str]):
568
+ for name in names:
569
+ if hasattr(self.model, name):
570
+ self._set_requires_grad(getattr(self.model, name), True)
571
+
572
+ def _freeze_modules(self, names: List[str]):
573
+ for name in names:
574
+ if hasattr(self.model, name):
575
+ self._set_requires_grad(getattr(self.model, name), False)
576
+
577
+ def _apply_train_stage_freezing(self):
578
+ stage = getattr(self.args, "train_stage", "gnn")
579
+
580
+ self._set_requires_grad(self.model, False)
581
+
582
+ if stage == "gnn":
583
+ modules = ["gnn", "global_edge_predictor", "ab_head", "conf_embedding", "fusion_weight_head"]
584
+ if bool(getattr(self.args, "mean_dispersion_prior", False)):
585
+ modules.append("dispersion_edge_predictor")
586
+ self._unfreeze_modules(modules)
587
+ logger.info("train_stage=gnn:训练 GNN / P_global / A-B。")
588
+ if bool(getattr(self.args, "mean_dispersion_prior", False)):
589
+ logger.info("mean_dispersion_prior=True:训练 dispersion_edge_predictor。")
590
+
591
+ elif stage == "multimode":
592
+ modules = ["conditioned_decoder", "coarse_decoder", "refiner", "geometry_refiner"]
593
+ if bool(getattr(self.args, "use_mode_specific_p", False)):
594
+ modules.append("mode_generator")
595
+ if bool(getattr(self.args, "use_torsion_residual", False)):
596
+ modules.append("torsion_residual_head")
597
+ if bool(getattr(self.args, "use_end2end_pglobal", False)):
598
+ modules.extend(["gnn", "global_edge_predictor", "ab_head", "conf_embedding"])
599
+ self._unfreeze_modules(modules)
600
+ logger.info("train_stage=multimode:训练 conditioned decoder + refiner(Z条件化解码)。")
601
+ if bool(getattr(self.args, "use_mode_specific_p", False)):
602
+ logger.info("use_mode_specific_p=True:额外训练 mode_generator,生成 P_k = f(P_global, z_k)。")
603
+ if bool(getattr(self.args, "use_torsion_residual", False)):
604
+ logger.info("use_torsion_residual=True:额外训练 torsion_residual_head,预测二面角残差。")
605
+ if bool(getattr(self.args, "use_end2end_pglobal", False)):
606
+ logger.info("use_end2end_pglobal=True:Stage 2 同时微调 GNN/P_global 分支。")
607
+
608
+ else:
609
+ raise ValueError(f"Unknown train_stage={stage}. Expected one of: gnn, multimode.")
610
+
611
+ # ========================================================
612
+ # Run loop
613
+ # ========================================================
614
+
615
+ def run(self, optuna_trial: Optional[Any] = None) -> Tuple[Optional[float], Optional[int]]:
616
+ best_valid_score = float(self.best_valid_score)
617
+ best_test_score = float(self.best_test_score)
618
+ best_epoch = self.best_epoch
619
+
620
+ try:
621
+ for epoch in range(self.resume_epoch, int(self.cfg.epoch)):
622
+ logger.info(f"Starting epoch {epoch}")
623
+ skip_batches = self.resume_batch_idx + 1 if epoch == self.resume_epoch else 0
624
+ self.train_epoch(epoch, skip_batches=skip_batches)
625
+ if epoch == self.resume_epoch:
626
+ self.resume_batch_idx = -1
627
+ if self.did_optimizer_step:
628
+ self.scheduler.step()
629
+ if self.cfg.metric.upper() == "RMSD":
630
+ train_score = safe_float(self._test_rmsd_limited(self.train_loader, max_samples=100, split_name="train"))
631
+ self.writer.add_scalar("train_rmsd", train_score, epoch)
632
+ valid_score = safe_float(self.test(self.valid_loader, val=True))
633
+ self.writer.add_scalar(f"valid_{self.cfg.metric.lower()}", valid_score, epoch)
634
+ logger.info(f"Epoch {epoch} valid {self.cfg.metric} = {valid_score:.6f}, best_test_so_far={best_test_score:.6f}")
635
+
636
+ if valid_score < best_valid_score:
637
+ test_score = safe_float(self.test(self.test_loader, val=False))
638
+ best_valid_score = valid_score
639
+ best_test_score = test_score
640
+ best_epoch = epoch
641
+ logger.info(f"New best valid {best_valid_score:.6f} (epoch {best_epoch}), test={best_test_score:.6f}")
642
+ try:
643
+ self.checkpoint_mgr.save_best(self.model, self.opt, epoch, best_test_score)
644
+ except Exception:
645
+ logger.exception("保存 best checkpoint 失败")
646
+
647
+ self.best_valid_score = best_valid_score
648
+ self.best_test_score = best_test_score
649
+ self.best_epoch = best_epoch
650
+
651
+ if self.cfg.early_stop:
652
+ self.early_stopper.step(valid_score, self.model)
653
+ if self.early_stopper.early_stop:
654
+ logger.info("Early stopping triggered; restoring best model state")
655
+ if self.early_stopper.best_model_state is not None:
656
+ self.model.load_state_dict(self.early_stopper.best_model_state)
657
+ if self.last_found.get("found", False):
658
+ now_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S.%f")[:-3]
659
+ out_dir = f"rdmol/{now_str}_{self.last_found.get('smiles_target')}"
660
+ try:
661
+ save_molecule_views(out_dir, self.last_found.get("mol_pred"), pos_gt_data=self.last_found.get("pos_gt_data"), smiles_target=self.last_found.get("smiles_target"), args_to_save=self.args)
662
+ except Exception:
663
+ logger.exception("保存 mid conformer 出错")
664
+ break
665
+
666
+ report_optuna_and_maybe_prune(optuna_trial, best_valid_score, epoch, self.logger_path)
667
+
668
+ write_final_metrics(self.logger_path, best_valid_score, best_test_score, best_epoch)
669
+ return None if best_valid_score == float("inf") else best_valid_score, best_epoch
670
+
671
+ except Exception as error:
672
+ logger.exception("训练过程中发生异常")
673
+ try:
674
+ handle_training_exception(self.logger_path, error, best_valid_score, best_test_score, best_epoch)
675
+ except Exception:
676
+ logger.exception("调用 handle_training_exception 失败")
677
+ raise
678
+
679
+ # ========================================================
680
+ # Train epoch / batch
681
+ # ========================================================
682
+
683
+ def train_epoch(self, epoch: int, skip_batches: int = 0) -> None:
684
+ self.model.train()
685
+ stage = getattr(self.args, "train_stage", "gnn")
686
+
687
+ if stage == "multimode" and not bool(getattr(self.args, "use_end2end_pglobal", False)):
688
+ for name in ["gnn", "global_edge_predictor", "ab_head"]:
689
+ if hasattr(self.model, name):
690
+ getattr(self.model, name).eval()
691
+
692
+ if hasattr(self.train_loader.batch_sampler, "set_epoch"):
693
+ self.train_loader.batch_sampler.set_epoch(epoch)
694
+ disable_bar = bool(getattr(self.args, "distributed_stage2", False)) and dist.get_rank() != 0
695
+ pbar = tqdm(self.train_loader, desc=f"Train E[{epoch}]", disable=disable_bar)
696
+ epoch_loss = 0.0
697
+ n_batches = 0
698
+
699
+ for batch_idx, batch in enumerate(pbar):
700
+ if batch_idx < int(skip_batches):
701
+ pbar.set_postfix_str(f"resume-skip {batch_idx + 1}/{skip_batches}")
702
+ continue
703
+ loss_val = self.train_batch(batch, epoch)
704
+ epoch_loss += loss_val
705
+ n_batches += 1
706
+ pbar.set_postfix_str(f"loss={loss_val:.4f}")
707
+ if (
708
+ self.checkpoint_interval_steps > 0
709
+ and self.total_step > 0
710
+ and self.total_step % self.checkpoint_interval_steps == 0
711
+ ):
712
+ self._save_progress_checkpoint("latest.pth", epoch, batch_idx)
713
+ if self.terminate_requested:
714
+ self._save_progress_checkpoint("interrupted.pth", epoch, batch_idx)
715
+ raise RuntimeError(
716
+ "Training interrupted by SIGTERM after saving interrupted.pth"
717
+ )
718
+
719
+ avg = epoch_loss / max(1, n_batches)
720
+ self.writer.add_scalar("loss_per_epoch", avg, epoch)
721
+ self.writer.add_scalar("lr_per_epoch", self.opt.param_groups[0]["lr"], epoch)
722
+ logger.info(f"Epoch {epoch} average loss {avg:.6f}")
723
+
724
+ def _normalize_loss_by_graphs(self, loss: torch.Tensor, batch: Any) -> torch.Tensor:
725
+ num_graphs = int(getattr(batch, "num_graphs", 1))
726
+ return loss / max(1, num_graphs)
727
+
728
+ def train_batch(self, batch: Any, epoch: int = 0) -> float:
729
+ batch = batch.to(self.device, non_blocking=(self.device.type == "cuda"))
730
+ stage = getattr(self.args, "train_stage", "gnn")
731
+ self.opt.zero_grad(set_to_none=True)
732
+
733
+ graph_feat = None # 用于后续 a/b 记录
734
+
735
+ # ---------------------------------------------------
736
+ # Stage 1: manifold learning
737
+ # ---------------------------------------------------
738
+ if stage == "gnn":
739
+ outputs = self.model(batch, epoch=epoch)
740
+ if len(outputs) == 6:
741
+ pred_pos, graph_feat, pos_loss, mani_loss, total_loss, mol_list = outputs
742
+ else:
743
+ pred_pos, graph_feat, pos_loss, mani_loss, mol_list = outputs
744
+ total_loss = mani_loss
745
+ # compute_stage1_loss already averages over molecule groups. A
746
+ # second division by data.num_graphs suppressed the probability-
747
+ # head gradients by roughly the number of conformers in a batch.
748
+ loss = total_loss
749
+
750
+ # ---------------------------------------------------
751
+ # Stage 2: multimode conformer generation
752
+ # ---------------------------------------------------
753
+ elif stage == "multimode":
754
+ node_feat = self.model.gnn(batch)
755
+ graph_feat = self.model.pool(node_feat, batch.batch)
756
+
757
+ loss, pos_loss, _ = self.model.compute_multimode_loss(batch, node_feat, graph_feat)
758
+
759
+ loss = torch.nan_to_num(loss, nan=0.0, posinf=1e6, neginf=0.0)
760
+ pos_loss = torch.nan_to_num(pos_loss, nan=0.0, posinf=1e6, neginf=0.0)
761
+
762
+ else:
763
+ raise ValueError(f"Unknown train_stage={stage}")
764
+
765
+ # ---------------------------------------------------
766
+ # Gradient check & backward
767
+ # ---------------------------------------------------
768
+ if not torch.isfinite(loss):
769
+ logger.warning("Skip batch because loss is NaN/Inf")
770
+ self.opt.zero_grad(set_to_none=True)
771
+ return 0.0
772
+
773
+ loss.backward()
774
+
775
+ trainable_params = [p for p in self.model.parameters() if p.requires_grad]
776
+ bad_grad = any(p.grad is not None and not torch.isfinite(p.grad).all() for p in trainable_params)
777
+ if bad_grad:
778
+ logger.warning("Skip optimizer step because gradient is NaN/Inf")
779
+ self.opt.zero_grad(set_to_none=True)
780
+ return 0.0
781
+
782
+ torch.nn.utils.clip_grad_norm_(trainable_params, 1.0)
783
+ self.opt.step()
784
+ self.did_optimizer_step = True
785
+ self.total_step += 1
786
+
787
+ # Optional save
788
+ if get_arg(self.args, "get_image", False) and stage == "multimode":
789
+ try:
790
+ yT = getattr(self.model, "_last_yT", None)
791
+ if yT is not None:
792
+ self._maybe_save_target_conformer(batch, yT)
793
+ except Exception:
794
+ pass
795
+
796
+ # ---------------------------------------------------
797
+ # Logging (enhanced with TensorBoard)
798
+ # ---------------------------------------------------
799
+ self.writer.add_scalar("loss/total", float(loss.detach().cpu()), self.total_step)
800
+ self.writer.add_scalar("lr", self.opt.param_groups[0]["lr"], self.total_step)
801
+
802
+ if stage == "multimode":
803
+ pos_val = float(pos_loss.detach().cpu()) if torch.is_tensor(pos_loss) else float(pos_loss)
804
+ self.writer.add_scalar("loss/pos", pos_val, self.total_step)
805
+
806
+ # 记录 a, b 参数
807
+ if graph_feat is not None and stage == "gnn":
808
+ try:
809
+ with torch.no_grad():
810
+ a_sample, b_sample = self.model.predict_ab(graph_feat[0])
811
+ self.writer.add_scalar("params/a", a_sample.item(), self.total_step)
812
+ self.writer.add_scalar("params/b", b_sample.item(), self.total_step)
813
+ except Exception:
814
+ pass
815
+
816
+ # 每 50 步记录梯度直方图
817
+ if self.total_step % 50 == 0:
818
+ for name, param in self.model.named_parameters():
819
+ if param.grad is not None and param.requires_grad:
820
+ self.writer.add_histogram(f"grad/{name}", param.grad, self.total_step)
821
+
822
+ return float(loss.detach().cpu())
823
+
824
+ # -------------------------------------------------------
825
+ # Visualisation helper
826
+ # -------------------------------------------------------
827
+ def _maybe_save_target_conformer(self, batch: Any, pred_pos: torch.Tensor) -> None:
828
+ smiles_target = get_arg(self.args, "smiles", None)
829
+ if smiles_target is None or self.last_found.get("found", False) or not hasattr(batch, "smiles") or pred_pos is None:
830
+ return
831
+ for graph_idx, smiles in enumerate(batch.smiles):
832
+ if smiles != smiles_target:
833
+ continue
834
+ try:
835
+ mol_pred = Chem.Mol(batch.rdmol[graph_idx])
836
+ mol_pred.RemoveAllConformers()
837
+ node_mask = batch.batch == graph_idx
838
+ pos_i = pred_pos[node_mask].detach().cpu().tolist()
839
+ conf = Chem.Conformer(mol_pred.GetNumAtoms())
840
+ for atom_idx, xyz in enumerate(pos_i):
841
+ x, y, z = xyz
842
+ conf.SetAtomPosition(atom_idx, Point3D(float(x), float(y), float(z)))
843
+ mol_pred.AddConformer(conf, assignId=True)
844
+ pos_gt_data = batch.pos[node_mask].detach().cpu().numpy()
845
+ self.last_found.update(
846
+ found=True,
847
+ mol_pred=mol_pred,
848
+ pos_gt_data=pos_gt_data,
849
+ smiles_target=smiles_target,
850
+ )
851
+ except Exception:
852
+ logger.exception("保存目标 conformer 出错")
853
+ break
854
+
855
+ # -------------------------------------------------------
856
+ # Helper functions
857
+ # -------------------------------------------------------
858
+ def _get_conf_idx(self, batch: Any, graph_idx: int) -> int:
859
+ if hasattr(self.model, "_get_conf_idx"):
860
+ return self.model._get_conf_idx(batch, graph_idx)
861
+ if hasattr(batch, "conf_id"):
862
+ return int(batch.conf_id[graph_idx].item()) % self.max_confs
863
+ return 0
864
+
865
+ def _get_edge_index_i(self, batch: Any, graph_idx: int):
866
+ mask = batch.batch == graph_idx
867
+ return self.model._get_local_edge_index(edge_index=batch.edge_index, mask=mask)
868
+
869
+ @torch.no_grad()
870
+ def _predict_single_conformer(self, batch: Any, graph_idx: int):
871
+ node_feat = self.model.gnn(batch)
872
+ mask = batch.batch == graph_idx
873
+ feat_i = node_feat[mask]
874
+ edge_index_i = self._get_edge_index_i(batch, graph_idx)
875
+
876
+ if hasattr(self.model, "predict_conformers"):
877
+ try:
878
+ outputs = self.model.predict_conformers(
879
+ feat_i=feat_i,
880
+ edge_index_i=edge_index_i,
881
+ num_modes=1,
882
+ deterministic=False,
883
+ )
884
+ except TypeError:
885
+ try:
886
+ outputs = self.model.predict_conformers(
887
+ feat_i=feat_i,
888
+ edge_index_i=edge_index_i,
889
+ )
890
+ except TypeError:
891
+ outputs = None
892
+
893
+ if isinstance(outputs, list) and len(outputs) > 0:
894
+ out0 = outputs[0]
895
+ if isinstance(out0, dict):
896
+ y = out0.get("Y", out0.get("y", out0.get("coords", None)))
897
+ if y is not None:
898
+ return center_and_rescale(y, target_rms=1.0)
899
+ if torch.is_tensor(out0):
900
+ return center_and_rescale(out0, target_rms=1.0)
901
+
902
+ if hasattr(self.model, "_predict_P_global"):
903
+ P_i, _ = self.model._predict_P_global(feat_i=feat_i, edge_index_i=edge_index_i)
904
+ if hasattr(self.model, "decode_probability_mode"):
905
+ try:
906
+ decoded = self.model.decode_probability_mode(
907
+ P=P_i, node_feat=feat_i, edge_index_i=edge_index_i
908
+ )
909
+ if isinstance(decoded, dict):
910
+ y = decoded.get("Y", decoded.get("y", decoded.get("coords", None)))
911
+ else:
912
+ y = decoded
913
+ if y is not None:
914
+ return center_and_rescale(y, target_rms=1.0)
915
+ except TypeError:
916
+ pass
917
+ if hasattr(self.model, "coarse_decoder"):
918
+ try:
919
+ y0 = self.model.coarse_decoder(node_feat=feat_i, edge_index=edge_index_i)
920
+ except TypeError:
921
+ y0 = self.model.coarse_decoder(feat_i, edge_index_i)
922
+ if hasattr(self.model, "refiner"):
923
+ try:
924
+ yT, _ = self.model.refiner(node_feat=feat_i, coords=y0, edge_index=edge_index_i)
925
+ except TypeError:
926
+ yT = self.model.refiner(feat_i, y0, edge_index_i)
927
+ return center_and_rescale(yT, target_rms=1.0)
928
+ return center_and_rescale(y0, target_rms=1.0)
929
+ return None
930
+
931
+ @torch.no_grad()
932
+ def _test_rmsd_limited(self, loader: DataLoader, max_samples: int = 100, split_name: str = "train") -> float:
933
+ self.model.eval()
934
+ total_rmsd = 0.0
935
+ total_valid = 0
936
+ for batch in loader:
937
+ batch = batch.to(self.device)
938
+ n_graphs = int(batch.num_graphs)
939
+ for graph_idx in range(n_graphs):
940
+ if total_valid >= max_samples:
941
+ avg_rmsd = total_rmsd / max(1, total_valid)
942
+ logger.info(f"RMSD eval {split_name} conformers = {total_valid}, RMSD = {avg_rmsd:.6f}")
943
+ return avg_rmsd
944
+ mask = batch.batch == graph_idx
945
+ pos_i = batch.pos[mask]
946
+ if pos_i.numel() == 0 or pos_i.shape[0] <= 1:
947
+ continue
948
+ y_i = self._predict_single_conformer(batch, graph_idx)
949
+ if y_i is None:
950
+ continue
951
+ if not torch.isfinite(y_i).all():
952
+ continue
953
+ if y_i.shape[0] != pos_i.shape[0]:
954
+ logger.warning(f"RMSD 跳过一个构象:y_i.shape={y_i.shape}, pos_i.shape={pos_i.shape}")
955
+ continue
956
+ atom_x = batch.x[mask] if hasattr(batch, "x") else None
957
+ heavy_mask = heavy_atom_mask_from_x(atom_x)
958
+ if heavy_mask is not None:
959
+ y_eval = y_i[heavy_mask]
960
+ pos_eval = pos_i[heavy_mask]
961
+ else:
962
+ y_eval = y_i
963
+ pos_eval = pos_i
964
+ rmsd_i = kabsch_rmsd(y_eval, pos_eval)
965
+ if not torch.isfinite(rmsd_i):
966
+ continue
967
+ total_rmsd += float(rmsd_i.detach().cpu())
968
+ total_valid += 1
969
+ avg_rmsd = total_rmsd / max(1, total_valid)
970
+ logger.info(f"RMSD eval {split_name} conformers = {total_valid}, RMSD = {avg_rmsd:.6f}")
971
+ return avg_rmsd
972
+
973
+ @torch.no_grad()
974
+ def test(self, loader: DataLoader, val: bool) -> float:
975
+ self.model.eval()
976
+ metric = self.cfg.metric.upper()
977
+ split_name = "valid" if val else "test"
978
+ if metric == "CE":
979
+ score = self._test_ce(loader)
980
+ elif metric == "RMSD":
981
+ score = self._test_rmsd_limited(loader, max_samples=100, split_name=split_name)
982
+ else:
983
+ logger.warning(f"未知 metric={self.cfg.metric},返回默认 10.0")
984
+ score = 10.0
985
+ tag = "val-score" if val else "test-score"
986
+ self.writer.add_scalar(tag, score, self.total_step)
987
+ return float(score)
988
+
989
+ @torch.no_grad()
990
+ def _test_ce(self, loader: DataLoader) -> float:
991
+ total_ce = 0.0
992
+ total_valid = 0
993
+ for batch in loader:
994
+ batch = batch.to(self.device)
995
+ node_feat = self.model.gnn(batch)
996
+ graph_feat = self.model.pool(node_feat, batch.batch)
997
+ matched_stage1_objective = (
998
+ bool(getattr(self.args, "stage1_nonlocal_heavy_only", False))
999
+ or (
1000
+ float(getattr(self.args, "stage1_fixed_kernel_a", 0.0)) > 0.0
1001
+ and float(getattr(self.args, "stage1_fixed_kernel_b", 0.0)) > 0.0
1002
+ )
1003
+ )
1004
+ if matched_stage1_objective:
1005
+ ce, _ = self.model.compute_stage1_loss(batch, node_feat, graph_feat)
1006
+ groups = self.model._group_graph_indices_by_mol_id(batch, int(batch.num_graphs))
1007
+ count = max(1, len(groups))
1008
+ if torch.isfinite(ce):
1009
+ total_ce += float(ce.detach().cpu()) * count
1010
+ total_valid += count
1011
+ continue
1012
+ n_graphs = int(batch.num_graphs)
1013
+ for graph_idx in range(n_graphs):
1014
+ mask = batch.batch == graph_idx
1015
+ feat_i = node_feat[mask]
1016
+ pos_i = batch.pos[mask]
1017
+ if pos_i.numel() == 0 or pos_i.shape[0] <= 1:
1018
+ continue
1019
+ edge_index_i = self._get_edge_index_i(batch, graph_idx)
1020
+ P_i, _ = self.model._predict_P_global(feat_i=feat_i, edge_index_i=edge_index_i)
1021
+ conf_idx = self._get_conf_idx(batch, graph_idx)
1022
+ try:
1023
+ a_i, b_i = self.model.predict_ab(graph_feat[graph_idx], conf_idx)
1024
+ except TypeError:
1025
+ a_i, b_i = self.model.predict_ab(graph_feat[graph_idx])
1026
+ Q_i = prob_low_dim(pos_i, a_i, b_i).clamp(1e-6, 1.0 - 1e-6)
1027
+ ce = cross_entropy_loss(P_i, Q_i)
1028
+ if not torch.isfinite(ce):
1029
+ continue
1030
+ total_ce += float(ce.detach().cpu())
1031
+ total_valid += 1
1032
+ avg_ce = total_ce / max(1, total_valid)
1033
+ logger.info(f"CE eval valid conformers = {total_valid}, CE = {avg_ce:.6f}")
1034
+ return avg_ce
1035
+
1036
+ @torch.no_grad()
1037
+ def _test_rmsd(self, loader: DataLoader) -> float:
1038
+ return self._test_rmsd_limited(loader, max_samples=10 ** 9, split_name="valid")