dualmapref 0.2.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- args_parse.py +593 -0
- candidate_corrector.py +161 -0
- candidate_scorer.py +359 -0
- conformend/__init__.py +36 -0
- conformend/refiner.py +25 -0
- dataset/__init__.py +1 -0
- dataset/drugdataset.py +644 -0
- dataset/manifold.py +237 -0
- dataset/sharded_drugdataset.py +124 -0
- dataset/smiles2graph.py +217 -0
- dualmapref/__init__.py +24 -0
- dualmapref/refiner.py +16 -0
- dualmapref-0.2.8.dist-info/LICENSE +21 -0
- dualmapref-0.2.8.dist-info/METADATA +167 -0
- dualmapref-0.2.8.dist-info/RECORD +186 -0
- dualmapref-0.2.8.dist-info/WHEEL +5 -0
- dualmapref-0.2.8.dist-info/entry_points.txt +10 -0
- dualmapref-0.2.8.dist-info/top_level.txt +29 -0
- exputils.py +289 -0
- infer.py +23 -0
- infer_base_conformers.py +5589 -0
- infer_mean_dispersion_torsion.py +1117 -0
- infer_torsion.py +752 -0
- infer_torsional_denoiser.py +1021 -0
- manimol/__init__.py +30 -0
- manimol/__main__.py +6 -0
- manimol/cli.py +36 -0
- manimol/full_cli.py +41 -0
- manimol/legacy_imports.py +43 -0
- manimol/manifest.py +56 -0
- manimol/postprocess.py +148 -0
- manimol/priors.py +60 -0
- manimol/protocols.py +119 -0
- manimol/refiner.py +645 -0
- manimol/runtime.py +194 -0
- manimol/selection.py +85 -0
- manimol/torsion.py +22 -0
- manimol/train_cli.py +14 -0
- mixture_flow/src/__init__.py +1 -0
- mixture_flow/src/_path.py +7 -0
- mixture_flow/src/infer28_mixture_torsion_flow.py +1093 -0
- mixture_flow/src/mixture_torsion_flow.py +291 -0
- mixture_flow/src/summarize_mixture_debug.py +80 -0
- mixture_flow/src/train28_mixture_torsion_flow.py +583 -0
- mixture_torsion_flow.py +7 -0
- models/ manifold_learning.py +86 -0
- models/__init__.py +8 -0
- models/dist.py +273 -0
- models/dist2coords.py +23 -0
- models/egnn.py +107 -0
- models/experimental_backbones.py +364 -0
- models/gnnconv.py +512 -0
- models/kernel_inversion.py +141 -0
- models/kernels.py +427 -0
- models/losses.py +154 -0
- models/mean_dispersion_bridge.py +119 -0
- models/mean_dispersion_torsion_context.py +155 -0
- models/model.py +73 -0
- models/model0.py +2375 -0
- models/vis.py +128 -0
- pgraph_controls.py +107 -0
- scripts/__init__.py +2 -0
- scripts/analyze_basin_alignment.py +63 -0
- scripts/analyze_hif2a_conformer_aggregation.py +267 -0
- scripts/analyze_hif2a_contact_pharmacophore.py +1111 -0
- scripts/analyze_hif2a_fixed_budget.py +202 -0
- scripts/analyze_hif2a_pharmacophore.py +405 -0
- scripts/analyze_hif2a_pharmacophore_match_metric.py +386 -0
- scripts/analyze_p_control_candidate_flow.py +154 -0
- scripts/analyze_scaffold_basin.py +83 -0
- scripts/audit_dispersion_proposals.py +208 -0
- scripts/audit_etflow_protocol.py +258 -0
- scripts/audit_external_overlap.py +150 -0
- scripts/audit_oldbest_fulltrainval_split.py +82 -0
- scripts/audit_pdbbind_contact_pharmacophores.py +146 -0
- scripts/audit_qm9_geomol_test_split.py +94 -0
- scripts/audit_stage1_stage2_cross.py +131 -0
- scripts/build_geom_drugs_lowref_diagnostic.py +84 -0
- scripts/build_qm9_backfill_manifest.py +99 -0
- scripts/check_etflow_runtime.py +56 -0
- scripts/compare_prior_sources.py +122 -0
- scripts/compose_binding_mode_figure.py +87 -0
- scripts/compose_hif2a_contact_pharmacophore_suppfig.py +255 -0
- scripts/compose_hif2a_functional_geometry_pipeline.py +266 -0
- scripts/compose_manimol_internal_architecture.py +291 -0
- scripts/diagnose_flow_basin_intervention.py +54 -0
- scripts/diagnose_hif2a_pharmacophore_budget.py +185 -0
- scripts/diagnose_route2_p_ranking.py +123 -0
- scripts/discover_binding_mode_inputs.py +30 -0
- scripts/evaluate_functional_prior_stage1.py +230 -0
- scripts/evaluate_geomanimol.py +74 -0
- scripts/evaluate_geomanimol_flow.py +23 -0
- scripts/evaluate_geomanimol_multiscaffold.py +46 -0
- scripts/evaluate_geomanimol_residual.py +45 -0
- scripts/evaluate_geomanimol_torsion.py +40 -0
- scripts/evaluate_hif2a_bioactive_enrichment.py +91 -0
- scripts/evaluate_hif2a_pms_suite.py +440 -0
- scripts/evaluate_pdbbind_contact_pms.py +155 -0
- scripts/evaluate_platinum_external.py +114 -0
- scripts/evaluate_platinum_pool.py +82 -0
- scripts/evaluate_route2_calibrated_p.py +104 -0
- scripts/evaluate_route2_library_components.py +596 -0
- scripts/evaluate_route2_p_basin_budget.py +288 -0
- scripts/evaluate_stage1_checkpoint_only.py +30 -0
- scripts/export_dispersion_bond_audit.py +136 -0
- scripts/export_top20_shards.py +207 -0
- scripts/external_validation_metrics.py +152 -0
- scripts/fit_topology_prior.py +117 -0
- scripts/frozen_dispersion_audit.py +281 -0
- scripts/geomanimol_debug.py +50 -0
- scripts/geomanimol_smoke.py +37 -0
- scripts/infer_etflow_matched.py +524 -0
- scripts/inspect_checkpoint.py +12 -0
- scripts/inspect_conformer_capacity.py +71 -0
- scripts/inspect_pool.py +12 -0
- scripts/make_figure3_selection_loss.py +474 -0
- scripts/make_figure3_selection_summary.py +275 -0
- scripts/make_geom_debug_subset.py +62 -0
- scripts/make_low_reference_test_subset.py +70 -0
- scripts/merge_stage1_into_stage2.py +82 -0
- scripts/place_hif2a_candidates_for_evaluation.py +66 -0
- scripts/plot_hif2a_low_budget_efficiency.py +100 -0
- scripts/plot_prior_validation.py +48 -0
- scripts/precompute_stage2_assignments.py +90 -0
- scripts/prepare_bace_activity.py +98 -0
- scripts/prepare_bace_etkdg_benchmark.py +90 -0
- scripts/prepare_figure5_4x4_cases.py +470 -0
- scripts/prepare_geom_drugs_geomol_top20.py +472 -0
- scripts/prepare_pdbbind_core_raw.py +90 -0
- scripts/prepare_platinum_raw.py +74 -0
- scripts/prepare_pubchem_etkdg.py +54 -0
- scripts/rebuild_hif2a_final_sdf_and_metrics.py +55 -0
- scripts/rebuild_hif2a_fixed_pool_sweep.py +175 -0
- scripts/rebuild_hif2a_strict.py +220 -0
- scripts/recompute_hif2a_bootstrap.py +63 -0
- scripts/reference_free_geometry.py +112 -0
- scripts/render_binding_modes.py +413 -0
- scripts/render_figure5_4x4.py +330 -0
- scripts/render_figure6_pharmacophore_pymol.py +162 -0
- scripts/render_hif2a_contact_pharmacophore_pymol.py +414 -0
- scripts/render_hif2a_figure11.py +397 -0
- scripts/run_bace_2d_baseline.py +73 -0
- scripts/run_drugs_validation_matrix.py +225 -0
- scripts/run_external_reference_free.py +53 -0
- scripts/run_manimol_finalk_sensitivity.py +106 -0
- scripts/run_manimol_oversampling_sensitivity.py +109 -0
- scripts/run_pdisp_matched_intervention.py +116 -0
- scripts/run_stage1_stage2_cross20.py +83 -0
- scripts/score_hif2a_candidates_with_pgraph.py +221 -0
- scripts/summarize_dispersion_intervention.py +88 -0
- scripts/test_mean_dispersion_bridge.py +50 -0
- scripts/train_dispersion_head_diagnostic.py +320 -0
- scripts/train_dispersion_head_streaming.py +519 -0
- scripts/train_geomanimol.py +100 -0
- scripts/train_geomanimol_flow.py +26 -0
- scripts/train_geomanimol_residual.py +70 -0
- scripts/train_geomanimol_torsion.py +50 -0
- scripts/train_quantile_representation_diagnostic.py +204 -0
- scripts/validate_mean_dispersion_prior.py +173 -0
- scripts/validate_packed_shard.py +41 -0
- scripts/validate_stage1_prior.py +221 -0
- scripts/validate_top20_shards.py +36 -0
- scripts/validation/__init__.py +1 -0
- scripts/validation/hif2a_downstream_baselines.py +105 -0
- scripts/validation/hif2a_pairwise_ranker.py +218 -0
- scripts/verify_binding_mode_figure.py +47 -0
- torsion_manifold.py +281 -0
- train.py +78 -0
- train_graph_encoder.py +206 -0
- train_local_torsion_corrector.py +511 -0
- train_mean_dispersion_torsion_prior.py +329 -0
- train_probability_basin.py +235 -0
- train_torsion_generator.py +1631 -0
- train_torsion_manifold_prior.py +258 -0
- train_torsional_denoiser.py +415 -0
- trainer.py +1038 -0
- utils/__init__.py +7 -0
- utils/checkpoint.py +40 -0
- utils/device.py +17 -0
- utils/helpers.py +98 -0
- utils/kabsch.py +29 -0
- utils/lookup_table.py +164 -0
- utils/metrics.py +50 -0
- utils/optuna.py +111 -0
- utils/save_mol.py +366 -0
- utils/util.py +36 -0
trainer.py
ADDED
|
@@ -0,0 +1,1038 @@
|
|
|
1
|
+
# trainer.py
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import random
|
|
7
|
+
import signal
|
|
8
|
+
from typing import Any, Optional, Tuple, Dict, List, Iterator
|
|
9
|
+
|
|
10
|
+
from rdkit import Chem
|
|
11
|
+
from rdkit.Geometry import Point3D
|
|
12
|
+
from torch import nn
|
|
13
|
+
from tqdm import tqdm
|
|
14
|
+
import torch
|
|
15
|
+
import torch.distributed as dist
|
|
16
|
+
import torch.nn.functional as F
|
|
17
|
+
from torch.optim.lr_scheduler import CosineAnnealingLR
|
|
18
|
+
from torch.utils.data import Sampler
|
|
19
|
+
from torch_geometric.loader import DataLoader
|
|
20
|
+
try:
|
|
21
|
+
from torch.utils.tensorboard import SummaryWriter
|
|
22
|
+
except ModuleNotFoundError:
|
|
23
|
+
class SummaryWriter: # type: ignore[no-redef]
|
|
24
|
+
"""No-op fallback for inference environments without tensorboard."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, *args, **kwargs):
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
def add_scalar(self, *args, **kwargs):
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
def add_histogram(self, *args, **kwargs):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
def flush(self):
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
def close(self):
|
|
39
|
+
pass
|
|
40
|
+
|
|
41
|
+
from dataset import QM9Dataset
|
|
42
|
+
from dataset.sharded_drugdataset import ShardedDrugDataset
|
|
43
|
+
from models import GNNEncoder
|
|
44
|
+
from exputils import describe_model
|
|
45
|
+
from utils.device import get_device
|
|
46
|
+
from utils.checkpoint import CheckpointManager
|
|
47
|
+
from utils.helpers import to_runconfig_from_args, get_arg, safe_float, save_molecule_views
|
|
48
|
+
from utils.optuna import report_optuna_and_maybe_prune, write_final_metrics, handle_training_exception
|
|
49
|
+
from models.model0 import prob_low_dim, cross_entropy_loss, center_and_rescale, safe_kabsch_rmsd, heavy_atom_mask_from_x
|
|
50
|
+
from utils.kabsch import kabsch_rmsd
|
|
51
|
+
|
|
52
|
+
logger = logging.getLogger(__name__)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class MoleculeGroupedBatchSampler(Sampler[List[int]]):
|
|
56
|
+
def __init__(self, dataset, batch_size: int, shuffle: bool = True, drop_last: bool = False):
|
|
57
|
+
self.dataset = dataset
|
|
58
|
+
self.batch_size = int(batch_size)
|
|
59
|
+
self.shuffle = bool(shuffle)
|
|
60
|
+
self.drop_last = bool(drop_last)
|
|
61
|
+
self.groups_by_storage = getattr(dataset, "molecule_groups_by_shard", [])
|
|
62
|
+
self.groups = self._build_groups()
|
|
63
|
+
|
|
64
|
+
def _build_groups(self) -> List[List[int]]:
|
|
65
|
+
if self.groups_by_storage:
|
|
66
|
+
return [group for block in self.groups_by_storage for group in block]
|
|
67
|
+
mol_to_indices: Dict[int, List[int]] = {}
|
|
68
|
+
for local_idx in range(len(self.dataset)):
|
|
69
|
+
data = self.dataset[local_idx]
|
|
70
|
+
mol_id = int(data.mol_id.item()) if hasattr(data, "mol_id") else local_idx
|
|
71
|
+
if mol_id not in mol_to_indices:
|
|
72
|
+
mol_to_indices[mol_id] = []
|
|
73
|
+
mol_to_indices[mol_id].append(local_idx)
|
|
74
|
+
groups = list(mol_to_indices.values())
|
|
75
|
+
for group in groups:
|
|
76
|
+
group.sort(key=lambda idx: int(self.dataset[idx].conf_id.item()) if hasattr(self.dataset[idx], "conf_id") else idx)
|
|
77
|
+
return groups
|
|
78
|
+
|
|
79
|
+
def __iter__(self) -> Iterator[List[int]]:
|
|
80
|
+
if self.groups_by_storage:
|
|
81
|
+
blocks = [list(block) for block in self.groups_by_storage]
|
|
82
|
+
if self.shuffle:
|
|
83
|
+
random.shuffle(blocks)
|
|
84
|
+
for block in blocks:
|
|
85
|
+
random.shuffle(block)
|
|
86
|
+
groups = [group for block in blocks for group in block]
|
|
87
|
+
else:
|
|
88
|
+
groups = list(self.groups)
|
|
89
|
+
if self.shuffle:
|
|
90
|
+
random.shuffle(groups)
|
|
91
|
+
current_batch: List[int] = []
|
|
92
|
+
for group in groups:
|
|
93
|
+
if len(group) >= self.batch_size:
|
|
94
|
+
if len(current_batch) > 0:
|
|
95
|
+
yield current_batch
|
|
96
|
+
current_batch = []
|
|
97
|
+
yield list(group)
|
|
98
|
+
continue
|
|
99
|
+
if len(current_batch) + len(group) > self.batch_size:
|
|
100
|
+
if len(current_batch) > 0:
|
|
101
|
+
yield current_batch
|
|
102
|
+
current_batch = list(group)
|
|
103
|
+
else:
|
|
104
|
+
current_batch.extend(group)
|
|
105
|
+
if len(current_batch) > 0 and not self.drop_last:
|
|
106
|
+
yield current_batch
|
|
107
|
+
|
|
108
|
+
def __len__(self) -> int:
|
|
109
|
+
count = 0
|
|
110
|
+
current_size = 0
|
|
111
|
+
for group in self.groups:
|
|
112
|
+
gsize = len(group)
|
|
113
|
+
if gsize >= self.batch_size:
|
|
114
|
+
if current_size > 0:
|
|
115
|
+
count += 1
|
|
116
|
+
current_size = 0
|
|
117
|
+
count += 1
|
|
118
|
+
continue
|
|
119
|
+
if current_size + gsize > self.batch_size:
|
|
120
|
+
count += 1
|
|
121
|
+
current_size = gsize
|
|
122
|
+
else:
|
|
123
|
+
current_size += gsize
|
|
124
|
+
if current_size > 0 and not self.drop_last:
|
|
125
|
+
count += 1
|
|
126
|
+
return count
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class DistributedMoleculeGroupedBatchSampler(MoleculeGroupedBatchSampler):
|
|
130
|
+
"""Deterministically shard complete molecule batches across ranks."""
|
|
131
|
+
|
|
132
|
+
def __init__(self, dataset, batch_size: int, rank: int, world_size: int, seed: int = 0):
|
|
133
|
+
super().__init__(dataset, batch_size=batch_size, shuffle=False, drop_last=False)
|
|
134
|
+
self.rank = int(rank)
|
|
135
|
+
self.world_size = int(world_size)
|
|
136
|
+
self.seed = int(seed)
|
|
137
|
+
self.epoch = 0
|
|
138
|
+
|
|
139
|
+
def set_epoch(self, epoch: int) -> None:
|
|
140
|
+
self.epoch = int(epoch)
|
|
141
|
+
|
|
142
|
+
@staticmethod
|
|
143
|
+
def _pack_groups(groups: List[List[int]], batch_size: int) -> List[List[int]]:
|
|
144
|
+
batches: List[List[int]] = []
|
|
145
|
+
current: List[int] = []
|
|
146
|
+
for group in groups:
|
|
147
|
+
if len(group) >= batch_size:
|
|
148
|
+
if current:
|
|
149
|
+
batches.append(current)
|
|
150
|
+
current = []
|
|
151
|
+
batches.append(list(group))
|
|
152
|
+
elif len(current) + len(group) > batch_size:
|
|
153
|
+
if current:
|
|
154
|
+
batches.append(current)
|
|
155
|
+
current = list(group)
|
|
156
|
+
else:
|
|
157
|
+
current.extend(group)
|
|
158
|
+
if current:
|
|
159
|
+
batches.append(current)
|
|
160
|
+
return batches
|
|
161
|
+
|
|
162
|
+
def _batches(self) -> List[List[int]]:
|
|
163
|
+
rng = random.Random(self.seed + self.epoch)
|
|
164
|
+
if self.groups_by_storage:
|
|
165
|
+
# Keep batches shard-local. Globally shuffling the flattened groups
|
|
166
|
+
# defeats the tiny shard cache and makes every rank repeatedly
|
|
167
|
+
# deserialize unrelated files while its GPU waits for data.
|
|
168
|
+
blocks = [list(block) for block in self.groups_by_storage]
|
|
169
|
+
rng.shuffle(blocks)
|
|
170
|
+
batches: List[List[int]] = []
|
|
171
|
+
for block in blocks:
|
|
172
|
+
rng.shuffle(block)
|
|
173
|
+
batches.extend(self._pack_groups(block, self.batch_size))
|
|
174
|
+
else:
|
|
175
|
+
groups = list(self.groups)
|
|
176
|
+
rng.shuffle(groups)
|
|
177
|
+
batches = self._pack_groups(groups, self.batch_size)
|
|
178
|
+
if batches:
|
|
179
|
+
target = ((len(batches) + self.world_size - 1) // self.world_size) * self.world_size
|
|
180
|
+
batches.extend([list(batches[i % len(batches)]) for i in range(target - len(batches))])
|
|
181
|
+
return batches
|
|
182
|
+
|
|
183
|
+
def __iter__(self) -> Iterator[List[int]]:
|
|
184
|
+
return iter(self._batches()[self.rank::self.world_size])
|
|
185
|
+
|
|
186
|
+
def __len__(self) -> int:
|
|
187
|
+
return len(self._batches()) // self.world_size
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class EarlyStopping:
|
|
191
|
+
def __init__(self, patience: int = 10, lower_better: bool = True):
|
|
192
|
+
self.patience = int(patience)
|
|
193
|
+
self.lower_better = bool(lower_better)
|
|
194
|
+
self.counter = 0
|
|
195
|
+
self.best_score = float("inf") if lower_better else -float("inf")
|
|
196
|
+
self.best_model_state = None
|
|
197
|
+
self.early_stop = False
|
|
198
|
+
|
|
199
|
+
def step(self, score: float, model: torch.nn.Module) -> None:
|
|
200
|
+
improved = score < self.best_score if self.lower_better else score > self.best_score
|
|
201
|
+
if improved:
|
|
202
|
+
self.best_score = score
|
|
203
|
+
self.best_model_state = {k: v.cpu().clone() if isinstance(v, torch.Tensor) else v for k, v in model.state_dict().items()}
|
|
204
|
+
self.counter = 0
|
|
205
|
+
else:
|
|
206
|
+
self.counter += 1
|
|
207
|
+
if self.counter >= self.patience:
|
|
208
|
+
self.early_stop = True
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class Trainer:
|
|
212
|
+
def __init__(self, args: Any, writer: SummaryWriter, logger_path: str):
|
|
213
|
+
self.args = args
|
|
214
|
+
self.cfg = to_runconfig_from_args(args)
|
|
215
|
+
self.device = get_device(self.cfg.gpu)
|
|
216
|
+
self.writer = writer
|
|
217
|
+
self.logger_path = str(logger_path)
|
|
218
|
+
self.checkpoint_mgr = CheckpointManager(self.logger_path)
|
|
219
|
+
self.total_step = 0
|
|
220
|
+
self.did_optimizer_step = False
|
|
221
|
+
self.terminate_requested = False
|
|
222
|
+
self.checkpoint_interval_steps = max(
|
|
223
|
+
0, int(getattr(args, "checkpoint_interval_steps", 1000))
|
|
224
|
+
)
|
|
225
|
+
self.max_confs = int(getattr(args, "max_confs", 5))
|
|
226
|
+
|
|
227
|
+
sharded_root = str(getattr(args, "sharded_data_root", "") or "")
|
|
228
|
+
if sharded_root:
|
|
229
|
+
logger.info("Using isolated tensor-only shard cache: %s", sharded_root)
|
|
230
|
+
dataset = None
|
|
231
|
+
assignment_root = str(getattr(args, "stage2_assignment_cache", "") or "")
|
|
232
|
+
self.train_set = ShardedDrugDataset(sharded_root, "train", getattr(args, "shard_cache_size", 2), assignment_root)
|
|
233
|
+
self.valid_set = ShardedDrugDataset(sharded_root, "val", getattr(args, "shard_cache_size", 2), assignment_root)
|
|
234
|
+
self.test_set = ShardedDrugDataset(sharded_root, "test", getattr(args, "shard_cache_size", 2), assignment_root)
|
|
235
|
+
self.max_confs = int(getattr(args, "max_confs", 20))
|
|
236
|
+
self.args.max_confs = self.max_confs
|
|
237
|
+
self.cfg.max_confs = self.max_confs
|
|
238
|
+
else:
|
|
239
|
+
dataset = QM9Dataset(
|
|
240
|
+
name=self.cfg.dataset,
|
|
241
|
+
root=self.cfg.data_root,
|
|
242
|
+
max_confs=self.max_confs,
|
|
243
|
+
raw_prefix=getattr(args, "raw_prefix", "converted"),
|
|
244
|
+
train_raw_filename=getattr(args, "train_raw_filename", None),
|
|
245
|
+
val_raw_filename=getattr(args, "val_raw_filename", None),
|
|
246
|
+
test_raw_filename=getattr(args, "test_raw_filename", None),
|
|
247
|
+
keep_full_test_conformers=bool(getattr(args, "full_test_conformers", False)),
|
|
248
|
+
)
|
|
249
|
+
# The dataset resolves auto mode before preprocessing. Propagate the
|
|
250
|
+
# resolved value into the encoder so its conformer embedding capacity
|
|
251
|
+
# exactly matches the cached dataset.
|
|
252
|
+
if dataset is not None:
|
|
253
|
+
self.max_confs = int(dataset.max_confs)
|
|
254
|
+
self.args.max_confs = self.max_confs
|
|
255
|
+
self.cfg.max_confs = self.max_confs
|
|
256
|
+
self.train_set = dataset[dataset.train_index]
|
|
257
|
+
self.valid_set = dataset[dataset.valid_index]
|
|
258
|
+
self.test_set = dataset[dataset.test_index]
|
|
259
|
+
self.train_set = self._limit_molecules(self.train_set, int(getattr(args, "train_mol_limit", 0)))
|
|
260
|
+
self.valid_set = self._limit_molecules(self.valid_set, int(getattr(args, "valid_mol_limit", 0)))
|
|
261
|
+
self.test_set = self._limit_molecules(self.test_set, int(getattr(args, "test_mol_limit", 0)))
|
|
262
|
+
|
|
263
|
+
loader_kwargs = dict(
|
|
264
|
+
num_workers=int(getattr(self.cfg, "num_workers", 0)),
|
|
265
|
+
pin_memory=bool(getattr(self.cfg, "pin_memory", False)) and self.device.type == "cuda",
|
|
266
|
+
)
|
|
267
|
+
if loader_kwargs["num_workers"] > 0:
|
|
268
|
+
loader_kwargs["persistent_workers"] = True
|
|
269
|
+
prefetch_factor = getattr(self.cfg, "prefetch_factor", None)
|
|
270
|
+
if prefetch_factor is not None:
|
|
271
|
+
loader_kwargs["prefetch_factor"] = int(prefetch_factor)
|
|
272
|
+
|
|
273
|
+
distributed_stage2 = bool(getattr(args, "distributed_stage2", False))
|
|
274
|
+
if distributed_stage2:
|
|
275
|
+
if not dist.is_available() or not dist.is_initialized():
|
|
276
|
+
raise RuntimeError("--distributed_stage2 requires torchrun with an initialized process group")
|
|
277
|
+
train_batch_sampler = DistributedMoleculeGroupedBatchSampler(
|
|
278
|
+
self.train_set,
|
|
279
|
+
batch_size=self.cfg.bs,
|
|
280
|
+
rank=dist.get_rank(),
|
|
281
|
+
world_size=dist.get_world_size(),
|
|
282
|
+
seed=int(getattr(args, "random_seed", 0)),
|
|
283
|
+
)
|
|
284
|
+
else:
|
|
285
|
+
train_batch_sampler = MoleculeGroupedBatchSampler(
|
|
286
|
+
self.train_set, batch_size=self.cfg.bs, shuffle=True, drop_last=False
|
|
287
|
+
)
|
|
288
|
+
self.train_loader = DataLoader(
|
|
289
|
+
self.train_set,
|
|
290
|
+
batch_sampler=train_batch_sampler,
|
|
291
|
+
**loader_kwargs,
|
|
292
|
+
)
|
|
293
|
+
self.valid_loader = DataLoader(
|
|
294
|
+
self.valid_set,
|
|
295
|
+
batch_sampler=MoleculeGroupedBatchSampler(self.valid_set, batch_size=self.cfg.bs, shuffle=False, drop_last=False),
|
|
296
|
+
**loader_kwargs,
|
|
297
|
+
)
|
|
298
|
+
self.test_loader = DataLoader(
|
|
299
|
+
self.test_set,
|
|
300
|
+
batch_sampler=MoleculeGroupedBatchSampler(self.test_set, batch_size=self.cfg.bs, shuffle=False, drop_last=False),
|
|
301
|
+
**loader_kwargs,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
self.model = GNNEncoder(args=args, config={"model": {"model_level": "graph"}}).to(self.device)
|
|
305
|
+
|
|
306
|
+
self._load_checkpoint_if_needed()
|
|
307
|
+
self._apply_train_stage_freezing()
|
|
308
|
+
|
|
309
|
+
trainable_params = [p for p in self.model.parameters() if p.requires_grad]
|
|
310
|
+
if len(trainable_params) == 0:
|
|
311
|
+
raise RuntimeError("没有可训练参数,请检查 train_stage 和 requires_grad 设置。")
|
|
312
|
+
|
|
313
|
+
self.opt = self._build_optimizer()
|
|
314
|
+
|
|
315
|
+
trainable_count = sum(p.numel() for p in self.model.parameters() if p.requires_grad)
|
|
316
|
+
frozen_count = sum(p.numel() for p in self.model.parameters() if not p.requires_grad)
|
|
317
|
+
logger.info(f"Trainable parameters: {trainable_count}")
|
|
318
|
+
logger.info(f"Frozen parameters: {frozen_count}")
|
|
319
|
+
|
|
320
|
+
eta_min = float(self.cfg.eta_min)
|
|
321
|
+
if eta_min >= float(self.cfg.lr):
|
|
322
|
+
logger.warning(f"eta_min={eta_min} >= lr={self.cfg.lr},已自动改为 lr * 0.01 = {float(self.cfg.lr) * 0.01}")
|
|
323
|
+
eta_min = float(self.cfg.lr) * 0.01
|
|
324
|
+
self.scheduler = CosineAnnealingLR(self.opt, T_max=max(1, int(self.cfg.epoch)), eta_min=eta_min)
|
|
325
|
+
self.resume_epoch = 0
|
|
326
|
+
self.resume_batch_idx = -1
|
|
327
|
+
self.best_valid_score = float("inf")
|
|
328
|
+
self.best_test_score = float("inf")
|
|
329
|
+
self.best_epoch: Optional[int] = None
|
|
330
|
+
self.early_stopper = EarlyStopping(patience=self.cfg.patience, lower_better=True)
|
|
331
|
+
self._load_progress_checkpoint_if_needed()
|
|
332
|
+
|
|
333
|
+
self.last_found: Dict[str, Any] = dict(found=False, mol_pred=None, pos_gt_data=None, smiles_target=None)
|
|
334
|
+
try:
|
|
335
|
+
describe_model(self.model, path=self.logger_path)
|
|
336
|
+
except Exception:
|
|
337
|
+
logger.exception("describe_model 出错,可忽略")
|
|
338
|
+
logger.info(f"Trainer initialized on device {self.device}")
|
|
339
|
+
logger.info("Data unit = conformer; training batch is grouped by mol_id.")
|
|
340
|
+
signal.signal(signal.SIGTERM, self._request_termination)
|
|
341
|
+
|
|
342
|
+
def _request_termination(self, signum, frame):
|
|
343
|
+
self.terminate_requested = True
|
|
344
|
+
logger.warning(
|
|
345
|
+
"Received SIGTERM; saving an interruption checkpoint after the current batch."
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
def _save_progress_checkpoint(self, filename: str, epoch: int, batch_idx: int) -> None:
|
|
349
|
+
path = os.path.join(self.logger_path, filename)
|
|
350
|
+
tmp_path = f"{path}.tmp"
|
|
351
|
+
torch.save(
|
|
352
|
+
{
|
|
353
|
+
"model_state_dict": self.model.state_dict(),
|
|
354
|
+
"optimizer_state_dict": self.opt.state_dict(),
|
|
355
|
+
"epoch": int(epoch),
|
|
356
|
+
"batch_idx": int(batch_idx),
|
|
357
|
+
"total_step": int(self.total_step),
|
|
358
|
+
"scheduler_state_dict": self.scheduler.state_dict(),
|
|
359
|
+
"early_stopper_state": {
|
|
360
|
+
"counter": int(self.early_stopper.counter),
|
|
361
|
+
"best_score": float(self.early_stopper.best_score),
|
|
362
|
+
"best_model_state": self.early_stopper.best_model_state,
|
|
363
|
+
"early_stop": bool(self.early_stopper.early_stop),
|
|
364
|
+
},
|
|
365
|
+
"training_state": {
|
|
366
|
+
"best_valid_score": float(self.best_valid_score),
|
|
367
|
+
"best_test_score": float(self.best_test_score),
|
|
368
|
+
"best_epoch": self.best_epoch,
|
|
369
|
+
},
|
|
370
|
+
},
|
|
371
|
+
tmp_path,
|
|
372
|
+
)
|
|
373
|
+
os.replace(tmp_path, path)
|
|
374
|
+
logger.info("Saved progress checkpoint: %s", path)
|
|
375
|
+
|
|
376
|
+
def _limit_molecules(self, subset, mol_limit: int):
|
|
377
|
+
if mol_limit <= 0:
|
|
378
|
+
return subset
|
|
379
|
+
keep_indices = []
|
|
380
|
+
seen = set()
|
|
381
|
+
for idx in range(len(subset)):
|
|
382
|
+
data = subset[idx]
|
|
383
|
+
mol_id = int(data.mol_id.item()) if hasattr(data, "mol_id") else idx
|
|
384
|
+
if mol_id not in seen:
|
|
385
|
+
if len(seen) >= int(mol_limit):
|
|
386
|
+
break
|
|
387
|
+
seen.add(mol_id)
|
|
388
|
+
if mol_id in seen:
|
|
389
|
+
keep_indices.append(idx)
|
|
390
|
+
logger.info(f"Limited dataset split to {len(seen)} molecules / {len(keep_indices)} conformers.")
|
|
391
|
+
return subset[keep_indices]
|
|
392
|
+
|
|
393
|
+
def _build_optimizer(self):
|
|
394
|
+
if not bool(getattr(self.args, "use_end2end_pglobal", False)):
|
|
395
|
+
trainable_params = [p for p in self.model.parameters() if p.requires_grad]
|
|
396
|
+
return torch.optim.Adam(trainable_params, lr=self.cfg.lr)
|
|
397
|
+
|
|
398
|
+
pglobal_prefixes = ("gnn.", "global_edge_predictor.", "ab_head.", "conf_embedding.")
|
|
399
|
+
pglobal_params = []
|
|
400
|
+
other_params = []
|
|
401
|
+
for name, param in self.model.named_parameters():
|
|
402
|
+
if not param.requires_grad:
|
|
403
|
+
continue
|
|
404
|
+
if name.startswith(pglobal_prefixes):
|
|
405
|
+
pglobal_params.append(param)
|
|
406
|
+
else:
|
|
407
|
+
other_params.append(param)
|
|
408
|
+
groups = []
|
|
409
|
+
if other_params:
|
|
410
|
+
groups.append({"params": other_params, "lr": self.cfg.lr})
|
|
411
|
+
if pglobal_params:
|
|
412
|
+
groups.append({"params": pglobal_params, "lr": float(getattr(self.args, "pglobal_lr", 3e-5))})
|
|
413
|
+
if not groups:
|
|
414
|
+
raise RuntimeError("没有可训练参数,请检查 train_stage 和 requires_grad 设置。")
|
|
415
|
+
logger.info(
|
|
416
|
+
f"End-to-end optimizer groups: decoder/refiner lr={self.cfg.lr}, "
|
|
417
|
+
f"P_global lr={float(getattr(self.args, 'pglobal_lr', 3e-5))}"
|
|
418
|
+
)
|
|
419
|
+
return torch.optim.Adam(groups)
|
|
420
|
+
|
|
421
|
+
# --------------------------------------------------------
|
|
422
|
+
# Checkpoint loading
|
|
423
|
+
# --------------------------------------------------------
|
|
424
|
+
def _load_checkpoint_if_needed(self):
|
|
425
|
+
checkpoint_path = getattr(self.args, "checkpoint_path", None)
|
|
426
|
+
if not checkpoint_path:
|
|
427
|
+
return
|
|
428
|
+
try:
|
|
429
|
+
ckpt = torch.load(checkpoint_path, map_location=self.device)
|
|
430
|
+
if isinstance(ckpt, dict) and "model_state_dict" in ckpt:
|
|
431
|
+
state = ckpt["model_state_dict"]
|
|
432
|
+
epoch = ckpt.get("epoch", "unknown")
|
|
433
|
+
elif isinstance(ckpt, dict) and "state_dict" in ckpt:
|
|
434
|
+
state = ckpt["state_dict"]
|
|
435
|
+
epoch = ckpt.get("epoch", "unknown")
|
|
436
|
+
else:
|
|
437
|
+
state = ckpt
|
|
438
|
+
epoch = "unknown"
|
|
439
|
+
model_state = self.model.state_dict()
|
|
440
|
+
compatible_state = {}
|
|
441
|
+
skipped = []
|
|
442
|
+
|
|
443
|
+
stage = getattr(self.args, "train_stage", "gnn")
|
|
444
|
+
skip_prefixes = []
|
|
445
|
+
if stage == "multimode":
|
|
446
|
+
skip_prefixes = ["mode_generator", "coarse_decoder", "conditioned_decoder",
|
|
447
|
+
"refiner", "torsion_residual_head"]
|
|
448
|
+
if not bool(getattr(self.args, "use_end2end_pglobal", False)):
|
|
449
|
+
skip_prefixes.extend(["ab_head", "fusion_weight_head"])
|
|
450
|
+
|
|
451
|
+
for key, value in state.items():
|
|
452
|
+
should_skip = any(key.startswith(prefix + ".") or key == prefix for prefix in skip_prefixes)
|
|
453
|
+
if should_skip:
|
|
454
|
+
skipped.append(key)
|
|
455
|
+
continue
|
|
456
|
+
if key in model_state and model_state[key].shape == value.shape:
|
|
457
|
+
compatible_state[key] = value
|
|
458
|
+
else:
|
|
459
|
+
skipped.append(key)
|
|
460
|
+
model_state.update(compatible_state)
|
|
461
|
+
self.model.load_state_dict(model_state, strict=True)
|
|
462
|
+
logger.info(f"Loaded compatible checkpoint from {checkpoint_path} (epoch {epoch}), loaded={len(compatible_state)}, skipped={len(skipped)}")
|
|
463
|
+
if skipped:
|
|
464
|
+
logger.warning(f"Skipped incompatible keys: {skipped}")
|
|
465
|
+
except Exception:
|
|
466
|
+
logger.exception("加载 checkpoint 失败,继续从头开始")
|
|
467
|
+
|
|
468
|
+
def _load_progress_checkpoint_if_needed(self):
|
|
469
|
+
"""Restore an interruption/latest checkpoint when explicitly requested.
|
|
470
|
+
|
|
471
|
+
This is separate from ``checkpoint_path`` so legacy runs keep their
|
|
472
|
+
historical weight-only loading behavior. Progress checkpoints also
|
|
473
|
+
restore optimizer state and resume at the first unprocessed batch.
|
|
474
|
+
"""
|
|
475
|
+
checkpoint_path = getattr(self.args, "resume_progress_checkpoint", None)
|
|
476
|
+
if not checkpoint_path:
|
|
477
|
+
return
|
|
478
|
+
if not os.path.exists(checkpoint_path):
|
|
479
|
+
raise FileNotFoundError(f"Progress checkpoint not found: {checkpoint_path}")
|
|
480
|
+
ckpt = torch.load(checkpoint_path, map_location=self.device)
|
|
481
|
+
required = ("model_state_dict", "optimizer_state_dict", "epoch", "batch_idx", "total_step")
|
|
482
|
+
missing = [key for key in required if key not in ckpt]
|
|
483
|
+
if missing:
|
|
484
|
+
raise ValueError(f"Not a resumable progress checkpoint; missing keys: {missing}")
|
|
485
|
+
self.model.load_state_dict(ckpt["model_state_dict"], strict=True)
|
|
486
|
+
self.opt.load_state_dict(ckpt["optimizer_state_dict"])
|
|
487
|
+
self.resume_epoch = int(ckpt["epoch"])
|
|
488
|
+
self.resume_batch_idx = int(ckpt["batch_idx"])
|
|
489
|
+
self.total_step = int(ckpt["total_step"])
|
|
490
|
+
scheduler_state = ckpt.get("scheduler_state_dict")
|
|
491
|
+
if scheduler_state:
|
|
492
|
+
saved_t_max = int(scheduler_state.get("T_max", 0))
|
|
493
|
+
if int(self.cfg.epoch) > saved_t_max:
|
|
494
|
+
# The prior short run used a smaller cosine horizon. Preserve
|
|
495
|
+
# the current optimizer LR and decay it over the added epochs.
|
|
496
|
+
for group in self.opt.param_groups:
|
|
497
|
+
group["initial_lr"] = group["lr"]
|
|
498
|
+
remaining_epochs = max(1, int(self.cfg.epoch) - self.resume_epoch)
|
|
499
|
+
self.scheduler = CosineAnnealingLR(
|
|
500
|
+
self.opt,
|
|
501
|
+
T_max=remaining_epochs,
|
|
502
|
+
eta_min=float(self.cfg.eta_min),
|
|
503
|
+
)
|
|
504
|
+
logger.info(
|
|
505
|
+
"Extended scheduler horizon from T_max=%d to %d remaining epochs.",
|
|
506
|
+
saved_t_max,
|
|
507
|
+
remaining_epochs,
|
|
508
|
+
)
|
|
509
|
+
else:
|
|
510
|
+
self.scheduler.load_state_dict(scheduler_state)
|
|
511
|
+
elif int(self.cfg.epoch) > self.resume_epoch:
|
|
512
|
+
# Older progress checkpoints did not serialize the scheduler.
|
|
513
|
+
# Start an extended decay from the optimizer LR that was restored.
|
|
514
|
+
for group in self.opt.param_groups:
|
|
515
|
+
group["initial_lr"] = group["lr"]
|
|
516
|
+
remaining_epochs = max(1, int(self.cfg.epoch) - self.resume_epoch)
|
|
517
|
+
self.scheduler = CosineAnnealingLR(
|
|
518
|
+
self.opt,
|
|
519
|
+
T_max=remaining_epochs,
|
|
520
|
+
eta_min=float(self.cfg.eta_min),
|
|
521
|
+
)
|
|
522
|
+
logger.info(
|
|
523
|
+
"Rebased scheduler for legacy progress checkpoint over %d remaining epochs.",
|
|
524
|
+
remaining_epochs,
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
early_state = ckpt.get("early_stopper_state")
|
|
528
|
+
if early_state:
|
|
529
|
+
self.early_stopper.counter = int(early_state.get("counter", 0))
|
|
530
|
+
self.early_stopper.best_score = float(
|
|
531
|
+
early_state.get("best_score", self.early_stopper.best_score)
|
|
532
|
+
)
|
|
533
|
+
self.early_stopper.best_model_state = early_state.get("best_model_state")
|
|
534
|
+
self.early_stopper.early_stop = bool(early_state.get("early_stop", False))
|
|
535
|
+
|
|
536
|
+
training_state = ckpt.get("training_state", {})
|
|
537
|
+
self.best_valid_score = float(training_state.get("best_valid_score", self.best_valid_score))
|
|
538
|
+
self.best_test_score = float(training_state.get("best_test_score", self.best_test_score))
|
|
539
|
+
self.best_epoch = training_state.get("best_epoch", self.best_epoch)
|
|
540
|
+
if bool(getattr(self.args, "resume_next_epoch", False)) and self.resume_batch_idx >= 0:
|
|
541
|
+
logger.warning(
|
|
542
|
+
"Fast resume enabled: dropping the unfinished tail of epoch %d "
|
|
543
|
+
"and starting epoch %d to avoid replaying %d data-loader batches.",
|
|
544
|
+
self.resume_epoch,
|
|
545
|
+
self.resume_epoch + 1,
|
|
546
|
+
self.resume_batch_idx + 1,
|
|
547
|
+
)
|
|
548
|
+
self.resume_epoch += 1
|
|
549
|
+
self.resume_batch_idx = -1
|
|
550
|
+
logger.info(
|
|
551
|
+
"Resumed progress checkpoint %s: epoch=%d, last_batch=%d, "
|
|
552
|
+
"next_batch=%d, total_step=%d",
|
|
553
|
+
checkpoint_path,
|
|
554
|
+
self.resume_epoch,
|
|
555
|
+
self.resume_batch_idx,
|
|
556
|
+
self.resume_batch_idx + 1,
|
|
557
|
+
self.total_step,
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
# --------------------------------------------------------
|
|
561
|
+
# Freezing
|
|
562
|
+
# --------------------------------------------------------
|
|
563
|
+
def _set_requires_grad(self, module: torch.nn.Module, flag: bool):
|
|
564
|
+
for param in module.parameters():
|
|
565
|
+
param.requires_grad = flag
|
|
566
|
+
|
|
567
|
+
def _unfreeze_modules(self, names: List[str]):
|
|
568
|
+
for name in names:
|
|
569
|
+
if hasattr(self.model, name):
|
|
570
|
+
self._set_requires_grad(getattr(self.model, name), True)
|
|
571
|
+
|
|
572
|
+
def _freeze_modules(self, names: List[str]):
|
|
573
|
+
for name in names:
|
|
574
|
+
if hasattr(self.model, name):
|
|
575
|
+
self._set_requires_grad(getattr(self.model, name), False)
|
|
576
|
+
|
|
577
|
+
def _apply_train_stage_freezing(self):
|
|
578
|
+
stage = getattr(self.args, "train_stage", "gnn")
|
|
579
|
+
|
|
580
|
+
self._set_requires_grad(self.model, False)
|
|
581
|
+
|
|
582
|
+
if stage == "gnn":
|
|
583
|
+
modules = ["gnn", "global_edge_predictor", "ab_head", "conf_embedding", "fusion_weight_head"]
|
|
584
|
+
if bool(getattr(self.args, "mean_dispersion_prior", False)):
|
|
585
|
+
modules.append("dispersion_edge_predictor")
|
|
586
|
+
self._unfreeze_modules(modules)
|
|
587
|
+
logger.info("train_stage=gnn:训练 GNN / P_global / A-B。")
|
|
588
|
+
if bool(getattr(self.args, "mean_dispersion_prior", False)):
|
|
589
|
+
logger.info("mean_dispersion_prior=True:训练 dispersion_edge_predictor。")
|
|
590
|
+
|
|
591
|
+
elif stage == "multimode":
|
|
592
|
+
modules = ["conditioned_decoder", "coarse_decoder", "refiner", "geometry_refiner"]
|
|
593
|
+
if bool(getattr(self.args, "use_mode_specific_p", False)):
|
|
594
|
+
modules.append("mode_generator")
|
|
595
|
+
if bool(getattr(self.args, "use_torsion_residual", False)):
|
|
596
|
+
modules.append("torsion_residual_head")
|
|
597
|
+
if bool(getattr(self.args, "use_end2end_pglobal", False)):
|
|
598
|
+
modules.extend(["gnn", "global_edge_predictor", "ab_head", "conf_embedding"])
|
|
599
|
+
self._unfreeze_modules(modules)
|
|
600
|
+
logger.info("train_stage=multimode:训练 conditioned decoder + refiner(Z条件化解码)。")
|
|
601
|
+
if bool(getattr(self.args, "use_mode_specific_p", False)):
|
|
602
|
+
logger.info("use_mode_specific_p=True:额外训练 mode_generator,生成 P_k = f(P_global, z_k)。")
|
|
603
|
+
if bool(getattr(self.args, "use_torsion_residual", False)):
|
|
604
|
+
logger.info("use_torsion_residual=True:额外训练 torsion_residual_head,预测二面角残差。")
|
|
605
|
+
if bool(getattr(self.args, "use_end2end_pglobal", False)):
|
|
606
|
+
logger.info("use_end2end_pglobal=True:Stage 2 同时微调 GNN/P_global 分支。")
|
|
607
|
+
|
|
608
|
+
else:
|
|
609
|
+
raise ValueError(f"Unknown train_stage={stage}. Expected one of: gnn, multimode.")
|
|
610
|
+
|
|
611
|
+
# ========================================================
|
|
612
|
+
# Run loop
|
|
613
|
+
# ========================================================
|
|
614
|
+
|
|
615
|
+
def run(self, optuna_trial: Optional[Any] = None) -> Tuple[Optional[float], Optional[int]]:
|
|
616
|
+
best_valid_score = float(self.best_valid_score)
|
|
617
|
+
best_test_score = float(self.best_test_score)
|
|
618
|
+
best_epoch = self.best_epoch
|
|
619
|
+
|
|
620
|
+
try:
|
|
621
|
+
for epoch in range(self.resume_epoch, int(self.cfg.epoch)):
|
|
622
|
+
logger.info(f"Starting epoch {epoch}")
|
|
623
|
+
skip_batches = self.resume_batch_idx + 1 if epoch == self.resume_epoch else 0
|
|
624
|
+
self.train_epoch(epoch, skip_batches=skip_batches)
|
|
625
|
+
if epoch == self.resume_epoch:
|
|
626
|
+
self.resume_batch_idx = -1
|
|
627
|
+
if self.did_optimizer_step:
|
|
628
|
+
self.scheduler.step()
|
|
629
|
+
if self.cfg.metric.upper() == "RMSD":
|
|
630
|
+
train_score = safe_float(self._test_rmsd_limited(self.train_loader, max_samples=100, split_name="train"))
|
|
631
|
+
self.writer.add_scalar("train_rmsd", train_score, epoch)
|
|
632
|
+
valid_score = safe_float(self.test(self.valid_loader, val=True))
|
|
633
|
+
self.writer.add_scalar(f"valid_{self.cfg.metric.lower()}", valid_score, epoch)
|
|
634
|
+
logger.info(f"Epoch {epoch} valid {self.cfg.metric} = {valid_score:.6f}, best_test_so_far={best_test_score:.6f}")
|
|
635
|
+
|
|
636
|
+
if valid_score < best_valid_score:
|
|
637
|
+
test_score = safe_float(self.test(self.test_loader, val=False))
|
|
638
|
+
best_valid_score = valid_score
|
|
639
|
+
best_test_score = test_score
|
|
640
|
+
best_epoch = epoch
|
|
641
|
+
logger.info(f"New best valid {best_valid_score:.6f} (epoch {best_epoch}), test={best_test_score:.6f}")
|
|
642
|
+
try:
|
|
643
|
+
self.checkpoint_mgr.save_best(self.model, self.opt, epoch, best_test_score)
|
|
644
|
+
except Exception:
|
|
645
|
+
logger.exception("保存 best checkpoint 失败")
|
|
646
|
+
|
|
647
|
+
self.best_valid_score = best_valid_score
|
|
648
|
+
self.best_test_score = best_test_score
|
|
649
|
+
self.best_epoch = best_epoch
|
|
650
|
+
|
|
651
|
+
if self.cfg.early_stop:
|
|
652
|
+
self.early_stopper.step(valid_score, self.model)
|
|
653
|
+
if self.early_stopper.early_stop:
|
|
654
|
+
logger.info("Early stopping triggered; restoring best model state")
|
|
655
|
+
if self.early_stopper.best_model_state is not None:
|
|
656
|
+
self.model.load_state_dict(self.early_stopper.best_model_state)
|
|
657
|
+
if self.last_found.get("found", False):
|
|
658
|
+
now_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S.%f")[:-3]
|
|
659
|
+
out_dir = f"rdmol/{now_str}_{self.last_found.get('smiles_target')}"
|
|
660
|
+
try:
|
|
661
|
+
save_molecule_views(out_dir, self.last_found.get("mol_pred"), pos_gt_data=self.last_found.get("pos_gt_data"), smiles_target=self.last_found.get("smiles_target"), args_to_save=self.args)
|
|
662
|
+
except Exception:
|
|
663
|
+
logger.exception("保存 mid conformer 出错")
|
|
664
|
+
break
|
|
665
|
+
|
|
666
|
+
report_optuna_and_maybe_prune(optuna_trial, best_valid_score, epoch, self.logger_path)
|
|
667
|
+
|
|
668
|
+
write_final_metrics(self.logger_path, best_valid_score, best_test_score, best_epoch)
|
|
669
|
+
return None if best_valid_score == float("inf") else best_valid_score, best_epoch
|
|
670
|
+
|
|
671
|
+
except Exception as error:
|
|
672
|
+
logger.exception("训练过程中发生异常")
|
|
673
|
+
try:
|
|
674
|
+
handle_training_exception(self.logger_path, error, best_valid_score, best_test_score, best_epoch)
|
|
675
|
+
except Exception:
|
|
676
|
+
logger.exception("调用 handle_training_exception 失败")
|
|
677
|
+
raise
|
|
678
|
+
|
|
679
|
+
# ========================================================
|
|
680
|
+
# Train epoch / batch
|
|
681
|
+
# ========================================================
|
|
682
|
+
|
|
683
|
+
def train_epoch(self, epoch: int, skip_batches: int = 0) -> None:
|
|
684
|
+
self.model.train()
|
|
685
|
+
stage = getattr(self.args, "train_stage", "gnn")
|
|
686
|
+
|
|
687
|
+
if stage == "multimode" and not bool(getattr(self.args, "use_end2end_pglobal", False)):
|
|
688
|
+
for name in ["gnn", "global_edge_predictor", "ab_head"]:
|
|
689
|
+
if hasattr(self.model, name):
|
|
690
|
+
getattr(self.model, name).eval()
|
|
691
|
+
|
|
692
|
+
if hasattr(self.train_loader.batch_sampler, "set_epoch"):
|
|
693
|
+
self.train_loader.batch_sampler.set_epoch(epoch)
|
|
694
|
+
disable_bar = bool(getattr(self.args, "distributed_stage2", False)) and dist.get_rank() != 0
|
|
695
|
+
pbar = tqdm(self.train_loader, desc=f"Train E[{epoch}]", disable=disable_bar)
|
|
696
|
+
epoch_loss = 0.0
|
|
697
|
+
n_batches = 0
|
|
698
|
+
|
|
699
|
+
for batch_idx, batch in enumerate(pbar):
|
|
700
|
+
if batch_idx < int(skip_batches):
|
|
701
|
+
pbar.set_postfix_str(f"resume-skip {batch_idx + 1}/{skip_batches}")
|
|
702
|
+
continue
|
|
703
|
+
loss_val = self.train_batch(batch, epoch)
|
|
704
|
+
epoch_loss += loss_val
|
|
705
|
+
n_batches += 1
|
|
706
|
+
pbar.set_postfix_str(f"loss={loss_val:.4f}")
|
|
707
|
+
if (
|
|
708
|
+
self.checkpoint_interval_steps > 0
|
|
709
|
+
and self.total_step > 0
|
|
710
|
+
and self.total_step % self.checkpoint_interval_steps == 0
|
|
711
|
+
):
|
|
712
|
+
self._save_progress_checkpoint("latest.pth", epoch, batch_idx)
|
|
713
|
+
if self.terminate_requested:
|
|
714
|
+
self._save_progress_checkpoint("interrupted.pth", epoch, batch_idx)
|
|
715
|
+
raise RuntimeError(
|
|
716
|
+
"Training interrupted by SIGTERM after saving interrupted.pth"
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
avg = epoch_loss / max(1, n_batches)
|
|
720
|
+
self.writer.add_scalar("loss_per_epoch", avg, epoch)
|
|
721
|
+
self.writer.add_scalar("lr_per_epoch", self.opt.param_groups[0]["lr"], epoch)
|
|
722
|
+
logger.info(f"Epoch {epoch} average loss {avg:.6f}")
|
|
723
|
+
|
|
724
|
+
def _normalize_loss_by_graphs(self, loss: torch.Tensor, batch: Any) -> torch.Tensor:
|
|
725
|
+
num_graphs = int(getattr(batch, "num_graphs", 1))
|
|
726
|
+
return loss / max(1, num_graphs)
|
|
727
|
+
|
|
728
|
+
def train_batch(self, batch: Any, epoch: int = 0) -> float:
|
|
729
|
+
batch = batch.to(self.device, non_blocking=(self.device.type == "cuda"))
|
|
730
|
+
stage = getattr(self.args, "train_stage", "gnn")
|
|
731
|
+
self.opt.zero_grad(set_to_none=True)
|
|
732
|
+
|
|
733
|
+
graph_feat = None # 用于后续 a/b 记录
|
|
734
|
+
|
|
735
|
+
# ---------------------------------------------------
|
|
736
|
+
# Stage 1: manifold learning
|
|
737
|
+
# ---------------------------------------------------
|
|
738
|
+
if stage == "gnn":
|
|
739
|
+
outputs = self.model(batch, epoch=epoch)
|
|
740
|
+
if len(outputs) == 6:
|
|
741
|
+
pred_pos, graph_feat, pos_loss, mani_loss, total_loss, mol_list = outputs
|
|
742
|
+
else:
|
|
743
|
+
pred_pos, graph_feat, pos_loss, mani_loss, mol_list = outputs
|
|
744
|
+
total_loss = mani_loss
|
|
745
|
+
# compute_stage1_loss already averages over molecule groups. A
|
|
746
|
+
# second division by data.num_graphs suppressed the probability-
|
|
747
|
+
# head gradients by roughly the number of conformers in a batch.
|
|
748
|
+
loss = total_loss
|
|
749
|
+
|
|
750
|
+
# ---------------------------------------------------
|
|
751
|
+
# Stage 2: multimode conformer generation
|
|
752
|
+
# ---------------------------------------------------
|
|
753
|
+
elif stage == "multimode":
|
|
754
|
+
node_feat = self.model.gnn(batch)
|
|
755
|
+
graph_feat = self.model.pool(node_feat, batch.batch)
|
|
756
|
+
|
|
757
|
+
loss, pos_loss, _ = self.model.compute_multimode_loss(batch, node_feat, graph_feat)
|
|
758
|
+
|
|
759
|
+
loss = torch.nan_to_num(loss, nan=0.0, posinf=1e6, neginf=0.0)
|
|
760
|
+
pos_loss = torch.nan_to_num(pos_loss, nan=0.0, posinf=1e6, neginf=0.0)
|
|
761
|
+
|
|
762
|
+
else:
|
|
763
|
+
raise ValueError(f"Unknown train_stage={stage}")
|
|
764
|
+
|
|
765
|
+
# ---------------------------------------------------
|
|
766
|
+
# Gradient check & backward
|
|
767
|
+
# ---------------------------------------------------
|
|
768
|
+
if not torch.isfinite(loss):
|
|
769
|
+
logger.warning("Skip batch because loss is NaN/Inf")
|
|
770
|
+
self.opt.zero_grad(set_to_none=True)
|
|
771
|
+
return 0.0
|
|
772
|
+
|
|
773
|
+
loss.backward()
|
|
774
|
+
|
|
775
|
+
trainable_params = [p for p in self.model.parameters() if p.requires_grad]
|
|
776
|
+
bad_grad = any(p.grad is not None and not torch.isfinite(p.grad).all() for p in trainable_params)
|
|
777
|
+
if bad_grad:
|
|
778
|
+
logger.warning("Skip optimizer step because gradient is NaN/Inf")
|
|
779
|
+
self.opt.zero_grad(set_to_none=True)
|
|
780
|
+
return 0.0
|
|
781
|
+
|
|
782
|
+
torch.nn.utils.clip_grad_norm_(trainable_params, 1.0)
|
|
783
|
+
self.opt.step()
|
|
784
|
+
self.did_optimizer_step = True
|
|
785
|
+
self.total_step += 1
|
|
786
|
+
|
|
787
|
+
# Optional save
|
|
788
|
+
if get_arg(self.args, "get_image", False) and stage == "multimode":
|
|
789
|
+
try:
|
|
790
|
+
yT = getattr(self.model, "_last_yT", None)
|
|
791
|
+
if yT is not None:
|
|
792
|
+
self._maybe_save_target_conformer(batch, yT)
|
|
793
|
+
except Exception:
|
|
794
|
+
pass
|
|
795
|
+
|
|
796
|
+
# ---------------------------------------------------
|
|
797
|
+
# Logging (enhanced with TensorBoard)
|
|
798
|
+
# ---------------------------------------------------
|
|
799
|
+
self.writer.add_scalar("loss/total", float(loss.detach().cpu()), self.total_step)
|
|
800
|
+
self.writer.add_scalar("lr", self.opt.param_groups[0]["lr"], self.total_step)
|
|
801
|
+
|
|
802
|
+
if stage == "multimode":
|
|
803
|
+
pos_val = float(pos_loss.detach().cpu()) if torch.is_tensor(pos_loss) else float(pos_loss)
|
|
804
|
+
self.writer.add_scalar("loss/pos", pos_val, self.total_step)
|
|
805
|
+
|
|
806
|
+
# 记录 a, b 参数
|
|
807
|
+
if graph_feat is not None and stage == "gnn":
|
|
808
|
+
try:
|
|
809
|
+
with torch.no_grad():
|
|
810
|
+
a_sample, b_sample = self.model.predict_ab(graph_feat[0])
|
|
811
|
+
self.writer.add_scalar("params/a", a_sample.item(), self.total_step)
|
|
812
|
+
self.writer.add_scalar("params/b", b_sample.item(), self.total_step)
|
|
813
|
+
except Exception:
|
|
814
|
+
pass
|
|
815
|
+
|
|
816
|
+
# 每 50 步记录梯度直方图
|
|
817
|
+
if self.total_step % 50 == 0:
|
|
818
|
+
for name, param in self.model.named_parameters():
|
|
819
|
+
if param.grad is not None and param.requires_grad:
|
|
820
|
+
self.writer.add_histogram(f"grad/{name}", param.grad, self.total_step)
|
|
821
|
+
|
|
822
|
+
return float(loss.detach().cpu())
|
|
823
|
+
|
|
824
|
+
# -------------------------------------------------------
|
|
825
|
+
# Visualisation helper
|
|
826
|
+
# -------------------------------------------------------
|
|
827
|
+
def _maybe_save_target_conformer(self, batch: Any, pred_pos: torch.Tensor) -> None:
|
|
828
|
+
smiles_target = get_arg(self.args, "smiles", None)
|
|
829
|
+
if smiles_target is None or self.last_found.get("found", False) or not hasattr(batch, "smiles") or pred_pos is None:
|
|
830
|
+
return
|
|
831
|
+
for graph_idx, smiles in enumerate(batch.smiles):
|
|
832
|
+
if smiles != smiles_target:
|
|
833
|
+
continue
|
|
834
|
+
try:
|
|
835
|
+
mol_pred = Chem.Mol(batch.rdmol[graph_idx])
|
|
836
|
+
mol_pred.RemoveAllConformers()
|
|
837
|
+
node_mask = batch.batch == graph_idx
|
|
838
|
+
pos_i = pred_pos[node_mask].detach().cpu().tolist()
|
|
839
|
+
conf = Chem.Conformer(mol_pred.GetNumAtoms())
|
|
840
|
+
for atom_idx, xyz in enumerate(pos_i):
|
|
841
|
+
x, y, z = xyz
|
|
842
|
+
conf.SetAtomPosition(atom_idx, Point3D(float(x), float(y), float(z)))
|
|
843
|
+
mol_pred.AddConformer(conf, assignId=True)
|
|
844
|
+
pos_gt_data = batch.pos[node_mask].detach().cpu().numpy()
|
|
845
|
+
self.last_found.update(
|
|
846
|
+
found=True,
|
|
847
|
+
mol_pred=mol_pred,
|
|
848
|
+
pos_gt_data=pos_gt_data,
|
|
849
|
+
smiles_target=smiles_target,
|
|
850
|
+
)
|
|
851
|
+
except Exception:
|
|
852
|
+
logger.exception("保存目标 conformer 出错")
|
|
853
|
+
break
|
|
854
|
+
|
|
855
|
+
# -------------------------------------------------------
|
|
856
|
+
# Helper functions
|
|
857
|
+
# -------------------------------------------------------
|
|
858
|
+
def _get_conf_idx(self, batch: Any, graph_idx: int) -> int:
|
|
859
|
+
if hasattr(self.model, "_get_conf_idx"):
|
|
860
|
+
return self.model._get_conf_idx(batch, graph_idx)
|
|
861
|
+
if hasattr(batch, "conf_id"):
|
|
862
|
+
return int(batch.conf_id[graph_idx].item()) % self.max_confs
|
|
863
|
+
return 0
|
|
864
|
+
|
|
865
|
+
def _get_edge_index_i(self, batch: Any, graph_idx: int):
|
|
866
|
+
mask = batch.batch == graph_idx
|
|
867
|
+
return self.model._get_local_edge_index(edge_index=batch.edge_index, mask=mask)
|
|
868
|
+
|
|
869
|
+
@torch.no_grad()
|
|
870
|
+
def _predict_single_conformer(self, batch: Any, graph_idx: int):
|
|
871
|
+
node_feat = self.model.gnn(batch)
|
|
872
|
+
mask = batch.batch == graph_idx
|
|
873
|
+
feat_i = node_feat[mask]
|
|
874
|
+
edge_index_i = self._get_edge_index_i(batch, graph_idx)
|
|
875
|
+
|
|
876
|
+
if hasattr(self.model, "predict_conformers"):
|
|
877
|
+
try:
|
|
878
|
+
outputs = self.model.predict_conformers(
|
|
879
|
+
feat_i=feat_i,
|
|
880
|
+
edge_index_i=edge_index_i,
|
|
881
|
+
num_modes=1,
|
|
882
|
+
deterministic=False,
|
|
883
|
+
)
|
|
884
|
+
except TypeError:
|
|
885
|
+
try:
|
|
886
|
+
outputs = self.model.predict_conformers(
|
|
887
|
+
feat_i=feat_i,
|
|
888
|
+
edge_index_i=edge_index_i,
|
|
889
|
+
)
|
|
890
|
+
except TypeError:
|
|
891
|
+
outputs = None
|
|
892
|
+
|
|
893
|
+
if isinstance(outputs, list) and len(outputs) > 0:
|
|
894
|
+
out0 = outputs[0]
|
|
895
|
+
if isinstance(out0, dict):
|
|
896
|
+
y = out0.get("Y", out0.get("y", out0.get("coords", None)))
|
|
897
|
+
if y is not None:
|
|
898
|
+
return center_and_rescale(y, target_rms=1.0)
|
|
899
|
+
if torch.is_tensor(out0):
|
|
900
|
+
return center_and_rescale(out0, target_rms=1.0)
|
|
901
|
+
|
|
902
|
+
if hasattr(self.model, "_predict_P_global"):
|
|
903
|
+
P_i, _ = self.model._predict_P_global(feat_i=feat_i, edge_index_i=edge_index_i)
|
|
904
|
+
if hasattr(self.model, "decode_probability_mode"):
|
|
905
|
+
try:
|
|
906
|
+
decoded = self.model.decode_probability_mode(
|
|
907
|
+
P=P_i, node_feat=feat_i, edge_index_i=edge_index_i
|
|
908
|
+
)
|
|
909
|
+
if isinstance(decoded, dict):
|
|
910
|
+
y = decoded.get("Y", decoded.get("y", decoded.get("coords", None)))
|
|
911
|
+
else:
|
|
912
|
+
y = decoded
|
|
913
|
+
if y is not None:
|
|
914
|
+
return center_and_rescale(y, target_rms=1.0)
|
|
915
|
+
except TypeError:
|
|
916
|
+
pass
|
|
917
|
+
if hasattr(self.model, "coarse_decoder"):
|
|
918
|
+
try:
|
|
919
|
+
y0 = self.model.coarse_decoder(node_feat=feat_i, edge_index=edge_index_i)
|
|
920
|
+
except TypeError:
|
|
921
|
+
y0 = self.model.coarse_decoder(feat_i, edge_index_i)
|
|
922
|
+
if hasattr(self.model, "refiner"):
|
|
923
|
+
try:
|
|
924
|
+
yT, _ = self.model.refiner(node_feat=feat_i, coords=y0, edge_index=edge_index_i)
|
|
925
|
+
except TypeError:
|
|
926
|
+
yT = self.model.refiner(feat_i, y0, edge_index_i)
|
|
927
|
+
return center_and_rescale(yT, target_rms=1.0)
|
|
928
|
+
return center_and_rescale(y0, target_rms=1.0)
|
|
929
|
+
return None
|
|
930
|
+
|
|
931
|
+
@torch.no_grad()
|
|
932
|
+
def _test_rmsd_limited(self, loader: DataLoader, max_samples: int = 100, split_name: str = "train") -> float:
|
|
933
|
+
self.model.eval()
|
|
934
|
+
total_rmsd = 0.0
|
|
935
|
+
total_valid = 0
|
|
936
|
+
for batch in loader:
|
|
937
|
+
batch = batch.to(self.device)
|
|
938
|
+
n_graphs = int(batch.num_graphs)
|
|
939
|
+
for graph_idx in range(n_graphs):
|
|
940
|
+
if total_valid >= max_samples:
|
|
941
|
+
avg_rmsd = total_rmsd / max(1, total_valid)
|
|
942
|
+
logger.info(f"RMSD eval {split_name} conformers = {total_valid}, RMSD = {avg_rmsd:.6f}")
|
|
943
|
+
return avg_rmsd
|
|
944
|
+
mask = batch.batch == graph_idx
|
|
945
|
+
pos_i = batch.pos[mask]
|
|
946
|
+
if pos_i.numel() == 0 or pos_i.shape[0] <= 1:
|
|
947
|
+
continue
|
|
948
|
+
y_i = self._predict_single_conformer(batch, graph_idx)
|
|
949
|
+
if y_i is None:
|
|
950
|
+
continue
|
|
951
|
+
if not torch.isfinite(y_i).all():
|
|
952
|
+
continue
|
|
953
|
+
if y_i.shape[0] != pos_i.shape[0]:
|
|
954
|
+
logger.warning(f"RMSD 跳过一个构象:y_i.shape={y_i.shape}, pos_i.shape={pos_i.shape}")
|
|
955
|
+
continue
|
|
956
|
+
atom_x = batch.x[mask] if hasattr(batch, "x") else None
|
|
957
|
+
heavy_mask = heavy_atom_mask_from_x(atom_x)
|
|
958
|
+
if heavy_mask is not None:
|
|
959
|
+
y_eval = y_i[heavy_mask]
|
|
960
|
+
pos_eval = pos_i[heavy_mask]
|
|
961
|
+
else:
|
|
962
|
+
y_eval = y_i
|
|
963
|
+
pos_eval = pos_i
|
|
964
|
+
rmsd_i = kabsch_rmsd(y_eval, pos_eval)
|
|
965
|
+
if not torch.isfinite(rmsd_i):
|
|
966
|
+
continue
|
|
967
|
+
total_rmsd += float(rmsd_i.detach().cpu())
|
|
968
|
+
total_valid += 1
|
|
969
|
+
avg_rmsd = total_rmsd / max(1, total_valid)
|
|
970
|
+
logger.info(f"RMSD eval {split_name} conformers = {total_valid}, RMSD = {avg_rmsd:.6f}")
|
|
971
|
+
return avg_rmsd
|
|
972
|
+
|
|
973
|
+
@torch.no_grad()
|
|
974
|
+
def test(self, loader: DataLoader, val: bool) -> float:
|
|
975
|
+
self.model.eval()
|
|
976
|
+
metric = self.cfg.metric.upper()
|
|
977
|
+
split_name = "valid" if val else "test"
|
|
978
|
+
if metric == "CE":
|
|
979
|
+
score = self._test_ce(loader)
|
|
980
|
+
elif metric == "RMSD":
|
|
981
|
+
score = self._test_rmsd_limited(loader, max_samples=100, split_name=split_name)
|
|
982
|
+
else:
|
|
983
|
+
logger.warning(f"未知 metric={self.cfg.metric},返回默认 10.0")
|
|
984
|
+
score = 10.0
|
|
985
|
+
tag = "val-score" if val else "test-score"
|
|
986
|
+
self.writer.add_scalar(tag, score, self.total_step)
|
|
987
|
+
return float(score)
|
|
988
|
+
|
|
989
|
+
@torch.no_grad()
|
|
990
|
+
def _test_ce(self, loader: DataLoader) -> float:
|
|
991
|
+
total_ce = 0.0
|
|
992
|
+
total_valid = 0
|
|
993
|
+
for batch in loader:
|
|
994
|
+
batch = batch.to(self.device)
|
|
995
|
+
node_feat = self.model.gnn(batch)
|
|
996
|
+
graph_feat = self.model.pool(node_feat, batch.batch)
|
|
997
|
+
matched_stage1_objective = (
|
|
998
|
+
bool(getattr(self.args, "stage1_nonlocal_heavy_only", False))
|
|
999
|
+
or (
|
|
1000
|
+
float(getattr(self.args, "stage1_fixed_kernel_a", 0.0)) > 0.0
|
|
1001
|
+
and float(getattr(self.args, "stage1_fixed_kernel_b", 0.0)) > 0.0
|
|
1002
|
+
)
|
|
1003
|
+
)
|
|
1004
|
+
if matched_stage1_objective:
|
|
1005
|
+
ce, _ = self.model.compute_stage1_loss(batch, node_feat, graph_feat)
|
|
1006
|
+
groups = self.model._group_graph_indices_by_mol_id(batch, int(batch.num_graphs))
|
|
1007
|
+
count = max(1, len(groups))
|
|
1008
|
+
if torch.isfinite(ce):
|
|
1009
|
+
total_ce += float(ce.detach().cpu()) * count
|
|
1010
|
+
total_valid += count
|
|
1011
|
+
continue
|
|
1012
|
+
n_graphs = int(batch.num_graphs)
|
|
1013
|
+
for graph_idx in range(n_graphs):
|
|
1014
|
+
mask = batch.batch == graph_idx
|
|
1015
|
+
feat_i = node_feat[mask]
|
|
1016
|
+
pos_i = batch.pos[mask]
|
|
1017
|
+
if pos_i.numel() == 0 or pos_i.shape[0] <= 1:
|
|
1018
|
+
continue
|
|
1019
|
+
edge_index_i = self._get_edge_index_i(batch, graph_idx)
|
|
1020
|
+
P_i, _ = self.model._predict_P_global(feat_i=feat_i, edge_index_i=edge_index_i)
|
|
1021
|
+
conf_idx = self._get_conf_idx(batch, graph_idx)
|
|
1022
|
+
try:
|
|
1023
|
+
a_i, b_i = self.model.predict_ab(graph_feat[graph_idx], conf_idx)
|
|
1024
|
+
except TypeError:
|
|
1025
|
+
a_i, b_i = self.model.predict_ab(graph_feat[graph_idx])
|
|
1026
|
+
Q_i = prob_low_dim(pos_i, a_i, b_i).clamp(1e-6, 1.0 - 1e-6)
|
|
1027
|
+
ce = cross_entropy_loss(P_i, Q_i)
|
|
1028
|
+
if not torch.isfinite(ce):
|
|
1029
|
+
continue
|
|
1030
|
+
total_ce += float(ce.detach().cpu())
|
|
1031
|
+
total_valid += 1
|
|
1032
|
+
avg_ce = total_ce / max(1, total_valid)
|
|
1033
|
+
logger.info(f"CE eval valid conformers = {total_valid}, CE = {avg_ce:.6f}")
|
|
1034
|
+
return avg_ce
|
|
1035
|
+
|
|
1036
|
+
@torch.no_grad()
|
|
1037
|
+
def _test_rmsd(self, loader: DataLoader) -> float:
|
|
1038
|
+
return self._test_rmsd_limited(loader, max_samples=10 ** 9, split_name="valid")
|