dualmapref 0.2.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- args_parse.py +593 -0
- candidate_corrector.py +161 -0
- candidate_scorer.py +359 -0
- conformend/__init__.py +36 -0
- conformend/refiner.py +25 -0
- dataset/__init__.py +1 -0
- dataset/drugdataset.py +644 -0
- dataset/manifold.py +237 -0
- dataset/sharded_drugdataset.py +124 -0
- dataset/smiles2graph.py +217 -0
- dualmapref/__init__.py +24 -0
- dualmapref/refiner.py +16 -0
- dualmapref-0.2.8.dist-info/LICENSE +21 -0
- dualmapref-0.2.8.dist-info/METADATA +167 -0
- dualmapref-0.2.8.dist-info/RECORD +186 -0
- dualmapref-0.2.8.dist-info/WHEEL +5 -0
- dualmapref-0.2.8.dist-info/entry_points.txt +10 -0
- dualmapref-0.2.8.dist-info/top_level.txt +29 -0
- exputils.py +289 -0
- infer.py +23 -0
- infer_base_conformers.py +5589 -0
- infer_mean_dispersion_torsion.py +1117 -0
- infer_torsion.py +752 -0
- infer_torsional_denoiser.py +1021 -0
- manimol/__init__.py +30 -0
- manimol/__main__.py +6 -0
- manimol/cli.py +36 -0
- manimol/full_cli.py +41 -0
- manimol/legacy_imports.py +43 -0
- manimol/manifest.py +56 -0
- manimol/postprocess.py +148 -0
- manimol/priors.py +60 -0
- manimol/protocols.py +119 -0
- manimol/refiner.py +645 -0
- manimol/runtime.py +194 -0
- manimol/selection.py +85 -0
- manimol/torsion.py +22 -0
- manimol/train_cli.py +14 -0
- mixture_flow/src/__init__.py +1 -0
- mixture_flow/src/_path.py +7 -0
- mixture_flow/src/infer28_mixture_torsion_flow.py +1093 -0
- mixture_flow/src/mixture_torsion_flow.py +291 -0
- mixture_flow/src/summarize_mixture_debug.py +80 -0
- mixture_flow/src/train28_mixture_torsion_flow.py +583 -0
- mixture_torsion_flow.py +7 -0
- models/ manifold_learning.py +86 -0
- models/__init__.py +8 -0
- models/dist.py +273 -0
- models/dist2coords.py +23 -0
- models/egnn.py +107 -0
- models/experimental_backbones.py +364 -0
- models/gnnconv.py +512 -0
- models/kernel_inversion.py +141 -0
- models/kernels.py +427 -0
- models/losses.py +154 -0
- models/mean_dispersion_bridge.py +119 -0
- models/mean_dispersion_torsion_context.py +155 -0
- models/model.py +73 -0
- models/model0.py +2375 -0
- models/vis.py +128 -0
- pgraph_controls.py +107 -0
- scripts/__init__.py +2 -0
- scripts/analyze_basin_alignment.py +63 -0
- scripts/analyze_hif2a_conformer_aggregation.py +267 -0
- scripts/analyze_hif2a_contact_pharmacophore.py +1111 -0
- scripts/analyze_hif2a_fixed_budget.py +202 -0
- scripts/analyze_hif2a_pharmacophore.py +405 -0
- scripts/analyze_hif2a_pharmacophore_match_metric.py +386 -0
- scripts/analyze_p_control_candidate_flow.py +154 -0
- scripts/analyze_scaffold_basin.py +83 -0
- scripts/audit_dispersion_proposals.py +208 -0
- scripts/audit_etflow_protocol.py +258 -0
- scripts/audit_external_overlap.py +150 -0
- scripts/audit_oldbest_fulltrainval_split.py +82 -0
- scripts/audit_pdbbind_contact_pharmacophores.py +146 -0
- scripts/audit_qm9_geomol_test_split.py +94 -0
- scripts/audit_stage1_stage2_cross.py +131 -0
- scripts/build_geom_drugs_lowref_diagnostic.py +84 -0
- scripts/build_qm9_backfill_manifest.py +99 -0
- scripts/check_etflow_runtime.py +56 -0
- scripts/compare_prior_sources.py +122 -0
- scripts/compose_binding_mode_figure.py +87 -0
- scripts/compose_hif2a_contact_pharmacophore_suppfig.py +255 -0
- scripts/compose_hif2a_functional_geometry_pipeline.py +266 -0
- scripts/compose_manimol_internal_architecture.py +291 -0
- scripts/diagnose_flow_basin_intervention.py +54 -0
- scripts/diagnose_hif2a_pharmacophore_budget.py +185 -0
- scripts/diagnose_route2_p_ranking.py +123 -0
- scripts/discover_binding_mode_inputs.py +30 -0
- scripts/evaluate_functional_prior_stage1.py +230 -0
- scripts/evaluate_geomanimol.py +74 -0
- scripts/evaluate_geomanimol_flow.py +23 -0
- scripts/evaluate_geomanimol_multiscaffold.py +46 -0
- scripts/evaluate_geomanimol_residual.py +45 -0
- scripts/evaluate_geomanimol_torsion.py +40 -0
- scripts/evaluate_hif2a_bioactive_enrichment.py +91 -0
- scripts/evaluate_hif2a_pms_suite.py +440 -0
- scripts/evaluate_pdbbind_contact_pms.py +155 -0
- scripts/evaluate_platinum_external.py +114 -0
- scripts/evaluate_platinum_pool.py +82 -0
- scripts/evaluate_route2_calibrated_p.py +104 -0
- scripts/evaluate_route2_library_components.py +596 -0
- scripts/evaluate_route2_p_basin_budget.py +288 -0
- scripts/evaluate_stage1_checkpoint_only.py +30 -0
- scripts/export_dispersion_bond_audit.py +136 -0
- scripts/export_top20_shards.py +207 -0
- scripts/external_validation_metrics.py +152 -0
- scripts/fit_topology_prior.py +117 -0
- scripts/frozen_dispersion_audit.py +281 -0
- scripts/geomanimol_debug.py +50 -0
- scripts/geomanimol_smoke.py +37 -0
- scripts/infer_etflow_matched.py +524 -0
- scripts/inspect_checkpoint.py +12 -0
- scripts/inspect_conformer_capacity.py +71 -0
- scripts/inspect_pool.py +12 -0
- scripts/make_figure3_selection_loss.py +474 -0
- scripts/make_figure3_selection_summary.py +275 -0
- scripts/make_geom_debug_subset.py +62 -0
- scripts/make_low_reference_test_subset.py +70 -0
- scripts/merge_stage1_into_stage2.py +82 -0
- scripts/place_hif2a_candidates_for_evaluation.py +66 -0
- scripts/plot_hif2a_low_budget_efficiency.py +100 -0
- scripts/plot_prior_validation.py +48 -0
- scripts/precompute_stage2_assignments.py +90 -0
- scripts/prepare_bace_activity.py +98 -0
- scripts/prepare_bace_etkdg_benchmark.py +90 -0
- scripts/prepare_figure5_4x4_cases.py +470 -0
- scripts/prepare_geom_drugs_geomol_top20.py +472 -0
- scripts/prepare_pdbbind_core_raw.py +90 -0
- scripts/prepare_platinum_raw.py +74 -0
- scripts/prepare_pubchem_etkdg.py +54 -0
- scripts/rebuild_hif2a_final_sdf_and_metrics.py +55 -0
- scripts/rebuild_hif2a_fixed_pool_sweep.py +175 -0
- scripts/rebuild_hif2a_strict.py +220 -0
- scripts/recompute_hif2a_bootstrap.py +63 -0
- scripts/reference_free_geometry.py +112 -0
- scripts/render_binding_modes.py +413 -0
- scripts/render_figure5_4x4.py +330 -0
- scripts/render_figure6_pharmacophore_pymol.py +162 -0
- scripts/render_hif2a_contact_pharmacophore_pymol.py +414 -0
- scripts/render_hif2a_figure11.py +397 -0
- scripts/run_bace_2d_baseline.py +73 -0
- scripts/run_drugs_validation_matrix.py +225 -0
- scripts/run_external_reference_free.py +53 -0
- scripts/run_manimol_finalk_sensitivity.py +106 -0
- scripts/run_manimol_oversampling_sensitivity.py +109 -0
- scripts/run_pdisp_matched_intervention.py +116 -0
- scripts/run_stage1_stage2_cross20.py +83 -0
- scripts/score_hif2a_candidates_with_pgraph.py +221 -0
- scripts/summarize_dispersion_intervention.py +88 -0
- scripts/test_mean_dispersion_bridge.py +50 -0
- scripts/train_dispersion_head_diagnostic.py +320 -0
- scripts/train_dispersion_head_streaming.py +519 -0
- scripts/train_geomanimol.py +100 -0
- scripts/train_geomanimol_flow.py +26 -0
- scripts/train_geomanimol_residual.py +70 -0
- scripts/train_geomanimol_torsion.py +50 -0
- scripts/train_quantile_representation_diagnostic.py +204 -0
- scripts/validate_mean_dispersion_prior.py +173 -0
- scripts/validate_packed_shard.py +41 -0
- scripts/validate_stage1_prior.py +221 -0
- scripts/validate_top20_shards.py +36 -0
- scripts/validation/__init__.py +1 -0
- scripts/validation/hif2a_downstream_baselines.py +105 -0
- scripts/validation/hif2a_pairwise_ranker.py +218 -0
- scripts/verify_binding_mode_figure.py +47 -0
- torsion_manifold.py +281 -0
- train.py +78 -0
- train_graph_encoder.py +206 -0
- train_local_torsion_corrector.py +511 -0
- train_mean_dispersion_torsion_prior.py +329 -0
- train_probability_basin.py +235 -0
- train_torsion_generator.py +1631 -0
- train_torsion_manifold_prior.py +258 -0
- train_torsional_denoiser.py +415 -0
- trainer.py +1038 -0
- utils/__init__.py +7 -0
- utils/checkpoint.py +40 -0
- utils/device.py +17 -0
- utils/helpers.py +98 -0
- utils/kabsch.py +29 -0
- utils/lookup_table.py +164 -0
- utils/metrics.py +50 -0
- utils/optuna.py +111 -0
- utils/save_mol.py +366 -0
- utils/util.py +36 -0
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fixed-pool diagnostic: P allocates basin budgets, density selects locally.
|
|
3
|
+
|
|
4
|
+
The candidate pool is never regenerated. Candidate basins are obtained from
|
|
5
|
+
torsion-space geometry, so Normal/Shuffled/Random differ only through the
|
|
6
|
+
stored Stage-I probability graph used to allocate the final 2R budget.
|
|
7
|
+
Reference conformers are accessed only after selection for COV/AMR reporting.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import csv
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import sys
|
|
17
|
+
from argparse import Namespace
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import torch
|
|
22
|
+
from rdkit import Chem
|
|
23
|
+
from tqdm import tqdm
|
|
24
|
+
|
|
25
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
26
|
+
|
|
27
|
+
import infer_base_conformers
|
|
28
|
+
from pgraph_controls import controlled_pglobal
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def parse_args():
|
|
32
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
33
|
+
parser.add_argument("--fixed_dump_name", action="store_true")
|
|
34
|
+
parser.add_argument("--pool_pt", required=True)
|
|
35
|
+
parser.add_argument("--max_molecules", type=int, default=0)
|
|
36
|
+
parser.add_argument("--modes", type=int, default=3)
|
|
37
|
+
parser.add_argument("--budget_multiplier", type=float, default=2.0)
|
|
38
|
+
parser.add_argument("--p_weights", default="0.25,0.5,0.75")
|
|
39
|
+
parser.add_argument("--candidate_p_weights", default="0.25,0.5,0.75")
|
|
40
|
+
parser.add_argument("--controls", default="normal,shuffle,random")
|
|
41
|
+
parser.add_argument("--temperature", type=float, default=1.0)
|
|
42
|
+
parser.add_argument("--threshold", type=float, default=0.75)
|
|
43
|
+
parser.add_argument("--density_k", type=int, default=20)
|
|
44
|
+
parser.add_argument("--pair_weighting", choices=["uniform", "variance"], default="uniform")
|
|
45
|
+
parser.add_argument("--seed", type=int, default=20260825)
|
|
46
|
+
parser.add_argument("--rmsd_backend", choices=["kabsch", "kabsch_atom", "rdkit_best"], default="rdkit_best")
|
|
47
|
+
parser.add_argument("--rdkit_max_matches", type=int, default=1000000)
|
|
48
|
+
parser.add_argument("--output_prefix", required=True)
|
|
49
|
+
return parser.parse_args()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def farthest_point_basins(distance: torch.Tensor, modes: int) -> torch.Tensor:
|
|
53
|
+
n = int(distance.size(0))
|
|
54
|
+
modes = max(1, min(int(modes), n))
|
|
55
|
+
centers = [int(torch.argmin(distance.mean(dim=1)).item())]
|
|
56
|
+
nearest = distance[centers[0]].clone()
|
|
57
|
+
while len(centers) < modes:
|
|
58
|
+
index = int(torch.argmax(nearest).item())
|
|
59
|
+
if index in centers:
|
|
60
|
+
break
|
|
61
|
+
centers.append(index)
|
|
62
|
+
nearest = torch.minimum(nearest, distance[index])
|
|
63
|
+
return distance[:, torch.tensor(centers, dtype=torch.long)].argmin(dim=1)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def nonlocal_pair_mask(mol, n_atoms: int) -> torch.Tensor:
|
|
67
|
+
mask = torch.triu(torch.ones((n_atoms, n_atoms), dtype=torch.bool), diagonal=1)
|
|
68
|
+
if mol is None or int(mol.GetNumAtoms()) != n_atoms:
|
|
69
|
+
return mask
|
|
70
|
+
topo = torch.as_tensor(Chem.GetDistanceMatrix(mol), dtype=torch.float32)
|
|
71
|
+
heavy = torch.tensor([atom.GetAtomicNum() > 1 for atom in mol.GetAtoms()], dtype=torch.bool)
|
|
72
|
+
return mask & (topo >= 3.0) & heavy[:, None] & heavy[None, :]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def candidate_p_bce(pool: torch.Tensor, pglobal: torch.Tensor, kernel, mol, pair_weighting="uniform") -> torch.Tensor:
|
|
76
|
+
a, b = kernel
|
|
77
|
+
a = torch.as_tensor(a, dtype=torch.float32)
|
|
78
|
+
b = torch.as_tensor(b, dtype=torch.float32)
|
|
79
|
+
target = pglobal.float().clamp(1e-6, 1.0 - 1e-6)
|
|
80
|
+
mask = nonlocal_pair_mask(mol, int(pool.size(1)))
|
|
81
|
+
if not bool(mask.any()):
|
|
82
|
+
mask = torch.triu(torch.ones_like(target, dtype=torch.bool), diagonal=1)
|
|
83
|
+
relations = []
|
|
84
|
+
for conf in pool.float():
|
|
85
|
+
relation = infer_base_conformers.prob_low_dim_kernel(conf, a, b, kernel="umap").clamp(1e-6, 1.0 - 1e-6)
|
|
86
|
+
relations.append(relation[mask])
|
|
87
|
+
relation_values = torch.stack(relations)
|
|
88
|
+
weights = torch.ones(relation_values.size(1), dtype=relation_values.dtype)
|
|
89
|
+
if str(pair_weighting) == "variance":
|
|
90
|
+
weights = relation_values.var(dim=0, unbiased=False).sqrt()
|
|
91
|
+
weights = weights / weights.mean().clamp_min(1e-6)
|
|
92
|
+
weights = weights.clamp_min(0.05)
|
|
93
|
+
bce = -(target[mask].view(1, -1) * relation_values.log()
|
|
94
|
+
+ (1.0 - target[mask].view(1, -1)) * (1.0 - relation_values).log())
|
|
95
|
+
return (bce * weights.view(1, -1)).sum(dim=1) / weights.sum().clamp_min(1e-6)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def allocate(weights: torch.Tensor, available: torch.Tensor, budget: int) -> torch.Tensor:
|
|
99
|
+
quota = torch.zeros_like(available)
|
|
100
|
+
active = torch.where(available > 0)[0]
|
|
101
|
+
if active.numel() == 0 or budget <= 0:
|
|
102
|
+
return quota
|
|
103
|
+
for mode in active[: min(int(active.numel()), budget)]:
|
|
104
|
+
quota[mode] = 1
|
|
105
|
+
remaining = budget - int(quota.sum())
|
|
106
|
+
while remaining > 0:
|
|
107
|
+
capacity = available - quota
|
|
108
|
+
eligible = torch.where(capacity > 0)[0]
|
|
109
|
+
if eligible.numel() == 0:
|
|
110
|
+
break
|
|
111
|
+
desired = weights[eligible] * float(budget) - quota[eligible].float()
|
|
112
|
+
mode = int(eligible[torch.argmax(desired)].item())
|
|
113
|
+
quota[mode] += 1
|
|
114
|
+
remaining -= 1
|
|
115
|
+
return quota
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def mode_weights(assignment, bce, modes, p_weight, temperature):
|
|
119
|
+
available = torch.bincount(assignment, minlength=modes)
|
|
120
|
+
support = available.float() / available.sum().clamp_min(1)
|
|
121
|
+
compatibility = torch.full((modes,), float("inf"))
|
|
122
|
+
for mode in range(modes):
|
|
123
|
+
values = bce[assignment == mode]
|
|
124
|
+
if values.numel():
|
|
125
|
+
keep = max(1, int(math.ceil(values.numel() * 0.25)))
|
|
126
|
+
compatibility[mode] = torch.sort(values).values[:keep].mean()
|
|
127
|
+
finite = torch.isfinite(compatibility)
|
|
128
|
+
if finite.sum() <= 1:
|
|
129
|
+
p_mode = support.clone()
|
|
130
|
+
else:
|
|
131
|
+
centered = compatibility[finite] - compatibility[finite].median()
|
|
132
|
+
scale = centered.abs().median().clamp_min(1e-4)
|
|
133
|
+
logits = torch.full_like(compatibility, -1e6)
|
|
134
|
+
logits[finite] = -centered / (scale * max(float(temperature), 1e-4))
|
|
135
|
+
p_mode = torch.softmax(logits, dim=0)
|
|
136
|
+
weights = (1.0 - float(p_weight)) * support + float(p_weight) * p_mode
|
|
137
|
+
weights = weights * (available > 0)
|
|
138
|
+
return weights / weights.sum().clamp_min(1e-8), available, compatibility
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def local_density_selection(pool, assignment, quota, record, density_k):
|
|
142
|
+
selected = []
|
|
143
|
+
for mode, count in enumerate(quota.tolist()):
|
|
144
|
+
members = torch.where(assignment == mode)[0]
|
|
145
|
+
if count <= 0 or members.numel() == 0:
|
|
146
|
+
continue
|
|
147
|
+
local = infer_base_conformers.select_precision2r_conformers(
|
|
148
|
+
pool[members], min(int(count), int(members.numel())), mol=record.get("mol"),
|
|
149
|
+
heavy_mask=record.get("heavy_mask"), metric="kabsch", density_k=density_k,
|
|
150
|
+
prefilter_quantile=1.0, density_weight=1.0, centrality_weight=0.45,
|
|
151
|
+
duplicate_radius=0.0,
|
|
152
|
+
)
|
|
153
|
+
selected.append(local)
|
|
154
|
+
return torch.cat(selected, dim=0) if selected else pool[:0]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def local_p_density_selection(pool, assignment, quota, record, bce, density_k, p_weight):
|
|
158
|
+
"""Select within each basin using a calibrated P score plus local density.
|
|
159
|
+
|
|
160
|
+
Lower scores are preferred for both terms. The calibration is performed
|
|
161
|
+
within each basin so molecule size and absolute BCE scale do not dominate.
|
|
162
|
+
"""
|
|
163
|
+
selected = []
|
|
164
|
+
p_weight = min(1.0, max(0.0, float(p_weight)))
|
|
165
|
+
for mode, count in enumerate(quota.tolist()):
|
|
166
|
+
members = torch.where(assignment == mode)[0]
|
|
167
|
+
if count <= 0 or members.numel() == 0:
|
|
168
|
+
continue
|
|
169
|
+
local_pool = pool[members]
|
|
170
|
+
if members.numel() <= count:
|
|
171
|
+
selected.append(local_pool)
|
|
172
|
+
continue
|
|
173
|
+
dist = infer_base_conformers._pairwise_torsion_signature_distance(
|
|
174
|
+
local_pool, record.get("mol"), fallback_metric="kabsch",
|
|
175
|
+
heavy_mask=record.get("heavy_mask")
|
|
176
|
+
)
|
|
177
|
+
n = int(dist.size(0))
|
|
178
|
+
eye = torch.eye(n, dtype=dist.dtype)
|
|
179
|
+
k = max(1, min(int(density_k), max(1, n - 1)))
|
|
180
|
+
sorted_dist = torch.sort(dist + eye * 1e6, dim=1).values
|
|
181
|
+
local_radius = sorted_dist[:, :k].mean(dim=1)
|
|
182
|
+
centrality = dist.mean(dim=1)
|
|
183
|
+
density_score = (
|
|
184
|
+
infer_base_conformers._normalize_01(local_radius)
|
|
185
|
+
+ 0.45 * infer_base_conformers._normalize_01(centrality)
|
|
186
|
+
)
|
|
187
|
+
p_score = infer_base_conformers._normalize_01(bce[members])
|
|
188
|
+
score = (1.0 - p_weight) * density_score + p_weight * p_score
|
|
189
|
+
chosen = torch.argsort(score)[: int(count)]
|
|
190
|
+
selected.append(local_pool[chosen])
|
|
191
|
+
return torch.cat(selected, dim=0) if selected else pool[:0]
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def metrics(final, record, args):
|
|
195
|
+
rmsd = infer_base_conformers.compute_rmsd_matrix(
|
|
196
|
+
final, record["ref_confs"].float(), heavy_mask=record.get("heavy_mask"), mol=record.get("mol"),
|
|
197
|
+
backend=args.rmsd_backend, rdkit_max_matches=args.rdkit_max_matches,
|
|
198
|
+
)
|
|
199
|
+
cov_r, amr_r, cov_p, amr_p = infer_base_conformers.evaluate_metrics_from_rmsd(rmsd, args.threshold)
|
|
200
|
+
return {"cov_r": cov_r, "cov_p": cov_p, "amr_r": amr_r, "amr_p": amr_p}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def write_csv(path: Path, rows):
|
|
204
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
205
|
+
with path.open("w", newline="") as handle:
|
|
206
|
+
writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
|
|
207
|
+
writer.writeheader(); writer.writerows(rows)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def main():
|
|
211
|
+
args = parse_args()
|
|
212
|
+
payload = torch.load(args.pool_pt, map_location="cpu", weights_only=False)
|
|
213
|
+
records = list(payload["records"])
|
|
214
|
+
if args.max_molecules > 0:
|
|
215
|
+
records = records[: args.max_molecules]
|
|
216
|
+
controls = [x.strip() for x in args.controls.split(",") if x.strip()]
|
|
217
|
+
p_weights = [float(x) for x in args.p_weights.split(",") if x.strip()]
|
|
218
|
+
candidate_p_weights = [float(x) for x in args.candidate_p_weights.split(",") if x.strip()]
|
|
219
|
+
rows = []
|
|
220
|
+
for record in tqdm(records, desc="fixed-pool P basin budget", unit="molecule"):
|
|
221
|
+
pool = record["raw_pool"].float()
|
|
222
|
+
budget = min(int(pool.size(0)), max(1, int(round(args.budget_multiplier * record["ref_confs"].size(0)))))
|
|
223
|
+
distance = infer_base_conformers._pairwise_torsion_signature_distance(
|
|
224
|
+
pool, record.get("mol"), fallback_metric="kabsch", heavy_mask=record.get("heavy_mask")
|
|
225
|
+
)
|
|
226
|
+
assignment = farthest_point_basins(distance, args.modes)
|
|
227
|
+
modes = int(assignment.max().item()) + 1
|
|
228
|
+
|
|
229
|
+
global_density = infer_base_conformers.select_precision2r_conformers(
|
|
230
|
+
pool, budget, mol=record.get("mol"), heavy_mask=record.get("heavy_mask"), metric="kabsch",
|
|
231
|
+
density_k=args.density_k, prefilter_quantile=1.0, density_weight=1.0,
|
|
232
|
+
centrality_weight=0.45, duplicate_radius=0.0,
|
|
233
|
+
)
|
|
234
|
+
rows.append({"mol_id": record["mol_id"], "control": "density_global", "p_weight": 0.0,
|
|
235
|
+
"candidate_p_weight": 0.0,
|
|
236
|
+
"pool_size": int(pool.size(0)), "budget": budget, **metrics(global_density, record, args)})
|
|
237
|
+
|
|
238
|
+
learned = record.get("pglobal")
|
|
239
|
+
kernel = record.get("pgraph_kernel")
|
|
240
|
+
if learned is None or kernel is None:
|
|
241
|
+
continue
|
|
242
|
+
for control in controls:
|
|
243
|
+
control_args = Namespace(pglobal_control=control, pglobal_seed=args.seed, disable_stage1_pglobal=False)
|
|
244
|
+
target = controlled_pglobal(learned.float(), control_args, mol=record.get("mol"))
|
|
245
|
+
bce = candidate_p_bce(
|
|
246
|
+
pool, target, kernel, record.get("mol"),
|
|
247
|
+
pair_weighting=args.pair_weighting,
|
|
248
|
+
)
|
|
249
|
+
for p_weight in p_weights:
|
|
250
|
+
weights, available, compatibility = mode_weights(
|
|
251
|
+
assignment, bce, modes, p_weight, args.temperature
|
|
252
|
+
)
|
|
253
|
+
quota = allocate(weights, available, budget)
|
|
254
|
+
for candidate_p_weight in candidate_p_weights:
|
|
255
|
+
final = local_p_density_selection(
|
|
256
|
+
pool, assignment, quota, record, bce, args.density_k,
|
|
257
|
+
candidate_p_weight,
|
|
258
|
+
)
|
|
259
|
+
rows.append({"mol_id": record["mol_id"], "control": control,
|
|
260
|
+
"p_weight": p_weight,
|
|
261
|
+
"candidate_p_weight": candidate_p_weight,
|
|
262
|
+
"pool_size": int(pool.size(0)), "budget": budget,
|
|
263
|
+
**metrics(final, record, args)})
|
|
264
|
+
|
|
265
|
+
summary = []
|
|
266
|
+
for control, p_weight, candidate_p_weight in sorted({
|
|
267
|
+
(r["control"], r["p_weight"], r.get("candidate_p_weight", 0.0)) for r in rows
|
|
268
|
+
}):
|
|
269
|
+
subset = [r for r in rows if r["control"] == control
|
|
270
|
+
and r["p_weight"] == p_weight
|
|
271
|
+
and r.get("candidate_p_weight", 0.0) == candidate_p_weight]
|
|
272
|
+
summary.append({"control": control, "p_weight": p_weight,
|
|
273
|
+
"candidate_p_weight": candidate_p_weight,
|
|
274
|
+
"molecules": len(subset),
|
|
275
|
+
**{key: float(np.mean([r[key] for r in subset]))
|
|
276
|
+
for key in ("cov_r", "cov_p", "amr_r", "amr_p")}})
|
|
277
|
+
prefix = Path(args.output_prefix)
|
|
278
|
+
write_csv(prefix.with_name(prefix.name + "_per_molecule.csv"), rows)
|
|
279
|
+
write_csv(prefix.with_name(prefix.name + "_summary.csv"), summary)
|
|
280
|
+
prefix.with_name(prefix.name + "_summary.json").write_text(
|
|
281
|
+
json.dumps({"protocol": vars(args), "summary": summary}, indent=2) + "\n"
|
|
282
|
+
)
|
|
283
|
+
for row in summary:
|
|
284
|
+
print(row)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
if __name__ == "__main__":
|
|
288
|
+
main()
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Evaluate a Stage-I checkpoint without performing any optimizer update."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from args_parse import args_parser
|
|
9
|
+
from exputils import initialize_exp, set_seed, get_dump_path
|
|
10
|
+
from train_graph_encoder import fix_args, build_writer
|
|
11
|
+
from trainer import Trainer
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main() -> None:
|
|
15
|
+
args = fix_args(args_parser())
|
|
16
|
+
initialize_exp(args)
|
|
17
|
+
set_seed(getattr(args, "random_seed", 0))
|
|
18
|
+
logger_path = get_dump_path(args)
|
|
19
|
+
writer = build_writer(logger_path)
|
|
20
|
+
try:
|
|
21
|
+
trainer = Trainer(args=args, writer=writer, logger_path=logger_path)
|
|
22
|
+
valid = trainer.test(trainer.valid_loader, val=True)
|
|
23
|
+
test = trainer.test(trainer.test_loader, val=False)
|
|
24
|
+
print({"checkpoint": args.checkpoint_path, "epoch0_valid": valid, "epoch0_test": test})
|
|
25
|
+
finally:
|
|
26
|
+
writer.close()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
if __name__ == "__main__":
|
|
30
|
+
main()
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Export bond-level evidence for matched dispersion interventions.
|
|
3
|
+
|
|
4
|
+
Each input is a raw-pool cache produced by infer_base_conformers.py. The script compares
|
|
5
|
+
the original molecule-specific dispersion-derived score/width with the
|
|
6
|
+
values actually used by the selected control, without recomputing either
|
|
7
|
+
quantity from the generated conformers.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import csv
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
import torch
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _as_list(value):
|
|
22
|
+
if value is None:
|
|
23
|
+
return None
|
|
24
|
+
if torch.is_tensor(value):
|
|
25
|
+
return value.detach().cpu().reshape(-1).tolist()
|
|
26
|
+
return list(value)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _float_at(value, i):
|
|
30
|
+
values = _as_list(value)
|
|
31
|
+
return None if values is None or i >= len(values) else float(values[i])
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _bool_at(value, i):
|
|
35
|
+
values = _as_list(value)
|
|
36
|
+
return None if values is None or i >= len(values) else bool(values[i])
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def main():
|
|
40
|
+
parser = argparse.ArgumentParser()
|
|
41
|
+
parser.add_argument("--input", action="append", required=True,
|
|
42
|
+
help="control=raw_pool.pt; repeat for normal/constant/shuffle")
|
|
43
|
+
parser.add_argument("--output-csv", required=True)
|
|
44
|
+
parser.add_argument("--summary-json", required=True)
|
|
45
|
+
args = parser.parse_args()
|
|
46
|
+
|
|
47
|
+
rows = []
|
|
48
|
+
for spec in args.input:
|
|
49
|
+
if "=" not in spec:
|
|
50
|
+
raise ValueError(f"Expected control=path, got {spec!r}")
|
|
51
|
+
control, path = spec.split("=", 1)
|
|
52
|
+
payload = torch.load(path, map_location="cpu", weights_only=False)
|
|
53
|
+
records = payload.get("records", payload) if isinstance(payload, dict) else payload
|
|
54
|
+
for record in records:
|
|
55
|
+
mol_id = record.get("mol_id", record.get("molecule_id"))
|
|
56
|
+
original_scores = record.get("dispersion_torsion_original_scores")
|
|
57
|
+
used_scores = record.get("dispersion_torsion_scores")
|
|
58
|
+
original_widths = record.get("dispersion_torsion_original_widths")
|
|
59
|
+
used_widths = record.get("dispersion_torsion_widths")
|
|
60
|
+
original_valid = record.get("dispersion_torsion_original_valid")
|
|
61
|
+
used_valid = record.get("dispersion_torsion_valid")
|
|
62
|
+
if original_scores is None or used_scores is None:
|
|
63
|
+
raise RuntimeError(
|
|
64
|
+
f"{path}: missing original/used score fields for mol_id={mol_id}; "
|
|
65
|
+
"rerun with the instrumented inference code"
|
|
66
|
+
)
|
|
67
|
+
n = max(len(_as_list(original_scores)), len(_as_list(used_scores)))
|
|
68
|
+
for bond_id in range(n):
|
|
69
|
+
o_score = _float_at(original_scores, bond_id)
|
|
70
|
+
u_score = _float_at(used_scores, bond_id)
|
|
71
|
+
o_width = _float_at(original_widths, bond_id)
|
|
72
|
+
u_width = _float_at(used_widths, bond_id)
|
|
73
|
+
rows.append({
|
|
74
|
+
"mol_id": mol_id,
|
|
75
|
+
"bond_id": bond_id,
|
|
76
|
+
"control": control,
|
|
77
|
+
"original_valid": _bool_at(original_valid, bond_id),
|
|
78
|
+
"used_valid": _bool_at(used_valid, bond_id),
|
|
79
|
+
"dispersion_original": o_score,
|
|
80
|
+
"dispersion_used": u_score,
|
|
81
|
+
"bond_score_original": o_score,
|
|
82
|
+
"bond_score_used": u_score,
|
|
83
|
+
"width_original": o_width,
|
|
84
|
+
"width_used": u_width,
|
|
85
|
+
"abs_delta_d": None if o_score is None or u_score is None else abs(o_score - u_score),
|
|
86
|
+
"abs_delta_w": None if o_width is None or u_width is None else abs(o_width - u_width),
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
output = Path(args.output_csv)
|
|
90
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
fields = list(rows[0]) if rows else [
|
|
92
|
+
"mol_id", "bond_id", "control", "original_valid", "used_valid",
|
|
93
|
+
"dispersion_original", "dispersion_used", "bond_score_original",
|
|
94
|
+
"bond_score_used", "width_original", "width_used", "abs_delta_d", "abs_delta_w",
|
|
95
|
+
]
|
|
96
|
+
with output.open("w", newline="") as handle:
|
|
97
|
+
writer = csv.DictWriter(handle, fieldnames=fields)
|
|
98
|
+
writer.writeheader()
|
|
99
|
+
writer.writerows(rows)
|
|
100
|
+
|
|
101
|
+
summary = {}
|
|
102
|
+
for control in sorted({row["control"] for row in rows}):
|
|
103
|
+
subset = [row for row in rows if row["control"] == control]
|
|
104
|
+
d = sorted(float(row["abs_delta_d"]) for row in subset if row["abs_delta_d"] is not None)
|
|
105
|
+
w = sorted(float(row["abs_delta_w"]) for row in subset if row["abs_delta_w"] is not None)
|
|
106
|
+
score_pairs_changed = sum(
|
|
107
|
+
row["abs_delta_d"] is not None and row["abs_delta_d"] > 1e-8 for row in subset
|
|
108
|
+
)
|
|
109
|
+
width_pairs_changed = sum(
|
|
110
|
+
row["abs_delta_w"] is not None and row["abs_delta_w"] > 1e-8 for row in subset
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
def stats(values):
|
|
114
|
+
if not values:
|
|
115
|
+
return {"mean": None, "median": None, "max": None}
|
|
116
|
+
mid = len(values) // 2
|
|
117
|
+
median = values[mid] if len(values) % 2 else (values[mid - 1] + values[mid]) / 2
|
|
118
|
+
return {"mean": sum(values) / len(values), "median": median, "max": values[-1]}
|
|
119
|
+
|
|
120
|
+
summary[control] = {
|
|
121
|
+
"rows": len(subset),
|
|
122
|
+
"molecules": len({row["mol_id"] for row in subset}),
|
|
123
|
+
"score_pairs_changed": score_pairs_changed,
|
|
124
|
+
"width_pairs_changed": width_pairs_changed,
|
|
125
|
+
"abs_delta_d": stats(d),
|
|
126
|
+
"abs_delta_w": stats(w),
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
summary_path = Path(args.summary_json)
|
|
130
|
+
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
|
131
|
+
summary_path.write_text(json.dumps(summary, indent=2) + "\n")
|
|
132
|
+
print(json.dumps(summary, indent=2))
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
if __name__ == "__main__":
|
|
136
|
+
main()
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Resumable streaming export of the full Top20 cache."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import pickle
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch_geometric.data import Data
|
|
13
|
+
from torch_geometric.data.separate import separate
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def atom_signature(item):
|
|
17
|
+
x = getattr(item, "x", None)
|
|
18
|
+
if x is None:
|
|
19
|
+
return ""
|
|
20
|
+
return hashlib.sha256(x.detach().cpu().contiguous().numpy().tobytes()).hexdigest()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def compact_item(item):
|
|
24
|
+
keep = {"x", "edge_index", "edge_attr", "pos", "torsion_index",
|
|
25
|
+
"mol_id", "conf_id", "num_atoms", "smiles"}
|
|
26
|
+
fields = {}
|
|
27
|
+
for key in keep:
|
|
28
|
+
if not hasattr(item, key):
|
|
29
|
+
continue
|
|
30
|
+
value = getattr(item, key)
|
|
31
|
+
if torch.is_tensor(value):
|
|
32
|
+
# PyG separation can return views backed by the full collated cache.
|
|
33
|
+
# Clone every retained tensor so a shard serializes only its records.
|
|
34
|
+
value = value.detach().cpu().clone()
|
|
35
|
+
fields[key] = value
|
|
36
|
+
return Data(**fields)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def atomic_save(payload, path):
|
|
40
|
+
tmp = path + ".tmp"
|
|
41
|
+
torch.save(payload, tmp, pickle_protocol=4)
|
|
42
|
+
os.replace(tmp, path)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def pack_molecule(group):
|
|
46
|
+
"""Store topology once and only coordinates per conformer."""
|
|
47
|
+
first = group[0]
|
|
48
|
+
static_keys = ("x", "edge_index", "edge_attr", "torsion_index")
|
|
49
|
+
for item in group[1:]:
|
|
50
|
+
for key in static_keys:
|
|
51
|
+
left, right = getattr(first, key, None), getattr(item, key, None)
|
|
52
|
+
if (left is None) != (right is None):
|
|
53
|
+
raise ValueError("Inconsistent %s within molecule %s" % (key, first.mol_id))
|
|
54
|
+
if left is not None and left.shape != right.shape:
|
|
55
|
+
raise ValueError("Non-shared %s shape within molecule %s" % (key, first.mol_id))
|
|
56
|
+
packed = {}
|
|
57
|
+
for key in static_keys + ("mol_id", "num_atoms", "smiles"):
|
|
58
|
+
if hasattr(first, key):
|
|
59
|
+
packed[key] = getattr(first, key)
|
|
60
|
+
packed["pos"] = torch.stack([item.pos for item in group], dim=0)
|
|
61
|
+
packed["conf_id"] = torch.stack([item.conf_id.reshape(()) for item in group], dim=0)
|
|
62
|
+
return packed
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def write_progress(root, progress):
|
|
66
|
+
path = os.path.join(root, "progress.json")
|
|
67
|
+
tmp = path + ".tmp"
|
|
68
|
+
with open(tmp, "w") as handle:
|
|
69
|
+
json.dump(progress, handle)
|
|
70
|
+
handle.flush()
|
|
71
|
+
os.fsync(handle.fileno())
|
|
72
|
+
os.replace(tmp, path)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def load_source(path):
|
|
76
|
+
try:
|
|
77
|
+
print("Loading source cache with mmap=True: %s" % path, flush=True)
|
|
78
|
+
return torch.load(path, map_location="cpu", weights_only=False, mmap=True)
|
|
79
|
+
except (TypeError, RuntimeError) as exc:
|
|
80
|
+
print("mmap unavailable (%s); falling back to regular load" % exc, flush=True)
|
|
81
|
+
return torch.load(path, map_location="cpu", weights_only=False)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def export_split(data, slices, indices, root, split, shard_molecules, progress, resume,
|
|
85
|
+
max_shards=0):
|
|
86
|
+
out_dir = os.path.join(root, split)
|
|
87
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
88
|
+
info = progress.setdefault("splits", {}).setdefault(split, {
|
|
89
|
+
"next_index": 0, "next_shard": 0, "samples": 0, "molecules": 0,
|
|
90
|
+
"shards": [], "identity_shards": []
|
|
91
|
+
})
|
|
92
|
+
start = int(info["next_index"]) if resume else 0
|
|
93
|
+
shard_id = int(info["next_shard"]) if resume else 0
|
|
94
|
+
pending, pending_hashes, pending_molecules = [], [], 0
|
|
95
|
+
current_mol, current_group = None, []
|
|
96
|
+
|
|
97
|
+
def flush_group():
|
|
98
|
+
nonlocal current_mol, current_group, pending_molecules
|
|
99
|
+
if current_group:
|
|
100
|
+
pending.append(current_group)
|
|
101
|
+
first = current_group[0]
|
|
102
|
+
pending_hashes.append([
|
|
103
|
+
int(current_mol), str(getattr(first, "smiles", "")),
|
|
104
|
+
int(first.num_atoms.item()), atom_signature(first), len(current_group)
|
|
105
|
+
])
|
|
106
|
+
pending_molecules += 1
|
|
107
|
+
current_mol, current_group = None, []
|
|
108
|
+
|
|
109
|
+
def flush_shard():
|
|
110
|
+
nonlocal pending, pending_hashes, pending_molecules, shard_id
|
|
111
|
+
if not pending:
|
|
112
|
+
return
|
|
113
|
+
group_sizes = [len(group) for group in pending]
|
|
114
|
+
sample_count = sum(group_sizes)
|
|
115
|
+
molecules = [pack_molecule(group) for group in pending]
|
|
116
|
+
name = "shard_%06d.pt" % shard_id
|
|
117
|
+
atomic_save({"molecules": molecules}, os.path.join(out_dir, name))
|
|
118
|
+
info["shards"].append({"file": os.path.join(split, name),
|
|
119
|
+
"start": int(info["samples"]),
|
|
120
|
+
"samples": sample_count,
|
|
121
|
+
"molecules": int(pending_molecules),
|
|
122
|
+
"group_sizes": group_sizes})
|
|
123
|
+
identity_digest = hashlib.sha256(json.dumps(
|
|
124
|
+
pending_hashes, separators=(",", ":"), ensure_ascii=True
|
|
125
|
+
).encode()).hexdigest()
|
|
126
|
+
info["identity_shards"].append({
|
|
127
|
+
"file": os.path.join(split, name), "sha256": identity_digest,
|
|
128
|
+
"samples": sample_count, "molecules": int(pending_molecules)
|
|
129
|
+
})
|
|
130
|
+
info["samples"] += sample_count
|
|
131
|
+
info["molecules"] += int(pending_molecules)
|
|
132
|
+
info["next_shard"] = shard_id + 1
|
|
133
|
+
shard_id += 1
|
|
134
|
+
pending, pending_hashes, pending_molecules = [], [], 0
|
|
135
|
+
write_progress(root, progress)
|
|
136
|
+
print("[%s] shard=%d molecules=%d samples=%d" %
|
|
137
|
+
(split, shard_id, info["molecules"], info["samples"]), flush=True)
|
|
138
|
+
|
|
139
|
+
for local_pos in range(start, len(indices)):
|
|
140
|
+
source_index = int(indices[local_pos])
|
|
141
|
+
item = separate(data.__class__, data, source_index, slices, decrement=False)
|
|
142
|
+
mol_id = int(item.mol_id.item())
|
|
143
|
+
if current_mol is None:
|
|
144
|
+
current_mol = mol_id
|
|
145
|
+
elif mol_id != current_mol:
|
|
146
|
+
flush_group()
|
|
147
|
+
if pending_molecules >= shard_molecules:
|
|
148
|
+
flush_shard()
|
|
149
|
+
if max_shards and shard_id >= max_shards:
|
|
150
|
+
break
|
|
151
|
+
current_mol = mol_id
|
|
152
|
+
current_group.append(compact_item(item))
|
|
153
|
+
info["next_index"] = local_pos + 1
|
|
154
|
+
flush_group()
|
|
155
|
+
flush_shard()
|
|
156
|
+
write_progress(root, progress)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def main():
|
|
160
|
+
parser = argparse.ArgumentParser()
|
|
161
|
+
parser.add_argument("--source", required=True)
|
|
162
|
+
parser.add_argument("--split-file", required=True)
|
|
163
|
+
parser.add_argument("--output", required=True)
|
|
164
|
+
parser.add_argument("--shard-molecules", type=int, default=64)
|
|
165
|
+
parser.add_argument("--max-shards-per-split", type=int, default=0,
|
|
166
|
+
help="Smoke-test limit; zero exports every shard.")
|
|
167
|
+
parser.add_argument("--resume", action="store_true")
|
|
168
|
+
args = parser.parse_args()
|
|
169
|
+
os.makedirs(args.output, exist_ok=True)
|
|
170
|
+
progress_path = os.path.join(args.output, "progress.json")
|
|
171
|
+
if args.resume and os.path.exists(progress_path):
|
|
172
|
+
with open(progress_path) as handle:
|
|
173
|
+
progress = json.load(handle)
|
|
174
|
+
else:
|
|
175
|
+
progress = {"format": "manimol-shards-v3-packed", "source": os.path.abspath(args.source), "splits": {}}
|
|
176
|
+
data, slices = load_source(args.source)
|
|
177
|
+
with open(args.split_file, "rb") as handle:
|
|
178
|
+
train_indices, valid_indices, test_indices = pickle.load(handle)
|
|
179
|
+
for split, indices in {"train": train_indices, "val": valid_indices, "test": test_indices}.items():
|
|
180
|
+
export_split(data, slices, indices, args.output, split, args.shard_molecules,
|
|
181
|
+
progress, args.resume, args.max_shards_per_split)
|
|
182
|
+
if args.max_shards_per_split:
|
|
183
|
+
print("Smoke export complete; manifest is intentionally not written.", flush=True)
|
|
184
|
+
return
|
|
185
|
+
manifest = {"format": "manimol-shards-v3-packed", "source": os.path.abspath(args.source), "splits": {}}
|
|
186
|
+
all_hashes = []
|
|
187
|
+
total_records = 0
|
|
188
|
+
total_molecules = 0
|
|
189
|
+
for split, info in progress["splits"].items():
|
|
190
|
+
manifest["splits"][split] = {k: info[k] for k in ("shards", "samples", "molecules")}
|
|
191
|
+
all_hashes.extend([[split, row["file"], row["sha256"], row["samples"], row["molecules"]]
|
|
192
|
+
for row in info["identity_shards"]])
|
|
193
|
+
total_records += int(info["samples"])
|
|
194
|
+
total_molecules += int(info["molecules"])
|
|
195
|
+
all_hashes.sort(key=lambda row: (row[0], row[1]))
|
|
196
|
+
digest = hashlib.sha256(json.dumps(all_hashes, separators=(",", ":"), ensure_ascii=True).encode()).hexdigest()
|
|
197
|
+
manifest["identity"] = {"records": total_records, "molecules": total_molecules,
|
|
198
|
+
"sha256": digest,
|
|
199
|
+
"fields": ["smiles", "num_atoms", "atom_feature_sequence", "mol_id", "conf_id"]}
|
|
200
|
+
with open(os.path.join(args.output, "manifest.json"), "w") as handle:
|
|
201
|
+
json.dump(manifest, handle, indent=2)
|
|
202
|
+
print("identity_sha256=%s records=%d molecules=%d" %
|
|
203
|
+
(digest, total_records, total_molecules), flush=True)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
if __name__ == "__main__":
|
|
207
|
+
main()
|