D4CMPP2 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- D4CMPP2/_Data/AGENTS.md +24 -0
- D4CMPP2/_Data/Aqsoldb.csv +9291 -0
- D4CMPP2/_Data/BradleyMP.csv +3042 -0
- D4CMPP2/_Data/Lipophilicity.csv +1131 -0
- D4CMPP2/_Data/README.md +8 -0
- D4CMPP2/_Data/__init__.py +26 -0
- D4CMPP2/_Data/optical.csv +20237 -0
- D4CMPP2/_Data/test.csv +190 -0
- D4CMPP2/__init__.py +16 -0
- D4CMPP2/__main__.py +5 -0
- D4CMPP2/_main.py +500 -0
- D4CMPP2/cli.py +7 -0
- D4CMPP2/exceptions.py +53 -0
- D4CMPP2/grid_search.py +259 -0
- D4CMPP2/network_refer.yaml +160 -0
- D4CMPP2/networks/AFP_model.py +72 -0
- D4CMPP2/networks/AFPwithSolv_model.py +72 -0
- D4CMPP2/networks/DMPNN_model.py +90 -0
- D4CMPP2/networks/DMPNNwithSolv_model.py +89 -0
- D4CMPP2/networks/GAT_model.py +48 -0
- D4CMPP2/networks/GATwithSolv_model.py +63 -0
- D4CMPP2/networks/GCN_model.py +113 -0
- D4CMPP2/networks/GCNwithSolv_model.py +103 -0
- D4CMPP2/networks/GC_model.py +122 -0
- D4CMPP2/networks/ISATPM_model.py +14 -0
- D4CMPP2/networks/ISATPN_model.py +199 -0
- D4CMPP2/networks/ISAT_model.py +90 -0
- D4CMPP2/networks/MPNN_model.py +56 -0
- D4CMPP2/networks/MPNNwithSolv_model.py +72 -0
- D4CMPP2/networks/__init__.py +25 -0
- D4CMPP2/networks/base.py +250 -0
- D4CMPP2/networks/registry.py +187 -0
- D4CMPP2/networks/src/AFP.py +118 -0
- D4CMPP2/networks/src/BiDropout.py +29 -0
- D4CMPP2/networks/src/DMPNN.py +35 -0
- D4CMPP2/networks/src/GAT.py +69 -0
- D4CMPP2/networks/src/GC.py +85 -0
- D4CMPP2/networks/src/GCN.py +71 -0
- D4CMPP2/networks/src/ISAT.py +153 -0
- D4CMPP2/networks/src/Linear.py +49 -0
- D4CMPP2/networks/src/MPNN.py +56 -0
- D4CMPP2/networks/src/SolventLayer.py +62 -0
- D4CMPP2/networks/src/__init__.py +0 -0
- D4CMPP2/networks/src/distGCN.py +21 -0
- D4CMPP2/networks/src/pyg_hetero.py +24 -0
- D4CMPP2/optimize.py +472 -0
- D4CMPP2/src/Analyzer/ISAAnalyzer.py +458 -0
- D4CMPP2/src/Analyzer/ISAPNAnalyzer.py +366 -0
- D4CMPP2/src/Analyzer/ISAwSAnalyzer.py +117 -0
- D4CMPP2/src/Analyzer/MolAnalyzer.py +319 -0
- D4CMPP2/src/Analyzer/__init__.py +54 -0
- D4CMPP2/src/Analyzer/core.py +480 -0
- D4CMPP2/src/Analyzer/factory.py +166 -0
- D4CMPP2/src/Analyzer/interpretation.py +232 -0
- D4CMPP2/src/Analyzer/results.py +101 -0
- D4CMPP2/src/DataManager/Dataset/GraphDataset.py +314 -0
- D4CMPP2/src/DataManager/Dataset/ISAGraphDataset.py +384 -0
- D4CMPP2/src/DataManager/Dataset/__init__.py +0 -0
- D4CMPP2/src/DataManager/GraphGenerator/ISAGraphGenerator.py +223 -0
- D4CMPP2/src/DataManager/GraphGenerator/MolGraphGenerator.py +73 -0
- D4CMPP2/src/DataManager/GraphGenerator/__init__.py +14 -0
- D4CMPP2/src/DataManager/ISADataManager.py +67 -0
- D4CMPP2/src/DataManager/MolDataManager.py +735 -0
- D4CMPP2/src/DataManager/__init__.py +14 -0
- D4CMPP2/src/DataManager/contracts.py +179 -0
- D4CMPP2/src/NetworkManager/ISANetworkManager.py +12 -0
- D4CMPP2/src/NetworkManager/NetworkManager.py +520 -0
- D4CMPP2/src/NetworkManager/__init__.py +14 -0
- D4CMPP2/src/PostProcessor.py +160 -0
- D4CMPP2/src/TrainManager/ISATrainManager.py +26 -0
- D4CMPP2/src/TrainManager/TrainManager.py +254 -0
- D4CMPP2/src/TrainManager/__init__.py +14 -0
- D4CMPP2/src/TrainManager/callbacks.py +119 -0
- D4CMPP2/src/__init__.py +0 -0
- D4CMPP2/src/utils/PATH.py +246 -0
- D4CMPP2/src/utils/__init__.py +0 -0
- D4CMPP2/src/utils/argparser.py +56 -0
- D4CMPP2/src/utils/checkpointing.py +90 -0
- D4CMPP2/src/utils/config_resolution.py +123 -0
- D4CMPP2/src/utils/config_validation.py +370 -0
- D4CMPP2/src/utils/csv_validation.py +105 -0
- D4CMPP2/src/utils/data_quality.py +181 -0
- D4CMPP2/src/utils/featureizer.py +202 -0
- D4CMPP2/src/utils/functional_group.csv +169 -0
- D4CMPP2/src/utils/graph_cache.py +213 -0
- D4CMPP2/src/utils/leaderboard.py +212 -0
- D4CMPP2/src/utils/metrics.py +31 -0
- D4CMPP2/src/utils/module_loader.py +147 -0
- D4CMPP2/src/utils/output.py +80 -0
- D4CMPP2/src/utils/reproducibility.py +70 -0
- D4CMPP2/src/utils/run_manifest.py +175 -0
- D4CMPP2/src/utils/scaler.py +40 -0
- D4CMPP2/src/utils/sculptor.py +713 -0
- D4CMPP2/src/utils/splitting.py +250 -0
- D4CMPP2/src/utils/supportfile_saver.py +94 -0
- D4CMPP2/src/utils/tools.py +156 -0
- D4CMPP2/src/utils/transfer_learning.py +111 -0
- d4cmpp2-0.4.0.dist-info/METADATA +420 -0
- d4cmpp2-0.4.0.dist-info/RECORD +103 -0
- d4cmpp2-0.4.0.dist-info/WHEEL +5 -0
- d4cmpp2-0.4.0.dist-info/entry_points.txt +2 -0
- d4cmpp2-0.4.0.dist-info/licenses/LICENSE +21 -0
- d4cmpp2-0.4.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""Early, dependency-light validation for training configuration values."""
|
|
2
|
+
|
|
3
|
+
import difflib
|
|
4
|
+
import importlib
|
|
5
|
+
import re
|
|
6
|
+
import warnings
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _dependency_error(message):
|
|
10
|
+
"""Keep dependency-light source tests while categorizing package imports."""
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
from D4CMPP2.exceptions import DependencyError
|
|
14
|
+
except ImportError:
|
|
15
|
+
return RuntimeError(message)
|
|
16
|
+
return DependencyError(message)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
SCALERS = ("standard", "minmax", "normalizer", "robust", "identity")
|
|
20
|
+
REGISTRY_FIELDS = (
|
|
21
|
+
"network",
|
|
22
|
+
"data_manager_module",
|
|
23
|
+
"data_manager_class",
|
|
24
|
+
"network_manager_module",
|
|
25
|
+
"network_manager_class",
|
|
26
|
+
"train_manager_module",
|
|
27
|
+
"train_manager_class",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _type_name(value):
|
|
32
|
+
return type(value).__name__
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def validate_entry_args(kwargs):
|
|
36
|
+
"""Validate arguments whose contract is known before config/path loading."""
|
|
37
|
+
modes = [
|
|
38
|
+
key for key in ("LOAD_PATH", "RESUME_PATH", "TRANSFER_PATH")
|
|
39
|
+
if kwargs.get(key) is not None
|
|
40
|
+
]
|
|
41
|
+
if len(modes) > 1:
|
|
42
|
+
raise ValueError(
|
|
43
|
+
f"{modes} are mutually exclusive. Use LOAD_PATH for weight-only continuation, "
|
|
44
|
+
"RESUME_PATH for full-state resume, or TRANSFER_PATH for parameter transfer."
|
|
45
|
+
)
|
|
46
|
+
if modes:
|
|
47
|
+
key = modes[0]
|
|
48
|
+
path = kwargs[key]
|
|
49
|
+
if not isinstance(path, str) or not path.strip():
|
|
50
|
+
raise TypeError(
|
|
51
|
+
f"{key} must be a non-empty string, got {path!r} "
|
|
52
|
+
f"({_type_name(path)}). Provide a saved model folder or checkpoint path."
|
|
53
|
+
)
|
|
54
|
+
if key in {"LOAD_PATH", "RESUME_PATH"}:
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
data = kwargs.get("data")
|
|
58
|
+
if not isinstance(data, str) or not data.strip():
|
|
59
|
+
raise TypeError(
|
|
60
|
+
f"data must be a non-empty CSV name or path string, got {data!r} "
|
|
61
|
+
f"({_type_name(data)}). Example: data='Aqsoldb.csv'."
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
target = kwargs.get("target")
|
|
65
|
+
if not isinstance(target, list) or not target or not all(
|
|
66
|
+
isinstance(column, str) and column.strip() for column in target
|
|
67
|
+
):
|
|
68
|
+
raise TypeError(
|
|
69
|
+
f"target must be a non-empty list of column-name strings, got {target!r} "
|
|
70
|
+
f"({_type_name(target)}). Example: target=['Solubility']."
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
network = kwargs.get("network")
|
|
74
|
+
if not isinstance(network, str) or not network.strip():
|
|
75
|
+
raise TypeError(
|
|
76
|
+
f"network must be a non-empty network ID string, got {network!r} "
|
|
77
|
+
f"({_type_name(network)}). Example: network='GCN'."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def validate_network_entry(network_id, entry, available_ids):
|
|
82
|
+
"""Validate one selected network registry entry before manager loading."""
|
|
83
|
+
if entry is None:
|
|
84
|
+
candidates = difflib.get_close_matches(network_id, available_ids, n=3, cutoff=0.5)
|
|
85
|
+
message = (
|
|
86
|
+
f"Network ID {network_id!r} was not found. "
|
|
87
|
+
f"Available network IDs: {', '.join(available_ids)}."
|
|
88
|
+
)
|
|
89
|
+
if candidates:
|
|
90
|
+
message += f" Did you mean {', '.join(repr(item) for item in candidates)}?"
|
|
91
|
+
raise ValueError(message)
|
|
92
|
+
|
|
93
|
+
missing = [
|
|
94
|
+
field
|
|
95
|
+
for field in REGISTRY_FIELDS
|
|
96
|
+
if not isinstance(entry.get(field), str) or not entry[field].strip()
|
|
97
|
+
]
|
|
98
|
+
if missing:
|
|
99
|
+
raise ValueError(
|
|
100
|
+
f"Registry entry for network {network_id!r} is missing required non-empty "
|
|
101
|
+
f"keys: {missing}. Check the selected network reference YAML file."
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def validate_training_config(config, optimizer_names=None):
|
|
106
|
+
"""Validate merged training values without mutating the configuration."""
|
|
107
|
+
scaler = config.get("scaler")
|
|
108
|
+
if scaler not in SCALERS:
|
|
109
|
+
raise ValueError(
|
|
110
|
+
f"scaler must be one of {list(SCALERS)}, got {scaler!r}. "
|
|
111
|
+
"Choose a supported target scaling method."
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
optimizer = config.get("optimizer")
|
|
115
|
+
if not isinstance(optimizer, str) or not optimizer.strip():
|
|
116
|
+
raise TypeError(
|
|
117
|
+
f"optimizer must be a torch.optim class name string, got {optimizer!r} "
|
|
118
|
+
f"({_type_name(optimizer)}). Example: optimizer='Adam'."
|
|
119
|
+
)
|
|
120
|
+
if optimizer_names is not None and optimizer not in optimizer_names:
|
|
121
|
+
candidates = difflib.get_close_matches(optimizer, sorted(optimizer_names), n=3, cutoff=0.5)
|
|
122
|
+
hint = f" Did you mean {', '.join(repr(item) for item in candidates)}?" if candidates else ""
|
|
123
|
+
raise ValueError(
|
|
124
|
+
f"Optimizer {optimizer!r} is not available in torch.optim.{hint} "
|
|
125
|
+
"Use a torch.optim optimizer class name such as 'Adam'."
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
_positive_int(config, "max_epoch")
|
|
129
|
+
_positive_int(config, "batch_size")
|
|
130
|
+
_non_negative_int(config, "lr_patience")
|
|
131
|
+
_non_negative_int(config, "early_stopping_patience")
|
|
132
|
+
_positive_number(config, "learning_rate")
|
|
133
|
+
_non_negative_number(config, "weight_decay")
|
|
134
|
+
_non_negative_number(config, "min_lr")
|
|
135
|
+
|
|
136
|
+
learning_rate = config["learning_rate"]
|
|
137
|
+
min_lr = config["min_lr"]
|
|
138
|
+
if min_lr > learning_rate:
|
|
139
|
+
raise ValueError(
|
|
140
|
+
f"min_lr ({min_lr!r}) must not exceed learning_rate ({learning_rate!r}). "
|
|
141
|
+
"Lower min_lr or increase learning_rate."
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
pin_memory = config.get("pin_memory")
|
|
145
|
+
if not isinstance(pin_memory, bool):
|
|
146
|
+
raise TypeError(
|
|
147
|
+
f"pin_memory must be bool, got {pin_memory!r} ({_type_name(pin_memory)}). "
|
|
148
|
+
"Use pin_memory=True or pin_memory=False."
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
legacy_silent_errors = config.get("legacy_silent_errors", False)
|
|
152
|
+
if not isinstance(legacy_silent_errors, bool):
|
|
153
|
+
raise TypeError(
|
|
154
|
+
f"legacy_silent_errors must be bool, got {legacy_silent_errors!r} "
|
|
155
|
+
f"({_type_name(legacy_silent_errors)}). Use True only for temporary legacy compatibility."
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
verbose = config.get("verbose", True)
|
|
159
|
+
if not isinstance(verbose, bool):
|
|
160
|
+
raise TypeError(
|
|
161
|
+
f"verbose must be bool, got {verbose!r} ({_type_name(verbose)}). "
|
|
162
|
+
"Use verbose=True to show status output or verbose=False to hide it."
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
device = config.get("device")
|
|
166
|
+
if not isinstance(device, str) or not re.fullmatch(r"cpu|cuda(?::\d+)?", device):
|
|
167
|
+
raise ValueError(
|
|
168
|
+
f"device must be 'cpu', 'cuda', or 'cuda:<non-negative index>', got {device!r}. "
|
|
169
|
+
"Example: device='cpu' or device='cuda:0'."
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
scaler_scope = config.get("target_scaler_fit_scope", "train")
|
|
173
|
+
if scaler_scope not in {"train", "all"}:
|
|
174
|
+
raise ValueError(
|
|
175
|
+
f"target_scaler_fit_scope must be 'train' or 'all', got {scaler_scope!r}. "
|
|
176
|
+
"Use 'train' to avoid validation/test leakage or 'all' only for legacy numerical compatibility."
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
cache_policy = config.get("graph_cache_policy", "v2")
|
|
180
|
+
if cache_policy not in {"v2", "legacy", "regenerate"}:
|
|
181
|
+
raise ValueError(
|
|
182
|
+
f"graph_cache_policy must be 'v2', 'legacy', or 'regenerate', got {cache_policy!r}. "
|
|
183
|
+
"Use 'v2' for verified caches, 'legacy' only for explicit schema-v1 compatibility, "
|
|
184
|
+
"or 'regenerate' to rebuild an invalid v2 cache."
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
split_strategy = config.get("split_strategy", "auto")
|
|
188
|
+
if split_strategy not in {"auto", "random", "predefined", "scaffold"}:
|
|
189
|
+
raise ValueError(
|
|
190
|
+
f"split_strategy must be 'auto', 'random', 'predefined', or "
|
|
191
|
+
f"'scaffold', got {split_strategy!r}."
|
|
192
|
+
)
|
|
193
|
+
scaffold_column = config.get("scaffold_column")
|
|
194
|
+
if scaffold_column is not None and (
|
|
195
|
+
not isinstance(scaffold_column, str) or not scaffold_column.strip()
|
|
196
|
+
):
|
|
197
|
+
raise TypeError(
|
|
198
|
+
f"scaffold_column must be a non-empty molecule-column string or None, "
|
|
199
|
+
f"got {scaffold_column!r}."
|
|
200
|
+
)
|
|
201
|
+
include_chirality = config.get("scaffold_include_chirality", False)
|
|
202
|
+
if not isinstance(include_chirality, bool):
|
|
203
|
+
raise TypeError(
|
|
204
|
+
f"scaffold_include_chirality must be bool, got "
|
|
205
|
+
f"{include_chirality!r} ({_type_name(include_chirality)})."
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
random_seed = config.get("random_seed")
|
|
209
|
+
if random_seed is not None and (
|
|
210
|
+
isinstance(random_seed, bool) or not isinstance(random_seed, int) or random_seed < 0
|
|
211
|
+
):
|
|
212
|
+
raise ValueError(
|
|
213
|
+
f"random_seed must be a non-negative integer or None, got {random_seed!r}. "
|
|
214
|
+
"Example: random_seed=42."
|
|
215
|
+
)
|
|
216
|
+
deterministic = config.get("deterministic_algorithms", False)
|
|
217
|
+
if not isinstance(deterministic, bool):
|
|
218
|
+
raise TypeError(
|
|
219
|
+
f"deterministic_algorithms must be bool, got {deterministic!r} "
|
|
220
|
+
f"({_type_name(deterministic)})."
|
|
221
|
+
)
|
|
222
|
+
if deterministic and random_seed is None:
|
|
223
|
+
raise ValueError(
|
|
224
|
+
"deterministic_algorithms=True requires random_seed to be set. "
|
|
225
|
+
"Example: random_seed=42, deterministic_algorithms=True."
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
lr_dict = config.get("lr_dict", {})
|
|
229
|
+
if not isinstance(lr_dict, dict):
|
|
230
|
+
raise TypeError(
|
|
231
|
+
f"lr_dict must be a mapping of layer-name components to learning rates, "
|
|
232
|
+
f"got {lr_dict!r} ({_type_name(lr_dict)})."
|
|
233
|
+
)
|
|
234
|
+
invalid_lr_entries = {
|
|
235
|
+
key: value for key, value in lr_dict.items()
|
|
236
|
+
if (
|
|
237
|
+
not isinstance(key, str)
|
|
238
|
+
or not key.strip()
|
|
239
|
+
or isinstance(value, bool)
|
|
240
|
+
or not isinstance(value, (int, float))
|
|
241
|
+
or value <= 0
|
|
242
|
+
)
|
|
243
|
+
}
|
|
244
|
+
if invalid_lr_entries:
|
|
245
|
+
raise ValueError(
|
|
246
|
+
"lr_dict keys must be non-empty layer-name strings and values must be "
|
|
247
|
+
f"positive learning rates; invalid entries: {invalid_lr_entries!r}."
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
_validate_sculptor_config(config)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def validate_runtime_environment(config, backend="pyg", torch_module=None, importer=None):
|
|
254
|
+
"""Validate device availability and graph backend imports before model creation."""
|
|
255
|
+
importer = importer or importlib.import_module
|
|
256
|
+
if torch_module is None:
|
|
257
|
+
try:
|
|
258
|
+
torch_module = importer("torch")
|
|
259
|
+
except (ImportError, OSError) as exc:
|
|
260
|
+
raise _dependency_error(
|
|
261
|
+
"PyTorch could not be imported, so training cannot start. "
|
|
262
|
+
"Install a PyTorch build compatible with your Python and requested CPU/CUDA environment. "
|
|
263
|
+
f"Original import error: {exc}"
|
|
264
|
+
) from exc
|
|
265
|
+
|
|
266
|
+
device = config["device"]
|
|
267
|
+
if device.startswith("cuda"):
|
|
268
|
+
cuda = torch_module.cuda
|
|
269
|
+
if not cuda.is_available():
|
|
270
|
+
raise _dependency_error(
|
|
271
|
+
f"Requested device {device!r}, but CUDA is not available to PyTorch "
|
|
272
|
+
f"{getattr(torch_module, '__version__', 'unknown')!r}. "
|
|
273
|
+
"Check the NVIDIA driver and CUDA-enabled PyTorch installation, or explicitly set device='cpu'. "
|
|
274
|
+
"D4CMPP2 does not switch devices automatically."
|
|
275
|
+
)
|
|
276
|
+
index = 0 if device == "cuda" else int(device.split(":", 1)[1])
|
|
277
|
+
count = cuda.device_count()
|
|
278
|
+
if index >= count:
|
|
279
|
+
raise _dependency_error(
|
|
280
|
+
f"Requested device {device!r}, but PyTorch reports {count} CUDA device(s) "
|
|
281
|
+
f"with valid indices 0..{count - 1}. Choose an available index or set device='cpu'."
|
|
282
|
+
)
|
|
283
|
+
elif config.get("pin_memory", False):
|
|
284
|
+
warnings.warn(
|
|
285
|
+
"pin_memory=True was requested with device='cpu'. Pinned host memory normally benefits "
|
|
286
|
+
"CUDA transfers and may add overhead for CPU-only training; consider pin_memory=False.",
|
|
287
|
+
UserWarning,
|
|
288
|
+
stacklevel=2,
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
if backend != "pyg":
|
|
292
|
+
raise ValueError(f"Unknown graph backend {backend!r}; this version supports only 'pyg'.")
|
|
293
|
+
module_name = "torch_geometric"
|
|
294
|
+
try:
|
|
295
|
+
backend_module = importer(module_name)
|
|
296
|
+
except (ImportError, OSError) as exc:
|
|
297
|
+
torch_version = getattr(torch_module, "__version__", "unknown")
|
|
298
|
+
raise _dependency_error(
|
|
299
|
+
f"Graph backend {backend!r} could not be imported with PyTorch {torch_version!r}. "
|
|
300
|
+
f"Install compatible torch/{module_name} builds for the same Python and CPU/CUDA environment. "
|
|
301
|
+
f"Original import error: {exc}"
|
|
302
|
+
) from exc
|
|
303
|
+
return {
|
|
304
|
+
"torch": str(getattr(torch_module, "__version__", "unknown")),
|
|
305
|
+
backend: str(getattr(backend_module, "__version__", "unknown")),
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def validate_sculptor_index_argument(value):
|
|
310
|
+
"""Validate the legacy tuple form before it is expanded into config keys."""
|
|
311
|
+
if not isinstance(value, tuple):
|
|
312
|
+
return
|
|
313
|
+
if len(value) != 3 or any(
|
|
314
|
+
isinstance(item, bool) or not isinstance(item, int) or item < 0 for item in value
|
|
315
|
+
):
|
|
316
|
+
raise ValueError(
|
|
317
|
+
f"sculptor_index must be a tuple of three non-negative integers, got {value!r}. "
|
|
318
|
+
"Example: sculptor_index=(6, 2, 0)."
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _positive_int(config, key):
|
|
323
|
+
value = config.get(key)
|
|
324
|
+
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
|
325
|
+
raise ValueError(f"{key} must be a positive integer, got {value!r} ({_type_name(value)}).")
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _non_negative_int(config, key):
|
|
329
|
+
value = config.get(key)
|
|
330
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
331
|
+
raise ValueError(f"{key} must be a non-negative integer, got {value!r} ({_type_name(value)}).")
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _positive_number(config, key):
|
|
335
|
+
value = config.get(key)
|
|
336
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
|
|
337
|
+
raise ValueError(f"{key} must be a positive number, got {value!r} ({_type_name(value)}).")
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _non_negative_number(config, key):
|
|
341
|
+
value = config.get(key)
|
|
342
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0:
|
|
343
|
+
raise ValueError(f"{key} must be a non-negative number, got {value!r} ({_type_name(value)}).")
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _validate_sculptor_config(config):
|
|
347
|
+
isa_manager = config.get("data_manager_module") == "ISADataManager"
|
|
348
|
+
if not isa_manager:
|
|
349
|
+
return
|
|
350
|
+
|
|
351
|
+
keys = ("sculptor_s", "sculptor_c", "sculptor_a")
|
|
352
|
+
missing = [key for key in keys if key not in config]
|
|
353
|
+
if missing:
|
|
354
|
+
original = config.get("sculptor_index")
|
|
355
|
+
if isinstance(original, list):
|
|
356
|
+
raise TypeError(
|
|
357
|
+
f"sculptor_index must be a tuple of three non-negative integers, got list {original!r}. "
|
|
358
|
+
"Example: sculptor_index=(6, 2, 0)."
|
|
359
|
+
)
|
|
360
|
+
raise ValueError(
|
|
361
|
+
f"ISA network {config.get('network')!r} requires sculptor_index=(split, combine, absorb); "
|
|
362
|
+
f"missing normalized keys: {missing}. Example: sculptor_index=(6, 2, 0)."
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
invalid = {key: config[key] for key in keys if isinstance(config[key], bool) or not isinstance(config[key], int) or config[key] < 0}
|
|
366
|
+
if invalid:
|
|
367
|
+
raise ValueError(
|
|
368
|
+
f"ISA sculptor values must be non-negative integers, got {invalid!r}. "
|
|
369
|
+
"Example: sculptor_index=(6, 2, 0)."
|
|
370
|
+
)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Dependency-light CSV schema and split validation helpers."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
ALLOWED_SET_LABELS = ("train", "val", "test")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def validate_csv_schema(path, columns, row_count, molecule_columns, target_columns, numeric_columns):
|
|
10
|
+
"""Validate required columns and basic table shape without mutating data."""
|
|
11
|
+
if row_count == 0:
|
|
12
|
+
raise ValueError(
|
|
13
|
+
f"CSV file {path!r} contains no data rows. Add at least one row and the required columns."
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
groups = {
|
|
17
|
+
"molecule": list(molecule_columns),
|
|
18
|
+
"target": list(target_columns),
|
|
19
|
+
"numeric input": list(numeric_columns),
|
|
20
|
+
}
|
|
21
|
+
duplicates = {
|
|
22
|
+
name: sorted({column for column in requested if requested.count(column) > 1})
|
|
23
|
+
for name, requested in groups.items()
|
|
24
|
+
}
|
|
25
|
+
duplicates = {name: values for name, values in duplicates.items() if values}
|
|
26
|
+
if duplicates:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
f"Duplicate configured CSV columns were found for {path!r}: {duplicates}. "
|
|
29
|
+
"List each molecule, target, and numeric input column once."
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
available = list(columns)
|
|
33
|
+
missing = {
|
|
34
|
+
name: [column for column in requested if column not in available]
|
|
35
|
+
for name, requested in groups.items()
|
|
36
|
+
}
|
|
37
|
+
missing = {name: values for name, values in missing.items() if values}
|
|
38
|
+
if missing:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"Required CSV columns were not found in {path!r}: {missing}. "
|
|
41
|
+
f"Available columns: {available}. Check molecule_columns, target, and "
|
|
42
|
+
"numeric_input_columns spelling."
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def validate_numeric_columns(path, invalid_rows_by_column, kind):
|
|
47
|
+
"""Report non-numeric cells with original row indices."""
|
|
48
|
+
invalid = {column: list(rows) for column, rows in invalid_rows_by_column.items() if rows}
|
|
49
|
+
if invalid:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
f"{kind} columns in {path!r} contain non-numeric values at row indices: {invalid}. "
|
|
52
|
+
"Replace those values with numbers or blank/NaN values."
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def validate_nonempty_targets(path, all_nan_by_column):
|
|
57
|
+
"""Reject target columns that contain no usable values."""
|
|
58
|
+
empty = [column for column, all_nan in all_nan_by_column.items() if all_nan]
|
|
59
|
+
if empty:
|
|
60
|
+
raise ValueError(
|
|
61
|
+
f"Target columns {empty} in {path!r} contain only NaN/blank values. "
|
|
62
|
+
"Provide at least one numeric target value in each target column."
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def validate_set_labels(path, values, require_train_val=True):
|
|
67
|
+
"""Validate explicit split labels and return their counts."""
|
|
68
|
+
labels = list(values)
|
|
69
|
+
invalid = sorted({repr(value) for value in labels if _is_missing(value) or value not in ALLOWED_SET_LABELS})
|
|
70
|
+
counts = {label: labels.count(label) for label in ALLOWED_SET_LABELS}
|
|
71
|
+
if invalid:
|
|
72
|
+
raise ValueError(
|
|
73
|
+
f"Column 'set' in {path!r} contains invalid labels {invalid}. "
|
|
74
|
+
f"Allowed labels: {list(ALLOWED_SET_LABELS)}. Split counts: {counts}."
|
|
75
|
+
)
|
|
76
|
+
if require_train_val and (counts["train"] == 0 or counts["val"] == 0):
|
|
77
|
+
missing = [label for label in ("train", "val") if counts[label] == 0]
|
|
78
|
+
raise ValueError(
|
|
79
|
+
f"Column 'set' in {path!r} is missing required split labels {missing}. "
|
|
80
|
+
f"Split counts: {counts}. Add at least one train and one val row."
|
|
81
|
+
)
|
|
82
|
+
return counts
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def validate_automatic_split_size(path, row_count, minimum=10):
|
|
86
|
+
"""Reject datasets too small for the legacy 80/10/10 automatic split."""
|
|
87
|
+
if row_count < minimum:
|
|
88
|
+
raise ValueError(
|
|
89
|
+
f"Dataset {path!r} has {row_count} usable rows, but automatic splitting requires "
|
|
90
|
+
f"at least {minimum}. Add rows or provide a 'set' column with train/val labels."
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def validate_aligned_lengths(path, lengths):
|
|
95
|
+
"""Assert that extracted CSV arrays still share one row count."""
|
|
96
|
+
normalized = {name: int(length) for name, length in lengths.items()}
|
|
97
|
+
if len(set(normalized.values())) > 1:
|
|
98
|
+
raise ValueError(
|
|
99
|
+
f"CSV-derived arrays from {path!r} have inconsistent lengths: {normalized}. "
|
|
100
|
+
"Check that all molecule, numeric input, target, and set values use the same rows."
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _is_missing(value):
|
|
105
|
+
return value is None or (isinstance(value, float) and math.isnan(value))
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Non-mutating data quality summaries for molecular CSV inputs."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import uuid
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
from rdkit import Chem
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
REPORT_SCHEMA_VERSION = 1
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _canonical_smiles(value):
|
|
17
|
+
if pd.isna(value) or not str(value).strip():
|
|
18
|
+
return None, "empty_smiles"
|
|
19
|
+
molecule = Chem.MolFromSmiles(str(value))
|
|
20
|
+
if molecule is None:
|
|
21
|
+
return None, "invalid_smiles"
|
|
22
|
+
return Chem.MolToSmiles(molecule, canonical=True), None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _target_summary(frame, targets):
|
|
26
|
+
result = {}
|
|
27
|
+
for target in targets:
|
|
28
|
+
if target not in frame:
|
|
29
|
+
result[target] = {"missing_column": True}
|
|
30
|
+
continue
|
|
31
|
+
numeric = pd.to_numeric(frame[target], errors="coerce")
|
|
32
|
+
usable = numeric.dropna()
|
|
33
|
+
result[target] = {
|
|
34
|
+
"count": int(len(numeric)),
|
|
35
|
+
"usable_count": int(usable.size),
|
|
36
|
+
"missing_count": int(numeric.isna().sum()),
|
|
37
|
+
"min": float(usable.min()) if not usable.empty else None,
|
|
38
|
+
"max": float(usable.max()) if not usable.empty else None,
|
|
39
|
+
"mean": float(usable.mean()) if not usable.empty else None,
|
|
40
|
+
"std": float(usable.std()) if usable.size > 1 else None,
|
|
41
|
+
}
|
|
42
|
+
return result
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def build_data_quality_report(frame, config, graph_errors=None):
|
|
46
|
+
"""Build summary and issue rows without changing the input DataFrame."""
|
|
47
|
+
molecule_columns = list(config.get("molecule_columns", ["compound"]))
|
|
48
|
+
targets = list(config.get("target", []))
|
|
49
|
+
issues = []
|
|
50
|
+
canonical_columns = {}
|
|
51
|
+
|
|
52
|
+
for column in molecule_columns:
|
|
53
|
+
if column not in frame:
|
|
54
|
+
issues.append({
|
|
55
|
+
"issue": "missing_molecule_column",
|
|
56
|
+
"row_index": None,
|
|
57
|
+
"column": column,
|
|
58
|
+
"value": None,
|
|
59
|
+
"detail": "Configured molecule column is absent.",
|
|
60
|
+
})
|
|
61
|
+
continue
|
|
62
|
+
canonical = []
|
|
63
|
+
for row_index, value in frame[column].items():
|
|
64
|
+
normalized, issue = _canonical_smiles(value)
|
|
65
|
+
canonical.append(normalized)
|
|
66
|
+
if issue:
|
|
67
|
+
issues.append({
|
|
68
|
+
"issue": issue,
|
|
69
|
+
"row_index": row_index,
|
|
70
|
+
"column": column,
|
|
71
|
+
"value": None if pd.isna(value) else str(value),
|
|
72
|
+
"detail": "RDKit could not produce a canonical molecule." if issue == "invalid_smiles" else "SMILES is empty.",
|
|
73
|
+
})
|
|
74
|
+
canonical_columns[column] = pd.Series(canonical, index=frame.index)
|
|
75
|
+
|
|
76
|
+
signature = None
|
|
77
|
+
if len(canonical_columns) == len(molecule_columns):
|
|
78
|
+
signature_frame = pd.DataFrame(canonical_columns)
|
|
79
|
+
valid_signature = signature_frame.notna().all(axis=1)
|
|
80
|
+
signature = signature_frame.astype(str).agg("||".join, axis=1).where(valid_signature)
|
|
81
|
+
duplicate_mask = signature.notna() & signature.duplicated(keep=False)
|
|
82
|
+
for row_index in frame.index[duplicate_mask]:
|
|
83
|
+
issues.append({
|
|
84
|
+
"issue": "duplicate_molecule",
|
|
85
|
+
"row_index": row_index,
|
|
86
|
+
"column": "|".join(molecule_columns),
|
|
87
|
+
"value": signature.loc[row_index],
|
|
88
|
+
"detail": "Canonical molecule input appears in multiple rows.",
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
if "set" in frame:
|
|
92
|
+
grouped = pd.DataFrame({"signature": signature, "set": frame["set"]}).dropna()
|
|
93
|
+
overlap = grouped.groupby("signature")["set"].agg(lambda values: sorted(set(values)))
|
|
94
|
+
for molecule, labels in overlap.items():
|
|
95
|
+
if len(labels) > 1:
|
|
96
|
+
issues.append({
|
|
97
|
+
"issue": "split_molecule_overlap",
|
|
98
|
+
"row_index": None,
|
|
99
|
+
"column": "set",
|
|
100
|
+
"value": molecule,
|
|
101
|
+
"detail": f"Canonical molecule occurs across splits {labels}.",
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
for target in targets:
|
|
105
|
+
if target not in frame:
|
|
106
|
+
continue
|
|
107
|
+
numeric = pd.to_numeric(frame[target], errors="coerce")
|
|
108
|
+
for row_index in frame.index[numeric.isna()]:
|
|
109
|
+
issues.append({
|
|
110
|
+
"issue": "missing_target",
|
|
111
|
+
"row_index": row_index,
|
|
112
|
+
"column": target,
|
|
113
|
+
"value": None if pd.isna(frame.at[row_index, target]) else str(frame.at[row_index, target]),
|
|
114
|
+
"detail": "Target is blank, NaN, or non-numeric.",
|
|
115
|
+
})
|
|
116
|
+
if signature is not None:
|
|
117
|
+
grouped = pd.DataFrame({"signature": signature, "target": numeric}).dropna()
|
|
118
|
+
conflicts = grouped.groupby("signature")["target"].nunique()
|
|
119
|
+
for molecule in conflicts[conflicts > 1].index:
|
|
120
|
+
issues.append({
|
|
121
|
+
"issue": "conflicting_duplicate_target",
|
|
122
|
+
"row_index": None,
|
|
123
|
+
"column": target,
|
|
124
|
+
"value": molecule,
|
|
125
|
+
"detail": "Duplicate canonical molecule has differing target values.",
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
for error in graph_errors or []:
|
|
129
|
+
issues.append({
|
|
130
|
+
"issue": "graph_generation_failure",
|
|
131
|
+
"row_index": error.get("row_index"),
|
|
132
|
+
"column": error.get("type"),
|
|
133
|
+
"value": str(error.get("smiles")),
|
|
134
|
+
"detail": str(error.get("reason")),
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
issue_counts = {}
|
|
138
|
+
for issue in issues:
|
|
139
|
+
issue_counts[issue["issue"]] = issue_counts.get(issue["issue"], 0) + 1
|
|
140
|
+
split_counts = (
|
|
141
|
+
{str(key): int(value) for key, value in frame["set"].value_counts(dropna=False).items()}
|
|
142
|
+
if "set" in frame else {"automatic": int(len(frame))}
|
|
143
|
+
)
|
|
144
|
+
report = {
|
|
145
|
+
"report_schema_version": REPORT_SCHEMA_VERSION,
|
|
146
|
+
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
147
|
+
"data_path": str(config.get("DATA_PATH", "")),
|
|
148
|
+
"row_count": int(len(frame)),
|
|
149
|
+
"molecule_columns": molecule_columns,
|
|
150
|
+
"target_columns": targets,
|
|
151
|
+
"split_counts": split_counts,
|
|
152
|
+
"target_summary": _target_summary(frame, targets),
|
|
153
|
+
"issue_counts": issue_counts,
|
|
154
|
+
"issue_count": len(issues),
|
|
155
|
+
}
|
|
156
|
+
return report, pd.DataFrame(
|
|
157
|
+
issues,
|
|
158
|
+
columns=["issue", "row_index", "column", "value", "detail"],
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def write_data_quality_report(frame, config, graph_errors=None):
|
|
163
|
+
"""Atomically write JSON summary and issue CSV; return both paths."""
|
|
164
|
+
model_path = Path(config["MODEL_PATH"])
|
|
165
|
+
model_path.mkdir(parents=True, exist_ok=True)
|
|
166
|
+
report, issues = build_data_quality_report(frame, config, graph_errors)
|
|
167
|
+
json_path = model_path / "data_quality_report.json"
|
|
168
|
+
csv_path = model_path / "data_quality_issues.csv"
|
|
169
|
+
token = uuid.uuid4().hex
|
|
170
|
+
json_staging = json_path.with_name(f".{json_path.name}.{token}.tmp")
|
|
171
|
+
csv_staging = csv_path.with_name(f".{csv_path.name}.{token}.tmp")
|
|
172
|
+
try:
|
|
173
|
+
json_staging.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
|
|
174
|
+
issues.to_csv(csv_staging, index=False)
|
|
175
|
+
os.replace(json_staging, json_path)
|
|
176
|
+
os.replace(csv_staging, csv_path)
|
|
177
|
+
finally:
|
|
178
|
+
for staging in (json_staging, csv_staging):
|
|
179
|
+
if staging.exists():
|
|
180
|
+
staging.unlink()
|
|
181
|
+
return json_path, csv_path
|