nsppk 1.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graph_io.py +538 -0
- hash.py +109 -0
- importance_nsppk.py +229 -0
- nsppk-1.3.1.dist-info/METADATA +142 -0
- nsppk-1.3.1.dist-info/RECORD +10 -0
- nsppk-1.3.1.dist-info/WHEEL +5 -0
- nsppk-1.3.1.dist-info/licenses/LICENSE +21 -0
- nsppk-1.3.1.dist-info/top_level.txt +5 -0
- nsppk.py +1918 -0
- utils.py +272 -0
graph_io.py
ADDED
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import io
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
import urllib.request
|
|
6
|
+
from contextlib import contextmanager
|
|
7
|
+
from numbers import Integral, Real
|
|
8
|
+
from urllib.parse import urlparse
|
|
9
|
+
|
|
10
|
+
import networkx as nx
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def ensure_graph_labels(graph, default_label=0):
|
|
15
|
+
"""
|
|
16
|
+
Ensure every node and edge has a discrete label.
|
|
17
|
+
|
|
18
|
+
NSPPK fundamentally hashes node and edge labels. For plain NetworkX graphs
|
|
19
|
+
with unlabeled nodes or edges, fall back to a shared default label so the
|
|
20
|
+
estimator remains usable without extra preprocessing.
|
|
21
|
+
"""
|
|
22
|
+
for node_idx in graph.nodes():
|
|
23
|
+
graph.nodes[node_idx].setdefault("label", default_label)
|
|
24
|
+
for u, v in graph.edges():
|
|
25
|
+
graph.edges[u, v].setdefault("label", default_label)
|
|
26
|
+
return graph
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _is_url(uri):
|
|
30
|
+
scheme = urlparse(str(uri)).scheme
|
|
31
|
+
return scheme in {"http", "https"}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@contextmanager
|
|
35
|
+
def _open_binary_uri(uri):
|
|
36
|
+
if _is_url(uri):
|
|
37
|
+
stream = urllib.request.urlopen(str(uri))
|
|
38
|
+
try:
|
|
39
|
+
yield stream
|
|
40
|
+
finally:
|
|
41
|
+
stream.close()
|
|
42
|
+
else:
|
|
43
|
+
with open(uri, "rb") as stream:
|
|
44
|
+
yield stream
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@contextmanager
|
|
48
|
+
def _open_text_uri(uri, encoding="utf-8"):
|
|
49
|
+
with _open_binary_uri(uri) as stream:
|
|
50
|
+
text_stream = io.TextIOWrapper(stream, encoding=encoding)
|
|
51
|
+
try:
|
|
52
|
+
yield text_stream
|
|
53
|
+
finally:
|
|
54
|
+
text_stream.close()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _require_rdkit(reader_name):
|
|
58
|
+
try:
|
|
59
|
+
from rdkit import Chem
|
|
60
|
+
except ImportError as exc:
|
|
61
|
+
raise ImportError(
|
|
62
|
+
f"RDKit is required for '{reader_name}' input. Install rdkit to use this reader."
|
|
63
|
+
) from exc
|
|
64
|
+
return Chem
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _require_torch_geometric():
|
|
68
|
+
try:
|
|
69
|
+
import torch
|
|
70
|
+
from torch_geometric.data import Batch, Data
|
|
71
|
+
from torch_geometric.utils import to_networkx
|
|
72
|
+
except ImportError as exc:
|
|
73
|
+
raise ImportError(
|
|
74
|
+
"torch and torch_geometric are required for 'pyg_pt' input. "
|
|
75
|
+
"Install both packages to use this reader."
|
|
76
|
+
) from exc
|
|
77
|
+
return torch, Data, Batch, to_networkx
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _atom_to_discrete_label(atom):
|
|
81
|
+
return (
|
|
82
|
+
atom.GetAtomicNum(),
|
|
83
|
+
atom.GetFormalCharge(),
|
|
84
|
+
int(atom.GetChiralTag()),
|
|
85
|
+
atom.GetTotalNumHs(),
|
|
86
|
+
int(atom.GetHybridization()),
|
|
87
|
+
int(atom.GetIsAromatic()),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _bond_to_discrete_label(bond):
|
|
92
|
+
return (
|
|
93
|
+
str(bond.GetBondType()),
|
|
94
|
+
int(bond.GetStereo()),
|
|
95
|
+
int(bond.GetIsConjugated()),
|
|
96
|
+
int(bond.IsInRing()),
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _rdkit_mol_to_nx_graph(mol):
|
|
101
|
+
graph = nx.Graph()
|
|
102
|
+
|
|
103
|
+
for atom in mol.GetAtoms():
|
|
104
|
+
graph.add_node(atom.GetIdx(), label=_atom_to_discrete_label(atom))
|
|
105
|
+
|
|
106
|
+
for bond in mol.GetBonds():
|
|
107
|
+
graph.add_edge(
|
|
108
|
+
bond.GetBeginAtomIdx(),
|
|
109
|
+
bond.GetEndAtomIdx(),
|
|
110
|
+
label=_bond_to_discrete_label(bond),
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
return graph
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _hashable_value(value):
|
|
117
|
+
if hasattr(value, "tolist"):
|
|
118
|
+
value = value.tolist()
|
|
119
|
+
if isinstance(value, np.ndarray):
|
|
120
|
+
value = value.tolist()
|
|
121
|
+
if isinstance(value, list):
|
|
122
|
+
return tuple(_hashable_value(item) for item in value)
|
|
123
|
+
if isinstance(value, tuple):
|
|
124
|
+
return tuple(_hashable_value(item) for item in value)
|
|
125
|
+
if isinstance(value, dict):
|
|
126
|
+
return tuple(sorted((key, _hashable_value(val)) for key, val in value.items()))
|
|
127
|
+
return value
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _parse_smiles_line(line):
|
|
131
|
+
stripped = line.strip()
|
|
132
|
+
if not stripped or stripped.startswith("#"):
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
if "," in stripped:
|
|
136
|
+
row = next(csv.reader([stripped]))
|
|
137
|
+
return [value.strip() for value in row]
|
|
138
|
+
|
|
139
|
+
return stripped.split()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _iter_smiles_graphs(uri):
|
|
143
|
+
return _iter_smiles_graphs_with_stats(uri, stats=None)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _iter_smiles_graphs_with_stats(uri, stats=None):
|
|
147
|
+
Chem = _require_rdkit("smiles")
|
|
148
|
+
|
|
149
|
+
with _open_text_uri(uri) as stream:
|
|
150
|
+
csv_header = None
|
|
151
|
+
for line_number, line in enumerate(stream, start=1):
|
|
152
|
+
parts = _parse_smiles_line(line)
|
|
153
|
+
if not parts:
|
|
154
|
+
continue
|
|
155
|
+
if parts[0].lower() == "smiles":
|
|
156
|
+
csv_header = parts
|
|
157
|
+
continue
|
|
158
|
+
|
|
159
|
+
smiles = parts[0]
|
|
160
|
+
mol = Chem.MolFromSmiles(smiles)
|
|
161
|
+
if mol is None:
|
|
162
|
+
if stats is not None:
|
|
163
|
+
stats["errors"] += 1
|
|
164
|
+
continue
|
|
165
|
+
|
|
166
|
+
graph = _rdkit_mol_to_nx_graph(mol)
|
|
167
|
+
graph.graph["smiles"] = smiles
|
|
168
|
+
graph.graph["source_uri"] = str(uri)
|
|
169
|
+
graph.graph["source_line"] = line_number
|
|
170
|
+
if csv_header is not None:
|
|
171
|
+
for key, value in zip(csv_header[1:], parts[1:]):
|
|
172
|
+
if key:
|
|
173
|
+
graph.graph[key] = value
|
|
174
|
+
if len(parts) > 1 and parts[1]:
|
|
175
|
+
graph.graph["name"] = parts[1]
|
|
176
|
+
elif len(parts) > 1 and parts[1]:
|
|
177
|
+
graph.graph["name"] = parts[1]
|
|
178
|
+
yield graph
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _iter_sdf_graphs(uri):
|
|
182
|
+
return _iter_sdf_graphs_with_stats(uri, stats=None)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _iter_sdf_graphs_with_stats(uri, stats=None):
|
|
186
|
+
Chem = _require_rdkit("sdf")
|
|
187
|
+
|
|
188
|
+
with _open_binary_uri(uri) as stream:
|
|
189
|
+
supplier = Chem.ForwardSDMolSupplier(stream)
|
|
190
|
+
for mol in supplier:
|
|
191
|
+
if mol is None:
|
|
192
|
+
if stats is not None:
|
|
193
|
+
stats["errors"] += 1
|
|
194
|
+
continue
|
|
195
|
+
|
|
196
|
+
graph = _rdkit_mol_to_nx_graph(mol)
|
|
197
|
+
if mol.HasProp("_Name"):
|
|
198
|
+
graph.graph["name"] = mol.GetProp("_Name")
|
|
199
|
+
graph.graph["source_uri"] = str(uri)
|
|
200
|
+
yield graph
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _iter_pyg_pt_graphs(uri):
|
|
204
|
+
return _iter_pyg_pt_graphs_with_stats(uri, stats=None)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _iter_pyg_pt_graphs_with_stats(uri, stats=None):
|
|
208
|
+
torch, Data, Batch, to_networkx = _require_torch_geometric()
|
|
209
|
+
|
|
210
|
+
with _open_binary_uri(uri) as stream:
|
|
211
|
+
payload = io.BytesIO(stream.read())
|
|
212
|
+
|
|
213
|
+
obj = torch.load(payload, map_location="cpu")
|
|
214
|
+
if isinstance(obj, Batch):
|
|
215
|
+
data_objects = obj.to_data_list()
|
|
216
|
+
elif isinstance(obj, Data):
|
|
217
|
+
data_objects = [obj]
|
|
218
|
+
else:
|
|
219
|
+
try:
|
|
220
|
+
data_objects = list(obj)
|
|
221
|
+
except TypeError as exc:
|
|
222
|
+
raise TypeError(
|
|
223
|
+
"The 'pyg_pt' reader expects a torch_geometric Data, Batch, or iterable of Data objects."
|
|
224
|
+
) from exc
|
|
225
|
+
|
|
226
|
+
for data in data_objects:
|
|
227
|
+
if not isinstance(data, Data):
|
|
228
|
+
raise TypeError("The 'pyg_pt' reader expects each item to be a torch_geometric.data.Data instance.")
|
|
229
|
+
|
|
230
|
+
keys = list(data.keys()) if callable(getattr(data, "keys", None)) else list(data.keys)
|
|
231
|
+
node_attrs = [key for key in keys if key not in {"batch", "ptr"} and data.is_node_attr(key)]
|
|
232
|
+
edge_attrs = [key for key in keys if data.is_edge_attr(key)]
|
|
233
|
+
graph = to_networkx(
|
|
234
|
+
data,
|
|
235
|
+
node_attrs=node_attrs,
|
|
236
|
+
edge_attrs=edge_attrs,
|
|
237
|
+
to_undirected=True,
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
for node_idx in graph.nodes():
|
|
241
|
+
if "label" not in graph.nodes[node_idx]:
|
|
242
|
+
if "x" in graph.nodes[node_idx]:
|
|
243
|
+
graph.nodes[node_idx]["label"] = _hashable_value(graph.nodes[node_idx]["x"])
|
|
244
|
+
else:
|
|
245
|
+
graph.nodes[node_idx]["label"] = 0
|
|
246
|
+
else:
|
|
247
|
+
graph.nodes[node_idx]["label"] = _hashable_value(graph.nodes[node_idx]["label"])
|
|
248
|
+
|
|
249
|
+
for u, v in graph.edges():
|
|
250
|
+
if "label" not in graph.edges[u, v]:
|
|
251
|
+
if "edge_attr" in graph.edges[u, v]:
|
|
252
|
+
graph.edges[u, v]["label"] = _hashable_value(graph.edges[u, v]["edge_attr"])
|
|
253
|
+
else:
|
|
254
|
+
graph.edges[u, v]["label"] = 0
|
|
255
|
+
else:
|
|
256
|
+
graph.edges[u, v]["label"] = _hashable_value(graph.edges[u, v]["label"])
|
|
257
|
+
|
|
258
|
+
graph.graph["source_uri"] = str(uri)
|
|
259
|
+
yield graph
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
_BUILTIN_READERS = {
|
|
263
|
+
"smiles": _iter_smiles_graphs_with_stats,
|
|
264
|
+
"sdf": _iter_sdf_graphs_with_stats,
|
|
265
|
+
"pyg_pt": _iter_pyg_pt_graphs_with_stats,
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _validate_graph(graph):
|
|
270
|
+
if not isinstance(graph, nx.Graph):
|
|
271
|
+
raise TypeError(
|
|
272
|
+
"Readers must yield networkx.Graph instances. "
|
|
273
|
+
f"Received {type(graph).__name__}."
|
|
274
|
+
)
|
|
275
|
+
return graph
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _validate_limit(limit):
|
|
279
|
+
if limit is None:
|
|
280
|
+
return
|
|
281
|
+
|
|
282
|
+
if isinstance(limit, bool):
|
|
283
|
+
raise ValueError("limit must be None, a non-negative integer, or a float strictly between 0 and 1.")
|
|
284
|
+
|
|
285
|
+
if isinstance(limit, Integral):
|
|
286
|
+
if limit < 0:
|
|
287
|
+
raise ValueError("Integer limit must be non-negative.")
|
|
288
|
+
return
|
|
289
|
+
|
|
290
|
+
if isinstance(limit, Real):
|
|
291
|
+
if not 0 < float(limit) < 1:
|
|
292
|
+
raise ValueError("Float limit must be strictly between 0 and 1.")
|
|
293
|
+
return
|
|
294
|
+
|
|
295
|
+
raise ValueError("limit must be None, a non-negative integer, or a float strictly between 0 and 1.")
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _validate_start_after_instance(start_after_instance):
|
|
299
|
+
if isinstance(start_after_instance, bool):
|
|
300
|
+
raise ValueError("start_after_instance must be a non-negative integer.")
|
|
301
|
+
if not isinstance(start_after_instance, Integral):
|
|
302
|
+
raise ValueError("start_after_instance must be a non-negative integer.")
|
|
303
|
+
if start_after_instance < 0:
|
|
304
|
+
raise ValueError("start_after_instance must be a non-negative integer.")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _make_rng(random_state):
|
|
308
|
+
if isinstance(random_state, np.random.Generator):
|
|
309
|
+
return random_state
|
|
310
|
+
return np.random.default_rng(random_state)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _apply_start_offset(graph_iterable, start_after_instance=0):
|
|
314
|
+
_validate_start_after_instance(start_after_instance)
|
|
315
|
+
remaining = int(start_after_instance)
|
|
316
|
+
for graph in graph_iterable:
|
|
317
|
+
if remaining > 0:
|
|
318
|
+
remaining -= 1
|
|
319
|
+
continue
|
|
320
|
+
yield graph
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _apply_limit(graph_iterable, limit=None, random_state=None):
|
|
324
|
+
_validate_limit(limit)
|
|
325
|
+
|
|
326
|
+
if limit is None:
|
|
327
|
+
yield from graph_iterable
|
|
328
|
+
return
|
|
329
|
+
|
|
330
|
+
if isinstance(limit, Integral):
|
|
331
|
+
remaining = int(limit)
|
|
332
|
+
for graph in graph_iterable:
|
|
333
|
+
if remaining <= 0:
|
|
334
|
+
break
|
|
335
|
+
yield graph
|
|
336
|
+
remaining -= 1
|
|
337
|
+
return
|
|
338
|
+
|
|
339
|
+
probability = float(limit)
|
|
340
|
+
rng = _make_rng(random_state)
|
|
341
|
+
for graph in graph_iterable:
|
|
342
|
+
if rng.random() < probability:
|
|
343
|
+
yield graph
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _normalize_label_extractor(label_extractor):
|
|
347
|
+
if label_extractor is None:
|
|
348
|
+
return lambda graph: graph.graph["name"]
|
|
349
|
+
if not callable(label_extractor):
|
|
350
|
+
raise TypeError("label_extractor must be callable when provided.")
|
|
351
|
+
return label_extractor
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _select_balanced_indices(labels, target_size, rng):
|
|
355
|
+
labels = np.asarray(labels)
|
|
356
|
+
unique_labels = np.unique(labels)
|
|
357
|
+
if unique_labels.size < 2:
|
|
358
|
+
raise ValueError("balance=True requires at least two classes.")
|
|
359
|
+
|
|
360
|
+
class_indices = {label: np.flatnonzero(labels == label) for label in unique_labels}
|
|
361
|
+
minority_count = min(len(indices) for indices in class_indices.values())
|
|
362
|
+
max_balanced_size = minority_count * unique_labels.size
|
|
363
|
+
|
|
364
|
+
if target_size is None:
|
|
365
|
+
target_size = max_balanced_size
|
|
366
|
+
else:
|
|
367
|
+
target_size = min(int(target_size), len(labels))
|
|
368
|
+
|
|
369
|
+
if target_size <= 0:
|
|
370
|
+
return np.empty(0, dtype=int)
|
|
371
|
+
|
|
372
|
+
per_class = min(minority_count, target_size // unique_labels.size)
|
|
373
|
+
remainder = target_size - per_class * unique_labels.size
|
|
374
|
+
|
|
375
|
+
selected_parts = []
|
|
376
|
+
leftover_parts = []
|
|
377
|
+
for label in unique_labels:
|
|
378
|
+
indices = np.array(class_indices[label], copy=True)
|
|
379
|
+
chosen = rng.choice(indices, size=per_class, replace=False) if per_class else np.empty(0, dtype=int)
|
|
380
|
+
selected_parts.append(chosen)
|
|
381
|
+
if remainder > 0:
|
|
382
|
+
remaining = np.setdiff1d(indices, chosen, assume_unique=False)
|
|
383
|
+
leftover_parts.append(remaining)
|
|
384
|
+
|
|
385
|
+
selected_idx = np.concatenate(selected_parts) if selected_parts else np.empty(0, dtype=int)
|
|
386
|
+
|
|
387
|
+
if remainder > 0:
|
|
388
|
+
leftovers = np.concatenate([part for part in leftover_parts if len(part)]) if leftover_parts else np.empty(0, dtype=int)
|
|
389
|
+
if leftovers.size:
|
|
390
|
+
extra_size = min(remainder, leftovers.size)
|
|
391
|
+
extra_idx = rng.choice(leftovers, size=extra_size, replace=False)
|
|
392
|
+
selected_idx = np.concatenate([selected_idx, extra_idx])
|
|
393
|
+
|
|
394
|
+
return np.sort(selected_idx)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _compute_balanced_class_weight(labels):
|
|
398
|
+
labels = np.asarray(labels)
|
|
399
|
+
unique_labels, counts = np.unique(labels, return_counts=True)
|
|
400
|
+
if unique_labels.size == 0:
|
|
401
|
+
return {}, {}
|
|
402
|
+
n_samples = labels.shape[0]
|
|
403
|
+
n_classes = unique_labels.size
|
|
404
|
+
label_counts = {}
|
|
405
|
+
class_weight = {}
|
|
406
|
+
for label, count in zip(unique_labels.tolist(), counts.tolist()):
|
|
407
|
+
key = _hashable_value(label)
|
|
408
|
+
label_counts[key] = int(count)
|
|
409
|
+
class_weight[key] = n_samples / (n_classes * count)
|
|
410
|
+
return label_counts, class_weight
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _materialize_loaded_graphs(
|
|
414
|
+
uri,
|
|
415
|
+
source_type,
|
|
416
|
+
reader=None,
|
|
417
|
+
limit=None,
|
|
418
|
+
random_state=None,
|
|
419
|
+
verbose=False,
|
|
420
|
+
balance=False,
|
|
421
|
+
label_extractor=None,
|
|
422
|
+
start_after_instance=0,
|
|
423
|
+
return_metadata=False,
|
|
424
|
+
):
|
|
425
|
+
fractional_limit = isinstance(limit, Real) and not isinstance(limit, Integral)
|
|
426
|
+
load_limit = limit if fractional_limit else (None if balance else limit)
|
|
427
|
+
target_size = None if fractional_limit else limit
|
|
428
|
+
|
|
429
|
+
graph_iterable = _iter_loaded_graphs(
|
|
430
|
+
uri,
|
|
431
|
+
source_type,
|
|
432
|
+
reader=reader,
|
|
433
|
+
limit=load_limit,
|
|
434
|
+
random_state=random_state,
|
|
435
|
+
verbose=verbose,
|
|
436
|
+
mode="load",
|
|
437
|
+
start_after_instance=start_after_instance,
|
|
438
|
+
)
|
|
439
|
+
graphs = list(graph_iterable)
|
|
440
|
+
|
|
441
|
+
if not balance:
|
|
442
|
+
if return_metadata:
|
|
443
|
+
return graphs, None
|
|
444
|
+
return graphs
|
|
445
|
+
|
|
446
|
+
rng = _make_rng(random_state)
|
|
447
|
+
extractor = _normalize_label_extractor(label_extractor)
|
|
448
|
+
labels = np.asarray([extractor(graph) for graph in graphs])
|
|
449
|
+
selected_idx = _select_balanced_indices(labels, target_size=target_size, rng=rng)
|
|
450
|
+
balanced_graphs = [graphs[idx] for idx in selected_idx]
|
|
451
|
+
label_counts, class_weight = _compute_balanced_class_weight(labels)
|
|
452
|
+
metadata = {
|
|
453
|
+
"candidate_size": len(graphs),
|
|
454
|
+
"label_counts": label_counts,
|
|
455
|
+
"class_weight": class_weight,
|
|
456
|
+
}
|
|
457
|
+
if return_metadata:
|
|
458
|
+
return balanced_graphs, metadata
|
|
459
|
+
return balanced_graphs
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _iter_loaded_graphs(uri, source_type, reader=None, limit=None, random_state=None, verbose=False, mode="load", log_every=100, log_stream=None, start_after_instance=0):
|
|
463
|
+
stats = {
|
|
464
|
+
"seen": 0,
|
|
465
|
+
"loaded": 0,
|
|
466
|
+
"errors": 0,
|
|
467
|
+
"verbose": verbose,
|
|
468
|
+
"mode": mode,
|
|
469
|
+
"stream": sys.stderr if log_stream is None else log_stream,
|
|
470
|
+
}
|
|
471
|
+
start_time = time.perf_counter()
|
|
472
|
+
stats["timer"] = lambda: time.perf_counter() - start_time
|
|
473
|
+
|
|
474
|
+
if reader is None:
|
|
475
|
+
normalized_type = str(source_type).strip().lower()
|
|
476
|
+
if normalized_type not in _BUILTIN_READERS:
|
|
477
|
+
available = ", ".join(sorted(_BUILTIN_READERS))
|
|
478
|
+
raise ValueError(f"Unsupported source type '{source_type}'. Available built-in types: {available}.")
|
|
479
|
+
graph_iterable = _BUILTIN_READERS[normalized_type](uri, stats=stats)
|
|
480
|
+
else:
|
|
481
|
+
graph_iterable = reader(uri)
|
|
482
|
+
|
|
483
|
+
validated_graphs = (_validate_graph(graph) for graph in graph_iterable)
|
|
484
|
+
offset_graphs = _apply_start_offset(validated_graphs, start_after_instance=start_after_instance)
|
|
485
|
+
limited_graphs = _apply_limit(offset_graphs, limit=limit, random_state=random_state)
|
|
486
|
+
|
|
487
|
+
try:
|
|
488
|
+
for graph in limited_graphs:
|
|
489
|
+
stats["seen"] += 1
|
|
490
|
+
stats["loaded"] += 1
|
|
491
|
+
if verbose and (stats["loaded"] == 1 or stats["loaded"] % log_every == 0):
|
|
492
|
+
_emit_stream_stats(stats, final=False)
|
|
493
|
+
yield graph
|
|
494
|
+
finally:
|
|
495
|
+
if verbose:
|
|
496
|
+
_emit_stream_stats(stats, final=True)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _batched_graphs(graph_iterable, batch_size):
|
|
500
|
+
if not isinstance(batch_size, Integral) or isinstance(batch_size, bool) or batch_size <= 0:
|
|
501
|
+
raise ValueError("batch_size must be a positive integer.")
|
|
502
|
+
|
|
503
|
+
batch = []
|
|
504
|
+
for graph in graph_iterable:
|
|
505
|
+
batch.append(graph)
|
|
506
|
+
if len(batch) == batch_size:
|
|
507
|
+
yield batch
|
|
508
|
+
batch = []
|
|
509
|
+
|
|
510
|
+
if batch:
|
|
511
|
+
yield batch
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _format_stream_stats(stats):
|
|
515
|
+
elapsed = max(stats["timer"](), 1e-12)
|
|
516
|
+
loaded = stats["loaded"]
|
|
517
|
+
seen = stats["seen"]
|
|
518
|
+
errors = stats["errors"]
|
|
519
|
+
sec_per_graph = elapsed / loaded if loaded else float("nan")
|
|
520
|
+
graphs_per_sec = loaded / elapsed if loaded else 0.0
|
|
521
|
+
return (
|
|
522
|
+
f"seen={seen:>7d} "
|
|
523
|
+
f"loaded={loaded:>7d} "
|
|
524
|
+
f"errors={errors:>5d} "
|
|
525
|
+
f"elapsed={elapsed:>8.2f}s "
|
|
526
|
+
f"graphs/sec={graphs_per_sec:>8.2f} "
|
|
527
|
+
f"sec/graph={sec_per_graph:>10.6f}"
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _emit_stream_stats(stats, final=False):
|
|
532
|
+
if not stats["verbose"]:
|
|
533
|
+
return
|
|
534
|
+
|
|
535
|
+
prefix = "[load_from]" if stats["mode"] == "load" else "[stream_from]"
|
|
536
|
+
stage = "final" if final else "progress"
|
|
537
|
+
print(f"{prefix} {stage}: {_format_stream_stats(stats)}", file=stats["stream"])
|
|
538
|
+
stats["stream"].flush()
|
hash.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def _hash(value):
|
|
6
|
+
digest = hashlib.blake2s(str(value).encode(), digest_size=4).digest()
|
|
7
|
+
return int.from_bytes(digest, "big")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def masked_hash_value(value, bitmask=(2**20 - 1)):
|
|
11
|
+
return _hash(value) & bitmask
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def hash_value(value, nbits=10):
|
|
15
|
+
max_index = 2 ** nbits
|
|
16
|
+
h = masked_hash_value(value, max_index - 3)
|
|
17
|
+
h += 2
|
|
18
|
+
return h
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def hash_sequence(iterable):
|
|
22
|
+
return _hash(tuple(iterable))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def hash_set(iterable):
|
|
26
|
+
sorted_iterable = sorted(iterable)
|
|
27
|
+
tuple_representation = tuple(sorted_iterable)
|
|
28
|
+
return _hash(tuple_representation)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _hash_unordered_triplet(first, second, third):
|
|
32
|
+
"""Hash three unordered values without allocating a sorted list."""
|
|
33
|
+
if first > second:
|
|
34
|
+
first, second = second, first
|
|
35
|
+
if second > third:
|
|
36
|
+
second, third = third, second
|
|
37
|
+
if first > second:
|
|
38
|
+
first, second = second, first
|
|
39
|
+
return _hash((first, second, third))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def node_hash(node_idx, graph):
|
|
43
|
+
uh = _hash(graph.nodes[node_idx]['label'])
|
|
44
|
+
nh = hash_set(
|
|
45
|
+
_hash((_hash(graph.nodes[v]['label']), _hash(graph.edges[node_idx, v]['label'])))
|
|
46
|
+
for v in graph.neighbors(node_idx)
|
|
47
|
+
)
|
|
48
|
+
ext_node_h = _hash((uh, nh))
|
|
49
|
+
return ext_node_h
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def invert_dict(mydict):
|
|
53
|
+
reversed_dict = defaultdict(list)
|
|
54
|
+
for key, value in mydict.items():
|
|
55
|
+
reversed_dict[value].append(key)
|
|
56
|
+
return reversed_dict
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def rooted_graph_hashes(
|
|
60
|
+
node_idx,
|
|
61
|
+
graph,
|
|
62
|
+
radius=1,
|
|
63
|
+
node_idxs_to_dist_dict=None,
|
|
64
|
+
node_label_hashes=None,
|
|
65
|
+
node_hashes=None,
|
|
66
|
+
):
|
|
67
|
+
if node_idxs_to_dist_dict is None:
|
|
68
|
+
raise ValueError("node_idxs_to_dist_dict must be provided for rooted_graph_hashes().")
|
|
69
|
+
if node_label_hashes is None:
|
|
70
|
+
node_label_hashes = {
|
|
71
|
+
curr_node_idx: graph.nodes[curr_node_idx]['node_label_hash']
|
|
72
|
+
for curr_node_idx in graph.nodes()
|
|
73
|
+
}
|
|
74
|
+
if node_hashes is None:
|
|
75
|
+
node_hashes = {
|
|
76
|
+
curr_node_idx: graph.nodes[curr_node_idx]['node_hash']
|
|
77
|
+
for curr_node_idx in graph.nodes()
|
|
78
|
+
}
|
|
79
|
+
dist_to_node_idxs_dict = invert_dict({
|
|
80
|
+
curr_node_idx: dist
|
|
81
|
+
for curr_node_idx, dist in node_idxs_to_dist_dict.items()
|
|
82
|
+
if dist <= radius
|
|
83
|
+
})
|
|
84
|
+
iso_distance_codes = tuple(
|
|
85
|
+
hash_set(
|
|
86
|
+
node_label_hashes[curr_node_idx]
|
|
87
|
+
if dist == 0 else node_hashes[curr_node_idx]
|
|
88
|
+
for curr_node_idx in node_idxs
|
|
89
|
+
)
|
|
90
|
+
for dist, node_idxs in sorted(dist_to_node_idxs_dict.items())
|
|
91
|
+
)
|
|
92
|
+
h_list = [
|
|
93
|
+
hash_sequence(iso_distance_codes[:i])
|
|
94
|
+
for i in range(1, len(iso_distance_codes) + 1)
|
|
95
|
+
]
|
|
96
|
+
return h_list
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def edge_triplet_hash(u, v, graph):
|
|
100
|
+
return _hash_unordered_triplet(
|
|
101
|
+
_hash(graph.nodes[u]['label']),
|
|
102
|
+
_hash(graph.edges[u, v]['label']),
|
|
103
|
+
_hash(graph.nodes[v]['label']),
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def precompute_edge_triplet_hashes(graph):
|
|
108
|
+
for u, v in graph.edges():
|
|
109
|
+
graph.edges[u, v]['triplet_hash'] = edge_triplet_hash(u, v, graph)
|