emscan 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
emscan/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ """package description."""
2
+ from importlib.metadata import PackageNotFoundError, version
3
+
4
+ try:
5
+ __version__ = version("emscan")
6
+ except PackageNotFoundError:
7
+ __version__ = "uninstalled"
8
+
9
+ __author__ = "Lorenzo Gaifas"
10
+ __email__ = "brisvag@gmail.com"
emscan/_functions.py ADDED
@@ -0,0 +1,661 @@
1
+ import gc
2
+ import inspect
3
+ import os
4
+ import re
5
+ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
6
+ from functools import lru_cache, reduce
7
+ from itertools import chain
8
+ from math import ceil, floor
9
+ from operator import mul
10
+ from pathlib import Path
11
+
12
+ import mrcfile
13
+ import numpy as np
14
+ import pandas as pd
15
+ import requests
16
+ import sh
17
+ import torch
18
+ import xmltodict
19
+ from morphosamplers.sampler import sample_subvolumes
20
+ from rich import print
21
+ from scipy.signal.windows import gaussian
22
+ from scipy.spatial.transform import Rotation
23
+ from torch.fft import fftn, fftshift, ifftn, ifftshift
24
+ from torch.multiprocessing import set_start_method
25
+ from torchvision.transforms import InterpolationMode
26
+ from torchvision.transforms.functional import rotate
27
+
28
+ BIN_RESOLUTION = 4
29
+ GOLDEN_RATIO = (1 + np.sqrt(5)) / 2
30
+
31
+
32
+ def _print_tensors(depth=2):
33
+ """Print info about tensor variables locally and nonlocally."""
34
+ print("=" * 80)
35
+ for obj in gc.get_objects():
36
+ frame = inspect.currentframe()
37
+ frames = []
38
+ for _i in range(depth):
39
+ frame = frame.f_back
40
+ frames.append(frame)
41
+ try:
42
+ if torch.is_tensor(obj) or (
43
+ hasattr(obj, "data") and torch.is_tensor(obj.data)
44
+ ):
45
+ for gobj, gobjid in globals().items():
46
+ if gobjid is obj and gobj not in ("obj", "gobjid"):
47
+ print("GLOBAL:\t", gobj, obj.size(), obj.data_ptr())
48
+ for frame in frames:
49
+ fname = frame.f_code.co_name
50
+ locs = frame.f_locals
51
+ for gobj, gobjid in locs.items():
52
+ if gobjid is obj and gobj not in ("obj", "gobjid"):
53
+ print(f"{fname}():\t", gobj, obj.size(), obj.data_ptr())
54
+ except: # noqa
55
+ pass
56
+ print("=" * 80)
57
+
58
+
59
+ def normalize(img, dim=None, inplace=True):
60
+ """Normalize images to std=1 and mean=0."""
61
+ if not torch.is_complex(img):
62
+ img = img.to(torch.float32, copy=not inplace)
63
+ elif not inplace:
64
+ img = img.copy()
65
+ img -= img.mean(dim=dim, keepdim=True)
66
+ img /= img.std(dim=dim, keepdim=True)
67
+ if img.isnan().any():
68
+ img = torch.nan_to_num(img, out=img)
69
+ return img
70
+
71
+
72
+ def ft_and_shift(img, dim=None):
73
+ # first move the image to be center of the origin (ifftshift), which improves
74
+ # interpolation artifacts later (ftshift is equivalent to multiplying
75
+ # with a checkerboard, which messes up interpolation real bad)
76
+ return fftshift(fftn(ifftshift(img, dim=dim), dim=dim), dim=dim)
77
+
78
+
79
+ def ift_and_shift(img, dim=None):
80
+ # undo the above (note the order is the same cause it's actually inverted)
81
+ return fftshift(ifftn(ifftshift(img, dim=dim), dim=dim), dim=dim)
82
+
83
+
84
+ def rotate_by(arr, ang, center=None):
85
+ if center is None:
86
+ # torchvision expects the center as (x, y), i.e. (col, row)
87
+ center = list(np.array(arr.shape)[::-1] // 2)
88
+
89
+ return rotate(
90
+ arr[None],
91
+ float(ang),
92
+ center=center,
93
+ interpolation=InterpolationMode.BILINEAR,
94
+ )[0]
95
+
96
+
97
+ def rotations(img, degree_range, center=None):
98
+ """Generate a number of rotations from an image and a list of angles.
99
+
100
+ degree range: iterable of degrees (counterclockwise).
101
+ """
102
+ gauss = gaussian_window(img.shape, device=img.device)
103
+
104
+ for angle in degree_range:
105
+ if torch.is_complex(img):
106
+ real = rotate_by(img.real, angle, center)
107
+ imag = rotate_by(img.imag, angle, center)
108
+ rot = real + (1j * imag)
109
+ del real, imag
110
+ else:
111
+ rot = rotate_by(img, angle, center)
112
+ yield angle, rot * gauss
113
+ del rot
114
+
115
+
116
+ def coerce_ndim(img, ndim):
117
+ """Coerce image dimensionality if smaller than ndim."""
118
+ if img.ndim > ndim:
119
+ raise ValueError(f"image has too high dimensionality ({img.ndim})")
120
+ while img.ndim < ndim:
121
+ img = img[None]
122
+ return img
123
+
124
+
125
+ @lru_cache
126
+ def _rotations_hemisphere_fibonacci():
127
+ # http://extremelearning.com.au/how-to-evenly-distribute-points-on-a-
128
+ # sphere-more-effectively-than-the-canonical-fibonacci-lattice/
129
+ # Note: not using epsilon we ensure we get the Z axis, which is a common
130
+ # aligment for maps. Packing is slightly worse, but still better than healpix
131
+
132
+ N = 300 # THIS IS FIXED ONCE GENERATED THE DB!
133
+ i = np.arange(N)[: N // 2] # only take half cause projection symmetry
134
+ phi = 2 * np.pi * i / GOLDEN_RATIO
135
+ theta = np.arccos(1 - 2 * i / (N - 1))
136
+ angles = np.stack([phi, theta], axis=1)
137
+ return Rotation.from_euler("ZX", angles)
138
+
139
+
140
+ def rotated_projection_fts(ft):
141
+ """Generate rotated projections fts of a map."""
142
+ # TODO: sinc function to avoid edge artifacts
143
+
144
+ rot = _rotations_hemisphere_fibonacci()
145
+ pos = np.array(ft.shape) // 2 # using // is needed to center ctf for odd images
146
+ grid_size = (ft.shape[0], ft.shape[0], 1)
147
+ slices = sample_subvolumes(
148
+ np.array(ft), positions=pos, orientations=rot, grid_shape=grid_size, fill_value=0,
149
+ ).squeeze()
150
+
151
+ return torch.from_numpy(slices)
152
+
153
+
154
+ def gaussian_window(shape, sigmas=None, device=None):
155
+ """Generate a gaussian_window of the given shape and sigmas, ensuring it goes to zero."""
156
+ if sigmas is None:
157
+ sigmas = np.array(shape)
158
+ sigmas = np.broadcast_to(sigmas, len(shape))
159
+ windows = [gaussian(n, s) for n, s in zip(shape, sigmas, strict=True)]
160
+ mins = np.array([min(w) for w in windows])
161
+ # true minimum of the separable product, so the window reaches exactly 0
162
+ zero = np.prod(mins)
163
+ tensors = [torch.tensor(w, device=device) for w in np.ix_(*windows)]
164
+ window = reduce(mul, tensors)
165
+ window -= zero
166
+ window = torch.clip(window, 0, 1)
167
+ window /= window.max()
168
+ return window
169
+
170
+
171
+ def crop_to(img, target_shape, dim=None):
172
+ if dim is None:
173
+ dim = tuple(range(img.ndim))
174
+ # ensure we round the cropping correctly to keeo the center in the right place
175
+ edge_crop = (np.array(img.shape) - np.array(target_shape)) / 2
176
+ crop_left = []
177
+ crop_right = []
178
+ for d in range(img.ndim):
179
+ if d not in dim:
180
+ crop_left.append(None)
181
+ crop_right.append(None)
182
+ continue
183
+ # rounding must depend on the parity of the dimension being cropped
184
+ if img.shape[d] % 2:
185
+ left, right = floor(edge_crop[d]), ceil(edge_crop[d])
186
+ else:
187
+ left, right = ceil(edge_crop[d]), floor(edge_crop[d])
188
+ crop_left.append(left or None)
189
+ crop_right.append(-right or None)
190
+ crop_slice = tuple(
191
+ slice(*crops) for crops in zip(crop_left, crop_right, strict=True)
192
+ )
193
+ return img[crop_slice]
194
+
195
+
196
+ def pad_to(img, target_shape, dim=None, value=0):
197
+ if dim is None:
198
+ dim = tuple(range(img.ndim))
199
+ # ensure we round the padding correctly to keeo the center in the right place
200
+ edge_pad = (np.array(target_shape) - np.array(img.shape)) / 2
201
+ pad_left = []
202
+ pad_right = []
203
+ for d in range(img.ndim):
204
+ if d not in dim:
205
+ pad_left.append(0)
206
+ pad_right.append(0)
207
+ continue
208
+ # rounding must depend on the parity of the dimension being padded
209
+ if img.shape[d] % 2:
210
+ left, right = ceil(edge_pad[d]), floor(edge_pad[d])
211
+ else:
212
+ left, right = floor(edge_pad[d]), ceil(edge_pad[d])
213
+ pad_left.append(left)
214
+ pad_right.append(right)
215
+ padding = tuple(chain.from_iterable(zip(pad_right, pad_left, strict=True)))[::-1]
216
+ padded = torch.nn.functional.pad(img, padding, value=value)
217
+ return padded
218
+
219
+
220
+ def resize(ft, target_shape, dim=None):
221
+ """Bin or unbin image(s) ft by fourier cropping or padding."""
222
+ if dim is None:
223
+ dim = tuple(range(ft.ndim))
224
+
225
+ target_shape = np.array(target_shape)
226
+ input_shape = np.array(ft.shape)
227
+ target_shape_dims = target_shape[list(dim)]
228
+ input_shape_dims = input_shape[list(dim)]
229
+ if np.all(target_shape_dims == input_shape_dims):
230
+ return ft
231
+ elif np.all(target_shape_dims < input_shape_dims):
232
+ return crop_to(ft, target_shape, dim=dim)
233
+ elif np.all(target_shape_dims > input_shape_dims):
234
+ return pad_to(ft, target_shape, dim=dim)
235
+ else:
236
+ raise NotImplementedError("cannot pad and crop at the same time")
237
+
238
+
239
+ def crop_or_pad_from_px_sizes(ft, px_size_in, px_size_out, dim=None):
240
+ """Rescale an image's ft given before/after pixel sizes."""
241
+ ratio = np.array(px_size_in) / np.array(px_size_out)
242
+ target_shape = np.round(np.array(ft.shape) * ratio / 2) * 2
243
+ dim = tuple(range(ft.ndim)) if dim is None else dim
244
+ return resize(ft, target_shape, dim=dim)
245
+
246
+
247
+ def _ncc(S, L, angle_step):
248
+ """Fast cross correlation of all elements of projections and the image.
249
+
250
+ Performs also rotations and mirroring to cover all orientations.
251
+
252
+ Based on eq (9) from:
253
+ https://w.imagemagick.org/docs/AcceleratedTemplateMatchingUsingLocalStatisticsAndFourierTransforms.pdf
254
+ Optimized to reduce operations (FFTs are precomputed and images are pre-normalized).
255
+ Some stuff is also easier because L[0] and S are forced to have the same shape if S is big.
256
+ """
257
+ # see link above for what is L, LL, S, U, and the equation
258
+ corr_results = {"values": {}, "indices": {}, "angles": {}}
259
+
260
+ if S.shape[-1] < L.shape[-1]:
261
+ # crop L
262
+ L = crop_to(L, S.shape, dim=(1, 2))
263
+ L = normalize(L, dim=(1, 2), inplace=False)
264
+ else:
265
+ # cropping, so assume "same size" of S and L
266
+ S = crop_to(S, L.shape, dim=(1, 2))
267
+ S = normalize(S, dim=(1, 2), inplace=False)
268
+
269
+ L_ = ft_and_shift(L, dim=(1, 2))
270
+ L_conj = L_.conj()
271
+ LL_ = ft_and_shift(L**2, dim=(1, 2))
272
+ N = S[0].nelement()
273
+ # TODO this is just zeros with N in the center
274
+ U_ = ft_and_shift(torch.ones_like(S[0], device=S.device))
275
+
276
+ def _cc(ft1, ft2, ft2_is_conj=False):
277
+ conj = ft2 if ft2_is_conj else ft2.conj()
278
+ return ift_and_shift(ft1 * conj, dim=(1, 2)).real
279
+
280
+ denom_inner = (N * _cc(U_, LL_)) - (_cc(U_, L_) ** 2)
281
+ denom = torch.sqrt(torch.clamp_min(denom_inner, 0))
282
+ n_proj = len(L)
283
+
284
+ del L, L_, LL_, U_
285
+
286
+ def ncc(S_):
287
+ nonlocal denom, L_conj
288
+ # NOTE: this assume S is ft of normalized image (std=1, mean=0)
289
+ # NOTE: L_conj is precomputed to save computations!
290
+ return _cc(S_, L_conj, ft2_is_conj=True) / denom
291
+
292
+ for cls_idx, cls in enumerate(S):
293
+ best_ccs = torch.zeros(n_proj, device=S.device)
294
+ best_ang = torch.zeros(n_proj, device=S.device)
295
+
296
+ for angle, cls_rot in rotations(cls, np.arange(0, 360, angle_step)):
297
+ # normalize to avoid needing std and mean in ncc calculation
298
+ cls_rot = normalize(cls_rot)
299
+ # also correlate transposed image, because we only have half a sphere of projections
300
+ for cls_flip, flip_sign in ((cls_rot, 1), (cls_rot.T, -1)):
301
+ S_ = ft_and_shift(cls_flip)
302
+ ccs = ncc(S_).amax(dim=(1, 2))
303
+ best_ccs, replaced = torch.max(torch.stack([best_ccs, ccs]), dim=0)
304
+ best_ang[replaced == 1] = angle * flip_sign
305
+ del S_, ccs
306
+ del cls_rot
307
+
308
+ best_cc, best_idx = best_ccs.max(dim=0)
309
+ corr_results["values"][cls_idx] = best_cc.item()
310
+ corr_results["indices"][cls_idx] = best_idx.item()
311
+ corr_results["angles"][cls_idx] = best_ang[best_idx.item()].item()
312
+ del cls, best_ccs, best_cc, best_ang
313
+ del L_conj, S, denom
314
+ return corr_results
315
+
316
+
317
+ _CLASS_CACHE: dict = {}
318
+
319
+
320
+ def compute_ncc(cls_path, entry_path, angle_step=5, device=None, fraction=1):
321
+ """Small wrapper for _ncc to load entry in the proper gpu.
322
+
323
+ Class data is loaded once per worker process and reused across entries, so only a
324
+ small key crosses the process boundary instead of the full tensor.
325
+ """
326
+ key = (str(cls_path), device, fraction)
327
+ if key not in _CLASS_CACHE:
328
+ _CLASS_CACHE[key] = load_class_data(cls_path, device=device, fraction=fraction)
329
+ S = _CLASS_CACHE[key]
330
+ L = torch.load(entry_path, map_location=S.device, weights_only=True)
331
+ return _ncc(S, L, angle_step)
332
+
333
+
334
+ def load_class_data(cls_path, device=None, fraction=1):
335
+ with mrcfile.mmap(cls_path) as mrc:
336
+ class_data = coerce_ndim(torch.tensor(mrc.data, device=device), 3)
337
+ class_px_size = mrc.voxel_size.x.item()
338
+
339
+ class_data = normalize(class_data, dim=(1, 2))
340
+ class_data *= gaussian_window(class_data.shape[1:], device=device)
341
+ class_data = normalize(class_data, dim=(1, 2))
342
+
343
+ # crop/pad ft to match pixel size
344
+ class_ft = ft_and_shift(class_data, dim=(1, 2))
345
+
346
+ class_ft = crop_or_pad_from_px_sizes(
347
+ class_ft, class_px_size, BIN_RESOLUTION, dim=(1, 2)
348
+ )
349
+ class_data = ift_and_shift(class_ft, dim=(1, 2)).real
350
+
351
+ if fraction < 1:
352
+ cropped_shape = (np.array(class_data.shape[1:]) * fraction).round().astype(int)
353
+ class_data = crop_to(
354
+ class_data, (class_data.shape[0], *cropped_shape), dim=(1, 2)
355
+ )
356
+ class_data *= gaussian_window(class_data.shape[1:], device=device)
357
+
358
+ class_data = normalize(class_data, dim=(1, 2))
359
+ del class_ft
360
+ return class_data
361
+
362
+
363
+ def get_entries_to_process(prog, db_path, log, overwrite):
364
+ db = pd.read_csv(db_path / "database_summary.csv", sep="\t", index_col=0)
365
+ task = prog.add_task(description="Selecting valid entries...", total=len(db))
366
+
367
+ to_download = []
368
+ to_extract = []
369
+ to_project = []
370
+ to_delete = []
371
+ for entry_id in db.index:
372
+ prog.update(task, advance=1)
373
+ proj_path = db_path / f"{entry_id:04}.pt"
374
+ map_path = db_path / f"emd_{entry_id:04}.map"
375
+ gz_path = db_path / f"emd_{entry_id:04}.map.gz"
376
+
377
+ # only projection should ever need overwriting
378
+ if proj_path.exists() and not overwrite:
379
+ log.info(f"{entry_id:04} was already projected")
380
+ if map_path.exists():
381
+ to_delete.append(map_path)
382
+ elif map_path.exists():
383
+ log.info(f"{entry_id:04} was already extracted")
384
+ to_project.append(map_path)
385
+ elif gz_path.exists():
386
+ log.info(f"{entry_id:04} was already downloaded")
387
+ to_extract.append(gz_path)
388
+ to_project.append(map_path)
389
+ else:
390
+ log.info(f"{entry_id:04} is new")
391
+ to_download.append(entry_id)
392
+ to_extract.append(gz_path)
393
+ to_project.append(map_path)
394
+
395
+ log.warning(f"Will download {len(to_download)}.")
396
+ log.warning(f"Will extract {len(to_extract)}.")
397
+ log.warning(f"Will project {len(to_project)}.")
398
+ return to_download, to_extract, to_project, to_delete
399
+
400
+
401
+ def download_headers(prog, db_path, dry_run):
402
+ if dry_run:
403
+ print(
404
+ f"Will update header list (currently have {len(list(db_path.glob('*.xml')))} headers)."
405
+ )
406
+ return
407
+
408
+ task = prog.add_task(description="Downloading headers...", total=None)
409
+
410
+ rsync = sh.rsync.bake("-rltpvzhu", "--info=progress2")
411
+
412
+ def done(cmd, success, exit_code):
413
+ prog.update(task, advance=1, refresh=True)
414
+
415
+ full_list_url = "https://ftp.ebi.ac.uk/pub/databases/emdb/status/latest/emdb_released_holdings.json"
416
+
417
+ response = requests.get(full_list_url, timeout=60)
418
+ response.raise_for_status()
419
+
420
+ entries = response.json().keys()
421
+
422
+ failed_download = []
423
+ for entry_id in entries:
424
+ sync_path = f"rsync.ebi.ac.uk::pub/databases/emdb/structures/{entry_id}/header/{entry_id.lower()}-v30.xml"
425
+ try:
426
+ rsync(sync_path, db_path, _done=done)
427
+ except sh.ErrorReturnCode:
428
+ failed_download.append(entry_id)
429
+
430
+ return failed_download
431
+
432
+
433
+ def fetch_maps(*, prog, db_path, to_download, to_extract, dry_run, threads, failed):
434
+ """Download and extract maps, yielding map paths as soon as they're ready.
435
+
436
+ Failed entries are appended to failed and skipped.
437
+ """
438
+ if dry_run:
439
+ print(f"Will download {len(to_download)} maps")
440
+ print(f"Will extract {len(to_extract)} maps")
441
+ return iter(())
442
+
443
+ dl_task = prog.add_task(description="Downloading maps...", total=len(to_download))
444
+ ex_task = prog.add_task(description="Extracting...", total=len(to_extract))
445
+
446
+ rsync = sh.rsync.bake("-rltpvzhu", "--info=progress2")
447
+ download_ids = set(to_download)
448
+
449
+ def fetch(gz_path):
450
+ entry_id = int(re.search(r"\d+", gz_path.stem).group())
451
+ if entry_id in download_ids:
452
+ sync_path = f"rsync.ebi.ac.uk::pub/databases/emdb/structures/EMD-{entry_id:04}/map/emd_{entry_id:04}.map.gz"
453
+ try:
454
+ rsync(sync_path, db_path)
455
+ except sh.ErrorReturnCode:
456
+ failed.append(entry_id)
457
+ return None
458
+ prog.update(dl_task, advance=1)
459
+ sh.gzip("-d", str(gz_path))
460
+ prog.update(ex_task, advance=1)
461
+ return gz_path.with_suffix("")
462
+
463
+ def ready():
464
+ with ThreadPoolExecutor(max_workers=threads) as pool:
465
+ futures = [pool.submit(fetch, gz) for gz in to_extract]
466
+ for fut in as_completed(futures):
467
+ map_path = fut.result()
468
+ if map_path is not None:
469
+ yield map_path
470
+
471
+ return ready()
472
+
473
+
474
+ def _project_map(map_path, proj_path, save_as_mrc=False):
475
+ with mrcfile.open(map_path) as mrc:
476
+ img = torch.from_numpy(mrc.data.astype(np.float32, copy=True))
477
+ px_size = mrc.voxel_size.x.item()
478
+
479
+ # apply gaussian window to avoid edge issues
480
+ img = normalize(img)
481
+ img *= gaussian_window(img.shape)
482
+
483
+ # pad if needed
484
+ if not np.all(img.shape[0] == np.array(img.shape)):
485
+ # not square, pad to square before projecting
486
+ img = pad_to(img, [np.max(img.shape)] * 3)
487
+
488
+ img = normalize(img)
489
+
490
+ ft = crop_or_pad_from_px_sizes(ft_and_shift(img), px_size, BIN_RESOLUTION)
491
+ # # gaussian filter in fourier space as well to help with rotations and whatnot
492
+ # ft *= gaussian_window(ft.shape)
493
+ proj_ft = rotated_projection_fts(ft)
494
+ proj = ift_and_shift(proj_ft, dim=(1, 2)).real
495
+ proj = normalize(proj, dim=(1, 2))
496
+
497
+ if save_as_mrc:
498
+ with mrcfile.new(proj_path, np.asarray(proj), overwrite=True) as mrc:
499
+ mrc.voxel_size = BIN_RESOLUTION
500
+ else:
501
+ torch.save(proj, proj_path)
502
+ return proj_path
503
+
504
+
505
+ def _project_map_file(map_path):
506
+ """Project a map into its sibling <entry_id>.pt projection file."""
507
+ entry_id = re.search(r"\d+", map_path.stem).group()
508
+ return _project_map(map_path, map_path.with_name(f"{entry_id}.pt"))
509
+
510
+
511
+ def project_maps(*, prog, maps, log, dry_run, threads, keep_maps, total):
512
+ if dry_run:
513
+ print(f"Will project {total} maps")
514
+ return
515
+
516
+ task = prog.add_task(description="Projecting...", total=total)
517
+
518
+ torch.set_default_dtype(torch.float32)
519
+ set_start_method("spawn", force=True)
520
+
521
+ errors = []
522
+ with ProcessPoolExecutor(
523
+ max_workers=threads, initializer=os.nice, initargs=(10,)
524
+ ) as pool:
525
+ results = {}
526
+
527
+ def finish(fut):
528
+ map_path = results.pop(fut)
529
+ if fut.exception():
530
+ err = fut.exception()
531
+ err.add_note(map_path.stem)
532
+ errors.append(err)
533
+ log.warning(f"failed projecting {map_path.stem}: {err}")
534
+ else:
535
+ proj_path = fut.result()
536
+ log.info(f"finished projecting {proj_path.stem}")
537
+ if not keep_maps:
538
+ # keep the map on failure so a retry need not re-download it
539
+ map_path.unlink(missing_ok=True)
540
+ prog.update(task, advance=1)
541
+
542
+ # drain finished projections while the download/extract stream still feeds us
543
+ for map_path in maps:
544
+ for fut in tuple(results):
545
+ if fut.done():
546
+ finish(fut)
547
+ results[pool.submit(_project_map_file, map_path)] = map_path
548
+ for fut in as_completed(results):
549
+ finish(fut)
550
+
551
+
552
+ def generate_db_summary(prog, db_path, log, dry_run):
553
+ entries = list(Path(db_path).glob("*.xml"))
554
+ task = prog.add_task(
555
+ description="Generating database summary...", total=len(entries)
556
+ )
557
+
558
+ rows = {}
559
+ bad = 0
560
+ for xml in entries:
561
+ prog.update(task, advance=1)
562
+ with open(xml, "rb") as f:
563
+ data = xmltodict.parse(f)
564
+ entry_id = int(re.search(r"emd-(\d+)", xml.stem).group(1))
565
+ shape = [int(i) for i in data["emd"]["map"]["dimensions"].values()]
566
+ px = [float(v["#text"]) for v in data["emd"]["map"]["pixel_spacing"].values()]
567
+ size = int(data["emd"]["map"]["@size_kbytes"])
568
+ structure_det = data["emd"]["structure_determination_list"][
569
+ "structure_determination"
570
+ ]
571
+ # resolution
572
+ methods = (
573
+ "singleparticle",
574
+ "subtomogram_averaging",
575
+ "crystallography",
576
+ "helical",
577
+ "tomography",
578
+ )
579
+ if any(f"{method}_processing" in structure_det for method in methods):
580
+ pass
581
+ else:
582
+ log.info(f"{entry_id:04} is bad (method = {structure_det['method']})")
583
+ continue
584
+
585
+ errors = []
586
+ for method in methods:
587
+ try:
588
+ processing = structure_det[f"{method}_processing"]
589
+ if isinstance(processing, list):
590
+ processing = processing[0] # ugly but ok
591
+ resolution = float(
592
+ processing["final_reconstruction"]["resolution"]["#text"]
593
+ )
594
+ break
595
+ except Exception as e:
596
+ errors.append(e)
597
+ continue
598
+ else:
599
+ bad += 1
600
+ log.info(
601
+ f"{entry_id:04} is bad:\n"
602
+ + ", ".join(f"{e.__class__.__name__}({e})" for e in errors)
603
+ )
604
+ resolution = np.nan
605
+
606
+ rows[entry_id] = [*shape, *px, size, resolution]
607
+
608
+ db = pd.DataFrame.from_dict(
609
+ rows,
610
+ orient="index",
611
+ columns=["x", "y", "z", "px_x", "px_y", "px_z", "size_kb", "resolution"],
612
+ )
613
+ db = db.astype(
614
+ {
615
+ "x": int,
616
+ "y": int,
617
+ "z": int,
618
+ "px_x": float,
619
+ "px_y": float,
620
+ "px_z": float,
621
+ "size_kb": int,
622
+ "resolution": float,
623
+ }
624
+ )
625
+
626
+ shape = db[["x", "y", "z"]]
627
+ max_dim = shape.max(axis=1)
628
+ size_mb = db["size_kb"] / 1e3
629
+ # convert shape to square to check for sizes (will have to happen in processing)
630
+ volume_when_cube = max_dim**3
631
+ volume_padded_GB = volume_when_cube * 4 / 1e9
632
+ volume_nm = volume_when_cube * db["px_x"] ** 3 / 1e3 # nm^3
633
+ rescale_ratio = (db["px_x"] / BIN_RESOLUTION) ** 3
634
+ volume_rescaled_GB = volume_padded_GB * rescale_ratio
635
+
636
+ too_big_or_small = (
637
+ (volume_nm < 1e3) | (volume_padded_GB > 1) | (volume_rescaled_GB > 1)
638
+ )
639
+ too_heavy = size_mb > 400
640
+ anisotropic = (db.px_x != db.px_y) | (db.px_x != db.px_z)
641
+
642
+ # discard bad stuff
643
+ to_discard = too_big_or_small | too_heavy | anisotropic
644
+ db = db[~to_discard]
645
+
646
+ log.warning(
647
+ f"Found {bad} entries ({100*bad/len(entries):.2f}% of headers) with no single particle info."
648
+ )
649
+ log.warning(
650
+ f"{to_discard.sum()} entries were discarded because too heavy, big, or small to process."
651
+ )
652
+ db.sort_index(inplace=True)
653
+ if not dry_run:
654
+ db.to_csv(
655
+ db_path / "database_summary.csv",
656
+ sep="\t",
657
+ header=True,
658
+ index=True,
659
+ na_rep="NaN",
660
+ )
661
+ return db