emscan 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- emscan/__init__.py +10 -0
- emscan/_functions.py +661 -0
- emscan/main.py +591 -0
- emscan-1.0.0.dist-info/METADATA +121 -0
- emscan-1.0.0.dist-info/RECORD +8 -0
- emscan-1.0.0.dist-info/WHEEL +4 -0
- emscan-1.0.0.dist-info/entry_points.txt +2 -0
- emscan-1.0.0.dist-info/licenses/LICENSE +674 -0
emscan/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""package description."""
|
|
2
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
__version__ = version("emscan")
|
|
6
|
+
except PackageNotFoundError:
|
|
7
|
+
__version__ = "uninstalled"
|
|
8
|
+
|
|
9
|
+
__author__ = "Lorenzo Gaifas"
|
|
10
|
+
__email__ = "brisvag@gmail.com"
|
emscan/_functions.py
ADDED
|
@@ -0,0 +1,661 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import inspect
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
|
|
6
|
+
from functools import lru_cache, reduce
|
|
7
|
+
from itertools import chain
|
|
8
|
+
from math import ceil, floor
|
|
9
|
+
from operator import mul
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import mrcfile
|
|
13
|
+
import numpy as np
|
|
14
|
+
import pandas as pd
|
|
15
|
+
import requests
|
|
16
|
+
import sh
|
|
17
|
+
import torch
|
|
18
|
+
import xmltodict
|
|
19
|
+
from morphosamplers.sampler import sample_subvolumes
|
|
20
|
+
from rich import print
|
|
21
|
+
from scipy.signal.windows import gaussian
|
|
22
|
+
from scipy.spatial.transform import Rotation
|
|
23
|
+
from torch.fft import fftn, fftshift, ifftn, ifftshift
|
|
24
|
+
from torch.multiprocessing import set_start_method
|
|
25
|
+
from torchvision.transforms import InterpolationMode
|
|
26
|
+
from torchvision.transforms.functional import rotate
|
|
27
|
+
|
|
28
|
+
BIN_RESOLUTION = 4
|
|
29
|
+
GOLDEN_RATIO = (1 + np.sqrt(5)) / 2
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _print_tensors(depth=2):
|
|
33
|
+
"""Print info about tensor variables locally and nonlocally."""
|
|
34
|
+
print("=" * 80)
|
|
35
|
+
for obj in gc.get_objects():
|
|
36
|
+
frame = inspect.currentframe()
|
|
37
|
+
frames = []
|
|
38
|
+
for _i in range(depth):
|
|
39
|
+
frame = frame.f_back
|
|
40
|
+
frames.append(frame)
|
|
41
|
+
try:
|
|
42
|
+
if torch.is_tensor(obj) or (
|
|
43
|
+
hasattr(obj, "data") and torch.is_tensor(obj.data)
|
|
44
|
+
):
|
|
45
|
+
for gobj, gobjid in globals().items():
|
|
46
|
+
if gobjid is obj and gobj not in ("obj", "gobjid"):
|
|
47
|
+
print("GLOBAL:\t", gobj, obj.size(), obj.data_ptr())
|
|
48
|
+
for frame in frames:
|
|
49
|
+
fname = frame.f_code.co_name
|
|
50
|
+
locs = frame.f_locals
|
|
51
|
+
for gobj, gobjid in locs.items():
|
|
52
|
+
if gobjid is obj and gobj not in ("obj", "gobjid"):
|
|
53
|
+
print(f"{fname}():\t", gobj, obj.size(), obj.data_ptr())
|
|
54
|
+
except: # noqa
|
|
55
|
+
pass
|
|
56
|
+
print("=" * 80)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def normalize(img, dim=None, inplace=True):
|
|
60
|
+
"""Normalize images to std=1 and mean=0."""
|
|
61
|
+
if not torch.is_complex(img):
|
|
62
|
+
img = img.to(torch.float32, copy=not inplace)
|
|
63
|
+
elif not inplace:
|
|
64
|
+
img = img.copy()
|
|
65
|
+
img -= img.mean(dim=dim, keepdim=True)
|
|
66
|
+
img /= img.std(dim=dim, keepdim=True)
|
|
67
|
+
if img.isnan().any():
|
|
68
|
+
img = torch.nan_to_num(img, out=img)
|
|
69
|
+
return img
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def ft_and_shift(img, dim=None):
|
|
73
|
+
# first move the image to be center of the origin (ifftshift), which improves
|
|
74
|
+
# interpolation artifacts later (ftshift is equivalent to multiplying
|
|
75
|
+
# with a checkerboard, which messes up interpolation real bad)
|
|
76
|
+
return fftshift(fftn(ifftshift(img, dim=dim), dim=dim), dim=dim)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def ift_and_shift(img, dim=None):
|
|
80
|
+
# undo the above (note the order is the same cause it's actually inverted)
|
|
81
|
+
return fftshift(ifftn(ifftshift(img, dim=dim), dim=dim), dim=dim)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def rotate_by(arr, ang, center=None):
|
|
85
|
+
if center is None:
|
|
86
|
+
# torchvision expects the center as (x, y), i.e. (col, row)
|
|
87
|
+
center = list(np.array(arr.shape)[::-1] // 2)
|
|
88
|
+
|
|
89
|
+
return rotate(
|
|
90
|
+
arr[None],
|
|
91
|
+
float(ang),
|
|
92
|
+
center=center,
|
|
93
|
+
interpolation=InterpolationMode.BILINEAR,
|
|
94
|
+
)[0]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def rotations(img, degree_range, center=None):
|
|
98
|
+
"""Generate a number of rotations from an image and a list of angles.
|
|
99
|
+
|
|
100
|
+
degree range: iterable of degrees (counterclockwise).
|
|
101
|
+
"""
|
|
102
|
+
gauss = gaussian_window(img.shape, device=img.device)
|
|
103
|
+
|
|
104
|
+
for angle in degree_range:
|
|
105
|
+
if torch.is_complex(img):
|
|
106
|
+
real = rotate_by(img.real, angle, center)
|
|
107
|
+
imag = rotate_by(img.imag, angle, center)
|
|
108
|
+
rot = real + (1j * imag)
|
|
109
|
+
del real, imag
|
|
110
|
+
else:
|
|
111
|
+
rot = rotate_by(img, angle, center)
|
|
112
|
+
yield angle, rot * gauss
|
|
113
|
+
del rot
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def coerce_ndim(img, ndim):
|
|
117
|
+
"""Coerce image dimensionality if smaller than ndim."""
|
|
118
|
+
if img.ndim > ndim:
|
|
119
|
+
raise ValueError(f"image has too high dimensionality ({img.ndim})")
|
|
120
|
+
while img.ndim < ndim:
|
|
121
|
+
img = img[None]
|
|
122
|
+
return img
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@lru_cache
|
|
126
|
+
def _rotations_hemisphere_fibonacci():
|
|
127
|
+
# http://extremelearning.com.au/how-to-evenly-distribute-points-on-a-
|
|
128
|
+
# sphere-more-effectively-than-the-canonical-fibonacci-lattice/
|
|
129
|
+
# Note: not using epsilon we ensure we get the Z axis, which is a common
|
|
130
|
+
# aligment for maps. Packing is slightly worse, but still better than healpix
|
|
131
|
+
|
|
132
|
+
N = 300 # THIS IS FIXED ONCE GENERATED THE DB!
|
|
133
|
+
i = np.arange(N)[: N // 2] # only take half cause projection symmetry
|
|
134
|
+
phi = 2 * np.pi * i / GOLDEN_RATIO
|
|
135
|
+
theta = np.arccos(1 - 2 * i / (N - 1))
|
|
136
|
+
angles = np.stack([phi, theta], axis=1)
|
|
137
|
+
return Rotation.from_euler("ZX", angles)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def rotated_projection_fts(ft):
|
|
141
|
+
"""Generate rotated projections fts of a map."""
|
|
142
|
+
# TODO: sinc function to avoid edge artifacts
|
|
143
|
+
|
|
144
|
+
rot = _rotations_hemisphere_fibonacci()
|
|
145
|
+
pos = np.array(ft.shape) // 2 # using // is needed to center ctf for odd images
|
|
146
|
+
grid_size = (ft.shape[0], ft.shape[0], 1)
|
|
147
|
+
slices = sample_subvolumes(
|
|
148
|
+
np.array(ft), positions=pos, orientations=rot, grid_shape=grid_size, fill_value=0,
|
|
149
|
+
).squeeze()
|
|
150
|
+
|
|
151
|
+
return torch.from_numpy(slices)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def gaussian_window(shape, sigmas=None, device=None):
|
|
155
|
+
"""Generate a gaussian_window of the given shape and sigmas, ensuring it goes to zero."""
|
|
156
|
+
if sigmas is None:
|
|
157
|
+
sigmas = np.array(shape)
|
|
158
|
+
sigmas = np.broadcast_to(sigmas, len(shape))
|
|
159
|
+
windows = [gaussian(n, s) for n, s in zip(shape, sigmas, strict=True)]
|
|
160
|
+
mins = np.array([min(w) for w in windows])
|
|
161
|
+
# true minimum of the separable product, so the window reaches exactly 0
|
|
162
|
+
zero = np.prod(mins)
|
|
163
|
+
tensors = [torch.tensor(w, device=device) for w in np.ix_(*windows)]
|
|
164
|
+
window = reduce(mul, tensors)
|
|
165
|
+
window -= zero
|
|
166
|
+
window = torch.clip(window, 0, 1)
|
|
167
|
+
window /= window.max()
|
|
168
|
+
return window
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def crop_to(img, target_shape, dim=None):
|
|
172
|
+
if dim is None:
|
|
173
|
+
dim = tuple(range(img.ndim))
|
|
174
|
+
# ensure we round the cropping correctly to keeo the center in the right place
|
|
175
|
+
edge_crop = (np.array(img.shape) - np.array(target_shape)) / 2
|
|
176
|
+
crop_left = []
|
|
177
|
+
crop_right = []
|
|
178
|
+
for d in range(img.ndim):
|
|
179
|
+
if d not in dim:
|
|
180
|
+
crop_left.append(None)
|
|
181
|
+
crop_right.append(None)
|
|
182
|
+
continue
|
|
183
|
+
# rounding must depend on the parity of the dimension being cropped
|
|
184
|
+
if img.shape[d] % 2:
|
|
185
|
+
left, right = floor(edge_crop[d]), ceil(edge_crop[d])
|
|
186
|
+
else:
|
|
187
|
+
left, right = ceil(edge_crop[d]), floor(edge_crop[d])
|
|
188
|
+
crop_left.append(left or None)
|
|
189
|
+
crop_right.append(-right or None)
|
|
190
|
+
crop_slice = tuple(
|
|
191
|
+
slice(*crops) for crops in zip(crop_left, crop_right, strict=True)
|
|
192
|
+
)
|
|
193
|
+
return img[crop_slice]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def pad_to(img, target_shape, dim=None, value=0):
|
|
197
|
+
if dim is None:
|
|
198
|
+
dim = tuple(range(img.ndim))
|
|
199
|
+
# ensure we round the padding correctly to keeo the center in the right place
|
|
200
|
+
edge_pad = (np.array(target_shape) - np.array(img.shape)) / 2
|
|
201
|
+
pad_left = []
|
|
202
|
+
pad_right = []
|
|
203
|
+
for d in range(img.ndim):
|
|
204
|
+
if d not in dim:
|
|
205
|
+
pad_left.append(0)
|
|
206
|
+
pad_right.append(0)
|
|
207
|
+
continue
|
|
208
|
+
# rounding must depend on the parity of the dimension being padded
|
|
209
|
+
if img.shape[d] % 2:
|
|
210
|
+
left, right = ceil(edge_pad[d]), floor(edge_pad[d])
|
|
211
|
+
else:
|
|
212
|
+
left, right = floor(edge_pad[d]), ceil(edge_pad[d])
|
|
213
|
+
pad_left.append(left)
|
|
214
|
+
pad_right.append(right)
|
|
215
|
+
padding = tuple(chain.from_iterable(zip(pad_right, pad_left, strict=True)))[::-1]
|
|
216
|
+
padded = torch.nn.functional.pad(img, padding, value=value)
|
|
217
|
+
return padded
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def resize(ft, target_shape, dim=None):
|
|
221
|
+
"""Bin or unbin image(s) ft by fourier cropping or padding."""
|
|
222
|
+
if dim is None:
|
|
223
|
+
dim = tuple(range(ft.ndim))
|
|
224
|
+
|
|
225
|
+
target_shape = np.array(target_shape)
|
|
226
|
+
input_shape = np.array(ft.shape)
|
|
227
|
+
target_shape_dims = target_shape[list(dim)]
|
|
228
|
+
input_shape_dims = input_shape[list(dim)]
|
|
229
|
+
if np.all(target_shape_dims == input_shape_dims):
|
|
230
|
+
return ft
|
|
231
|
+
elif np.all(target_shape_dims < input_shape_dims):
|
|
232
|
+
return crop_to(ft, target_shape, dim=dim)
|
|
233
|
+
elif np.all(target_shape_dims > input_shape_dims):
|
|
234
|
+
return pad_to(ft, target_shape, dim=dim)
|
|
235
|
+
else:
|
|
236
|
+
raise NotImplementedError("cannot pad and crop at the same time")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def crop_or_pad_from_px_sizes(ft, px_size_in, px_size_out, dim=None):
|
|
240
|
+
"""Rescale an image's ft given before/after pixel sizes."""
|
|
241
|
+
ratio = np.array(px_size_in) / np.array(px_size_out)
|
|
242
|
+
target_shape = np.round(np.array(ft.shape) * ratio / 2) * 2
|
|
243
|
+
dim = tuple(range(ft.ndim)) if dim is None else dim
|
|
244
|
+
return resize(ft, target_shape, dim=dim)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _ncc(S, L, angle_step):
|
|
248
|
+
"""Fast cross correlation of all elements of projections and the image.
|
|
249
|
+
|
|
250
|
+
Performs also rotations and mirroring to cover all orientations.
|
|
251
|
+
|
|
252
|
+
Based on eq (9) from:
|
|
253
|
+
https://w.imagemagick.org/docs/AcceleratedTemplateMatchingUsingLocalStatisticsAndFourierTransforms.pdf
|
|
254
|
+
Optimized to reduce operations (FFTs are precomputed and images are pre-normalized).
|
|
255
|
+
Some stuff is also easier because L[0] and S are forced to have the same shape if S is big.
|
|
256
|
+
"""
|
|
257
|
+
# see link above for what is L, LL, S, U, and the equation
|
|
258
|
+
corr_results = {"values": {}, "indices": {}, "angles": {}}
|
|
259
|
+
|
|
260
|
+
if S.shape[-1] < L.shape[-1]:
|
|
261
|
+
# crop L
|
|
262
|
+
L = crop_to(L, S.shape, dim=(1, 2))
|
|
263
|
+
L = normalize(L, dim=(1, 2), inplace=False)
|
|
264
|
+
else:
|
|
265
|
+
# cropping, so assume "same size" of S and L
|
|
266
|
+
S = crop_to(S, L.shape, dim=(1, 2))
|
|
267
|
+
S = normalize(S, dim=(1, 2), inplace=False)
|
|
268
|
+
|
|
269
|
+
L_ = ft_and_shift(L, dim=(1, 2))
|
|
270
|
+
L_conj = L_.conj()
|
|
271
|
+
LL_ = ft_and_shift(L**2, dim=(1, 2))
|
|
272
|
+
N = S[0].nelement()
|
|
273
|
+
# TODO this is just zeros with N in the center
|
|
274
|
+
U_ = ft_and_shift(torch.ones_like(S[0], device=S.device))
|
|
275
|
+
|
|
276
|
+
def _cc(ft1, ft2, ft2_is_conj=False):
|
|
277
|
+
conj = ft2 if ft2_is_conj else ft2.conj()
|
|
278
|
+
return ift_and_shift(ft1 * conj, dim=(1, 2)).real
|
|
279
|
+
|
|
280
|
+
denom_inner = (N * _cc(U_, LL_)) - (_cc(U_, L_) ** 2)
|
|
281
|
+
denom = torch.sqrt(torch.clamp_min(denom_inner, 0))
|
|
282
|
+
n_proj = len(L)
|
|
283
|
+
|
|
284
|
+
del L, L_, LL_, U_
|
|
285
|
+
|
|
286
|
+
def ncc(S_):
|
|
287
|
+
nonlocal denom, L_conj
|
|
288
|
+
# NOTE: this assume S is ft of normalized image (std=1, mean=0)
|
|
289
|
+
# NOTE: L_conj is precomputed to save computations!
|
|
290
|
+
return _cc(S_, L_conj, ft2_is_conj=True) / denom
|
|
291
|
+
|
|
292
|
+
for cls_idx, cls in enumerate(S):
|
|
293
|
+
best_ccs = torch.zeros(n_proj, device=S.device)
|
|
294
|
+
best_ang = torch.zeros(n_proj, device=S.device)
|
|
295
|
+
|
|
296
|
+
for angle, cls_rot in rotations(cls, np.arange(0, 360, angle_step)):
|
|
297
|
+
# normalize to avoid needing std and mean in ncc calculation
|
|
298
|
+
cls_rot = normalize(cls_rot)
|
|
299
|
+
# also correlate transposed image, because we only have half a sphere of projections
|
|
300
|
+
for cls_flip, flip_sign in ((cls_rot, 1), (cls_rot.T, -1)):
|
|
301
|
+
S_ = ft_and_shift(cls_flip)
|
|
302
|
+
ccs = ncc(S_).amax(dim=(1, 2))
|
|
303
|
+
best_ccs, replaced = torch.max(torch.stack([best_ccs, ccs]), dim=0)
|
|
304
|
+
best_ang[replaced == 1] = angle * flip_sign
|
|
305
|
+
del S_, ccs
|
|
306
|
+
del cls_rot
|
|
307
|
+
|
|
308
|
+
best_cc, best_idx = best_ccs.max(dim=0)
|
|
309
|
+
corr_results["values"][cls_idx] = best_cc.item()
|
|
310
|
+
corr_results["indices"][cls_idx] = best_idx.item()
|
|
311
|
+
corr_results["angles"][cls_idx] = best_ang[best_idx.item()].item()
|
|
312
|
+
del cls, best_ccs, best_cc, best_ang
|
|
313
|
+
del L_conj, S, denom
|
|
314
|
+
return corr_results
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
_CLASS_CACHE: dict = {}
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def compute_ncc(cls_path, entry_path, angle_step=5, device=None, fraction=1):
|
|
321
|
+
"""Small wrapper for _ncc to load entry in the proper gpu.
|
|
322
|
+
|
|
323
|
+
Class data is loaded once per worker process and reused across entries, so only a
|
|
324
|
+
small key crosses the process boundary instead of the full tensor.
|
|
325
|
+
"""
|
|
326
|
+
key = (str(cls_path), device, fraction)
|
|
327
|
+
if key not in _CLASS_CACHE:
|
|
328
|
+
_CLASS_CACHE[key] = load_class_data(cls_path, device=device, fraction=fraction)
|
|
329
|
+
S = _CLASS_CACHE[key]
|
|
330
|
+
L = torch.load(entry_path, map_location=S.device, weights_only=True)
|
|
331
|
+
return _ncc(S, L, angle_step)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def load_class_data(cls_path, device=None, fraction=1):
|
|
335
|
+
with mrcfile.mmap(cls_path) as mrc:
|
|
336
|
+
class_data = coerce_ndim(torch.tensor(mrc.data, device=device), 3)
|
|
337
|
+
class_px_size = mrc.voxel_size.x.item()
|
|
338
|
+
|
|
339
|
+
class_data = normalize(class_data, dim=(1, 2))
|
|
340
|
+
class_data *= gaussian_window(class_data.shape[1:], device=device)
|
|
341
|
+
class_data = normalize(class_data, dim=(1, 2))
|
|
342
|
+
|
|
343
|
+
# crop/pad ft to match pixel size
|
|
344
|
+
class_ft = ft_and_shift(class_data, dim=(1, 2))
|
|
345
|
+
|
|
346
|
+
class_ft = crop_or_pad_from_px_sizes(
|
|
347
|
+
class_ft, class_px_size, BIN_RESOLUTION, dim=(1, 2)
|
|
348
|
+
)
|
|
349
|
+
class_data = ift_and_shift(class_ft, dim=(1, 2)).real
|
|
350
|
+
|
|
351
|
+
if fraction < 1:
|
|
352
|
+
cropped_shape = (np.array(class_data.shape[1:]) * fraction).round().astype(int)
|
|
353
|
+
class_data = crop_to(
|
|
354
|
+
class_data, (class_data.shape[0], *cropped_shape), dim=(1, 2)
|
|
355
|
+
)
|
|
356
|
+
class_data *= gaussian_window(class_data.shape[1:], device=device)
|
|
357
|
+
|
|
358
|
+
class_data = normalize(class_data, dim=(1, 2))
|
|
359
|
+
del class_ft
|
|
360
|
+
return class_data
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def get_entries_to_process(prog, db_path, log, overwrite):
|
|
364
|
+
db = pd.read_csv(db_path / "database_summary.csv", sep="\t", index_col=0)
|
|
365
|
+
task = prog.add_task(description="Selecting valid entries...", total=len(db))
|
|
366
|
+
|
|
367
|
+
to_download = []
|
|
368
|
+
to_extract = []
|
|
369
|
+
to_project = []
|
|
370
|
+
to_delete = []
|
|
371
|
+
for entry_id in db.index:
|
|
372
|
+
prog.update(task, advance=1)
|
|
373
|
+
proj_path = db_path / f"{entry_id:04}.pt"
|
|
374
|
+
map_path = db_path / f"emd_{entry_id:04}.map"
|
|
375
|
+
gz_path = db_path / f"emd_{entry_id:04}.map.gz"
|
|
376
|
+
|
|
377
|
+
# only projection should ever need overwriting
|
|
378
|
+
if proj_path.exists() and not overwrite:
|
|
379
|
+
log.info(f"{entry_id:04} was already projected")
|
|
380
|
+
if map_path.exists():
|
|
381
|
+
to_delete.append(map_path)
|
|
382
|
+
elif map_path.exists():
|
|
383
|
+
log.info(f"{entry_id:04} was already extracted")
|
|
384
|
+
to_project.append(map_path)
|
|
385
|
+
elif gz_path.exists():
|
|
386
|
+
log.info(f"{entry_id:04} was already downloaded")
|
|
387
|
+
to_extract.append(gz_path)
|
|
388
|
+
to_project.append(map_path)
|
|
389
|
+
else:
|
|
390
|
+
log.info(f"{entry_id:04} is new")
|
|
391
|
+
to_download.append(entry_id)
|
|
392
|
+
to_extract.append(gz_path)
|
|
393
|
+
to_project.append(map_path)
|
|
394
|
+
|
|
395
|
+
log.warning(f"Will download {len(to_download)}.")
|
|
396
|
+
log.warning(f"Will extract {len(to_extract)}.")
|
|
397
|
+
log.warning(f"Will project {len(to_project)}.")
|
|
398
|
+
return to_download, to_extract, to_project, to_delete
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def download_headers(prog, db_path, dry_run):
|
|
402
|
+
if dry_run:
|
|
403
|
+
print(
|
|
404
|
+
f"Will update header list (currently have {len(list(db_path.glob('*.xml')))} headers)."
|
|
405
|
+
)
|
|
406
|
+
return
|
|
407
|
+
|
|
408
|
+
task = prog.add_task(description="Downloading headers...", total=None)
|
|
409
|
+
|
|
410
|
+
rsync = sh.rsync.bake("-rltpvzhu", "--info=progress2")
|
|
411
|
+
|
|
412
|
+
def done(cmd, success, exit_code):
|
|
413
|
+
prog.update(task, advance=1, refresh=True)
|
|
414
|
+
|
|
415
|
+
full_list_url = "https://ftp.ebi.ac.uk/pub/databases/emdb/status/latest/emdb_released_holdings.json"
|
|
416
|
+
|
|
417
|
+
response = requests.get(full_list_url, timeout=60)
|
|
418
|
+
response.raise_for_status()
|
|
419
|
+
|
|
420
|
+
entries = response.json().keys()
|
|
421
|
+
|
|
422
|
+
failed_download = []
|
|
423
|
+
for entry_id in entries:
|
|
424
|
+
sync_path = f"rsync.ebi.ac.uk::pub/databases/emdb/structures/{entry_id}/header/{entry_id.lower()}-v30.xml"
|
|
425
|
+
try:
|
|
426
|
+
rsync(sync_path, db_path, _done=done)
|
|
427
|
+
except sh.ErrorReturnCode:
|
|
428
|
+
failed_download.append(entry_id)
|
|
429
|
+
|
|
430
|
+
return failed_download
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def fetch_maps(*, prog, db_path, to_download, to_extract, dry_run, threads, failed):
|
|
434
|
+
"""Download and extract maps, yielding map paths as soon as they're ready.
|
|
435
|
+
|
|
436
|
+
Failed entries are appended to failed and skipped.
|
|
437
|
+
"""
|
|
438
|
+
if dry_run:
|
|
439
|
+
print(f"Will download {len(to_download)} maps")
|
|
440
|
+
print(f"Will extract {len(to_extract)} maps")
|
|
441
|
+
return iter(())
|
|
442
|
+
|
|
443
|
+
dl_task = prog.add_task(description="Downloading maps...", total=len(to_download))
|
|
444
|
+
ex_task = prog.add_task(description="Extracting...", total=len(to_extract))
|
|
445
|
+
|
|
446
|
+
rsync = sh.rsync.bake("-rltpvzhu", "--info=progress2")
|
|
447
|
+
download_ids = set(to_download)
|
|
448
|
+
|
|
449
|
+
def fetch(gz_path):
|
|
450
|
+
entry_id = int(re.search(r"\d+", gz_path.stem).group())
|
|
451
|
+
if entry_id in download_ids:
|
|
452
|
+
sync_path = f"rsync.ebi.ac.uk::pub/databases/emdb/structures/EMD-{entry_id:04}/map/emd_{entry_id:04}.map.gz"
|
|
453
|
+
try:
|
|
454
|
+
rsync(sync_path, db_path)
|
|
455
|
+
except sh.ErrorReturnCode:
|
|
456
|
+
failed.append(entry_id)
|
|
457
|
+
return None
|
|
458
|
+
prog.update(dl_task, advance=1)
|
|
459
|
+
sh.gzip("-d", str(gz_path))
|
|
460
|
+
prog.update(ex_task, advance=1)
|
|
461
|
+
return gz_path.with_suffix("")
|
|
462
|
+
|
|
463
|
+
def ready():
|
|
464
|
+
with ThreadPoolExecutor(max_workers=threads) as pool:
|
|
465
|
+
futures = [pool.submit(fetch, gz) for gz in to_extract]
|
|
466
|
+
for fut in as_completed(futures):
|
|
467
|
+
map_path = fut.result()
|
|
468
|
+
if map_path is not None:
|
|
469
|
+
yield map_path
|
|
470
|
+
|
|
471
|
+
return ready()
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _project_map(map_path, proj_path, save_as_mrc=False):
|
|
475
|
+
with mrcfile.open(map_path) as mrc:
|
|
476
|
+
img = torch.from_numpy(mrc.data.astype(np.float32, copy=True))
|
|
477
|
+
px_size = mrc.voxel_size.x.item()
|
|
478
|
+
|
|
479
|
+
# apply gaussian window to avoid edge issues
|
|
480
|
+
img = normalize(img)
|
|
481
|
+
img *= gaussian_window(img.shape)
|
|
482
|
+
|
|
483
|
+
# pad if needed
|
|
484
|
+
if not np.all(img.shape[0] == np.array(img.shape)):
|
|
485
|
+
# not square, pad to square before projecting
|
|
486
|
+
img = pad_to(img, [np.max(img.shape)] * 3)
|
|
487
|
+
|
|
488
|
+
img = normalize(img)
|
|
489
|
+
|
|
490
|
+
ft = crop_or_pad_from_px_sizes(ft_and_shift(img), px_size, BIN_RESOLUTION)
|
|
491
|
+
# # gaussian filter in fourier space as well to help with rotations and whatnot
|
|
492
|
+
# ft *= gaussian_window(ft.shape)
|
|
493
|
+
proj_ft = rotated_projection_fts(ft)
|
|
494
|
+
proj = ift_and_shift(proj_ft, dim=(1, 2)).real
|
|
495
|
+
proj = normalize(proj, dim=(1, 2))
|
|
496
|
+
|
|
497
|
+
if save_as_mrc:
|
|
498
|
+
with mrcfile.new(proj_path, np.asarray(proj), overwrite=True) as mrc:
|
|
499
|
+
mrc.voxel_size = BIN_RESOLUTION
|
|
500
|
+
else:
|
|
501
|
+
torch.save(proj, proj_path)
|
|
502
|
+
return proj_path
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _project_map_file(map_path):
|
|
506
|
+
"""Project a map into its sibling <entry_id>.pt projection file."""
|
|
507
|
+
entry_id = re.search(r"\d+", map_path.stem).group()
|
|
508
|
+
return _project_map(map_path, map_path.with_name(f"{entry_id}.pt"))
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def project_maps(*, prog, maps, log, dry_run, threads, keep_maps, total):
|
|
512
|
+
if dry_run:
|
|
513
|
+
print(f"Will project {total} maps")
|
|
514
|
+
return
|
|
515
|
+
|
|
516
|
+
task = prog.add_task(description="Projecting...", total=total)
|
|
517
|
+
|
|
518
|
+
torch.set_default_dtype(torch.float32)
|
|
519
|
+
set_start_method("spawn", force=True)
|
|
520
|
+
|
|
521
|
+
errors = []
|
|
522
|
+
with ProcessPoolExecutor(
|
|
523
|
+
max_workers=threads, initializer=os.nice, initargs=(10,)
|
|
524
|
+
) as pool:
|
|
525
|
+
results = {}
|
|
526
|
+
|
|
527
|
+
def finish(fut):
|
|
528
|
+
map_path = results.pop(fut)
|
|
529
|
+
if fut.exception():
|
|
530
|
+
err = fut.exception()
|
|
531
|
+
err.add_note(map_path.stem)
|
|
532
|
+
errors.append(err)
|
|
533
|
+
log.warning(f"failed projecting {map_path.stem}: {err}")
|
|
534
|
+
else:
|
|
535
|
+
proj_path = fut.result()
|
|
536
|
+
log.info(f"finished projecting {proj_path.stem}")
|
|
537
|
+
if not keep_maps:
|
|
538
|
+
# keep the map on failure so a retry need not re-download it
|
|
539
|
+
map_path.unlink(missing_ok=True)
|
|
540
|
+
prog.update(task, advance=1)
|
|
541
|
+
|
|
542
|
+
# drain finished projections while the download/extract stream still feeds us
|
|
543
|
+
for map_path in maps:
|
|
544
|
+
for fut in tuple(results):
|
|
545
|
+
if fut.done():
|
|
546
|
+
finish(fut)
|
|
547
|
+
results[pool.submit(_project_map_file, map_path)] = map_path
|
|
548
|
+
for fut in as_completed(results):
|
|
549
|
+
finish(fut)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def generate_db_summary(prog, db_path, log, dry_run):
|
|
553
|
+
entries = list(Path(db_path).glob("*.xml"))
|
|
554
|
+
task = prog.add_task(
|
|
555
|
+
description="Generating database summary...", total=len(entries)
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
rows = {}
|
|
559
|
+
bad = 0
|
|
560
|
+
for xml in entries:
|
|
561
|
+
prog.update(task, advance=1)
|
|
562
|
+
with open(xml, "rb") as f:
|
|
563
|
+
data = xmltodict.parse(f)
|
|
564
|
+
entry_id = int(re.search(r"emd-(\d+)", xml.stem).group(1))
|
|
565
|
+
shape = [int(i) for i in data["emd"]["map"]["dimensions"].values()]
|
|
566
|
+
px = [float(v["#text"]) for v in data["emd"]["map"]["pixel_spacing"].values()]
|
|
567
|
+
size = int(data["emd"]["map"]["@size_kbytes"])
|
|
568
|
+
structure_det = data["emd"]["structure_determination_list"][
|
|
569
|
+
"structure_determination"
|
|
570
|
+
]
|
|
571
|
+
# resolution
|
|
572
|
+
methods = (
|
|
573
|
+
"singleparticle",
|
|
574
|
+
"subtomogram_averaging",
|
|
575
|
+
"crystallography",
|
|
576
|
+
"helical",
|
|
577
|
+
"tomography",
|
|
578
|
+
)
|
|
579
|
+
if any(f"{method}_processing" in structure_det for method in methods):
|
|
580
|
+
pass
|
|
581
|
+
else:
|
|
582
|
+
log.info(f"{entry_id:04} is bad (method = {structure_det['method']})")
|
|
583
|
+
continue
|
|
584
|
+
|
|
585
|
+
errors = []
|
|
586
|
+
for method in methods:
|
|
587
|
+
try:
|
|
588
|
+
processing = structure_det[f"{method}_processing"]
|
|
589
|
+
if isinstance(processing, list):
|
|
590
|
+
processing = processing[0] # ugly but ok
|
|
591
|
+
resolution = float(
|
|
592
|
+
processing["final_reconstruction"]["resolution"]["#text"]
|
|
593
|
+
)
|
|
594
|
+
break
|
|
595
|
+
except Exception as e:
|
|
596
|
+
errors.append(e)
|
|
597
|
+
continue
|
|
598
|
+
else:
|
|
599
|
+
bad += 1
|
|
600
|
+
log.info(
|
|
601
|
+
f"{entry_id:04} is bad:\n"
|
|
602
|
+
+ ", ".join(f"{e.__class__.__name__}({e})" for e in errors)
|
|
603
|
+
)
|
|
604
|
+
resolution = np.nan
|
|
605
|
+
|
|
606
|
+
rows[entry_id] = [*shape, *px, size, resolution]
|
|
607
|
+
|
|
608
|
+
db = pd.DataFrame.from_dict(
|
|
609
|
+
rows,
|
|
610
|
+
orient="index",
|
|
611
|
+
columns=["x", "y", "z", "px_x", "px_y", "px_z", "size_kb", "resolution"],
|
|
612
|
+
)
|
|
613
|
+
db = db.astype(
|
|
614
|
+
{
|
|
615
|
+
"x": int,
|
|
616
|
+
"y": int,
|
|
617
|
+
"z": int,
|
|
618
|
+
"px_x": float,
|
|
619
|
+
"px_y": float,
|
|
620
|
+
"px_z": float,
|
|
621
|
+
"size_kb": int,
|
|
622
|
+
"resolution": float,
|
|
623
|
+
}
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
shape = db[["x", "y", "z"]]
|
|
627
|
+
max_dim = shape.max(axis=1)
|
|
628
|
+
size_mb = db["size_kb"] / 1e3
|
|
629
|
+
# convert shape to square to check for sizes (will have to happen in processing)
|
|
630
|
+
volume_when_cube = max_dim**3
|
|
631
|
+
volume_padded_GB = volume_when_cube * 4 / 1e9
|
|
632
|
+
volume_nm = volume_when_cube * db["px_x"] ** 3 / 1e3 # nm^3
|
|
633
|
+
rescale_ratio = (db["px_x"] / BIN_RESOLUTION) ** 3
|
|
634
|
+
volume_rescaled_GB = volume_padded_GB * rescale_ratio
|
|
635
|
+
|
|
636
|
+
too_big_or_small = (
|
|
637
|
+
(volume_nm < 1e3) | (volume_padded_GB > 1) | (volume_rescaled_GB > 1)
|
|
638
|
+
)
|
|
639
|
+
too_heavy = size_mb > 400
|
|
640
|
+
anisotropic = (db.px_x != db.px_y) | (db.px_x != db.px_z)
|
|
641
|
+
|
|
642
|
+
# discard bad stuff
|
|
643
|
+
to_discard = too_big_or_small | too_heavy | anisotropic
|
|
644
|
+
db = db[~to_discard]
|
|
645
|
+
|
|
646
|
+
log.warning(
|
|
647
|
+
f"Found {bad} entries ({100*bad/len(entries):.2f}% of headers) with no single particle info."
|
|
648
|
+
)
|
|
649
|
+
log.warning(
|
|
650
|
+
f"{to_discard.sum()} entries were discarded because too heavy, big, or small to process."
|
|
651
|
+
)
|
|
652
|
+
db.sort_index(inplace=True)
|
|
653
|
+
if not dry_run:
|
|
654
|
+
db.to_csv(
|
|
655
|
+
db_path / "database_summary.csv",
|
|
656
|
+
sep="\t",
|
|
657
|
+
header=True,
|
|
658
|
+
index=True,
|
|
659
|
+
na_rep="NaN",
|
|
660
|
+
)
|
|
661
|
+
return db
|