multixem 0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multixem/__init__.py +6 -0
- multixem/analyse_refinement.py +656 -0
- multixem/bootstrap_dataset.py +310 -0
- multixem/bootstrap_maps.py +558 -0
- multixem/bootstrap_statistics.py +783 -0
- multixem/bootstrap_structures.py +1569 -0
- multixem/run.py +2193 -0
- multixem/tools.py +521 -0
- multixem-0.3.dist-info/METADATA +175 -0
- multixem-0.3.dist-info/RECORD +14 -0
- multixem-0.3.dist-info/WHEEL +5 -0
- multixem-0.3.dist-info/entry_points.txt +2 -0
- multixem-0.3.dist-info/licenses/LICENSE +373 -0
- multixem-0.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,558 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
import concurrent.futures
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import warnings
|
|
7
|
+
import gemmi
|
|
8
|
+
import numpy
|
|
9
|
+
import pandas
|
|
10
|
+
from .tools import (
|
|
11
|
+
json_numpy_converter,
|
|
12
|
+
scale_reflections,
|
|
13
|
+
write_bin_stats,
|
|
14
|
+
write_mtz_from_df,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def bootstrap_mean_map(
|
|
19
|
+
refined_mtzs, idx=0, prefix="", binner=None, mtz_ref="", n_proc=4
|
|
20
|
+
):
|
|
21
|
+
"""
|
|
22
|
+
Calculate the mean 2Fo-Fc and Fo-Fc maps from refined MTZ files after bootstrapping.
|
|
23
|
+
The maps are expected to be after refinement against a bootstrapped
|
|
24
|
+
data set.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
refined_mtzs (list of str): List of MTZ filenames.
|
|
28
|
+
idx (int): Index for naming the output file (applies if not set to 0).
|
|
29
|
+
prefix (str): Prefix for the output filename.
|
|
30
|
+
binner (gemmi.Binner): Binner object for resolution bins (optional).
|
|
31
|
+
mtz_ref (str): Reference MTZ file for scaling (optional).
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
None: Writes the mean maps in
|
|
35
|
+
'{prefix}group{idx}_bootstrap_mean_map_all.mtz'
|
|
36
|
+
'{prefix}group{idx}_bootstrap_mean_map_llweight0.mtz'
|
|
37
|
+
'{prefix}group{idx}_bootstrap_mean_map_llweightpos.mtz'
|
|
38
|
+
'{prefix}group{idx}_bootstrap_mean_map_llweightposw.mtz'
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def merge_reflections_bootstrap(
|
|
42
|
+
df_master,
|
|
43
|
+
mtz_first=None,
|
|
44
|
+
prefix="",
|
|
45
|
+
suffix="",
|
|
46
|
+
idx=0,
|
|
47
|
+
binner=None,
|
|
48
|
+
mtz_ref="",
|
|
49
|
+
do_llweighting=False,
|
|
50
|
+
):
|
|
51
|
+
"""
|
|
52
|
+
Merge reflections from the master DataFrame and calculate mean maps.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
df_master (pandas.DataFrame): DataFrame containing reflections.
|
|
56
|
+
It must contain columns "H", "K", "L", "F_complex", "DEL_F_complex",
|
|
57
|
+
mtz_first (gemmi.Mtz): Reference MTZ object for cell and spacegroup.
|
|
58
|
+
prefix (str): Prefix for the output filename.
|
|
59
|
+
suffix (str): Suffix for the output filename.
|
|
60
|
+
idx (int): Index for naming the output file.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
pandas.DataFrame: DataFrame with mean maps.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
"""# noqa: E741
|
|
67
|
+
def is_centric_vectorized(h, k, l): # noqa: E741
|
|
68
|
+
return mtz_first.spacegroup.operations().is_reflection_centric(
|
|
69
|
+
(int(h), int(k), int(l)) # noqa: E741
|
|
70
|
+
)"""
|
|
71
|
+
|
|
72
|
+
def calculate_mean_std_count(df, do_llweighting=False):
|
|
73
|
+
"""Calculate mean and standard deviation and number of structure factors."""
|
|
74
|
+
|
|
75
|
+
def stats_func(miller_index_df, column_name, do_llweighting=do_llweighting):
|
|
76
|
+
"""
|
|
77
|
+
Compute weighted or unweighted mean, std, and count
|
|
78
|
+
for one Miller index.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
miller_index_df (pandas.DataFrame): DataFrame for
|
|
82
|
+
a specific Miller index.
|
|
83
|
+
column_name (str): Column name to compute stats on.
|
|
84
|
+
do_llweighting (bool): Whether to apply llweighting.
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
pandas.Series: Series containing mean, std, and count.
|
|
88
|
+
"""
|
|
89
|
+
x = miller_index_df[column_name].values
|
|
90
|
+
|
|
91
|
+
if len(x) <= 1:
|
|
92
|
+
return pandas.Series([numpy.mean(x), 0.0, len(x)])
|
|
93
|
+
|
|
94
|
+
if do_llweighting and "llweight" in miller_index_df.columns:
|
|
95
|
+
# Weighted mean and variance
|
|
96
|
+
w = miller_index_df["llweight"].values
|
|
97
|
+
w = w / numpy.sum(w) # normalize weights
|
|
98
|
+
mean_val = numpy.sum(w * x)
|
|
99
|
+
|
|
100
|
+
real_mean = numpy.real(mean_val)
|
|
101
|
+
imag_mean = numpy.imag(mean_val)
|
|
102
|
+
real_part = numpy.real(x)
|
|
103
|
+
imag_part = numpy.imag(x)
|
|
104
|
+
real_var = numpy.sum(w * (real_part - real_mean) ** 2)
|
|
105
|
+
imag_var = numpy.sum(w * (imag_part - imag_mean) ** 2)
|
|
106
|
+
else:
|
|
107
|
+
# Unweighted mean and variance
|
|
108
|
+
mean_val = numpy.mean(x)
|
|
109
|
+
real_mean = numpy.real(mean_val)
|
|
110
|
+
imag_mean = numpy.imag(mean_val)
|
|
111
|
+
real_part = numpy.real(x)
|
|
112
|
+
imag_part = numpy.imag(x)
|
|
113
|
+
real_var = numpy.var(real_part, ddof=1, mean=real_mean)
|
|
114
|
+
imag_var = numpy.var(imag_part, ddof=1, mean=imag_mean)
|
|
115
|
+
|
|
116
|
+
std_val = numpy.sqrt(real_var + imag_var)
|
|
117
|
+
return pandas.Series([mean_val, std_val, len(x)])
|
|
118
|
+
|
|
119
|
+
# F_complex: apply stats_func to each Miller index
|
|
120
|
+
df_mean_fwt = df.groupby(["H", "K", "L"], as_index=False).apply(
|
|
121
|
+
lambda d: stats_func(d, "F_complex", do_llweighting=do_llweighting),
|
|
122
|
+
include_groups=False,
|
|
123
|
+
)
|
|
124
|
+
# This converts Series to DataFrame
|
|
125
|
+
# df_mean_fwt = df_mean_fwt.unstack(level=-1)
|
|
126
|
+
df_mean_fwt.columns = [
|
|
127
|
+
"H",
|
|
128
|
+
"K",
|
|
129
|
+
"L",
|
|
130
|
+
"F_complex_mean",
|
|
131
|
+
"SIGFWT",
|
|
132
|
+
"FWTcount",
|
|
133
|
+
]
|
|
134
|
+
with warnings.catch_warnings():
|
|
135
|
+
warnings.simplefilter("ignore", numpy.exceptions.ComplexWarning)
|
|
136
|
+
df_mean_fwt["SIGFWT"] = df_mean_fwt["SIGFWT"].astype(numpy.float32)
|
|
137
|
+
df_mean_fwt["FWTcount"] = df_mean_fwt["FWTcount"].astype(numpy.int32)
|
|
138
|
+
df_mean_fwt = df_mean_fwt.reset_index()
|
|
139
|
+
|
|
140
|
+
# DEL_F_complex: apply stats_func to each Miller index
|
|
141
|
+
df_mean_delfwt = df.groupby(["H", "K", "L"], as_index=False).apply(
|
|
142
|
+
lambda d: stats_func(d, "DEL_F_complex", do_llweighting=do_llweighting),
|
|
143
|
+
include_groups=False,
|
|
144
|
+
)
|
|
145
|
+
# This converts Series to DataFrame
|
|
146
|
+
# df_mean_delfwt = df_mean_delfwt.unstack(level=-1)
|
|
147
|
+
df_mean_delfwt.columns = [
|
|
148
|
+
"H",
|
|
149
|
+
"K",
|
|
150
|
+
"L",
|
|
151
|
+
"DEL_F_complex_mean",
|
|
152
|
+
"SIGDELFWT",
|
|
153
|
+
"DELFWTcount",
|
|
154
|
+
]
|
|
155
|
+
with warnings.catch_warnings():
|
|
156
|
+
warnings.simplefilter("ignore", numpy.exceptions.ComplexWarning)
|
|
157
|
+
df_mean_delfwt["SIGDELFWT"] = df_mean_delfwt["SIGDELFWT"].astype(
|
|
158
|
+
numpy.float32
|
|
159
|
+
)
|
|
160
|
+
df_mean_delfwt["DELFWTcount"] = df_mean_delfwt["DELFWTcount"].astype(
|
|
161
|
+
numpy.int32
|
|
162
|
+
)
|
|
163
|
+
df_mean_delfwt = df_mean_delfwt.reset_index()
|
|
164
|
+
|
|
165
|
+
df_mean_fwt_delfwt = df_mean_fwt.merge(
|
|
166
|
+
df_mean_delfwt, on=["H", "K", "L"], how="outer"
|
|
167
|
+
)
|
|
168
|
+
return df_mean_fwt_delfwt
|
|
169
|
+
|
|
170
|
+
"""
|
|
171
|
+
# old code which treated centric and acentric reflections differently
|
|
172
|
+
# and variance of acentric reflections was divided by two
|
|
173
|
+
if is_centric:
|
|
174
|
+
sigma_func = lambda x: ( # noqa: E731
|
|
175
|
+
numpy.sqrt(numpy.var(numpy.abs(x), ddof=1)) if len(x) > 1 else 0
|
|
176
|
+
)
|
|
177
|
+
else:
|
|
178
|
+
sigma_func = lambda x: ( # noqa: E731
|
|
179
|
+
numpy.sqrt(
|
|
180
|
+
(numpy.var(numpy.real(x), ddof=1)
|
|
181
|
+
+ numpy.var(numpy.imag(x), ddof=1)) / 2
|
|
182
|
+
)
|
|
183
|
+
if len(x) > 1
|
|
184
|
+
else 0
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
df_mean_f = (
|
|
188
|
+
df.groupby(["H", "K", "L"])["F_complex"]
|
|
189
|
+
.agg(
|
|
190
|
+
[
|
|
191
|
+
("F_complex_mean", lambda x: numpy.mean(x)),
|
|
192
|
+
("SIGFWT", sigma_func),
|
|
193
|
+
("FWTcount", "count"),
|
|
194
|
+
]
|
|
195
|
+
)
|
|
196
|
+
.reset_index()
|
|
197
|
+
)
|
|
198
|
+
df_mean_delf = (
|
|
199
|
+
df.groupby(["H", "K", "L"])["DEL_F_complex"]
|
|
200
|
+
.agg(
|
|
201
|
+
[
|
|
202
|
+
("DEL_F_complex_mean", lambda x: numpy.mean(x)),
|
|
203
|
+
("SIGDELFWT", sigma_func),
|
|
204
|
+
("DELFWTcount", "count"),
|
|
205
|
+
]
|
|
206
|
+
)
|
|
207
|
+
.reset_index()
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
return df_mean_f.merge(df_mean_delf, on=["H", "K", "L"], how="outer")"""
|
|
211
|
+
|
|
212
|
+
"""
|
|
213
|
+
is_centric_vec = numpy.vectorize(is_centric_vectorized)
|
|
214
|
+
centric_mask = is_centric_vec(df_master["H"], df_master["K"], df_master["L"])
|
|
215
|
+
acentric = df_master[~centric_mask]
|
|
216
|
+
centric = df_master[centric_mask]
|
|
217
|
+
stats_acentric = calculate_mean_std_count(acentric, is_centric=False)
|
|
218
|
+
stats_centric = calculate_mean_std_count(centric, is_centric=True)
|
|
219
|
+
df_mean = pandas.concat([stats_acentric, stats_centric], ignore_index=True)"""
|
|
220
|
+
|
|
221
|
+
df_master = df_master.astype({col: "int32" for col in ["H", "K", "L"]})
|
|
222
|
+
df_mean = calculate_mean_std_count(df_master, do_llweighting=do_llweighting)
|
|
223
|
+
|
|
224
|
+
# Convert to amplitude and phase
|
|
225
|
+
df_mean["FWT"] = numpy.abs(df_mean["F_complex_mean"])
|
|
226
|
+
df_mean["PHWT"] = numpy.rad2deg(numpy.angle(df_mean["F_complex_mean"]))
|
|
227
|
+
df_mean["DELFWT"] = numpy.abs(df_mean["DEL_F_complex_mean"])
|
|
228
|
+
df_mean["PHDELWT"] = numpy.rad2deg(numpy.angle(df_mean["DEL_F_complex_mean"]))
|
|
229
|
+
|
|
230
|
+
if mtz_first and prefix and suffix and idx:
|
|
231
|
+
# Save the mean maps as an MTZ file
|
|
232
|
+
columns = {
|
|
233
|
+
"FWT": "F",
|
|
234
|
+
"PHWT": "P",
|
|
235
|
+
"SIGFWT": "Q",
|
|
236
|
+
"FWTcount": "I",
|
|
237
|
+
"DELFWT": "F",
|
|
238
|
+
"PHDELWT": "P",
|
|
239
|
+
"SIGDELFWT": "Q",
|
|
240
|
+
"DELFWTcount": "I",
|
|
241
|
+
}
|
|
242
|
+
mtz_filename = (
|
|
243
|
+
f"{prefix}group{idx}_bootstrap_mean_map{suffix}.mtz"
|
|
244
|
+
if idx
|
|
245
|
+
else f"{prefix}bootstrap_mean_map{suffix}.mtz"
|
|
246
|
+
)
|
|
247
|
+
write_mtz_from_df(
|
|
248
|
+
df_mean[
|
|
249
|
+
[
|
|
250
|
+
"H",
|
|
251
|
+
"K",
|
|
252
|
+
"L",
|
|
253
|
+
"FWT",
|
|
254
|
+
"PHWT",
|
|
255
|
+
"SIGFWT",
|
|
256
|
+
"FWTcount",
|
|
257
|
+
"DELFWT",
|
|
258
|
+
"PHDELWT",
|
|
259
|
+
"SIGDELFWT",
|
|
260
|
+
"DELFWTcount",
|
|
261
|
+
]
|
|
262
|
+
],
|
|
263
|
+
mtz_first,
|
|
264
|
+
columns,
|
|
265
|
+
filename=mtz_filename,
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
# Calculate statistics per bin
|
|
269
|
+
if binner and mtz_first:
|
|
270
|
+
hkl_array = numpy.array(df_mean[["H", "K", "L"]].values, numpy.int32)
|
|
271
|
+
hkl_array = numpy.ascontiguousarray(hkl_array, dtype=numpy.int32)
|
|
272
|
+
df_mean["bin"] = binner.get_bins(hkl_array)
|
|
273
|
+
bin_stats = []
|
|
274
|
+
for b in range(binner.size):
|
|
275
|
+
df_bin = df_mean[df_mean["bin"] == b]
|
|
276
|
+
if not df_bin.empty:
|
|
277
|
+
mean_fwt = df_bin["FWT"].mean()
|
|
278
|
+
mean_sigfwt = df_bin["SIGFWT"].mean()
|
|
279
|
+
mean_fwt_sigfwt = mean_fwt / mean_sigfwt if mean_sigfwt else 0.0
|
|
280
|
+
fwt_count = df_bin["FWTcount"].sum()
|
|
281
|
+
mean_delfwt = df_bin["DELFWT"].mean()
|
|
282
|
+
mean_sigdelfwt = df_bin["SIGDELFWT"].mean()
|
|
283
|
+
mean_delfwt_sigdelfwt = (
|
|
284
|
+
mean_delfwt / mean_sigdelfwt if mean_sigdelfwt else 0.0
|
|
285
|
+
)
|
|
286
|
+
delfwt_count = df_bin["DELFWTcount"].sum()
|
|
287
|
+
bin_n_unique = len(df_bin)
|
|
288
|
+
bin_n_unique_expected = gemmi.count_reflections(
|
|
289
|
+
mtz_first.cell,
|
|
290
|
+
mtz_first.spacegroup,
|
|
291
|
+
binner.dmin_of_bin(b),
|
|
292
|
+
binner.dmax_of_bin(b),
|
|
293
|
+
unique=True,
|
|
294
|
+
)
|
|
295
|
+
completeness = bin_n_unique / bin_n_unique_expected
|
|
296
|
+
bin_stats.append(
|
|
297
|
+
{
|
|
298
|
+
"bin": b + 1,
|
|
299
|
+
"dmax": binner.dmax_of_bin(b),
|
|
300
|
+
"dmin": binner.dmin_of_bin(b),
|
|
301
|
+
"mean_FWT": mean_fwt,
|
|
302
|
+
"mean_SIGFWT": mean_sigfwt,
|
|
303
|
+
"mean_FWT_SIGFWT": mean_fwt_sigfwt,
|
|
304
|
+
"FWTcount": fwt_count,
|
|
305
|
+
"mean_DELFWT": mean_delfwt,
|
|
306
|
+
"mean_SIGDELFWT": mean_sigdelfwt,
|
|
307
|
+
"mean_DELFWT_SIGDELFWT": mean_delfwt_sigdelfwt,
|
|
308
|
+
"DELFWTcount": delfwt_count,
|
|
309
|
+
"count": bin_n_unique,
|
|
310
|
+
"completeness": completeness,
|
|
311
|
+
}
|
|
312
|
+
)
|
|
313
|
+
else:
|
|
314
|
+
bin_stats.append(
|
|
315
|
+
{
|
|
316
|
+
"bin": b + 1,
|
|
317
|
+
"dmax": binner.dmax_of_bin(b),
|
|
318
|
+
"dmin": binner.dmin_of_bin(b),
|
|
319
|
+
"mean_FWT": 0.0,
|
|
320
|
+
"mean_SIGFWT": 0.0,
|
|
321
|
+
"mean_FWT_SIGFWT": 0.0,
|
|
322
|
+
"FWTcount": 0,
|
|
323
|
+
"mean_DELFWT": 0.0,
|
|
324
|
+
"mean_SIGDELFWT": 0.0,
|
|
325
|
+
"mean_DELFWT_SIGDELFWT": 0.0,
|
|
326
|
+
"DELFWTcount": 0,
|
|
327
|
+
"count": 0,
|
|
328
|
+
"completeness": 0.0,
|
|
329
|
+
}
|
|
330
|
+
)
|
|
331
|
+
if prefix and suffix:
|
|
332
|
+
stats_filename = (
|
|
333
|
+
f"{prefix}group{idx}_bootstrap_mean_map{suffix}.txt"
|
|
334
|
+
if idx
|
|
335
|
+
else f"{prefix}bootstrap_mean_map{suffix}.txt"
|
|
336
|
+
)
|
|
337
|
+
write_bin_stats(bin_stats, stats_filename)
|
|
338
|
+
|
|
339
|
+
if mtz_ref and binner and prefix and suffix:
|
|
340
|
+
mtz_scaled_prefix = (
|
|
341
|
+
f"{prefix}group{idx}_bootstrap_mean_map{suffix}"
|
|
342
|
+
if idx
|
|
343
|
+
else f"{prefix}bootstrap_mean_map{suffix}"
|
|
344
|
+
)
|
|
345
|
+
df_scaled, bin_stats_scaled = scale_reflections(
|
|
346
|
+
mtz_ref,
|
|
347
|
+
df_mean.copy(),
|
|
348
|
+
binner,
|
|
349
|
+
output_mtz2_prefix=mtz_scaled_prefix,
|
|
350
|
+
)
|
|
351
|
+
"""if mtz_ref and prefix and suffix and idx:
|
|
352
|
+
# Save the scaled mean maps as an MTZ file
|
|
353
|
+
columns_scaled = {
|
|
354
|
+
"FWT": "F",
|
|
355
|
+
"PHWT": "P",
|
|
356
|
+
"DELFWT": "F",
|
|
357
|
+
"PHDELWT": "P",
|
|
358
|
+
}
|
|
359
|
+
write_mtz_from_df(
|
|
360
|
+
df_scaled[["H", "K", "L"] + list(columns_scaled.keys())],
|
|
361
|
+
mtz_ref,
|
|
362
|
+
columns_scaled,
|
|
363
|
+
mtz_scaled_filename,
|
|
364
|
+
)"""
|
|
365
|
+
|
|
366
|
+
# Calculate statistics per bin
|
|
367
|
+
# if binner and mtz_ref:
|
|
368
|
+
hkl_array = numpy.array(df_scaled[["H", "K", "L"]].values, numpy.int32)
|
|
369
|
+
hkl_array = numpy.ascontiguousarray(hkl_array, dtype=numpy.int32)
|
|
370
|
+
df_scaled["bin"] = binner.get_bins(hkl_array)
|
|
371
|
+
for b in range(binner.size):
|
|
372
|
+
df_bin = df_scaled[df_scaled["bin"] == b]
|
|
373
|
+
if not df_bin.empty:
|
|
374
|
+
mean_fwt = df_bin["FWT"].mean()
|
|
375
|
+
# mean_sigfwt = df_bin["SIGFWT"].mean()
|
|
376
|
+
# mean_fwt_sigfwt = mean_fwt / mean_sigfwt if mean_sigfwt else 0.0
|
|
377
|
+
# fwt_count = df_bin["FWTcount"].sum()
|
|
378
|
+
mean_delfwt = df_bin["DELFWT"].mean()
|
|
379
|
+
# mean_sigdelfwt = df_bin["SIGDELFWT"].mean()
|
|
380
|
+
# mean_delfwt_sigdelfwt = (
|
|
381
|
+
# # mean_delfwt / mean_sigdelfwt if mean_sigdelfwt else 0.0
|
|
382
|
+
# )
|
|
383
|
+
# delfwt_count = df_bin["DELFWTcount"].sum()
|
|
384
|
+
bin_n_unique = len(df_bin)
|
|
385
|
+
"""bin_n_unique_expected = gemmi.count_reflections(
|
|
386
|
+
mtz_first.cell,
|
|
387
|
+
mtz_first.spacegroup,
|
|
388
|
+
binner.dmin_of_bin(b),
|
|
389
|
+
binner.dmax_of_bin(b),
|
|
390
|
+
unique=True,
|
|
391
|
+
)
|
|
392
|
+
completeness = bin_n_unique / bin_n_unique_expected"""
|
|
393
|
+
bin_stats_scaled[b].update(
|
|
394
|
+
{
|
|
395
|
+
"bin": b + 1,
|
|
396
|
+
"dmax": binner.dmax_of_bin(b),
|
|
397
|
+
"dmin": binner.dmin_of_bin(b),
|
|
398
|
+
"mean_FWT": mean_fwt,
|
|
399
|
+
# "mean_SIGFWT": mean_sigfwt,
|
|
400
|
+
# "mean_FWT_SIGFWT": mean_fwt_sigfwt,
|
|
401
|
+
# "FWTcount": fwt_count,
|
|
402
|
+
"mean_DELFWT": mean_delfwt,
|
|
403
|
+
# "mean_SIGDELFWT": mean_sigdelfwt,
|
|
404
|
+
# "mean_DELFWT_SIGDELFWT": mean_delfwt_sigdelfwt,
|
|
405
|
+
# "DELFWTcount": delfwt_count,
|
|
406
|
+
"count": bin_n_unique,
|
|
407
|
+
# "completeness": completeness,
|
|
408
|
+
}
|
|
409
|
+
)
|
|
410
|
+
else:
|
|
411
|
+
bin_stats_scaled[b].update(
|
|
412
|
+
{
|
|
413
|
+
"bin": b + 1,
|
|
414
|
+
"dmax": binner.dmax_of_bin(b),
|
|
415
|
+
"dmin": binner.dmin_of_bin(b),
|
|
416
|
+
"mean_FWT": 0.0,
|
|
417
|
+
# "mean_SIGFWT": 0.0,
|
|
418
|
+
# "mean_FWT_SIGFWT": 0.0,
|
|
419
|
+
# "FWTcount": 0,
|
|
420
|
+
"mean_DELFWT": 0.0,
|
|
421
|
+
# "mean_SIGDELFWT": 0.0,
|
|
422
|
+
# "mean_DELFWT_SIGDELFWT": 0.0,
|
|
423
|
+
# "DELFWTcount": 0,
|
|
424
|
+
"count": 0,
|
|
425
|
+
# "completeness": 0.0,
|
|
426
|
+
}
|
|
427
|
+
)
|
|
428
|
+
stats_scaled_filename = mtz_scaled_prefix + "_scaled_stats.txt"
|
|
429
|
+
write_bin_stats(bin_stats_scaled, stats_scaled_filename)
|
|
430
|
+
|
|
431
|
+
return df_scaled
|
|
432
|
+
|
|
433
|
+
logging.info(f"Loading {len(refined_mtzs)} density maps...")
|
|
434
|
+
columns_selected = ["H", "K", "L", "FWT", "PHWT", "DELFWT", "PHDELWT", "llweight"]
|
|
435
|
+
if binner:
|
|
436
|
+
if mtz_ref:
|
|
437
|
+
logging.info(f"Scaling reflections to {mtz_ref}")
|
|
438
|
+
else:
|
|
439
|
+
mtz_first = gemmi.read_mtz_file(refined_mtzs[0])
|
|
440
|
+
col_labels_first = mtz_first.column_labels()
|
|
441
|
+
df_first = pandas.DataFrame(data=mtz_first.array, columns=col_labels_first)
|
|
442
|
+
df_first = df_first[columns_selected]
|
|
443
|
+
logging.info(f"Scaling reflections to {refined_mtzs[0]}")
|
|
444
|
+
|
|
445
|
+
# Process MTZ files in parallel
|
|
446
|
+
def _add_reflections(worker_args):
|
|
447
|
+
"""
|
|
448
|
+
Worker function to scale one MTZ file for bootstrap mean map calculation.
|
|
449
|
+
Returns (df, bin_stats) or (None, None) on failure.
|
|
450
|
+
"""
|
|
451
|
+
mtz_file, columns_selected, binner, mtz_ref = worker_args
|
|
452
|
+
try:
|
|
453
|
+
mtz = gemmi.read_mtz_file(mtz_file)
|
|
454
|
+
col_labels = mtz.column_labels()
|
|
455
|
+
df = pandas.DataFrame(data=mtz.array, columns=col_labels)
|
|
456
|
+
df = df[columns_selected]
|
|
457
|
+
if df.empty:
|
|
458
|
+
logging.warning(
|
|
459
|
+
f"No reflections in {mtz_file} for FWT/PHWT/DELFWT/PHDELWT."
|
|
460
|
+
)
|
|
461
|
+
return None, None
|
|
462
|
+
|
|
463
|
+
bin_stats = None
|
|
464
|
+
if binner and mtz_ref:
|
|
465
|
+
# scale per resolution bin
|
|
466
|
+
mtz_file_base = os.path.splitext(os.path.basename(mtz_file))[0]
|
|
467
|
+
df, bin_stats = scale_reflections(
|
|
468
|
+
mtz_ref, df, binner, output_mtz2_prefix=mtz_file_base
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
df = df.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
472
|
+
return df, bin_stats
|
|
473
|
+
|
|
474
|
+
except Exception as e:
|
|
475
|
+
logging.error(f"Error processing {mtz_file}: {e}")
|
|
476
|
+
import traceback
|
|
477
|
+
|
|
478
|
+
traceback.print_exc()
|
|
479
|
+
return None, None
|
|
480
|
+
|
|
481
|
+
worker_args_list = [
|
|
482
|
+
(mtz_file, columns_selected, binner, mtz_ref) for mtz_file in refined_mtzs
|
|
483
|
+
]
|
|
484
|
+
df_list = []
|
|
485
|
+
bin_stats_bootstrap_scale = []
|
|
486
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=n_proc) as executor:
|
|
487
|
+
futures = [
|
|
488
|
+
executor.submit(_add_reflections, worker_args)
|
|
489
|
+
for worker_args in worker_args_list
|
|
490
|
+
]
|
|
491
|
+
for future in concurrent.futures.as_completed(futures):
|
|
492
|
+
df, bin_stats = future.result()
|
|
493
|
+
if df is not None:
|
|
494
|
+
df_list.append(df)
|
|
495
|
+
if bin_stats is not None:
|
|
496
|
+
bin_stats_bootstrap_scale.append(bin_stats)
|
|
497
|
+
if df_list:
|
|
498
|
+
df_master = pandas.concat(df_list, ignore_index=True)
|
|
499
|
+
else:
|
|
500
|
+
logging.error("No valid MTZ files processed.")
|
|
501
|
+
return
|
|
502
|
+
|
|
503
|
+
if bin_stats_bootstrap_scale:
|
|
504
|
+
try:
|
|
505
|
+
json_filename = (
|
|
506
|
+
f"{prefix}group{idx}_bootstrap_map_scaling_stats.json"
|
|
507
|
+
if idx
|
|
508
|
+
else f"{prefix}bootstrap_map_scaling_stats.json"
|
|
509
|
+
)
|
|
510
|
+
with open(json_filename, "w") as f_json:
|
|
511
|
+
json.dump(
|
|
512
|
+
bin_stats_bootstrap_scale,
|
|
513
|
+
f_json,
|
|
514
|
+
indent=4,
|
|
515
|
+
default=json_numpy_converter,
|
|
516
|
+
)
|
|
517
|
+
logging.info(f"Saved bootstrap map scaling stats to {json_filename}")
|
|
518
|
+
except Exception as e:
|
|
519
|
+
logging.warning(
|
|
520
|
+
f"Could not write bootstrap map scaling stats JSON file: {e}"
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
# Convert FWT & PHWT and DELFWT & PHDELWT to complex numbers and calculate mean
|
|
524
|
+
df_master["F_complex"] = df_master["FWT"] * numpy.exp(
|
|
525
|
+
1j * numpy.deg2rad(df_master["PHWT"])
|
|
526
|
+
)
|
|
527
|
+
df_master["DEL_F_complex"] = df_master["DELFWT"] * numpy.exp(
|
|
528
|
+
1j * numpy.deg2rad(df_master["PHDELWT"])
|
|
529
|
+
)
|
|
530
|
+
# print(df_master.head(10))
|
|
531
|
+
# print(df_master[["H", "K", "L", "FWT", "PHWT"]].describe())
|
|
532
|
+
df_master_llweight_0 = df_master[df_master["llweight"] == 0].copy()
|
|
533
|
+
df_master_llweight_pos = df_master[df_master["llweight"] > 0].copy()
|
|
534
|
+
|
|
535
|
+
mtz_first = gemmi.read_mtz_file(refined_mtzs[0])
|
|
536
|
+
# save 4 mean maps: all reflections, llweight == 0,
|
|
537
|
+
# llweight > 0 and llweight > 0 weighted average
|
|
538
|
+
merge_reflections_bootstrap(
|
|
539
|
+
df_master, mtz_first, prefix, "_all", idx, binner, mtz_ref
|
|
540
|
+
)
|
|
541
|
+
merge_reflections_bootstrap(
|
|
542
|
+
df_master_llweight_0, mtz_first, prefix, "_llweight0", idx, binner, mtz_ref
|
|
543
|
+
)
|
|
544
|
+
merge_reflections_bootstrap(
|
|
545
|
+
df_master_llweight_pos, mtz_first, prefix, "_llweightpos", idx, binner, mtz_ref
|
|
546
|
+
)
|
|
547
|
+
merge_reflections_bootstrap(
|
|
548
|
+
df_master_llweight_pos,
|
|
549
|
+
mtz_first,
|
|
550
|
+
prefix,
|
|
551
|
+
"_llweightposw",
|
|
552
|
+
idx,
|
|
553
|
+
binner,
|
|
554
|
+
mtz_ref,
|
|
555
|
+
do_llweighting=True,
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
return
|