multixem 0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multixem/__init__.py +6 -0
- multixem/analyse_refinement.py +656 -0
- multixem/bootstrap_dataset.py +310 -0
- multixem/bootstrap_maps.py +558 -0
- multixem/bootstrap_statistics.py +783 -0
- multixem/bootstrap_structures.py +1569 -0
- multixem/run.py +2193 -0
- multixem/tools.py +521 -0
- multixem-0.3.dist-info/METADATA +175 -0
- multixem-0.3.dist-info/RECORD +14 -0
- multixem-0.3.dist-info/WHEEL +5 -0
- multixem-0.3.dist-info/entry_points.txt +2 -0
- multixem-0.3.dist-info/licenses/LICENSE +373 -0
- multixem-0.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
import os
|
|
3
|
+
import logging
|
|
4
|
+
import warnings
|
|
5
|
+
import gemmi
|
|
6
|
+
import numpy
|
|
7
|
+
import pandas
|
|
8
|
+
from .tools import write_mtz_from_df
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def bootstrap_dataset(
|
|
12
|
+
mtz_file: str,
|
|
13
|
+
binner: gemmi.Binner,
|
|
14
|
+
seeds=[1001, 1002, 1003],
|
|
15
|
+
labin: str = "",
|
|
16
|
+
draw_factor: float = 1.0,
|
|
17
|
+
random_weights: bool = False,
|
|
18
|
+
col_free: str = "",
|
|
19
|
+
fraction_zero: float = 0.05,
|
|
20
|
+
):
|
|
21
|
+
"""
|
|
22
|
+
Bootstrap the dataset from an MTZ file and save the results in new MTZ files.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
mtz_file (str): Path to the input MTZ file.
|
|
26
|
+
binner (gemmi.Binner): gemmi.Binner object for resolution binning.
|
|
27
|
+
seeds (list of int): List of random seeds for bootstrapping.
|
|
28
|
+
labin (str): Column label (e.g. `IMEAN,SIGIMEAN`)
|
|
29
|
+
to apply `df.dropna(subset=[labin.split(",")[0]])`.
|
|
30
|
+
draw_factor (float): Factor for a number of draws in resampling.
|
|
31
|
+
By default, the number of draws is equal to the number of
|
|
32
|
+
reflections in each bin (draw_factor==1.0).
|
|
33
|
+
random_weights (bool): Assign random weights instead of
|
|
34
|
+
resampling with replacement.
|
|
35
|
+
col_free (str): Label for free R flag in `mtz_file` which would be used to set
|
|
36
|
+
zero weight for reflections of the free set
|
|
37
|
+
if `fraction_zero` is zero
|
|
38
|
+
fraction zero (float): Sets fraction of reflections with zero weight [0.0, 1.0)
|
|
39
|
+
Returns:
|
|
40
|
+
list of str: List of output MTZ filenames created during bootstrapping.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def resample(
|
|
44
|
+
n: int,
|
|
45
|
+
seed: int = 1001,
|
|
46
|
+
draw_factor: float = 1.0,
|
|
47
|
+
column_name: str = "llweight",
|
|
48
|
+
):
|
|
49
|
+
"""
|
|
50
|
+
Create a DataFrame`llweight` column using resampling with replacement.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
n (int): Number of items to resample.
|
|
54
|
+
seed (int): Random seed for reproducibility.
|
|
55
|
+
draw_factor (float): Factor for a number of draws in resampling.
|
|
56
|
+
By default, the number of draws is equal to the
|
|
57
|
+
number of reflections in each bin (draw_factor==1.0).
|
|
58
|
+
column_name (str): Name of the column to create in the DataFrame.
|
|
59
|
+
Returns:
|
|
60
|
+
pandas.Series: Series with the bootstrap weights for each reflection.
|
|
61
|
+
"""
|
|
62
|
+
n_draws = int(n * draw_factor)
|
|
63
|
+
rng = numpy.random.default_rng(seed)
|
|
64
|
+
df_random = pandas.DataFrame(
|
|
65
|
+
rng.integers(1, n + 1, size=n_draws), columns=["index_resample"]
|
|
66
|
+
)
|
|
67
|
+
df_weight = (
|
|
68
|
+
df_random.groupby(["index_resample"])
|
|
69
|
+
.size()
|
|
70
|
+
.reindex(range(1, n + 1), fill_value=0)
|
|
71
|
+
)
|
|
72
|
+
if draw_factor != 1.0:
|
|
73
|
+
df_weight = df_weight / draw_factor
|
|
74
|
+
|
|
75
|
+
return df_weight.rename(column_name)
|
|
76
|
+
|
|
77
|
+
def resample_random(
|
|
78
|
+
n: int,
|
|
79
|
+
zero_mask: numpy.ndarray,
|
|
80
|
+
seed: int = 1001,
|
|
81
|
+
column_name: str = "llweight",
|
|
82
|
+
):
|
|
83
|
+
"""
|
|
84
|
+
Create a DataFrame`llweight` column using random resampling and keeping
|
|
85
|
+
a fraction of zero weights.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
n (int): Number of items to resample.
|
|
89
|
+
seed (int): Random seed for reproducibility.
|
|
90
|
+
zero_mask (numpy.ndarray): Optional boolean mask;
|
|
91
|
+
rows set to True are forced to zero.
|
|
92
|
+
Applied if `fraction_zero` is zero.
|
|
93
|
+
column_name (str): Name of the column to create in the DataFrame.
|
|
94
|
+
Returns:
|
|
95
|
+
pandas.Series: Series with the weights for each reflection.
|
|
96
|
+
"""
|
|
97
|
+
rng = numpy.random.default_rng(seed)
|
|
98
|
+
df_random = pandas.DataFrame(rng.random(size=n), columns=["index_resample"])
|
|
99
|
+
df_weight = df_random["index_resample"].copy()
|
|
100
|
+
# Set a fraction of weights to zero based on the provided mask
|
|
101
|
+
zero_mask_array = numpy.asarray(zero_mask, dtype=bool)
|
|
102
|
+
if zero_mask_array.shape[0] != n:
|
|
103
|
+
raise ValueError(
|
|
104
|
+
"Length mismatch between zero_mask and bin size in "
|
|
105
|
+
f"random resampling: {zero_mask_array.shape[0]} != {n}"
|
|
106
|
+
)
|
|
107
|
+
df_weight.loc[zero_mask_array] = 0.0
|
|
108
|
+
# Renormalize so that the total weight equals n
|
|
109
|
+
weight_sum = df_weight.sum()
|
|
110
|
+
assert weight_sum > 0
|
|
111
|
+
df_weight = df_weight * (n / weight_sum)
|
|
112
|
+
|
|
113
|
+
return df_weight.rename(column_name)
|
|
114
|
+
|
|
115
|
+
if random_weights:
|
|
116
|
+
if abs(fraction_zero) > 1e-6:
|
|
117
|
+
logging.info(
|
|
118
|
+
f"\nBootstrapping dataset {mtz_file} (using random resampling"
|
|
119
|
+
f" and keeping fraction of zero weights {fraction_zero})",
|
|
120
|
+
)
|
|
121
|
+
elif col_free:
|
|
122
|
+
logging.info(
|
|
123
|
+
f"\nBootstrapping dataset {mtz_file} (using random resampling,"
|
|
124
|
+
" free reflections will be excluded)."
|
|
125
|
+
)
|
|
126
|
+
else:
|
|
127
|
+
raise RuntimeError(
|
|
128
|
+
f"Free reflections were not found in {mtz_file} and --fraction_zero"
|
|
129
|
+
" is not set. Cannot assign reflections with zero weight. Aborting."
|
|
130
|
+
)
|
|
131
|
+
else:
|
|
132
|
+
logging.info(
|
|
133
|
+
"\n"
|
|
134
|
+
f"Bootstrapping dataset {mtz_file} (using a draw factor of {draw_factor})",
|
|
135
|
+
)
|
|
136
|
+
mtzs_out = []
|
|
137
|
+
mtz = gemmi.read_mtz_file(mtz_file)
|
|
138
|
+
df = pandas.DataFrame(data=mtz.array, columns=mtz.column_labels())
|
|
139
|
+
df = df.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
140
|
+
columns_dict = {
|
|
141
|
+
col.label: col.type for col in mtz.columns if col.label not in ["H", "K", "L"]
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
n_unique_orig = 0
|
|
145
|
+
# i_col = "IMEAN" # can be just "I" after servalcat fw or sigmaa, or IMEAN?
|
|
146
|
+
# dropping reflections can cause problems, let's save the filtered dataset as MTZ
|
|
147
|
+
if labin and labin.split(",")[0] in df.columns:
|
|
148
|
+
df = df.dropna(subset=[labin.split(",")[0]])
|
|
149
|
+
# Save the filtered dataset as MTZ, preserving all original columns
|
|
150
|
+
mtz_filtered_name = (
|
|
151
|
+
f"{os.path.splitext(os.path.basename(mtz_file))[0]}_filtered.mtz"
|
|
152
|
+
)
|
|
153
|
+
write_mtz_from_df(df, mtz, columns=columns_dict, filename=mtz_filtered_name)
|
|
154
|
+
n_unique_orig = df.shape[0]
|
|
155
|
+
else:
|
|
156
|
+
warnings.warn(
|
|
157
|
+
f"Column {labin} not found in MTZ file {mtz_file}. "
|
|
158
|
+
f"Using all reflections for bootstrapping."
|
|
159
|
+
)
|
|
160
|
+
n_unique_expected = gemmi.count_reflections(
|
|
161
|
+
mtz.cell,
|
|
162
|
+
mtz.spacegroup,
|
|
163
|
+
mtz.resolution_high(),
|
|
164
|
+
mtz.resolution_low(),
|
|
165
|
+
unique=True,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
hkl_array = numpy.array(df[["H", "K", "L"]].values, numpy.int32)
|
|
169
|
+
hkl_array = numpy.ascontiguousarray(hkl_array, dtype=numpy.int32)
|
|
170
|
+
df["bin"] = binner.get_bins(hkl_array)
|
|
171
|
+
# print("No. unique reflections:", len(df))
|
|
172
|
+
# print(df.head(10))
|
|
173
|
+
# print(df.describe())
|
|
174
|
+
|
|
175
|
+
bins = [bin for _, bin in df.groupby("bin")]
|
|
176
|
+
|
|
177
|
+
# Create per-bin masks once and keep them for all bootstrap samples
|
|
178
|
+
zero_mask_bins = []
|
|
179
|
+
if random_weights:
|
|
180
|
+
if abs(fraction_zero) > 1e-6:
|
|
181
|
+
# Create a random mask for reflections with zero weight
|
|
182
|
+
rng = numpy.random.default_rng(seeds[0])
|
|
183
|
+
zero_mask_bins = [rng.random(len(bin)) < fraction_zero for bin in bins]
|
|
184
|
+
elif col_free:
|
|
185
|
+
# Use free reflections to create a mask for refls with zero weight
|
|
186
|
+
zero_mask_bins = [bin[col_free].eq(0).to_numpy() for bin in bins]
|
|
187
|
+
else:
|
|
188
|
+
raise RuntimeError(
|
|
189
|
+
f"Free reflections were not found in {mtz_file} and --fraction_zero"
|
|
190
|
+
" is not set. Cannot assign reflections with zero weight. Aborting."
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
completeness_list = []
|
|
194
|
+
for i, seed in enumerate(seeds):
|
|
195
|
+
if random_weights:
|
|
196
|
+
assert len(zero_mask_bins) == len(bins)
|
|
197
|
+
parts = []
|
|
198
|
+
for b, bin in enumerate(bins):
|
|
199
|
+
w = resample_random(
|
|
200
|
+
len(bin),
|
|
201
|
+
zero_mask_bins[b],
|
|
202
|
+
seed=seed,
|
|
203
|
+
)
|
|
204
|
+
parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
|
|
205
|
+
df_bootstrap1_weight = pandas.concat(parts).sort_index()
|
|
206
|
+
else:
|
|
207
|
+
parts = []
|
|
208
|
+
for bin in bins:
|
|
209
|
+
w = resample(len(bin), seed, draw_factor)
|
|
210
|
+
parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
|
|
211
|
+
df_bootstrap1_weight = pandas.concat(parts).sort_index()
|
|
212
|
+
|
|
213
|
+
# Keep original reflection order to preserve HKL-to-weight mapping.
|
|
214
|
+
df_bootstrap1_weight_hkl = df[["H", "K", "L"]].copy()
|
|
215
|
+
df_bootstrap1_weight_hkl["llweight"] = df_bootstrap1_weight.reindex(
|
|
216
|
+
df.index
|
|
217
|
+
).values
|
|
218
|
+
weight_sum = df_bootstrap1_weight.sum()
|
|
219
|
+
if abs(weight_sum - len(df)) > 1e-6:
|
|
220
|
+
logging.warning(
|
|
221
|
+
f"Sum of weight coefficients ({weight_sum}) from bootstrap resampling"
|
|
222
|
+
f" {i} does not match the number of reflections ({len(df)})."
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
# Save the llweights in the MTZ file
|
|
226
|
+
mtz_out_name = (
|
|
227
|
+
f"{os.path.splitext(os.path.basename(mtz_file))[0]}_llweight{i}.mtz"
|
|
228
|
+
)
|
|
229
|
+
write_mtz_from_df(
|
|
230
|
+
df_bootstrap1_weight_hkl,
|
|
231
|
+
mtz,
|
|
232
|
+
columns={"llweight": "R"},
|
|
233
|
+
filename=mtz_out_name,
|
|
234
|
+
)
|
|
235
|
+
mtzs_out.append(mtz_out_name)
|
|
236
|
+
|
|
237
|
+
# Compute completeness
|
|
238
|
+
n_unique = len(
|
|
239
|
+
df_bootstrap1_weight_hkl[df_bootstrap1_weight_hkl["llweight"] > 0]
|
|
240
|
+
)
|
|
241
|
+
completeness = n_unique / n_unique_expected
|
|
242
|
+
completeness_list.append(completeness)
|
|
243
|
+
|
|
244
|
+
"""if i == 0:
|
|
245
|
+
df_bootstrap1_weight_master = df_bootstrap1_weight.copy()
|
|
246
|
+
df_bootstrap1_weight_master = df_bootstrap1_weight_master.rename(
|
|
247
|
+
"llweight_0"
|
|
248
|
+
).to_frame()
|
|
249
|
+
else:
|
|
250
|
+
df_bootstrap1_weight_master[f"llweight_{i}"] = (
|
|
251
|
+
df_bootstrap1_weight.values
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
columns = {i_col: "J", "SIG" + i_col: "Q", "llweight": "I"}
|
|
255
|
+
if "FreeR_flag" in df.columns:
|
|
256
|
+
columns["FreeR_flag"] = "I"
|
|
257
|
+
df_bootstrap1_data = df.merge(
|
|
258
|
+
df_bootstrap1_weight.copy(), left_index=True, right_index=True, how="left"
|
|
259
|
+
)
|
|
260
|
+
df_bootstrap1_data = df_bootstrap1_data[
|
|
261
|
+
df_bootstrap1_data["llweight"] > 0
|
|
262
|
+
]
|
|
263
|
+
mtz_out_name = f"{os.path.splitext(mtz_file)[0]}_bootstrap_data_{i}.mtz"
|
|
264
|
+
write_mtz_from_df(df_bootstrap1_data, mtz, columns, filename=mtz_out_name)
|
|
265
|
+
mtzs_out.append(mtz_out_name)"""
|
|
266
|
+
|
|
267
|
+
"""df_bootstrap1 = df.merge(
|
|
268
|
+
df_bootstrap1_weight_master, left_index=True, right_index=True, how="left"
|
|
269
|
+
)
|
|
270
|
+
# print(df_bootstrap1.head(10))
|
|
271
|
+
# print(df_bootstrap1.describe())
|
|
272
|
+
# TODO: other columns than I and FreeRflag?
|
|
273
|
+
columns = {i_col: "J", "SIG" + i_col: "Q"}
|
|
274
|
+
if "FreeR_flag" in df.columns:
|
|
275
|
+
columns["FreeR_flag"] = "I"
|
|
276
|
+
columns.update({f"llweight_{i}": "I" for i in range(len(seeds))})
|
|
277
|
+
write_mtz_from_df(
|
|
278
|
+
df_bootstrap1,
|
|
279
|
+
mtz,
|
|
280
|
+
columns,
|
|
281
|
+
filename=f"{os.path.splitext(mtz_file)[0]}_bootstrap.mtz",
|
|
282
|
+
)"""
|
|
283
|
+
|
|
284
|
+
completeness_mean = numpy.mean(completeness_list)
|
|
285
|
+
if random_weights:
|
|
286
|
+
if n_unique_orig:
|
|
287
|
+
logging.info(
|
|
288
|
+
"Completeness of bootstrap datasets"
|
|
289
|
+
" after exclusion of zero-weighted reflections:"
|
|
290
|
+
f" {completeness_mean:.2%}"
|
|
291
|
+
)
|
|
292
|
+
else:
|
|
293
|
+
logging.info(
|
|
294
|
+
"Completeness of bootstrap datasets"
|
|
295
|
+
" after exclusion of zero-weighted reflections:"
|
|
296
|
+
f" {completeness_mean:.2%}\n"
|
|
297
|
+
)
|
|
298
|
+
else:
|
|
299
|
+
completeness_std = numpy.std(completeness_list, ddof=1, mean=completeness_mean)
|
|
300
|
+
logging.info(
|
|
301
|
+
"Completeness of bootstrap datasets"
|
|
302
|
+
" after exclusion of zero-weighted reflections:"
|
|
303
|
+
f" {completeness_mean:.2%} ± {completeness_std:.2%}"
|
|
304
|
+
f" (using a draw factor of {draw_factor})\n"
|
|
305
|
+
)
|
|
306
|
+
if n_unique_orig:
|
|
307
|
+
completeness_orig = n_unique_orig / n_unique_expected
|
|
308
|
+
logging.info(f"Completeness of the original dataset: {completeness_orig:.2%}\n")
|
|
309
|
+
|
|
310
|
+
return mtzs_out
|