multixem 0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,310 @@
1
+ # coding: utf-8
2
+ import os
3
+ import logging
4
+ import warnings
5
+ import gemmi
6
+ import numpy
7
+ import pandas
8
+ from .tools import write_mtz_from_df
9
+
10
+
11
+ def bootstrap_dataset(
12
+ mtz_file: str,
13
+ binner: gemmi.Binner,
14
+ seeds=[1001, 1002, 1003],
15
+ labin: str = "",
16
+ draw_factor: float = 1.0,
17
+ random_weights: bool = False,
18
+ col_free: str = "",
19
+ fraction_zero: float = 0.05,
20
+ ):
21
+ """
22
+ Bootstrap the dataset from an MTZ file and save the results in new MTZ files.
23
+
24
+ Args:
25
+ mtz_file (str): Path to the input MTZ file.
26
+ binner (gemmi.Binner): gemmi.Binner object for resolution binning.
27
+ seeds (list of int): List of random seeds for bootstrapping.
28
+ labin (str): Column label (e.g. `IMEAN,SIGIMEAN`)
29
+ to apply `df.dropna(subset=[labin.split(",")[0]])`.
30
+ draw_factor (float): Factor for a number of draws in resampling.
31
+ By default, the number of draws is equal to the number of
32
+ reflections in each bin (draw_factor==1.0).
33
+ random_weights (bool): Assign random weights instead of
34
+ resampling with replacement.
35
+ col_free (str): Label for free R flag in `mtz_file` which would be used to set
36
+ zero weight for reflections of the free set
37
+ if `fraction_zero` is zero
38
+ fraction zero (float): Sets fraction of reflections with zero weight [0.0, 1.0)
39
+ Returns:
40
+ list of str: List of output MTZ filenames created during bootstrapping.
41
+ """
42
+
43
+ def resample(
44
+ n: int,
45
+ seed: int = 1001,
46
+ draw_factor: float = 1.0,
47
+ column_name: str = "llweight",
48
+ ):
49
+ """
50
+ Create a DataFrame`llweight` column using resampling with replacement.
51
+
52
+ Args:
53
+ n (int): Number of items to resample.
54
+ seed (int): Random seed for reproducibility.
55
+ draw_factor (float): Factor for a number of draws in resampling.
56
+ By default, the number of draws is equal to the
57
+ number of reflections in each bin (draw_factor==1.0).
58
+ column_name (str): Name of the column to create in the DataFrame.
59
+ Returns:
60
+ pandas.Series: Series with the bootstrap weights for each reflection.
61
+ """
62
+ n_draws = int(n * draw_factor)
63
+ rng = numpy.random.default_rng(seed)
64
+ df_random = pandas.DataFrame(
65
+ rng.integers(1, n + 1, size=n_draws), columns=["index_resample"]
66
+ )
67
+ df_weight = (
68
+ df_random.groupby(["index_resample"])
69
+ .size()
70
+ .reindex(range(1, n + 1), fill_value=0)
71
+ )
72
+ if draw_factor != 1.0:
73
+ df_weight = df_weight / draw_factor
74
+
75
+ return df_weight.rename(column_name)
76
+
77
+ def resample_random(
78
+ n: int,
79
+ zero_mask: numpy.ndarray,
80
+ seed: int = 1001,
81
+ column_name: str = "llweight",
82
+ ):
83
+ """
84
+ Create a DataFrame`llweight` column using random resampling and keeping
85
+ a fraction of zero weights.
86
+
87
+ Args:
88
+ n (int): Number of items to resample.
89
+ seed (int): Random seed for reproducibility.
90
+ zero_mask (numpy.ndarray): Optional boolean mask;
91
+ rows set to True are forced to zero.
92
+ Applied if `fraction_zero` is zero.
93
+ column_name (str): Name of the column to create in the DataFrame.
94
+ Returns:
95
+ pandas.Series: Series with the weights for each reflection.
96
+ """
97
+ rng = numpy.random.default_rng(seed)
98
+ df_random = pandas.DataFrame(rng.random(size=n), columns=["index_resample"])
99
+ df_weight = df_random["index_resample"].copy()
100
+ # Set a fraction of weights to zero based on the provided mask
101
+ zero_mask_array = numpy.asarray(zero_mask, dtype=bool)
102
+ if zero_mask_array.shape[0] != n:
103
+ raise ValueError(
104
+ "Length mismatch between zero_mask and bin size in "
105
+ f"random resampling: {zero_mask_array.shape[0]} != {n}"
106
+ )
107
+ df_weight.loc[zero_mask_array] = 0.0
108
+ # Renormalize so that the total weight equals n
109
+ weight_sum = df_weight.sum()
110
+ assert weight_sum > 0
111
+ df_weight = df_weight * (n / weight_sum)
112
+
113
+ return df_weight.rename(column_name)
114
+
115
+ if random_weights:
116
+ if abs(fraction_zero) > 1e-6:
117
+ logging.info(
118
+ f"\nBootstrapping dataset {mtz_file} (using random resampling"
119
+ f" and keeping fraction of zero weights {fraction_zero})",
120
+ )
121
+ elif col_free:
122
+ logging.info(
123
+ f"\nBootstrapping dataset {mtz_file} (using random resampling,"
124
+ " free reflections will be excluded)."
125
+ )
126
+ else:
127
+ raise RuntimeError(
128
+ f"Free reflections were not found in {mtz_file} and --fraction_zero"
129
+ " is not set. Cannot assign reflections with zero weight. Aborting."
130
+ )
131
+ else:
132
+ logging.info(
133
+ "\n"
134
+ f"Bootstrapping dataset {mtz_file} (using a draw factor of {draw_factor})",
135
+ )
136
+ mtzs_out = []
137
+ mtz = gemmi.read_mtz_file(mtz_file)
138
+ df = pandas.DataFrame(data=mtz.array, columns=mtz.column_labels())
139
+ df = df.astype({name: "int32" for name in ["H", "K", "L"]})
140
+ columns_dict = {
141
+ col.label: col.type for col in mtz.columns if col.label not in ["H", "K", "L"]
142
+ }
143
+
144
+ n_unique_orig = 0
145
+ # i_col = "IMEAN" # can be just "I" after servalcat fw or sigmaa, or IMEAN?
146
+ # dropping reflections can cause problems, let's save the filtered dataset as MTZ
147
+ if labin and labin.split(",")[0] in df.columns:
148
+ df = df.dropna(subset=[labin.split(",")[0]])
149
+ # Save the filtered dataset as MTZ, preserving all original columns
150
+ mtz_filtered_name = (
151
+ f"{os.path.splitext(os.path.basename(mtz_file))[0]}_filtered.mtz"
152
+ )
153
+ write_mtz_from_df(df, mtz, columns=columns_dict, filename=mtz_filtered_name)
154
+ n_unique_orig = df.shape[0]
155
+ else:
156
+ warnings.warn(
157
+ f"Column {labin} not found in MTZ file {mtz_file}. "
158
+ f"Using all reflections for bootstrapping."
159
+ )
160
+ n_unique_expected = gemmi.count_reflections(
161
+ mtz.cell,
162
+ mtz.spacegroup,
163
+ mtz.resolution_high(),
164
+ mtz.resolution_low(),
165
+ unique=True,
166
+ )
167
+
168
+ hkl_array = numpy.array(df[["H", "K", "L"]].values, numpy.int32)
169
+ hkl_array = numpy.ascontiguousarray(hkl_array, dtype=numpy.int32)
170
+ df["bin"] = binner.get_bins(hkl_array)
171
+ # print("No. unique reflections:", len(df))
172
+ # print(df.head(10))
173
+ # print(df.describe())
174
+
175
+ bins = [bin for _, bin in df.groupby("bin")]
176
+
177
+ # Create per-bin masks once and keep them for all bootstrap samples
178
+ zero_mask_bins = []
179
+ if random_weights:
180
+ if abs(fraction_zero) > 1e-6:
181
+ # Create a random mask for reflections with zero weight
182
+ rng = numpy.random.default_rng(seeds[0])
183
+ zero_mask_bins = [rng.random(len(bin)) < fraction_zero for bin in bins]
184
+ elif col_free:
185
+ # Use free reflections to create a mask for refls with zero weight
186
+ zero_mask_bins = [bin[col_free].eq(0).to_numpy() for bin in bins]
187
+ else:
188
+ raise RuntimeError(
189
+ f"Free reflections were not found in {mtz_file} and --fraction_zero"
190
+ " is not set. Cannot assign reflections with zero weight. Aborting."
191
+ )
192
+
193
+ completeness_list = []
194
+ for i, seed in enumerate(seeds):
195
+ if random_weights:
196
+ assert len(zero_mask_bins) == len(bins)
197
+ parts = []
198
+ for b, bin in enumerate(bins):
199
+ w = resample_random(
200
+ len(bin),
201
+ zero_mask_bins[b],
202
+ seed=seed,
203
+ )
204
+ parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
205
+ df_bootstrap1_weight = pandas.concat(parts).sort_index()
206
+ else:
207
+ parts = []
208
+ for bin in bins:
209
+ w = resample(len(bin), seed, draw_factor)
210
+ parts.append(pandas.Series(w.values, index=bin.index, name="llweight"))
211
+ df_bootstrap1_weight = pandas.concat(parts).sort_index()
212
+
213
+ # Keep original reflection order to preserve HKL-to-weight mapping.
214
+ df_bootstrap1_weight_hkl = df[["H", "K", "L"]].copy()
215
+ df_bootstrap1_weight_hkl["llweight"] = df_bootstrap1_weight.reindex(
216
+ df.index
217
+ ).values
218
+ weight_sum = df_bootstrap1_weight.sum()
219
+ if abs(weight_sum - len(df)) > 1e-6:
220
+ logging.warning(
221
+ f"Sum of weight coefficients ({weight_sum}) from bootstrap resampling"
222
+ f" {i} does not match the number of reflections ({len(df)})."
223
+ )
224
+
225
+ # Save the llweights in the MTZ file
226
+ mtz_out_name = (
227
+ f"{os.path.splitext(os.path.basename(mtz_file))[0]}_llweight{i}.mtz"
228
+ )
229
+ write_mtz_from_df(
230
+ df_bootstrap1_weight_hkl,
231
+ mtz,
232
+ columns={"llweight": "R"},
233
+ filename=mtz_out_name,
234
+ )
235
+ mtzs_out.append(mtz_out_name)
236
+
237
+ # Compute completeness
238
+ n_unique = len(
239
+ df_bootstrap1_weight_hkl[df_bootstrap1_weight_hkl["llweight"] > 0]
240
+ )
241
+ completeness = n_unique / n_unique_expected
242
+ completeness_list.append(completeness)
243
+
244
+ """if i == 0:
245
+ df_bootstrap1_weight_master = df_bootstrap1_weight.copy()
246
+ df_bootstrap1_weight_master = df_bootstrap1_weight_master.rename(
247
+ "llweight_0"
248
+ ).to_frame()
249
+ else:
250
+ df_bootstrap1_weight_master[f"llweight_{i}"] = (
251
+ df_bootstrap1_weight.values
252
+ )
253
+
254
+ columns = {i_col: "J", "SIG" + i_col: "Q", "llweight": "I"}
255
+ if "FreeR_flag" in df.columns:
256
+ columns["FreeR_flag"] = "I"
257
+ df_bootstrap1_data = df.merge(
258
+ df_bootstrap1_weight.copy(), left_index=True, right_index=True, how="left"
259
+ )
260
+ df_bootstrap1_data = df_bootstrap1_data[
261
+ df_bootstrap1_data["llweight"] > 0
262
+ ]
263
+ mtz_out_name = f"{os.path.splitext(mtz_file)[0]}_bootstrap_data_{i}.mtz"
264
+ write_mtz_from_df(df_bootstrap1_data, mtz, columns, filename=mtz_out_name)
265
+ mtzs_out.append(mtz_out_name)"""
266
+
267
+ """df_bootstrap1 = df.merge(
268
+ df_bootstrap1_weight_master, left_index=True, right_index=True, how="left"
269
+ )
270
+ # print(df_bootstrap1.head(10))
271
+ # print(df_bootstrap1.describe())
272
+ # TODO: other columns than I and FreeRflag?
273
+ columns = {i_col: "J", "SIG" + i_col: "Q"}
274
+ if "FreeR_flag" in df.columns:
275
+ columns["FreeR_flag"] = "I"
276
+ columns.update({f"llweight_{i}": "I" for i in range(len(seeds))})
277
+ write_mtz_from_df(
278
+ df_bootstrap1,
279
+ mtz,
280
+ columns,
281
+ filename=f"{os.path.splitext(mtz_file)[0]}_bootstrap.mtz",
282
+ )"""
283
+
284
+ completeness_mean = numpy.mean(completeness_list)
285
+ if random_weights:
286
+ if n_unique_orig:
287
+ logging.info(
288
+ "Completeness of bootstrap datasets"
289
+ " after exclusion of zero-weighted reflections:"
290
+ f" {completeness_mean:.2%}"
291
+ )
292
+ else:
293
+ logging.info(
294
+ "Completeness of bootstrap datasets"
295
+ " after exclusion of zero-weighted reflections:"
296
+ f" {completeness_mean:.2%}\n"
297
+ )
298
+ else:
299
+ completeness_std = numpy.std(completeness_list, ddof=1, mean=completeness_mean)
300
+ logging.info(
301
+ "Completeness of bootstrap datasets"
302
+ " after exclusion of zero-weighted reflections:"
303
+ f" {completeness_mean:.2%} ± {completeness_std:.2%}"
304
+ f" (using a draw factor of {draw_factor})\n"
305
+ )
306
+ if n_unique_orig:
307
+ completeness_orig = n_unique_orig / n_unique_expected
308
+ logging.info(f"Completeness of the original dataset: {completeness_orig:.2%}\n")
309
+
310
+ return mtzs_out