multixem 0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multixem/__init__.py +6 -0
- multixem/analyse_refinement.py +656 -0
- multixem/bootstrap_dataset.py +310 -0
- multixem/bootstrap_maps.py +558 -0
- multixem/bootstrap_statistics.py +783 -0
- multixem/bootstrap_structures.py +1569 -0
- multixem/run.py +2193 -0
- multixem/tools.py +521 -0
- multixem-0.3.dist-info/METADATA +175 -0
- multixem-0.3.dist-info/RECORD +14 -0
- multixem-0.3.dist-info/WHEEL +5 -0
- multixem-0.3.dist-info/entry_points.txt +2 -0
- multixem-0.3.dist-info/licenses/LICENSE +373 -0
- multixem-0.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,783 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
import os
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
import gemmi
|
|
7
|
+
import matplotlib
|
|
8
|
+
import matplotlib.pyplot as plt
|
|
9
|
+
import matplotlib.ticker as ticker
|
|
10
|
+
import numpy
|
|
11
|
+
import pandas
|
|
12
|
+
from .tools import filename_replace_char, json_numpy_converter
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
matplotlib.use("Agg")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def match_sigfigs(value, ref):
|
|
19
|
+
"""Format `value` to have the same number of significant figures as `ref`."""
|
|
20
|
+
if ref == 0:
|
|
21
|
+
return f"{value:.2f}"
|
|
22
|
+
decimals = max(0, -int(numpy.floor(numpy.log10(abs(ref)))) + 1)
|
|
23
|
+
return f"{value:.{decimals}f}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def df_scatter_plot(
|
|
27
|
+
df, x_cols, y_cols_groups, filename="scatter_plot.png", per_element=False
|
|
28
|
+
):
|
|
29
|
+
"""
|
|
30
|
+
Create multiple scatter subplots from a DataFrame and save as PNG.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
df (pandas.DataFrame or str or file stream): DataFrame or a CSV file.
|
|
34
|
+
x_cols (list of str): Column names for x-axis for each subplot.
|
|
35
|
+
If per_element=True,
|
|
36
|
+
x_cols should be 'atomic_number' or 'atom_id'.
|
|
37
|
+
y_cols_groups (list of list of str): Each sublist is a group of
|
|
38
|
+
y columns for one subplot.
|
|
39
|
+
filename (str): Output PNG filename.
|
|
40
|
+
per_element (bool): If True, create separate plots for each atom.
|
|
41
|
+
Hydrogens will be excluded.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def extract_atomic_numbers(df, atom_id_col):
|
|
45
|
+
# atom_id can look like "A/00 1/H19A", I want to extract "H"
|
|
46
|
+
atom_labels = df[atom_id_col]
|
|
47
|
+
proton_numbers = []
|
|
48
|
+
for atom_id in atom_labels:
|
|
49
|
+
atom_id_part = atom_id.split("/")[-1]
|
|
50
|
+
match = re.match(r"^([A-Z][a-z]?)", atom_id_part)
|
|
51
|
+
element = match.group(1)
|
|
52
|
+
proton_number = gemmi.Element(element).atomic_number
|
|
53
|
+
proton_numbers.append(proton_number)
|
|
54
|
+
return numpy.array(proton_numbers)
|
|
55
|
+
|
|
56
|
+
if isinstance(df, str):
|
|
57
|
+
df = pandas.read_csv(df)
|
|
58
|
+
|
|
59
|
+
n_subplots = len(y_cols_groups)
|
|
60
|
+
ncols = min(n_subplots, 2)
|
|
61
|
+
nrows = int(numpy.ceil(n_subplots / ncols))
|
|
62
|
+
fig, axes = plt.subplots(
|
|
63
|
+
nrows=nrows, ncols=ncols, figsize=(6 * ncols, 5 * nrows), squeeze=False
|
|
64
|
+
)
|
|
65
|
+
axes = axes.flatten()
|
|
66
|
+
max_y = 0
|
|
67
|
+
max_y_b = 0
|
|
68
|
+
if not per_element:
|
|
69
|
+
assert len(x_cols) == len(
|
|
70
|
+
y_cols_groups
|
|
71
|
+
), "x and y_groups must have the same length"
|
|
72
|
+
|
|
73
|
+
# Create a combined mask: True only where all x columns are not null
|
|
74
|
+
# for finding x_max and y_max
|
|
75
|
+
combined_mask = numpy.logical_and.reduce([df[col].notnull() for col in x_cols])
|
|
76
|
+
df_filt = df[combined_mask]
|
|
77
|
+
|
|
78
|
+
max_x = 0
|
|
79
|
+
max_x_b = 0
|
|
80
|
+
for data in x_cols:
|
|
81
|
+
if "sigma_b" not in data:
|
|
82
|
+
max_x = max(max_x, df_filt[data].max())
|
|
83
|
+
for i, group in enumerate(y_cols_groups):
|
|
84
|
+
for data in group:
|
|
85
|
+
if "sigma_b" not in data:
|
|
86
|
+
max_y = max(max_y, df_filt[data].max())
|
|
87
|
+
else: # per_element
|
|
88
|
+
if x_cols[0] != "atomic_number": # typically x_cols[0] == "atom_id":
|
|
89
|
+
# get atomic numbers from atom_id and save it as a new column
|
|
90
|
+
df["atomic_number"] = extract_atomic_numbers(df, x_cols[0])
|
|
91
|
+
df = df[df["atomic_number"] > 1] # exclude hydrogens
|
|
92
|
+
for i, group in enumerate(y_cols_groups):
|
|
93
|
+
for data in group:
|
|
94
|
+
if "sigma_b" not in data:
|
|
95
|
+
max_y = max(max_y, df[data].max())
|
|
96
|
+
|
|
97
|
+
for i, y_group in enumerate(y_cols_groups):
|
|
98
|
+
ax = axes[i]
|
|
99
|
+
ax.set_axisbelow(True)
|
|
100
|
+
for j, y_col in enumerate(y_group):
|
|
101
|
+
if not per_element:
|
|
102
|
+
ax.scatter(df[x_cols[i]], df[y_col], marker=".", label=y_col, alpha=0.5)
|
|
103
|
+
ax.set_xlabel(x_cols[i])
|
|
104
|
+
ax.set_title(x_cols[i] + " vs " + ", ".join(y_group))
|
|
105
|
+
ax.legend(loc="upper left")
|
|
106
|
+
else: # per_element
|
|
107
|
+
ax.scatter(
|
|
108
|
+
df["atomic_number"] + j * 0.2,
|
|
109
|
+
df[y_col],
|
|
110
|
+
marker=".",
|
|
111
|
+
label=y_col,
|
|
112
|
+
alpha=0.5,
|
|
113
|
+
)
|
|
114
|
+
ax.set_title(", ".join(y_group) + " per element")
|
|
115
|
+
ax.legend()
|
|
116
|
+
if not per_element:
|
|
117
|
+
if "sigma_b" not in y_col:
|
|
118
|
+
ax.set_xlim(0, max_x * 1.05)
|
|
119
|
+
ax.set_ylim(0, max_y * 1.05)
|
|
120
|
+
else: # not per_element, "sigma_b"
|
|
121
|
+
if "sigma_b" in x_cols[i]:
|
|
122
|
+
max_x_b_candidate = df_filt[x_cols[i]].max()
|
|
123
|
+
max_x_b = max(max_x_b, max_x_b_candidate)
|
|
124
|
+
ax.set_xlim(0, max_x_b * 1.05)
|
|
125
|
+
max_y_b_candidate = df_filt[y_col].max()
|
|
126
|
+
max_y_b = max(max_y_b, max_y_b_candidate)
|
|
127
|
+
ax.set_ylim(0, max_y_b * 1.05)
|
|
128
|
+
else: # per_element # do not touch xlim
|
|
129
|
+
if "sigma_b" not in y_col:
|
|
130
|
+
ax.set_ylim(0, max_y * 1.05)
|
|
131
|
+
else: # per_element, "sigma_b"
|
|
132
|
+
max_y_b_candidate = df[y_col].max()
|
|
133
|
+
max_y_b = max(max_y_b, max_y_b_candidate)
|
|
134
|
+
ax.set_ylim(0, max_y_b * 1.05)
|
|
135
|
+
if len(y_group) == 1:
|
|
136
|
+
ax.set_ylabel(y_group[0])
|
|
137
|
+
ax.grid()
|
|
138
|
+
|
|
139
|
+
# Hide unused axes if any
|
|
140
|
+
for j in range(i + 1, len(axes)):
|
|
141
|
+
fig.delaxes(axes[j])
|
|
142
|
+
|
|
143
|
+
plt.tight_layout()
|
|
144
|
+
plt.savefig(filename)
|
|
145
|
+
plt.close()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def analyse_distribution(
|
|
149
|
+
values, xlabel, outlier_factor=3.0, idx=0, prefix="", filtered=False, save=True
|
|
150
|
+
):
|
|
151
|
+
"""
|
|
152
|
+
Analyse a distribution of values and return summary statistics.
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
values (list or numpy array): The values to be analysed.
|
|
156
|
+
xlabel (str): The label for the x-axis of the histogram.
|
|
157
|
+
outlier_factor (float): The factor to determine outliers based on IQR.
|
|
158
|
+
idx (int): Index for naming the output files (applies if not set to 0).
|
|
159
|
+
prefix (str): Prefix for the output filenames.
|
|
160
|
+
filtered (bool): Whether the values are already filtered,
|
|
161
|
+
which will be reflected in the output filenames.
|
|
162
|
+
save (bool): Whether to save the histogram and outliers to CSV files.
|
|
163
|
+
Returns:
|
|
164
|
+
dict: A dictionary containing summary statistics,
|
|
165
|
+
including histogram bins and counts.
|
|
166
|
+
dict: A dictionary containing histogram bins and counts.
|
|
167
|
+
pandas.DataFrame or None: A DataFrame containing outliers with their
|
|
168
|
+
indices and values, or None if there are no outliers.
|
|
169
|
+
"""
|
|
170
|
+
if len(values) == 0:
|
|
171
|
+
return {}, {}, None
|
|
172
|
+
mean = numpy.mean(values)
|
|
173
|
+
stdev = numpy.std(values, ddof=1)
|
|
174
|
+
median = numpy.median(values)
|
|
175
|
+
q1 = numpy.percentile(values, 25)
|
|
176
|
+
q3 = numpy.percentile(values, 75)
|
|
177
|
+
iqr = q3 - q1
|
|
178
|
+
# MAD to be consistent with stdev for normal distribution
|
|
179
|
+
mad = numpy.median(numpy.abs(values - median)) * 1.4826
|
|
180
|
+
min_val = numpy.min(values)
|
|
181
|
+
max_val = numpy.max(values)
|
|
182
|
+
n_values = len(values)
|
|
183
|
+
distr_analysis_dict = {
|
|
184
|
+
"mean": mean,
|
|
185
|
+
"stdev": stdev,
|
|
186
|
+
"median": median,
|
|
187
|
+
"q1": q1,
|
|
188
|
+
"q3": q3,
|
|
189
|
+
"iqr": iqr,
|
|
190
|
+
"mad": mad,
|
|
191
|
+
"min": min_val,
|
|
192
|
+
"max": max_val,
|
|
193
|
+
"n": n_values,
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if not save:
|
|
197
|
+
return distr_analysis_dict, {}, None
|
|
198
|
+
|
|
199
|
+
counts, bins = numpy.histogram(values, bins="auto")
|
|
200
|
+
|
|
201
|
+
values_arr = numpy.asarray(values)
|
|
202
|
+
|
|
203
|
+
# IQR-based criterion
|
|
204
|
+
threshold_low = q1 - outlier_factor * iqr
|
|
205
|
+
threshold_high = q3 + outlier_factor * iqr
|
|
206
|
+
mask_iqr = (values_arr < threshold_low) | (values_arr > threshold_high)
|
|
207
|
+
|
|
208
|
+
# Additional median-relative criterion only for R1 ?
|
|
209
|
+
# is_r1 = str(xlabel).strip().upper() == "R1"
|
|
210
|
+
# if is_r1 and median > 0:
|
|
211
|
+
# rel_low = 0.7 * median
|
|
212
|
+
# rel_high = 1.3 * median
|
|
213
|
+
# mask_rel = (values_arr < rel_low) | (values_arr > rel_high)
|
|
214
|
+
|
|
215
|
+
# # Outlier must violate BOTH criteria
|
|
216
|
+
# outlier_mask = mask_iqr & mask_rel
|
|
217
|
+
# else:
|
|
218
|
+
# outlier_mask = mask_iqr
|
|
219
|
+
outlier_mask = mask_iqr
|
|
220
|
+
|
|
221
|
+
outlier_idx = numpy.where(outlier_mask)[0]
|
|
222
|
+
outliers = (
|
|
223
|
+
None
|
|
224
|
+
if outlier_idx.size == 0
|
|
225
|
+
else pandas.DataFrame({"index": outlier_idx, "value": values_arr[outlier_idx]})
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
csv_values_filename = filename_replace_char(f"histogram_{xlabel}_values.csv")
|
|
229
|
+
csv_histogram_filename = filename_replace_char(f"histogram_{xlabel}.csv")
|
|
230
|
+
if filtered:
|
|
231
|
+
csv_values_filename = csv_values_filename.replace(
|
|
232
|
+
"histogram", "histogram_filtered"
|
|
233
|
+
)
|
|
234
|
+
csv_histogram_filename = csv_histogram_filename.replace(
|
|
235
|
+
"histogram", "histogram_filtered"
|
|
236
|
+
)
|
|
237
|
+
if idx:
|
|
238
|
+
csv_values_filename = f"{prefix}group{idx}_bootstrap_{csv_values_filename}"
|
|
239
|
+
csv_histogram_filename = (
|
|
240
|
+
f"{prefix}group{idx}_bootstrap_{csv_histogram_filename}"
|
|
241
|
+
)
|
|
242
|
+
else:
|
|
243
|
+
csv_values_filename = f"{prefix}{csv_values_filename}"
|
|
244
|
+
csv_histogram_filename = f"{prefix}{csv_histogram_filename}"
|
|
245
|
+
|
|
246
|
+
df_values = pandas.DataFrame(values)
|
|
247
|
+
df_values.to_csv(csv_values_filename, index=False, header=False)
|
|
248
|
+
logging.info(f"Saved raw values to {csv_values_filename}")
|
|
249
|
+
|
|
250
|
+
bin_centers = (bins[:-1] + bins[1:]) / 2
|
|
251
|
+
df_histogram = pandas.DataFrame(
|
|
252
|
+
{
|
|
253
|
+
"bin_start": bins[:-1],
|
|
254
|
+
"bin_center": bin_centers,
|
|
255
|
+
"bin_end": bins[1:],
|
|
256
|
+
"count": counts,
|
|
257
|
+
}
|
|
258
|
+
)
|
|
259
|
+
df_histogram.to_csv(csv_histogram_filename, index=False, header=False)
|
|
260
|
+
logging.info(f"Saved histogram bins and counts to {csv_histogram_filename}")
|
|
261
|
+
|
|
262
|
+
if outliers is not None and not filtered:
|
|
263
|
+
csv_outliers_filename = filename_replace_char(
|
|
264
|
+
f"histogram_{xlabel}_outliers.csv"
|
|
265
|
+
)
|
|
266
|
+
if idx:
|
|
267
|
+
csv_outliers_filename = (
|
|
268
|
+
f"{prefix}group{idx}_bootstrap_{csv_outliers_filename}"
|
|
269
|
+
)
|
|
270
|
+
else:
|
|
271
|
+
csv_outliers_filename = f"{prefix}{csv_outliers_filename}"
|
|
272
|
+
outliers.to_csv(csv_outliers_filename, index=False, header=False)
|
|
273
|
+
logging.info(
|
|
274
|
+
f"Saved {len(outliers)} outliers outside the interval"
|
|
275
|
+
f" [{threshold_low:.4f}, {threshold_high:.4f}] to {csv_outliers_filename}"
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
return (
|
|
279
|
+
distr_analysis_dict,
|
|
280
|
+
{
|
|
281
|
+
"bins": bins,
|
|
282
|
+
"counts": counts.astype(int),
|
|
283
|
+
},
|
|
284
|
+
outliers,
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def plot_histogram(values, xlabel, ref={}, idx=0, prefix="", outlier_factor=3.0):
|
|
289
|
+
"""
|
|
290
|
+
Plot a histogram of the data and save as PNG.
|
|
291
|
+
|
|
292
|
+
Args:
|
|
293
|
+
data (list or numpy array): Data to plot.
|
|
294
|
+
xlabel (str): Label for the x-axis and the output file.
|
|
295
|
+
ref (dict): Reference values for the plot {label: value}.
|
|
296
|
+
idx (int): Index for naming the output file (applies if not set to 0).
|
|
297
|
+
prefix (str): Prefix for the output filename.
|
|
298
|
+
outlier_factor (float): Factor for determining outliers.
|
|
299
|
+
"""
|
|
300
|
+
distr, hist, outliers = analyse_distribution(
|
|
301
|
+
values, xlabel, idx=idx, prefix=prefix, outlier_factor=outlier_factor
|
|
302
|
+
)
|
|
303
|
+
min_val = distr["min"]
|
|
304
|
+
max_val = distr["max"]
|
|
305
|
+
if ref:
|
|
306
|
+
for ref_value in ref.values():
|
|
307
|
+
min_val = min(min_val, ref_value)
|
|
308
|
+
max_val = max(max_val, ref_value)
|
|
309
|
+
|
|
310
|
+
plt.figure(figsize=(8, 6))
|
|
311
|
+
plt.bar(
|
|
312
|
+
hist["bins"][:-1],
|
|
313
|
+
hist["counts"],
|
|
314
|
+
width=numpy.diff(hist["bins"]),
|
|
315
|
+
align="edge",
|
|
316
|
+
alpha=0.7,
|
|
317
|
+
)
|
|
318
|
+
plt.xlabel(xlabel)
|
|
319
|
+
plt.ylabel("Frequency")
|
|
320
|
+
plt.gca().yaxis.set_major_locator(
|
|
321
|
+
ticker.MaxNLocator(integer=True)
|
|
322
|
+
) # Ensure integer y-axis labels
|
|
323
|
+
buffer = (max_val - min_val) * 0.05 # 5% buffer around the data range
|
|
324
|
+
plt.xlim(min_val - buffer, max_val + buffer)
|
|
325
|
+
plt.axvline(
|
|
326
|
+
distr["mean"],
|
|
327
|
+
color="blue",
|
|
328
|
+
linestyle="--",
|
|
329
|
+
label=(
|
|
330
|
+
"Mean ± St.Dev. ="
|
|
331
|
+
f" {match_sigfigs(distr['mean'], distr['stdev'])} ± {distr['stdev']:.2g}"
|
|
332
|
+
),
|
|
333
|
+
)
|
|
334
|
+
plt.axvline(
|
|
335
|
+
distr["median"],
|
|
336
|
+
color="green",
|
|
337
|
+
linestyle="--",
|
|
338
|
+
label=(
|
|
339
|
+
"Median ± MAD ="
|
|
340
|
+
f" {match_sigfigs(distr['median'], distr['mad'])} ± {distr['mad']:.2g}"
|
|
341
|
+
),
|
|
342
|
+
)
|
|
343
|
+
for stat_ref_label, stat_value in ref.items():
|
|
344
|
+
plt.axvline(
|
|
345
|
+
stat_value,
|
|
346
|
+
color="orange",
|
|
347
|
+
linestyle="--",
|
|
348
|
+
label=(
|
|
349
|
+
"From single refinement:"
|
|
350
|
+
f" {stat_ref_label} = {match_sigfigs(stat_value, distr['stdev'])}"
|
|
351
|
+
),
|
|
352
|
+
)
|
|
353
|
+
plt.grid(axis="y", alpha=0.7)
|
|
354
|
+
plt.tight_layout()
|
|
355
|
+
plt.legend(title=f"n = {len(values)}")
|
|
356
|
+
png_filename = filename_replace_char(f"histogram_{xlabel}.png")
|
|
357
|
+
if idx:
|
|
358
|
+
png_filename = f"{prefix}group{idx}_bootstrap_{png_filename}"
|
|
359
|
+
else:
|
|
360
|
+
png_filename = f"{prefix}{png_filename}"
|
|
361
|
+
plt.savefig(png_filename)
|
|
362
|
+
plt.close()
|
|
363
|
+
logging.info(f"Saved histogram to {png_filename}")
|
|
364
|
+
|
|
365
|
+
return distr, hist, outliers
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def scatter_plot_histogram(
|
|
369
|
+
x, y, label, stat_ref={}, idx=0, prefix="", filtered=True, outlier_factor=3.0
|
|
370
|
+
):
|
|
371
|
+
"""
|
|
372
|
+
Plot a scatter plot of x vs y including histograms and save as PNG.
|
|
373
|
+
|
|
374
|
+
Args:
|
|
375
|
+
x (list or numpy array): Data for the x-axis.
|
|
376
|
+
y (list or numpy array): Data for the y-axis.
|
|
377
|
+
stat_ref (dict): Reference statistic for the plot {label: value}.
|
|
378
|
+
label (str): Label for the axes and the output file.
|
|
379
|
+
idx (int): Index for naming the output file (applies if not set to 0).
|
|
380
|
+
prefix (str): Prefix for the output filename.
|
|
381
|
+
filtered (bool): Whether the values are already filtered.
|
|
382
|
+
outlier_factor (float): Factor for determining outliers.
|
|
383
|
+
"""
|
|
384
|
+
if len(x) != len(y):
|
|
385
|
+
raise ValueError("x and y must have the same length.")
|
|
386
|
+
|
|
387
|
+
distr, hist, outliers = analyse_distribution(
|
|
388
|
+
x,
|
|
389
|
+
label,
|
|
390
|
+
idx=idx,
|
|
391
|
+
prefix=prefix,
|
|
392
|
+
filtered=filtered,
|
|
393
|
+
outlier_factor=outlier_factor,
|
|
394
|
+
)
|
|
395
|
+
distr_init, _, _ = analyse_distribution(
|
|
396
|
+
y,
|
|
397
|
+
label,
|
|
398
|
+
idx=idx,
|
|
399
|
+
prefix=prefix,
|
|
400
|
+
filtered=filtered,
|
|
401
|
+
save=False,
|
|
402
|
+
outlier_factor=outlier_factor,
|
|
403
|
+
)
|
|
404
|
+
min_val = min(min(x), min(y))
|
|
405
|
+
max_val = max(max(x), max(y))
|
|
406
|
+
if stat_ref:
|
|
407
|
+
for ref_value in stat_ref.values():
|
|
408
|
+
min_val = min(min_val, ref_value)
|
|
409
|
+
max_val = max(max_val, ref_value)
|
|
410
|
+
buffer = (max_val - min_val) * 0.05 # 5% buffer around the data range
|
|
411
|
+
|
|
412
|
+
fig, axs = plt.subplot_mosaic(
|
|
413
|
+
[["histx"], ["scatter"]],
|
|
414
|
+
sharex=True,
|
|
415
|
+
figsize=(7, 9),
|
|
416
|
+
height_ratios=(1, 2),
|
|
417
|
+
layout="constrained",
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
ax = axs["scatter"]
|
|
421
|
+
ax_histx = axs["histx"]
|
|
422
|
+
|
|
423
|
+
"""
|
|
424
|
+
binwidth = 0.25
|
|
425
|
+
xymax = max(numpy.max(numpy.abs(x)), numpy.max(numpy.abs(y)))
|
|
426
|
+
lim = (int(xymax/binwidth) + 1) * binwidth
|
|
427
|
+
|
|
428
|
+
bins = numpy.arange(-lim, lim + binwidth, binwidth)
|
|
429
|
+
"""
|
|
430
|
+
|
|
431
|
+
ax.scatter(x, y, alpha=0.1)
|
|
432
|
+
ax.plot( # line y=x
|
|
433
|
+
[min_val - buffer, max_val + buffer],
|
|
434
|
+
[min_val - buffer, max_val + buffer],
|
|
435
|
+
color="gray",
|
|
436
|
+
linestyle="--",
|
|
437
|
+
alpha=0.7,
|
|
438
|
+
)
|
|
439
|
+
ax.axvline(
|
|
440
|
+
distr["mean"],
|
|
441
|
+
color="blue",
|
|
442
|
+
linestyle="--",
|
|
443
|
+
label=(
|
|
444
|
+
"Mean ± St.Dev. ="
|
|
445
|
+
f" {match_sigfigs(distr['mean'], distr['stdev'])} ± {distr['stdev']:.2g}"
|
|
446
|
+
),
|
|
447
|
+
)
|
|
448
|
+
ax.axvline(
|
|
449
|
+
distr["median"],
|
|
450
|
+
color="green",
|
|
451
|
+
linestyle="--",
|
|
452
|
+
label=(
|
|
453
|
+
"Median ± MAD ="
|
|
454
|
+
f" {match_sigfigs(distr['median'], distr['mad'])} ± {distr['mad']:.2g}"
|
|
455
|
+
),
|
|
456
|
+
)
|
|
457
|
+
for stat_ref_label, stat_value in stat_ref.items():
|
|
458
|
+
ax.axvline(
|
|
459
|
+
stat_value,
|
|
460
|
+
color="orange",
|
|
461
|
+
linestyle="--",
|
|
462
|
+
label=(
|
|
463
|
+
"From single refinement:"
|
|
464
|
+
f" {stat_ref_label} = {match_sigfigs(stat_value, distr['stdev'])}"
|
|
465
|
+
),
|
|
466
|
+
)
|
|
467
|
+
ax.set_xlabel(f"Refined {label}")
|
|
468
|
+
ax.set_ylabel(f"Initial {label}")
|
|
469
|
+
ax.set_xlim(min_val - buffer, max_val + buffer)
|
|
470
|
+
ax.set_ylim(min_val - buffer, max_val + buffer)
|
|
471
|
+
ax.grid(True, alpha=0.7)
|
|
472
|
+
ax.legend(title=f"n = {len(x)}")
|
|
473
|
+
|
|
474
|
+
ax_histx.bar(
|
|
475
|
+
hist["bins"][:-1],
|
|
476
|
+
hist["counts"],
|
|
477
|
+
width=numpy.diff(hist["bins"]),
|
|
478
|
+
align="edge",
|
|
479
|
+
alpha=0.7,
|
|
480
|
+
)
|
|
481
|
+
ax_histx.tick_params(axis="x", labelbottom=False)
|
|
482
|
+
ax_histx.set_ylabel("Frequency")
|
|
483
|
+
# plt.xlabel(xlabel)
|
|
484
|
+
ax_histx.yaxis.set_major_locator(
|
|
485
|
+
ticker.MaxNLocator(integer=True)
|
|
486
|
+
) # Ensure integer y-axis labels
|
|
487
|
+
ax_histx.axvline(
|
|
488
|
+
distr["mean"],
|
|
489
|
+
color="blue",
|
|
490
|
+
linestyle="--",
|
|
491
|
+
label=(
|
|
492
|
+
"Mean ± St.Dev. ="
|
|
493
|
+
f" {match_sigfigs(distr['mean'], distr['stdev'])} ± {distr['stdev']:.2g}"
|
|
494
|
+
),
|
|
495
|
+
)
|
|
496
|
+
ax_histx.axvline(
|
|
497
|
+
distr["median"],
|
|
498
|
+
color="green",
|
|
499
|
+
linestyle="--",
|
|
500
|
+
label=(
|
|
501
|
+
"Median ± MAD ="
|
|
502
|
+
f" {match_sigfigs(distr['median'], distr['mad'])} ± {distr['mad']:.2g}"
|
|
503
|
+
),
|
|
504
|
+
)
|
|
505
|
+
for stat_ref_label, stat_value in stat_ref.items():
|
|
506
|
+
ax_histx.axvline(
|
|
507
|
+
stat_value,
|
|
508
|
+
color="orange",
|
|
509
|
+
linestyle="--",
|
|
510
|
+
label=(
|
|
511
|
+
"From single refinement:"
|
|
512
|
+
f" {stat_ref_label} = {match_sigfigs(stat_value, distr['stdev'])}"
|
|
513
|
+
),
|
|
514
|
+
)
|
|
515
|
+
ax_histx.grid(axis="y", alpha=0.7)
|
|
516
|
+
|
|
517
|
+
png_filename_base = filename_replace_char(f"scatter_histogram_{label}")
|
|
518
|
+
if idx:
|
|
519
|
+
png_filename_base = f"{prefix}group{idx}_bootstrap_{png_filename_base}"
|
|
520
|
+
else:
|
|
521
|
+
png_filename_base = f"{prefix}{png_filename_base}"
|
|
522
|
+
if filtered:
|
|
523
|
+
png_filename_base = f"{png_filename_base}_filtered"
|
|
524
|
+
png_filename = f"{png_filename_base}.png"
|
|
525
|
+
fig.savefig(png_filename)
|
|
526
|
+
logging.info(f"Saved scatter plot with histogram to {png_filename}")
|
|
527
|
+
plt.close(fig)
|
|
528
|
+
|
|
529
|
+
return distr, distr_init, outliers
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def bootstrap_analyse_stats(jsons, json_ref, idx=0, prefix=""):
|
|
533
|
+
logging.info(f"Loading {len(jsons)} json files with statistics...")
|
|
534
|
+
# Find out which statistics are available
|
|
535
|
+
# and create related empty lists in data_overall_dict and data_overall[_init]_dict
|
|
536
|
+
with open(jsons[0]) as f:
|
|
537
|
+
data_first = json.load(f)
|
|
538
|
+
stats_data_avail = data_first[-1]["data"]["summary"].keys()
|
|
539
|
+
data_overall_dict = {stat: [] for stat in stats_data_avail}
|
|
540
|
+
data_overall_init_dict = {stat: [] for stat in stats_data_avail}
|
|
541
|
+
# Add keys for geometry statistics from geom section
|
|
542
|
+
geom_stats_avail = False
|
|
543
|
+
geom_stats = [
|
|
544
|
+
"Bond distances, non H",
|
|
545
|
+
"Bond angles, non H",
|
|
546
|
+
]
|
|
547
|
+
for stat in geom_stats:
|
|
548
|
+
if (
|
|
549
|
+
stat in data_first[-1]["geom"]["summary"]["r.m.s.d."].keys()
|
|
550
|
+
and stat in data_first[-1]["geom"]["summary"]["r.m.s.Z"].keys()
|
|
551
|
+
):
|
|
552
|
+
geom_stats_avail = True
|
|
553
|
+
data_overall_dict[f"Rmsd {stat}"] = []
|
|
554
|
+
data_overall_dict[f"RmsZ {stat}"] = []
|
|
555
|
+
data_overall_init_dict[f"Rmsd {stat}"] = []
|
|
556
|
+
data_overall_init_dict[f"RmsZ {stat}"] = []
|
|
557
|
+
if "weight" in data_first[-1] and len(data_first) >= 3 and geom_stats_avail:
|
|
558
|
+
# First available value for weight is in the 1st cycle, not 0th cycle
|
|
559
|
+
data_overall_dict["weight"] = []
|
|
560
|
+
data_overall_init_dict["weight"] = []
|
|
561
|
+
"""
|
|
562
|
+
stats_additional = []
|
|
563
|
+
for stat in stats_data_avail:
|
|
564
|
+
if "CC" in stat:
|
|
565
|
+
data_overall_dict[f"R2_{stat}"] = []
|
|
566
|
+
data_overall_init_dict[f"R2_{stat}"] = []
|
|
567
|
+
stats_additional.append(f"R2_{stat}")
|
|
568
|
+
"""
|
|
569
|
+
|
|
570
|
+
# Load statistics from json files and append to lists in data_overall[_init]_dict
|
|
571
|
+
for json_file in jsons:
|
|
572
|
+
with open(json_file) as f:
|
|
573
|
+
data_loaded = json.load(f)
|
|
574
|
+
for stat in stats_data_avail:
|
|
575
|
+
data_overall_init_dict[stat].append(
|
|
576
|
+
data_loaded[0]["data"]["summary"].get(stat, 0)
|
|
577
|
+
)
|
|
578
|
+
data_overall_dict[stat].append(
|
|
579
|
+
data_loaded[-1]["data"]["summary"].get(stat, 0)
|
|
580
|
+
)
|
|
581
|
+
if geom_stats_avail:
|
|
582
|
+
for stat in geom_stats:
|
|
583
|
+
data_overall_init_dict[f"Rmsd {stat}"].append(
|
|
584
|
+
data_loaded[0]["geom"]["summary"]["r.m.s.d."].get(stat, 0)
|
|
585
|
+
)
|
|
586
|
+
data_overall_dict[f"Rmsd {stat}"].append(
|
|
587
|
+
data_loaded[-1]["geom"]["summary"]["r.m.s.d."].get(stat, 0)
|
|
588
|
+
)
|
|
589
|
+
data_overall_init_dict[f"RmsZ {stat}"].append(
|
|
590
|
+
data_loaded[0]["geom"]["summary"]["r.m.s.Z"].get(stat, 0)
|
|
591
|
+
)
|
|
592
|
+
data_overall_dict[f"RmsZ {stat}"].append(
|
|
593
|
+
data_loaded[-1]["geom"]["summary"]["r.m.s.Z"].get(stat, 0)
|
|
594
|
+
)
|
|
595
|
+
if "weight" in data_overall_dict.keys():
|
|
596
|
+
data_overall_init_dict["weight"].append(data_loaded[1].get("weight", 0))
|
|
597
|
+
data_overall_dict["weight"].append(data_loaded[-1].get("weight", 0))
|
|
598
|
+
"""
|
|
599
|
+
if "CC" in stat:
|
|
600
|
+
# also calculate R = sqrt(1 - CC^2)
|
|
601
|
+
# per resolution bin and average it with weights
|
|
602
|
+
if "work" in stat:
|
|
603
|
+
n_label = "n_work"
|
|
604
|
+
elif "free" in stat:
|
|
605
|
+
n_label = "n_free"
|
|
606
|
+
else:
|
|
607
|
+
n_label = "n_obs"
|
|
608
|
+
stat_binned = stat.replace("avg", "")
|
|
609
|
+
|
|
610
|
+
CCvalues_init = []
|
|
611
|
+
n_obs_values_init = []
|
|
612
|
+
for bin_data in data_loaded[0]["data"]["binned"]:
|
|
613
|
+
CCvalues_init.append(bin_data.get(stat_binned, 0))
|
|
614
|
+
n_obs_values_init.append(bin_data.get(n_label, 0))
|
|
615
|
+
CCvalues_init = numpy.array(CCvalues_init)
|
|
616
|
+
n_obs_values_init = numpy.array(n_obs_values_init)
|
|
617
|
+
R2values_init = numpy.sqrt(1 - numpy.power(CCvalues_init, 2))
|
|
618
|
+
R2value_init = (
|
|
619
|
+
numpy.average(R2values_init, weights=n_obs_values_init)
|
|
620
|
+
if n_obs_values_init.size > 0 and n_obs_values_init.sum() > 0
|
|
621
|
+
else 0
|
|
622
|
+
)
|
|
623
|
+
data_overall_init_dict[f"R2_{stat}"].append(R2value_init)
|
|
624
|
+
|
|
625
|
+
CCvalues = []
|
|
626
|
+
n_obs_values = []
|
|
627
|
+
for bin_data in data_loaded[-1]["data"]["binned"]:
|
|
628
|
+
CCvalues.append(bin_data.get(stat_binned, 0))
|
|
629
|
+
n_obs_values.append(bin_data.get(n_label, 0))
|
|
630
|
+
CCvalues = numpy.array(CCvalues)
|
|
631
|
+
n_obs_values = numpy.array(n_obs_values)
|
|
632
|
+
R2values = numpy.sqrt(1 - numpy.power(CCvalues, 2))
|
|
633
|
+
R2value = (
|
|
634
|
+
numpy.average(R2values, weights=n_obs_values)
|
|
635
|
+
if n_obs_values.size > 0 and n_obs_values.sum() > 0
|
|
636
|
+
else 0
|
|
637
|
+
)
|
|
638
|
+
data_overall_dict[f"R2_{stat}"].append(R2value)
|
|
639
|
+
"""
|
|
640
|
+
|
|
641
|
+
if json_ref and os.path.isfile(json_ref):
|
|
642
|
+
# Find reference statistics for a structure refined in a classic way
|
|
643
|
+
stats_ref_equivalents = {
|
|
644
|
+
"R": "R",
|
|
645
|
+
"R1": "R1",
|
|
646
|
+
"CCFavg": "CCFavg",
|
|
647
|
+
"CCIavg": "CCIavg",
|
|
648
|
+
"R_llw>0": "Rwork",
|
|
649
|
+
"R_llw=0": "Rfree",
|
|
650
|
+
"CCF_llw>0_avg": "CCFworkavg",
|
|
651
|
+
"CCF_llw=0_avg": "CCFfreeavg",
|
|
652
|
+
"R1_llw>0": "R1work",
|
|
653
|
+
"R1_llw=0": "R1free",
|
|
654
|
+
"CCI_llw>0_avg": "CCIworkavg",
|
|
655
|
+
"CCI_llw=0_avg": "CCIfreeavg",
|
|
656
|
+
"Rmsd Bond distances, non H": "Rmsd Bond distances, non H",
|
|
657
|
+
"Rmsd Bond angles, non H": "Rmsd Bond angles, non H",
|
|
658
|
+
"RmsZ Bond distances, non H": "RmsZ Bond distances, non H",
|
|
659
|
+
"RmsZ Bond angles, non H": "RmsZ Bond angles, non H",
|
|
660
|
+
"weight": "weight",
|
|
661
|
+
}
|
|
662
|
+
data_ref = {}
|
|
663
|
+
|
|
664
|
+
with open(json_ref) as f:
|
|
665
|
+
data_ref_loaded = json.load(f)
|
|
666
|
+
data_ref = data_ref_loaded[-1]["data"]["summary"]
|
|
667
|
+
if geom_stats_avail:
|
|
668
|
+
for stat in geom_stats:
|
|
669
|
+
data_ref[f"Rmsd {stat}"] = data_ref_loaded[-1]["geom"]["summary"][
|
|
670
|
+
"r.m.s.d."
|
|
671
|
+
].get(stat, 0)
|
|
672
|
+
data_ref[f"RmsZ {stat}"] = data_ref_loaded[-1]["geom"]["summary"][
|
|
673
|
+
"r.m.s.Z"
|
|
674
|
+
].get(stat, 0)
|
|
675
|
+
if "weight" in data_overall_dict and len(data_ref_loaded) >= 2:
|
|
676
|
+
data_ref["weight"] = data_ref_loaded[-1].get("weight", 0)
|
|
677
|
+
if data_ref:
|
|
678
|
+
logging.info(f"Loaded reference statistics from {json_ref}")
|
|
679
|
+
else:
|
|
680
|
+
logging.warning(f"No summary statistics found in reference {json_ref}")
|
|
681
|
+
else:
|
|
682
|
+
logging.warning(
|
|
683
|
+
f"Reference file with statistics not found {json_ref if json_ref else ''}"
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
distrs = {"init": {}, "final": {}}
|
|
687
|
+
llweight_R1_outliers = []
|
|
688
|
+
for stat in data_overall_dict.keys():
|
|
689
|
+
stat_ref = {}
|
|
690
|
+
if (
|
|
691
|
+
json_ref
|
|
692
|
+
and os.path.isfile(json_ref)
|
|
693
|
+
and data_ref
|
|
694
|
+
and stat in stats_ref_equivalents.keys()
|
|
695
|
+
and data_ref.get(stats_ref_equivalents[stat], None) is not None
|
|
696
|
+
):
|
|
697
|
+
stat_ref = {
|
|
698
|
+
stats_ref_equivalents[stat]: data_ref.get(
|
|
699
|
+
stats_ref_equivalents[stat], None
|
|
700
|
+
)
|
|
701
|
+
}
|
|
702
|
+
distr, distr_init, llweight_outliers = scatter_plot_histogram(
|
|
703
|
+
data_overall_dict[stat],
|
|
704
|
+
data_overall_init_dict[stat],
|
|
705
|
+
stat,
|
|
706
|
+
stat_ref,
|
|
707
|
+
idx,
|
|
708
|
+
prefix,
|
|
709
|
+
filtered=False,
|
|
710
|
+
)
|
|
711
|
+
distrs["final"][stat] = distr
|
|
712
|
+
distrs["init"][stat] = distr_init
|
|
713
|
+
if stat == "R1":
|
|
714
|
+
if llweight_outliers is not None:
|
|
715
|
+
llweight_R1_outliers = llweight_outliers["index"].tolist()
|
|
716
|
+
else:
|
|
717
|
+
llweight_R1_outliers = []
|
|
718
|
+
|
|
719
|
+
json_filename = (
|
|
720
|
+
f"{prefix}group{idx}_bootstrap_statistics.json"
|
|
721
|
+
if idx
|
|
722
|
+
else f"{prefix}bootstrap_statistics.json"
|
|
723
|
+
)
|
|
724
|
+
with open(json_filename, "w") as f:
|
|
725
|
+
json.dump(distrs, f, indent=2, default=json_numpy_converter)
|
|
726
|
+
logging.info(f"Saved statistics to {json_filename}")
|
|
727
|
+
|
|
728
|
+
# Filter outliers based on R1 values and re-analyse the distributions
|
|
729
|
+
if llweight_R1_outliers:
|
|
730
|
+
distrs_filtered = {"init": {}, "final": {}}
|
|
731
|
+
data_overall_filtered_dict = {}
|
|
732
|
+
data_overall_init_filtered_dict = {}
|
|
733
|
+
logging.info(
|
|
734
|
+
f"\nIdentified {len(llweight_R1_outliers) if llweight_R1_outliers else 0}"
|
|
735
|
+
" R1 value outliers which will be exluded from the following analysis\n"
|
|
736
|
+
f" {llweight_R1_outliers if llweight_R1_outliers else ''}"
|
|
737
|
+
)
|
|
738
|
+
for stat in data_overall_dict.keys():
|
|
739
|
+
data_overall_filtered_dict[stat] = [
|
|
740
|
+
val
|
|
741
|
+
for i, val in enumerate(data_overall_dict[stat])
|
|
742
|
+
if i not in llweight_R1_outliers
|
|
743
|
+
]
|
|
744
|
+
data_overall_init_filtered_dict[stat] = [
|
|
745
|
+
val
|
|
746
|
+
for i, val in enumerate(data_overall_init_dict[stat])
|
|
747
|
+
if i not in llweight_R1_outliers
|
|
748
|
+
]
|
|
749
|
+
if (
|
|
750
|
+
json_ref
|
|
751
|
+
and os.path.isfile(json_ref)
|
|
752
|
+
and data_ref
|
|
753
|
+
and stat in stats_ref_equivalents.keys()
|
|
754
|
+
and data_ref.get(stats_ref_equivalents[stat], None) is not None
|
|
755
|
+
):
|
|
756
|
+
stat_ref = {
|
|
757
|
+
stats_ref_equivalents[stat]: data_ref.get(
|
|
758
|
+
stats_ref_equivalents[stat], None
|
|
759
|
+
)
|
|
760
|
+
}
|
|
761
|
+
distr_filtered, distr_init_filtered, _ = scatter_plot_histogram(
|
|
762
|
+
data_overall_filtered_dict[stat],
|
|
763
|
+
data_overall_init_filtered_dict[stat],
|
|
764
|
+
stat,
|
|
765
|
+
stat_ref,
|
|
766
|
+
idx,
|
|
767
|
+
prefix,
|
|
768
|
+
filtered=True,
|
|
769
|
+
)
|
|
770
|
+
distrs_filtered["final"][stat] = distr_filtered
|
|
771
|
+
distrs_filtered["init"][stat] = distr_init_filtered
|
|
772
|
+
|
|
773
|
+
json_filename_filtered = (
|
|
774
|
+
f"{prefix}group{idx}_bootstrap_statistics_filtered.json"
|
|
775
|
+
if idx
|
|
776
|
+
else f"{prefix}bootstrap_statistics_filtered.json"
|
|
777
|
+
)
|
|
778
|
+
with open(json_filename_filtered, "w") as f:
|
|
779
|
+
json.dump(distrs_filtered, f, indent=2, default=json_numpy_converter)
|
|
780
|
+
logging.info(f"Saved filtered statistics to {json_filename_filtered}")
|
|
781
|
+
|
|
782
|
+
logging.info("")
|
|
783
|
+
return llweight_R1_outliers
|