multixem 0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,783 @@
1
+ # coding: utf-8
2
+ import os
3
+ import json
4
+ import logging
5
+ import re
6
+ import gemmi
7
+ import matplotlib
8
+ import matplotlib.pyplot as plt
9
+ import matplotlib.ticker as ticker
10
+ import numpy
11
+ import pandas
12
+ from .tools import filename_replace_char, json_numpy_converter
13
+
14
+
15
+ matplotlib.use("Agg")
16
+
17
+
18
+ def match_sigfigs(value, ref):
19
+ """Format `value` to have the same number of significant figures as `ref`."""
20
+ if ref == 0:
21
+ return f"{value:.2f}"
22
+ decimals = max(0, -int(numpy.floor(numpy.log10(abs(ref)))) + 1)
23
+ return f"{value:.{decimals}f}"
24
+
25
+
26
+ def df_scatter_plot(
27
+ df, x_cols, y_cols_groups, filename="scatter_plot.png", per_element=False
28
+ ):
29
+ """
30
+ Create multiple scatter subplots from a DataFrame and save as PNG.
31
+
32
+ Args:
33
+ df (pandas.DataFrame or str or file stream): DataFrame or a CSV file.
34
+ x_cols (list of str): Column names for x-axis for each subplot.
35
+ If per_element=True,
36
+ x_cols should be 'atomic_number' or 'atom_id'.
37
+ y_cols_groups (list of list of str): Each sublist is a group of
38
+ y columns for one subplot.
39
+ filename (str): Output PNG filename.
40
+ per_element (bool): If True, create separate plots for each atom.
41
+ Hydrogens will be excluded.
42
+ """
43
+
44
+ def extract_atomic_numbers(df, atom_id_col):
45
+ # atom_id can look like "A/00 1/H19A", I want to extract "H"
46
+ atom_labels = df[atom_id_col]
47
+ proton_numbers = []
48
+ for atom_id in atom_labels:
49
+ atom_id_part = atom_id.split("/")[-1]
50
+ match = re.match(r"^([A-Z][a-z]?)", atom_id_part)
51
+ element = match.group(1)
52
+ proton_number = gemmi.Element(element).atomic_number
53
+ proton_numbers.append(proton_number)
54
+ return numpy.array(proton_numbers)
55
+
56
+ if isinstance(df, str):
57
+ df = pandas.read_csv(df)
58
+
59
+ n_subplots = len(y_cols_groups)
60
+ ncols = min(n_subplots, 2)
61
+ nrows = int(numpy.ceil(n_subplots / ncols))
62
+ fig, axes = plt.subplots(
63
+ nrows=nrows, ncols=ncols, figsize=(6 * ncols, 5 * nrows), squeeze=False
64
+ )
65
+ axes = axes.flatten()
66
+ max_y = 0
67
+ max_y_b = 0
68
+ if not per_element:
69
+ assert len(x_cols) == len(
70
+ y_cols_groups
71
+ ), "x and y_groups must have the same length"
72
+
73
+ # Create a combined mask: True only where all x columns are not null
74
+ # for finding x_max and y_max
75
+ combined_mask = numpy.logical_and.reduce([df[col].notnull() for col in x_cols])
76
+ df_filt = df[combined_mask]
77
+
78
+ max_x = 0
79
+ max_x_b = 0
80
+ for data in x_cols:
81
+ if "sigma_b" not in data:
82
+ max_x = max(max_x, df_filt[data].max())
83
+ for i, group in enumerate(y_cols_groups):
84
+ for data in group:
85
+ if "sigma_b" not in data:
86
+ max_y = max(max_y, df_filt[data].max())
87
+ else: # per_element
88
+ if x_cols[0] != "atomic_number": # typically x_cols[0] == "atom_id":
89
+ # get atomic numbers from atom_id and save it as a new column
90
+ df["atomic_number"] = extract_atomic_numbers(df, x_cols[0])
91
+ df = df[df["atomic_number"] > 1] # exclude hydrogens
92
+ for i, group in enumerate(y_cols_groups):
93
+ for data in group:
94
+ if "sigma_b" not in data:
95
+ max_y = max(max_y, df[data].max())
96
+
97
+ for i, y_group in enumerate(y_cols_groups):
98
+ ax = axes[i]
99
+ ax.set_axisbelow(True)
100
+ for j, y_col in enumerate(y_group):
101
+ if not per_element:
102
+ ax.scatter(df[x_cols[i]], df[y_col], marker=".", label=y_col, alpha=0.5)
103
+ ax.set_xlabel(x_cols[i])
104
+ ax.set_title(x_cols[i] + " vs " + ", ".join(y_group))
105
+ ax.legend(loc="upper left")
106
+ else: # per_element
107
+ ax.scatter(
108
+ df["atomic_number"] + j * 0.2,
109
+ df[y_col],
110
+ marker=".",
111
+ label=y_col,
112
+ alpha=0.5,
113
+ )
114
+ ax.set_title(", ".join(y_group) + " per element")
115
+ ax.legend()
116
+ if not per_element:
117
+ if "sigma_b" not in y_col:
118
+ ax.set_xlim(0, max_x * 1.05)
119
+ ax.set_ylim(0, max_y * 1.05)
120
+ else: # not per_element, "sigma_b"
121
+ if "sigma_b" in x_cols[i]:
122
+ max_x_b_candidate = df_filt[x_cols[i]].max()
123
+ max_x_b = max(max_x_b, max_x_b_candidate)
124
+ ax.set_xlim(0, max_x_b * 1.05)
125
+ max_y_b_candidate = df_filt[y_col].max()
126
+ max_y_b = max(max_y_b, max_y_b_candidate)
127
+ ax.set_ylim(0, max_y_b * 1.05)
128
+ else: # per_element # do not touch xlim
129
+ if "sigma_b" not in y_col:
130
+ ax.set_ylim(0, max_y * 1.05)
131
+ else: # per_element, "sigma_b"
132
+ max_y_b_candidate = df[y_col].max()
133
+ max_y_b = max(max_y_b, max_y_b_candidate)
134
+ ax.set_ylim(0, max_y_b * 1.05)
135
+ if len(y_group) == 1:
136
+ ax.set_ylabel(y_group[0])
137
+ ax.grid()
138
+
139
+ # Hide unused axes if any
140
+ for j in range(i + 1, len(axes)):
141
+ fig.delaxes(axes[j])
142
+
143
+ plt.tight_layout()
144
+ plt.savefig(filename)
145
+ plt.close()
146
+
147
+
148
+ def analyse_distribution(
149
+ values, xlabel, outlier_factor=3.0, idx=0, prefix="", filtered=False, save=True
150
+ ):
151
+ """
152
+ Analyse a distribution of values and return summary statistics.
153
+
154
+ Args:
155
+ values (list or numpy array): The values to be analysed.
156
+ xlabel (str): The label for the x-axis of the histogram.
157
+ outlier_factor (float): The factor to determine outliers based on IQR.
158
+ idx (int): Index for naming the output files (applies if not set to 0).
159
+ prefix (str): Prefix for the output filenames.
160
+ filtered (bool): Whether the values are already filtered,
161
+ which will be reflected in the output filenames.
162
+ save (bool): Whether to save the histogram and outliers to CSV files.
163
+ Returns:
164
+ dict: A dictionary containing summary statistics,
165
+ including histogram bins and counts.
166
+ dict: A dictionary containing histogram bins and counts.
167
+ pandas.DataFrame or None: A DataFrame containing outliers with their
168
+ indices and values, or None if there are no outliers.
169
+ """
170
+ if len(values) == 0:
171
+ return {}, {}, None
172
+ mean = numpy.mean(values)
173
+ stdev = numpy.std(values, ddof=1)
174
+ median = numpy.median(values)
175
+ q1 = numpy.percentile(values, 25)
176
+ q3 = numpy.percentile(values, 75)
177
+ iqr = q3 - q1
178
+ # MAD to be consistent with stdev for normal distribution
179
+ mad = numpy.median(numpy.abs(values - median)) * 1.4826
180
+ min_val = numpy.min(values)
181
+ max_val = numpy.max(values)
182
+ n_values = len(values)
183
+ distr_analysis_dict = {
184
+ "mean": mean,
185
+ "stdev": stdev,
186
+ "median": median,
187
+ "q1": q1,
188
+ "q3": q3,
189
+ "iqr": iqr,
190
+ "mad": mad,
191
+ "min": min_val,
192
+ "max": max_val,
193
+ "n": n_values,
194
+ }
195
+
196
+ if not save:
197
+ return distr_analysis_dict, {}, None
198
+
199
+ counts, bins = numpy.histogram(values, bins="auto")
200
+
201
+ values_arr = numpy.asarray(values)
202
+
203
+ # IQR-based criterion
204
+ threshold_low = q1 - outlier_factor * iqr
205
+ threshold_high = q3 + outlier_factor * iqr
206
+ mask_iqr = (values_arr < threshold_low) | (values_arr > threshold_high)
207
+
208
+ # Additional median-relative criterion only for R1 ?
209
+ # is_r1 = str(xlabel).strip().upper() == "R1"
210
+ # if is_r1 and median > 0:
211
+ # rel_low = 0.7 * median
212
+ # rel_high = 1.3 * median
213
+ # mask_rel = (values_arr < rel_low) | (values_arr > rel_high)
214
+
215
+ # # Outlier must violate BOTH criteria
216
+ # outlier_mask = mask_iqr & mask_rel
217
+ # else:
218
+ # outlier_mask = mask_iqr
219
+ outlier_mask = mask_iqr
220
+
221
+ outlier_idx = numpy.where(outlier_mask)[0]
222
+ outliers = (
223
+ None
224
+ if outlier_idx.size == 0
225
+ else pandas.DataFrame({"index": outlier_idx, "value": values_arr[outlier_idx]})
226
+ )
227
+
228
+ csv_values_filename = filename_replace_char(f"histogram_{xlabel}_values.csv")
229
+ csv_histogram_filename = filename_replace_char(f"histogram_{xlabel}.csv")
230
+ if filtered:
231
+ csv_values_filename = csv_values_filename.replace(
232
+ "histogram", "histogram_filtered"
233
+ )
234
+ csv_histogram_filename = csv_histogram_filename.replace(
235
+ "histogram", "histogram_filtered"
236
+ )
237
+ if idx:
238
+ csv_values_filename = f"{prefix}group{idx}_bootstrap_{csv_values_filename}"
239
+ csv_histogram_filename = (
240
+ f"{prefix}group{idx}_bootstrap_{csv_histogram_filename}"
241
+ )
242
+ else:
243
+ csv_values_filename = f"{prefix}{csv_values_filename}"
244
+ csv_histogram_filename = f"{prefix}{csv_histogram_filename}"
245
+
246
+ df_values = pandas.DataFrame(values)
247
+ df_values.to_csv(csv_values_filename, index=False, header=False)
248
+ logging.info(f"Saved raw values to {csv_values_filename}")
249
+
250
+ bin_centers = (bins[:-1] + bins[1:]) / 2
251
+ df_histogram = pandas.DataFrame(
252
+ {
253
+ "bin_start": bins[:-1],
254
+ "bin_center": bin_centers,
255
+ "bin_end": bins[1:],
256
+ "count": counts,
257
+ }
258
+ )
259
+ df_histogram.to_csv(csv_histogram_filename, index=False, header=False)
260
+ logging.info(f"Saved histogram bins and counts to {csv_histogram_filename}")
261
+
262
+ if outliers is not None and not filtered:
263
+ csv_outliers_filename = filename_replace_char(
264
+ f"histogram_{xlabel}_outliers.csv"
265
+ )
266
+ if idx:
267
+ csv_outliers_filename = (
268
+ f"{prefix}group{idx}_bootstrap_{csv_outliers_filename}"
269
+ )
270
+ else:
271
+ csv_outliers_filename = f"{prefix}{csv_outliers_filename}"
272
+ outliers.to_csv(csv_outliers_filename, index=False, header=False)
273
+ logging.info(
274
+ f"Saved {len(outliers)} outliers outside the interval"
275
+ f" [{threshold_low:.4f}, {threshold_high:.4f}] to {csv_outliers_filename}"
276
+ )
277
+
278
+ return (
279
+ distr_analysis_dict,
280
+ {
281
+ "bins": bins,
282
+ "counts": counts.astype(int),
283
+ },
284
+ outliers,
285
+ )
286
+
287
+
288
+ def plot_histogram(values, xlabel, ref={}, idx=0, prefix="", outlier_factor=3.0):
289
+ """
290
+ Plot a histogram of the data and save as PNG.
291
+
292
+ Args:
293
+ data (list or numpy array): Data to plot.
294
+ xlabel (str): Label for the x-axis and the output file.
295
+ ref (dict): Reference values for the plot {label: value}.
296
+ idx (int): Index for naming the output file (applies if not set to 0).
297
+ prefix (str): Prefix for the output filename.
298
+ outlier_factor (float): Factor for determining outliers.
299
+ """
300
+ distr, hist, outliers = analyse_distribution(
301
+ values, xlabel, idx=idx, prefix=prefix, outlier_factor=outlier_factor
302
+ )
303
+ min_val = distr["min"]
304
+ max_val = distr["max"]
305
+ if ref:
306
+ for ref_value in ref.values():
307
+ min_val = min(min_val, ref_value)
308
+ max_val = max(max_val, ref_value)
309
+
310
+ plt.figure(figsize=(8, 6))
311
+ plt.bar(
312
+ hist["bins"][:-1],
313
+ hist["counts"],
314
+ width=numpy.diff(hist["bins"]),
315
+ align="edge",
316
+ alpha=0.7,
317
+ )
318
+ plt.xlabel(xlabel)
319
+ plt.ylabel("Frequency")
320
+ plt.gca().yaxis.set_major_locator(
321
+ ticker.MaxNLocator(integer=True)
322
+ ) # Ensure integer y-axis labels
323
+ buffer = (max_val - min_val) * 0.05 # 5% buffer around the data range
324
+ plt.xlim(min_val - buffer, max_val + buffer)
325
+ plt.axvline(
326
+ distr["mean"],
327
+ color="blue",
328
+ linestyle="--",
329
+ label=(
330
+ "Mean ± St.Dev. ="
331
+ f" {match_sigfigs(distr['mean'], distr['stdev'])} ± {distr['stdev']:.2g}"
332
+ ),
333
+ )
334
+ plt.axvline(
335
+ distr["median"],
336
+ color="green",
337
+ linestyle="--",
338
+ label=(
339
+ "Median ± MAD ="
340
+ f" {match_sigfigs(distr['median'], distr['mad'])} ± {distr['mad']:.2g}"
341
+ ),
342
+ )
343
+ for stat_ref_label, stat_value in ref.items():
344
+ plt.axvline(
345
+ stat_value,
346
+ color="orange",
347
+ linestyle="--",
348
+ label=(
349
+ "From single refinement:"
350
+ f" {stat_ref_label} = {match_sigfigs(stat_value, distr['stdev'])}"
351
+ ),
352
+ )
353
+ plt.grid(axis="y", alpha=0.7)
354
+ plt.tight_layout()
355
+ plt.legend(title=f"n = {len(values)}")
356
+ png_filename = filename_replace_char(f"histogram_{xlabel}.png")
357
+ if idx:
358
+ png_filename = f"{prefix}group{idx}_bootstrap_{png_filename}"
359
+ else:
360
+ png_filename = f"{prefix}{png_filename}"
361
+ plt.savefig(png_filename)
362
+ plt.close()
363
+ logging.info(f"Saved histogram to {png_filename}")
364
+
365
+ return distr, hist, outliers
366
+
367
+
368
+ def scatter_plot_histogram(
369
+ x, y, label, stat_ref={}, idx=0, prefix="", filtered=True, outlier_factor=3.0
370
+ ):
371
+ """
372
+ Plot a scatter plot of x vs y including histograms and save as PNG.
373
+
374
+ Args:
375
+ x (list or numpy array): Data for the x-axis.
376
+ y (list or numpy array): Data for the y-axis.
377
+ stat_ref (dict): Reference statistic for the plot {label: value}.
378
+ label (str): Label for the axes and the output file.
379
+ idx (int): Index for naming the output file (applies if not set to 0).
380
+ prefix (str): Prefix for the output filename.
381
+ filtered (bool): Whether the values are already filtered.
382
+ outlier_factor (float): Factor for determining outliers.
383
+ """
384
+ if len(x) != len(y):
385
+ raise ValueError("x and y must have the same length.")
386
+
387
+ distr, hist, outliers = analyse_distribution(
388
+ x,
389
+ label,
390
+ idx=idx,
391
+ prefix=prefix,
392
+ filtered=filtered,
393
+ outlier_factor=outlier_factor,
394
+ )
395
+ distr_init, _, _ = analyse_distribution(
396
+ y,
397
+ label,
398
+ idx=idx,
399
+ prefix=prefix,
400
+ filtered=filtered,
401
+ save=False,
402
+ outlier_factor=outlier_factor,
403
+ )
404
+ min_val = min(min(x), min(y))
405
+ max_val = max(max(x), max(y))
406
+ if stat_ref:
407
+ for ref_value in stat_ref.values():
408
+ min_val = min(min_val, ref_value)
409
+ max_val = max(max_val, ref_value)
410
+ buffer = (max_val - min_val) * 0.05 # 5% buffer around the data range
411
+
412
+ fig, axs = plt.subplot_mosaic(
413
+ [["histx"], ["scatter"]],
414
+ sharex=True,
415
+ figsize=(7, 9),
416
+ height_ratios=(1, 2),
417
+ layout="constrained",
418
+ )
419
+
420
+ ax = axs["scatter"]
421
+ ax_histx = axs["histx"]
422
+
423
+ """
424
+ binwidth = 0.25
425
+ xymax = max(numpy.max(numpy.abs(x)), numpy.max(numpy.abs(y)))
426
+ lim = (int(xymax/binwidth) + 1) * binwidth
427
+
428
+ bins = numpy.arange(-lim, lim + binwidth, binwidth)
429
+ """
430
+
431
+ ax.scatter(x, y, alpha=0.1)
432
+ ax.plot( # line y=x
433
+ [min_val - buffer, max_val + buffer],
434
+ [min_val - buffer, max_val + buffer],
435
+ color="gray",
436
+ linestyle="--",
437
+ alpha=0.7,
438
+ )
439
+ ax.axvline(
440
+ distr["mean"],
441
+ color="blue",
442
+ linestyle="--",
443
+ label=(
444
+ "Mean ± St.Dev. ="
445
+ f" {match_sigfigs(distr['mean'], distr['stdev'])} ± {distr['stdev']:.2g}"
446
+ ),
447
+ )
448
+ ax.axvline(
449
+ distr["median"],
450
+ color="green",
451
+ linestyle="--",
452
+ label=(
453
+ "Median ± MAD ="
454
+ f" {match_sigfigs(distr['median'], distr['mad'])} ± {distr['mad']:.2g}"
455
+ ),
456
+ )
457
+ for stat_ref_label, stat_value in stat_ref.items():
458
+ ax.axvline(
459
+ stat_value,
460
+ color="orange",
461
+ linestyle="--",
462
+ label=(
463
+ "From single refinement:"
464
+ f" {stat_ref_label} = {match_sigfigs(stat_value, distr['stdev'])}"
465
+ ),
466
+ )
467
+ ax.set_xlabel(f"Refined {label}")
468
+ ax.set_ylabel(f"Initial {label}")
469
+ ax.set_xlim(min_val - buffer, max_val + buffer)
470
+ ax.set_ylim(min_val - buffer, max_val + buffer)
471
+ ax.grid(True, alpha=0.7)
472
+ ax.legend(title=f"n = {len(x)}")
473
+
474
+ ax_histx.bar(
475
+ hist["bins"][:-1],
476
+ hist["counts"],
477
+ width=numpy.diff(hist["bins"]),
478
+ align="edge",
479
+ alpha=0.7,
480
+ )
481
+ ax_histx.tick_params(axis="x", labelbottom=False)
482
+ ax_histx.set_ylabel("Frequency")
483
+ # plt.xlabel(xlabel)
484
+ ax_histx.yaxis.set_major_locator(
485
+ ticker.MaxNLocator(integer=True)
486
+ ) # Ensure integer y-axis labels
487
+ ax_histx.axvline(
488
+ distr["mean"],
489
+ color="blue",
490
+ linestyle="--",
491
+ label=(
492
+ "Mean ± St.Dev. ="
493
+ f" {match_sigfigs(distr['mean'], distr['stdev'])} ± {distr['stdev']:.2g}"
494
+ ),
495
+ )
496
+ ax_histx.axvline(
497
+ distr["median"],
498
+ color="green",
499
+ linestyle="--",
500
+ label=(
501
+ "Median ± MAD ="
502
+ f" {match_sigfigs(distr['median'], distr['mad'])} ± {distr['mad']:.2g}"
503
+ ),
504
+ )
505
+ for stat_ref_label, stat_value in stat_ref.items():
506
+ ax_histx.axvline(
507
+ stat_value,
508
+ color="orange",
509
+ linestyle="--",
510
+ label=(
511
+ "From single refinement:"
512
+ f" {stat_ref_label} = {match_sigfigs(stat_value, distr['stdev'])}"
513
+ ),
514
+ )
515
+ ax_histx.grid(axis="y", alpha=0.7)
516
+
517
+ png_filename_base = filename_replace_char(f"scatter_histogram_{label}")
518
+ if idx:
519
+ png_filename_base = f"{prefix}group{idx}_bootstrap_{png_filename_base}"
520
+ else:
521
+ png_filename_base = f"{prefix}{png_filename_base}"
522
+ if filtered:
523
+ png_filename_base = f"{png_filename_base}_filtered"
524
+ png_filename = f"{png_filename_base}.png"
525
+ fig.savefig(png_filename)
526
+ logging.info(f"Saved scatter plot with histogram to {png_filename}")
527
+ plt.close(fig)
528
+
529
+ return distr, distr_init, outliers
530
+
531
+
532
+ def bootstrap_analyse_stats(jsons, json_ref, idx=0, prefix=""):
533
+ logging.info(f"Loading {len(jsons)} json files with statistics...")
534
+ # Find out which statistics are available
535
+ # and create related empty lists in data_overall_dict and data_overall[_init]_dict
536
+ with open(jsons[0]) as f:
537
+ data_first = json.load(f)
538
+ stats_data_avail = data_first[-1]["data"]["summary"].keys()
539
+ data_overall_dict = {stat: [] for stat in stats_data_avail}
540
+ data_overall_init_dict = {stat: [] for stat in stats_data_avail}
541
+ # Add keys for geometry statistics from geom section
542
+ geom_stats_avail = False
543
+ geom_stats = [
544
+ "Bond distances, non H",
545
+ "Bond angles, non H",
546
+ ]
547
+ for stat in geom_stats:
548
+ if (
549
+ stat in data_first[-1]["geom"]["summary"]["r.m.s.d."].keys()
550
+ and stat in data_first[-1]["geom"]["summary"]["r.m.s.Z"].keys()
551
+ ):
552
+ geom_stats_avail = True
553
+ data_overall_dict[f"Rmsd {stat}"] = []
554
+ data_overall_dict[f"RmsZ {stat}"] = []
555
+ data_overall_init_dict[f"Rmsd {stat}"] = []
556
+ data_overall_init_dict[f"RmsZ {stat}"] = []
557
+ if "weight" in data_first[-1] and len(data_first) >= 3 and geom_stats_avail:
558
+ # First available value for weight is in the 1st cycle, not 0th cycle
559
+ data_overall_dict["weight"] = []
560
+ data_overall_init_dict["weight"] = []
561
+ """
562
+ stats_additional = []
563
+ for stat in stats_data_avail:
564
+ if "CC" in stat:
565
+ data_overall_dict[f"R2_{stat}"] = []
566
+ data_overall_init_dict[f"R2_{stat}"] = []
567
+ stats_additional.append(f"R2_{stat}")
568
+ """
569
+
570
+ # Load statistics from json files and append to lists in data_overall[_init]_dict
571
+ for json_file in jsons:
572
+ with open(json_file) as f:
573
+ data_loaded = json.load(f)
574
+ for stat in stats_data_avail:
575
+ data_overall_init_dict[stat].append(
576
+ data_loaded[0]["data"]["summary"].get(stat, 0)
577
+ )
578
+ data_overall_dict[stat].append(
579
+ data_loaded[-1]["data"]["summary"].get(stat, 0)
580
+ )
581
+ if geom_stats_avail:
582
+ for stat in geom_stats:
583
+ data_overall_init_dict[f"Rmsd {stat}"].append(
584
+ data_loaded[0]["geom"]["summary"]["r.m.s.d."].get(stat, 0)
585
+ )
586
+ data_overall_dict[f"Rmsd {stat}"].append(
587
+ data_loaded[-1]["geom"]["summary"]["r.m.s.d."].get(stat, 0)
588
+ )
589
+ data_overall_init_dict[f"RmsZ {stat}"].append(
590
+ data_loaded[0]["geom"]["summary"]["r.m.s.Z"].get(stat, 0)
591
+ )
592
+ data_overall_dict[f"RmsZ {stat}"].append(
593
+ data_loaded[-1]["geom"]["summary"]["r.m.s.Z"].get(stat, 0)
594
+ )
595
+ if "weight" in data_overall_dict.keys():
596
+ data_overall_init_dict["weight"].append(data_loaded[1].get("weight", 0))
597
+ data_overall_dict["weight"].append(data_loaded[-1].get("weight", 0))
598
+ """
599
+ if "CC" in stat:
600
+ # also calculate R = sqrt(1 - CC^2)
601
+ # per resolution bin and average it with weights
602
+ if "work" in stat:
603
+ n_label = "n_work"
604
+ elif "free" in stat:
605
+ n_label = "n_free"
606
+ else:
607
+ n_label = "n_obs"
608
+ stat_binned = stat.replace("avg", "")
609
+
610
+ CCvalues_init = []
611
+ n_obs_values_init = []
612
+ for bin_data in data_loaded[0]["data"]["binned"]:
613
+ CCvalues_init.append(bin_data.get(stat_binned, 0))
614
+ n_obs_values_init.append(bin_data.get(n_label, 0))
615
+ CCvalues_init = numpy.array(CCvalues_init)
616
+ n_obs_values_init = numpy.array(n_obs_values_init)
617
+ R2values_init = numpy.sqrt(1 - numpy.power(CCvalues_init, 2))
618
+ R2value_init = (
619
+ numpy.average(R2values_init, weights=n_obs_values_init)
620
+ if n_obs_values_init.size > 0 and n_obs_values_init.sum() > 0
621
+ else 0
622
+ )
623
+ data_overall_init_dict[f"R2_{stat}"].append(R2value_init)
624
+
625
+ CCvalues = []
626
+ n_obs_values = []
627
+ for bin_data in data_loaded[-1]["data"]["binned"]:
628
+ CCvalues.append(bin_data.get(stat_binned, 0))
629
+ n_obs_values.append(bin_data.get(n_label, 0))
630
+ CCvalues = numpy.array(CCvalues)
631
+ n_obs_values = numpy.array(n_obs_values)
632
+ R2values = numpy.sqrt(1 - numpy.power(CCvalues, 2))
633
+ R2value = (
634
+ numpy.average(R2values, weights=n_obs_values)
635
+ if n_obs_values.size > 0 and n_obs_values.sum() > 0
636
+ else 0
637
+ )
638
+ data_overall_dict[f"R2_{stat}"].append(R2value)
639
+ """
640
+
641
+ if json_ref and os.path.isfile(json_ref):
642
+ # Find reference statistics for a structure refined in a classic way
643
+ stats_ref_equivalents = {
644
+ "R": "R",
645
+ "R1": "R1",
646
+ "CCFavg": "CCFavg",
647
+ "CCIavg": "CCIavg",
648
+ "R_llw>0": "Rwork",
649
+ "R_llw=0": "Rfree",
650
+ "CCF_llw>0_avg": "CCFworkavg",
651
+ "CCF_llw=0_avg": "CCFfreeavg",
652
+ "R1_llw>0": "R1work",
653
+ "R1_llw=0": "R1free",
654
+ "CCI_llw>0_avg": "CCIworkavg",
655
+ "CCI_llw=0_avg": "CCIfreeavg",
656
+ "Rmsd Bond distances, non H": "Rmsd Bond distances, non H",
657
+ "Rmsd Bond angles, non H": "Rmsd Bond angles, non H",
658
+ "RmsZ Bond distances, non H": "RmsZ Bond distances, non H",
659
+ "RmsZ Bond angles, non H": "RmsZ Bond angles, non H",
660
+ "weight": "weight",
661
+ }
662
+ data_ref = {}
663
+
664
+ with open(json_ref) as f:
665
+ data_ref_loaded = json.load(f)
666
+ data_ref = data_ref_loaded[-1]["data"]["summary"]
667
+ if geom_stats_avail:
668
+ for stat in geom_stats:
669
+ data_ref[f"Rmsd {stat}"] = data_ref_loaded[-1]["geom"]["summary"][
670
+ "r.m.s.d."
671
+ ].get(stat, 0)
672
+ data_ref[f"RmsZ {stat}"] = data_ref_loaded[-1]["geom"]["summary"][
673
+ "r.m.s.Z"
674
+ ].get(stat, 0)
675
+ if "weight" in data_overall_dict and len(data_ref_loaded) >= 2:
676
+ data_ref["weight"] = data_ref_loaded[-1].get("weight", 0)
677
+ if data_ref:
678
+ logging.info(f"Loaded reference statistics from {json_ref}")
679
+ else:
680
+ logging.warning(f"No summary statistics found in reference {json_ref}")
681
+ else:
682
+ logging.warning(
683
+ f"Reference file with statistics not found {json_ref if json_ref else ''}"
684
+ )
685
+
686
+ distrs = {"init": {}, "final": {}}
687
+ llweight_R1_outliers = []
688
+ for stat in data_overall_dict.keys():
689
+ stat_ref = {}
690
+ if (
691
+ json_ref
692
+ and os.path.isfile(json_ref)
693
+ and data_ref
694
+ and stat in stats_ref_equivalents.keys()
695
+ and data_ref.get(stats_ref_equivalents[stat], None) is not None
696
+ ):
697
+ stat_ref = {
698
+ stats_ref_equivalents[stat]: data_ref.get(
699
+ stats_ref_equivalents[stat], None
700
+ )
701
+ }
702
+ distr, distr_init, llweight_outliers = scatter_plot_histogram(
703
+ data_overall_dict[stat],
704
+ data_overall_init_dict[stat],
705
+ stat,
706
+ stat_ref,
707
+ idx,
708
+ prefix,
709
+ filtered=False,
710
+ )
711
+ distrs["final"][stat] = distr
712
+ distrs["init"][stat] = distr_init
713
+ if stat == "R1":
714
+ if llweight_outliers is not None:
715
+ llweight_R1_outliers = llweight_outliers["index"].tolist()
716
+ else:
717
+ llweight_R1_outliers = []
718
+
719
+ json_filename = (
720
+ f"{prefix}group{idx}_bootstrap_statistics.json"
721
+ if idx
722
+ else f"{prefix}bootstrap_statistics.json"
723
+ )
724
+ with open(json_filename, "w") as f:
725
+ json.dump(distrs, f, indent=2, default=json_numpy_converter)
726
+ logging.info(f"Saved statistics to {json_filename}")
727
+
728
+ # Filter outliers based on R1 values and re-analyse the distributions
729
+ if llweight_R1_outliers:
730
+ distrs_filtered = {"init": {}, "final": {}}
731
+ data_overall_filtered_dict = {}
732
+ data_overall_init_filtered_dict = {}
733
+ logging.info(
734
+ f"\nIdentified {len(llweight_R1_outliers) if llweight_R1_outliers else 0}"
735
+ " R1 value outliers which will be exluded from the following analysis\n"
736
+ f" {llweight_R1_outliers if llweight_R1_outliers else ''}"
737
+ )
738
+ for stat in data_overall_dict.keys():
739
+ data_overall_filtered_dict[stat] = [
740
+ val
741
+ for i, val in enumerate(data_overall_dict[stat])
742
+ if i not in llweight_R1_outliers
743
+ ]
744
+ data_overall_init_filtered_dict[stat] = [
745
+ val
746
+ for i, val in enumerate(data_overall_init_dict[stat])
747
+ if i not in llweight_R1_outliers
748
+ ]
749
+ if (
750
+ json_ref
751
+ and os.path.isfile(json_ref)
752
+ and data_ref
753
+ and stat in stats_ref_equivalents.keys()
754
+ and data_ref.get(stats_ref_equivalents[stat], None) is not None
755
+ ):
756
+ stat_ref = {
757
+ stats_ref_equivalents[stat]: data_ref.get(
758
+ stats_ref_equivalents[stat], None
759
+ )
760
+ }
761
+ distr_filtered, distr_init_filtered, _ = scatter_plot_histogram(
762
+ data_overall_filtered_dict[stat],
763
+ data_overall_init_filtered_dict[stat],
764
+ stat,
765
+ stat_ref,
766
+ idx,
767
+ prefix,
768
+ filtered=True,
769
+ )
770
+ distrs_filtered["final"][stat] = distr_filtered
771
+ distrs_filtered["init"][stat] = distr_init_filtered
772
+
773
+ json_filename_filtered = (
774
+ f"{prefix}group{idx}_bootstrap_statistics_filtered.json"
775
+ if idx
776
+ else f"{prefix}bootstrap_statistics_filtered.json"
777
+ )
778
+ with open(json_filename_filtered, "w") as f:
779
+ json.dump(distrs_filtered, f, indent=2, default=json_numpy_converter)
780
+ logging.info(f"Saved filtered statistics to {json_filename_filtered}")
781
+
782
+ logging.info("")
783
+ return llweight_R1_outliers