multixem 0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
multixem/run.py ADDED
@@ -0,0 +1,2193 @@
1
+ # coding: utf-8
2
+ import os
3
+ import sys
4
+ import argparse
5
+ import subprocess
6
+ import numpy
7
+ import pandas
8
+ import matplotlib.pyplot as plt
9
+ import matplotlib
10
+ import logging
11
+ import json
12
+ import shlex
13
+ import glob
14
+ import re
15
+ import yaml
16
+ from collections import Counter
17
+ import concurrent.futures
18
+
19
+ # servalcat 0.4.145 must be imported before gemmi
20
+ try:
21
+ from servalcat import utils as servalcat_utils
22
+
23
+ servalcat_imported = True
24
+ except ModuleNotFoundError:
25
+ servalcat_imported = False
26
+ import gemmi
27
+ from . import __version__
28
+ from .tools import write_bin_stats, calc_scale_real, write_mtz_from_df
29
+ from .analyse_refinement import (
30
+ adp_analysis_histograms,
31
+ compute_difference_maps,
32
+ compute_structure_differences,
33
+ write_coot_script,
34
+ )
35
+ from .bootstrap_dataset import bootstrap_dataset
36
+ from .bootstrap_statistics import bootstrap_analyse_stats
37
+ from .bootstrap_structures import (
38
+ # unrestrain,
39
+ unrestrain_yaml,
40
+ geometry_analysis_load,
41
+ select_cids_for_geometry_analysis,
42
+ bootstrap_analyse_structures,
43
+ )
44
+ from .bootstrap_maps import bootstrap_mean_map
45
+
46
+ matplotlib.use("Agg")
47
+
48
+
49
+ def setup_logging():
50
+ """
51
+ Set up logging to output to both console and file.
52
+ """
53
+ # Create logger
54
+ logger = logging.getLogger()
55
+ logger.setLevel(logging.INFO)
56
+
57
+ # Remove any existing handlers
58
+ for handler in logger.handlers[:]:
59
+ logger.removeHandler(handler)
60
+
61
+ # Create formatters - simple format for INFO, detailed for others
62
+ class CustomFormatter(logging.Formatter):
63
+ def format(self, record):
64
+ if record.levelno in [logging.INFO, logging.DEBUG]:
65
+ return record.getMessage()
66
+ else:
67
+ formatted_time = self.formatTime(record, datefmt="%Y-%m-%d %H:%M:%S")
68
+ return f"{formatted_time} - {record.levelname} - {record.getMessage()}"
69
+
70
+ formatter = CustomFormatter()
71
+
72
+ # Console handler
73
+ console_handler = logging.StreamHandler(sys.stdout)
74
+ console_handler.setLevel(logging.INFO)
75
+ console_handler.setFormatter(formatter)
76
+ logger.addHandler(console_handler)
77
+
78
+ # File handler - capture everything including DEBUG
79
+ file_handler = logging.FileHandler("multixem.log", mode="a")
80
+ file_handler.setLevel(logging.DEBUG)
81
+ file_handler.setFormatter(formatter)
82
+ logger.addHandler(file_handler)
83
+
84
+ # Capture warnings and route them through logging
85
+ logging.captureWarnings(True)
86
+ warnings_logger = logging.getLogger("py.warnings")
87
+ warnings_logger.setLevel(logging.WARNING)
88
+
89
+ # Also capture uncaught exceptions
90
+ def handle_exception(exc_type, exc_value, exc_traceback):
91
+ if issubclass(exc_type, KeyboardInterrupt):
92
+ # Allow KeyboardInterrupt to be handled normally
93
+ sys.__excepthook__(exc_type, exc_value, exc_traceback)
94
+ return
95
+ # Log uncaught exceptions with full traceback
96
+ import traceback
97
+
98
+ tb_str = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback))
99
+ logger.error(f"Uncaught exception:\n{tb_str}")
100
+
101
+ sys.excepthook = handle_exception
102
+
103
+ return logger
104
+
105
+
106
+ def sort_llweight_files(file_list):
107
+ """
108
+ Sort files containing '_llweight<number>_' by the numerical value.
109
+
110
+ Args:
111
+ file_list: List of file paths containing '_llweight<number>_'
112
+
113
+ Returns:
114
+ Sorted list of file paths
115
+ """
116
+
117
+ def extract_llweight_number(filepath):
118
+ match = re.search(r"_llweight(\d+)_", filepath)
119
+ if match:
120
+ return int(match.group(1))
121
+ return -1 # Files without llweight number will be sorted first
122
+
123
+ return sorted(file_list, key=extract_llweight_number)
124
+
125
+
126
+ def filter_out_refined_files(mmcif_files, llweights):
127
+ """
128
+ Filter out refined mmcif files corresponding to outliers in R1 values.
129
+
130
+ Args:
131
+ mmcif_files: List of refined mmcif file paths.
132
+ llweights: List of llweight numbers to exclude.
133
+
134
+ Returns:
135
+ Filtered list of mmcif file paths.
136
+ """
137
+ if not llweights:
138
+ return mmcif_files
139
+ filtered_files = []
140
+ for file in mmcif_files:
141
+ match = re.search(r"_llweight(\d+)_", file)
142
+ if match:
143
+ llweight_num = int(match.group(1))
144
+ if llweight_num not in llweights:
145
+ filtered_files.append(file)
146
+ else:
147
+ logging.warning(
148
+ f"Could not get llweight number from {file}, including it by default."
149
+ )
150
+ filtered_files.append(file)
151
+ return filtered_files
152
+
153
+
154
+ def create_parser():
155
+ """
156
+ Create the argument parser for the command-line interface.
157
+
158
+ Returns:
159
+ argparse.ArgumentParser: The argument parser object.
160
+ """
161
+
162
+ def positive_int(value):
163
+ if isinstance(value, list):
164
+ positive_int(v for v in value)
165
+ else:
166
+ try:
167
+ ivalue = int(value)
168
+ except ValueError:
169
+ raise argparse.ArgumentTypeError(f"{value} is not an integer.")
170
+ if ivalue <= 0:
171
+ raise argparse.ArgumentTypeError(f"{value} is not a positive integer.")
172
+ return ivalue
173
+
174
+ def positive_float(value):
175
+ try:
176
+ fvalue = float(value)
177
+ except ValueError:
178
+ raise argparse.ArgumentTypeError(f"{value} is not a float.")
179
+ if fvalue <= 0:
180
+ raise argparse.ArgumentTypeError(f"{value} is not a positive float.")
181
+ return fvalue
182
+
183
+ def existing_file(path):
184
+ abs_norm_path = os.path.abspath(os.path.normpath(path))
185
+ if not os.path.isfile(abs_norm_path):
186
+ raise argparse.ArgumentTypeError(f"File does not exist: {abs_norm_path}")
187
+ return abs_norm_path
188
+
189
+ def existing_directory(path):
190
+ abs_norm_path = os.path.abspath(os.path.normpath(path))
191
+ if not os.path.isdir(abs_norm_path):
192
+ raise argparse.ArgumentTypeError(
193
+ f"Directory does not exist: {abs_norm_path}"
194
+ )
195
+ return abs_norm_path
196
+
197
+ class ArgumentDefaultsHelpFormatterCustom(argparse.ArgumentDefaultsHelpFormatter):
198
+ def _get_help_string(self, action):
199
+ help_str = action.help
200
+ # Only show default if it's not None and not suppressed
201
+ if action.default is not None and action.default != argparse.SUPPRESS:
202
+ if "%(default)" not in help_str:
203
+ help_str += f" (default: {action.default})"
204
+ return help_str
205
+
206
+ parser = argparse.ArgumentParser(
207
+ prog="multixem",
208
+ description="Refinement pipeline for multiple data sets in structure biology.",
209
+ formatter_class=ArgumentDefaultsHelpFormatterCustom,
210
+ )
211
+ parser.add_argument(
212
+ "-v",
213
+ "--version",
214
+ action="version",
215
+ version=__version__,
216
+ help="show version and exit",
217
+ )
218
+
219
+ subparsers = parser.add_subparsers(dest="command", help="Available commands")
220
+
221
+ # Parent parser for common options
222
+ common_parent = argparse.ArgumentParser(add_help=False)
223
+ common_parent.add_argument(
224
+ "--n_bins",
225
+ type=positive_int,
226
+ default=20,
227
+ help="Number of resolution bins. Must be a positive integer.",
228
+ )
229
+ common_parent.add_argument(
230
+ "-p", "--prefix", type=str, help="Prefix for the output files."
231
+ )
232
+ common_parent.add_argument(
233
+ "--n_proc",
234
+ type=positive_int,
235
+ default=4,
236
+ help="Number of processes to use for parallelisation."
237
+ + " Must be a positive integer.",
238
+ )
239
+ common_parent.add_argument(
240
+ "--geometry_cids",
241
+ type=existing_file,
242
+ help=(
243
+ "Input file with atomic CIDs defining bonds, angles or torsions to be"
244
+ " analysed while performing bootstrap."
245
+ ),
246
+ )
247
+ common_parent.add_argument(
248
+ "--no_origin_shift",
249
+ action="store_true",
250
+ default=False,
251
+ help=(
252
+ "In some point groups, origin is not fixed (floating origin problem)."
253
+ " The program automatically detects this and applies a shift to"
254
+ " account for it. Use this option to disable this automatic correction."
255
+ ),
256
+ )
257
+
258
+ common_refinement_parent = argparse.ArgumentParser(add_help=False)
259
+ # TODO does not work for single command line argument
260
+ common_refinement_parent.add_argument(
261
+ "--servalcat_args",
262
+ type=str,
263
+ default=[],
264
+ help="Command line arguments for Servalcat (quoted)."
265
+ + " Do not use options -s, --source, --config and --hout here.",
266
+ )
267
+ common_refinement_parent.add_argument(
268
+ "--model",
269
+ type=existing_file,
270
+ nargs="+",
271
+ help="Input atomic structure model file(s).",
272
+ required=False,
273
+ )
274
+ # TODO more files
275
+ common_refinement_parent.add_argument(
276
+ "--hklin",
277
+ type=existing_file,
278
+ nargs="+",
279
+ help="Input merged diffraction data file(s).",
280
+ )
281
+ common_refinement_parent.add_argument(
282
+ "--hklin_free", type=existing_file, help="Input MTZ file for test flags."
283
+ )
284
+ common_refinement_parent.add_argument(
285
+ "--amplitude",
286
+ action="store_true",
287
+ help="Use amplitude rather than intensities (not recommended).",
288
+ )
289
+ common_refinement_parent.add_argument(
290
+ "--model_dir",
291
+ type=existing_directory,
292
+ help=(
293
+ "Directory containing multiple input atomic structure model files"
294
+ " for bootstrap."
295
+ ),
296
+ )
297
+ common_refinement_parent.add_argument(
298
+ "--unre",
299
+ type=positive_int,
300
+ default=0,
301
+ help="Number of cycles for prior unrestrained refinement during bootstrap.",
302
+ )
303
+ common_refinement_parent.add_argument(
304
+ "--quick",
305
+ action="store_true",
306
+ help="Quick run (only for development).",
307
+ )
308
+ # TODO: if input has Friedel pairs but a user wants to merge them
309
+ common_refinement_parent.add_argument(
310
+ "-s",
311
+ "--source",
312
+ type=str,
313
+ default="xray",
314
+ choices=["xray", "electron", "neutron"],
315
+ help="Radiation source, xray or electron or neutron.",
316
+ )
317
+ common_refinement_parent.add_argument(
318
+ "--servalcat_config",
319
+ type=existing_file,
320
+ help=(
321
+ "YAML configuration file for Servalcat"
322
+ " (e.g. to specify occupancy refinement)"
323
+ ),
324
+ )
325
+ common_refinement_parent.add_argument(
326
+ "--draw_factor",
327
+ type=positive_float,
328
+ default=1.0,
329
+ help=(
330
+ "Factor for a number of draws in bootstrap resampling."
331
+ " (the default 1.0 means the number of draws is equal"
332
+ " to the number of reflections in each bin)."
333
+ ),
334
+ )
335
+ common_refinement_parent.add_argument(
336
+ "--random_weights",
337
+ action="store_true",
338
+ help=(
339
+ "Use fractional-random-weight bootstrap method, i.e. assign random weights"
340
+ " to reflection likelihood terms in the target log-likelihood function."
341
+ " A fraction of reflection will be assigned zero weights"
342
+ " based on the provided free flag in --hklin_free or --hklin,"
343
+ " or randomly if no free flag is provided, controlled by --fraction_zero."
344
+ ),
345
+ )
346
+ common_refinement_parent.add_argument(
347
+ "--fraction_zero",
348
+ type=float,
349
+ default=0.0,
350
+ help=(
351
+ "If random weighting is used, this specifies the fraction of"
352
+ " zero weights to include. Must be in [0.0, 1.0)."
353
+ " If a value of 0.0 is set (default), zero weights will be assigned"
354
+ " according to the free flags in --hklin_free or --hklin."
355
+ " Recommended value: 0.05."
356
+ ),
357
+ )
358
+
359
+ # Main pipeline subcommand (default behavior)
360
+ pipeline_parser = subparsers.add_parser(
361
+ "pipeline",
362
+ parents=[common_parent, common_refinement_parent],
363
+ help="Run the main refinement pipeline",
364
+ formatter_class=ArgumentDefaultsHelpFormatterCustom,
365
+ )
366
+ pipeline_parser.add_argument(
367
+ "-u",
368
+ "--hklin_unmerged",
369
+ type=existing_file,
370
+ nargs="+",
371
+ help="Input unmerged diffraction data file(s).",
372
+ )
373
+ pipeline_parser.add_argument(
374
+ "--n_batches",
375
+ type=positive_int,
376
+ nargs="+",
377
+ default=0, # default values set up in validate_args()
378
+ help="Number of batches per merging group, or list of batch edges"
379
+ + " where to split the data."
380
+ + " Must be a positive integer or space-separated list of positive integers.",
381
+ )
382
+ pipeline_parser.add_argument(
383
+ "--merge_whole_file",
384
+ action="store_true",
385
+ help="Merge all the batches in the input unmerged file(s).",
386
+ )
387
+ pipeline_parser.add_argument(
388
+ "--molrep",
389
+ action="store_true",
390
+ help="Run MolRep for molecular replacement before structure refinement.",
391
+ )
392
+ pipeline_parser.add_argument(
393
+ "--unify_cell",
394
+ action="store_true",
395
+ help=(
396
+ "Set the same unit cell parameters for all datasets, "
397
+ "based on the first dataset."
398
+ ),
399
+ )
400
+ pipeline_parser.add_argument(
401
+ "--bootstrap",
402
+ type=positive_int,
403
+ default=0,
404
+ help=(
405
+ "No. of bootstrap resampled sub data sets to be created and"
406
+ " used for refinement. Must be a positive integer higher than 1."
407
+ ),
408
+ )
409
+
410
+ # mean subcommand - mean and std for statistics, model and maps
411
+ mean_parser = subparsers.add_parser(
412
+ "mean",
413
+ parents=[common_parent],
414
+ help="Calculate mean maps from bootstrapped refinement results",
415
+ formatter_class=ArgumentDefaultsHelpFormatterCustom,
416
+ )
417
+ mean_parser.add_argument(
418
+ "file_name_template",
419
+ type=str,
420
+ help=(
421
+ "Template name for input files, e.g. put `dataset`"
422
+ " for `dataset_llweight*_refine.mtz` and `dataset_llweight*_refine.mmcif`."
423
+ ),
424
+ )
425
+ mean_parser.add_argument(
426
+ "--calculate_mean_map",
427
+ type=bool,
428
+ default=False,
429
+ help="Calculate mean maps.",
430
+ )
431
+ mean_parser.add_argument(
432
+ "--cif",
433
+ type=existing_file,
434
+ help="Path to a small molecule CIF file.",
435
+ )
436
+
437
+ # Bootstrap subcommand (optional new command)
438
+ bootstrap_parser = subparsers.add_parser(
439
+ "bootstrap",
440
+ parents=[common_parent, common_refinement_parent],
441
+ help="Run multiple refinement against bootstrap sub data sets",
442
+ formatter_class=ArgumentDefaultsHelpFormatterCustom,
443
+ )
444
+ bootstrap_parser.add_argument(
445
+ "n_samples",
446
+ type=positive_int,
447
+ default=5000,
448
+ help=(
449
+ "No. of bootstrap resampled sub data sets to be created and"
450
+ " used for refinement. Must be a positive integer higher than 1."
451
+ ),
452
+ )
453
+
454
+ def validate_common(args):
455
+ if args.n_proc <= 0:
456
+ parser.error("--n_proc must be positive")
457
+
458
+ def validate_model_dir(args, n_required):
459
+ if not args.model_dir:
460
+ return
461
+ model_files = []
462
+ for pattern in ["*.pdb", "*.cif", "*.mmcif"]:
463
+ model_files.extend(glob.glob(os.path.join(args.model_dir, pattern)))
464
+
465
+ if len(model_files) < n_required:
466
+ parser.error(
467
+ f"--model_dir contains {len(model_files)} structure model files, "
468
+ f"but at least {n_required} are required."
469
+ )
470
+ args.models = sorted(model_files)[:n_required]
471
+
472
+ def validate_pipeline(args):
473
+ validate_common(args)
474
+ if args.n_batches and not args.hklin_unmerged:
475
+ parser.error("--n_batches requires --hklin_unmerged to be provided.")
476
+ if args.merge_whole_file and not args.hklin_unmerged:
477
+ parser.error("--merge_whole_file requires --hklin_unmerged to be provided.")
478
+ if args.hklin_unmerged and not args.n_batches and not args.merge_whole_file:
479
+ args.n_batches = [60] # Default
480
+ if args.model and len(args.model) > 1:
481
+ if len(args.model or []) < (
482
+ len(args.hklin or []) + len(args.hklin_unmerged or [])
483
+ ):
484
+ parser.error(
485
+ "Just a single model can be provided for multiple data sets, "
486
+ "or the number of models must match the number of data sets."
487
+ )
488
+ if args.bootstrap and args.bootstrap < 2:
489
+ parser.error("--bootstrap must be at least 2.")
490
+ if args.geometry_cids and not args.bootstrap:
491
+ parser.error("--geometry_cids requires --bootstrap to be provided.")
492
+ if args.unre and not args.bootstrap:
493
+ parser.error("--unre requires --bootstrap to be provided.")
494
+ if args.bootstrap and args.model_dir:
495
+ validate_model_dir(args, args.bootstrap)
496
+ if args.random_weights and (
497
+ args.fraction_zero < 0.0 or args.fraction_zero >= 1.0
498
+ ):
499
+ parser.error("--fraction_zero must be in the range [0.0, 1.0).")
500
+
501
+ def validate_mean(args):
502
+ validate_common(args)
503
+
504
+ def validate_bootstrap(args):
505
+ validate_common(args)
506
+ # TODO
507
+ if not args.hklin:
508
+ parser.error("--hklin is required for bootstrap.")
509
+ if args.n_samples < 2:
510
+ parser.error("Number of samples must be at least 2.")
511
+ if args.model_dir:
512
+ model_files = glob.glob(os.path.join(args.model_dir, "*.pdb"))
513
+ model_files += glob.glob(os.path.join(args.model_dir, "*.cif"))
514
+ model_files += glob.glob(os.path.join(args.model_dir, "*.mmcif"))
515
+ if len(model_files) < args.n_samples:
516
+ parser.error(
517
+ f"--model_dir contains {len(model_files)} model files, "
518
+ f"but at least {args.n_samples} are required."
519
+ )
520
+ args.models = sorted(model_files)[: args.n_samples]
521
+ if args.model_dir:
522
+ validate_model_dir(args, args.n_samples)
523
+ if args.random_weights and (
524
+ args.fraction_zero < 0.0 or args.fraction_zero >= 1.0
525
+ ):
526
+ parser.error("--fraction_zero must be in the range [0.0, 1.0).")
527
+
528
+ pipeline_parser.set_defaults(func=validate_pipeline)
529
+ mean_parser.set_defaults(func=validate_mean)
530
+ bootstrap_parser.set_defaults(func=validate_bootstrap)
531
+
532
+ # Parse and validate immediately
533
+ original_parse_args = parser.parse_args
534
+
535
+ def parse_and_validate(cmd_args=None):
536
+ args = original_parse_args(cmd_args)
537
+ if hasattr(args, "func"):
538
+ args.func(args)
539
+ return args
540
+
541
+ parser.parse_args = parse_and_validate
542
+
543
+ return parser
544
+
545
+
546
+ def check_reflection_file_columns(
547
+ hklin: str,
548
+ unmerged: bool = False,
549
+ prefer_amplitude: bool = False,
550
+ search_col_free: bool = False,
551
+ ):
552
+ """
553
+ Check the input reflection file for the presence of intensities,
554
+ amplitudes and Friedel pairs. Find the column labels and decide which to use.
555
+
556
+ Args:
557
+ hklin (str): gemmi.Mtz object or path to the input reflection file.
558
+ unmerged (bool): Whether the input file is unmerged data.
559
+ prefer_amplitude (bool): If both intensities and amplitudes are found,
560
+ prefer amplitudes over intensities.
561
+ search_col_free (bool): Whether to search only for a free reflection column
562
+
563
+ Returns:
564
+ tuple: A tuple containing three boolean values:
565
+ - labin (str): Column labels for --labin option in Servalcat.
566
+ - anom: True if Friedel pairs are present.
567
+ """
568
+ if not isinstance(hklin, gemmi.Mtz):
569
+ m = gemmi.read_mtz_file(hklin)
570
+ else:
571
+ m = hklin
572
+ # TODO: CIF
573
+ # dmax = m.resolution_low()
574
+ # dmin = m.resolution_high()
575
+ unexpected_column_warning = (
576
+ "This is quite unusual for unmerged data file, are you sure about the file?"
577
+ )
578
+
579
+ anom = False
580
+
581
+ col_iplusminus = []
582
+ col_sigiplusminus = []
583
+ col_imean = []
584
+ col_fplusminus = []
585
+ col_sigfplusminus = []
586
+ col_fmean = []
587
+ col_sigxmean = []
588
+ col_free = ""
589
+
590
+ for column in m.columns:
591
+ if not col_free and column.type == "I":
592
+ if "free" in column.label.lower() and "flag" in column.label.lower():
593
+ col_free = column.label
594
+ logging.info(f"Column with free R flag (type I) found: {column.label}")
595
+ if search_col_free:
596
+ logging.info("Free R flag column found successfully.")
597
+ return None, None, col_free
598
+ elif column.type == "K":
599
+ logging.info(
600
+ f"Column with intensity (type K, Friedel pairs) found: {column.label}"
601
+ )
602
+ logging.info("Friedel pairs will be kept separately.")
603
+ anom = True
604
+ col_iplusminus.append(column.label)
605
+ elif column.type == "M":
606
+ logging.info(
607
+ f"Column with standard deviation associated to intensity column"
608
+ f" (type M, Friedel pairs) found: {column.label}"
609
+ )
610
+ col_sigiplusminus.append(column.label)
611
+ elif column.type == "J":
612
+ logging.info(
613
+ f"Column with intensity (type J, no Friedel pairs)"
614
+ f" found: {column.label}"
615
+ )
616
+ col_imean.append(column.label)
617
+ elif column.type == "Q":
618
+ logging.info(
619
+ f"Column with standard deviation associated to intensity/amplitude"
620
+ f" column (type Q, no Friedel pairs) found: {column.label}"
621
+ )
622
+ col_sigxmean.append(column.label)
623
+ elif column.type == "G":
624
+ logging.info(
625
+ f"Column with amplitude (type G, Friedel pairs) found: {column.label}"
626
+ )
627
+ anom = True
628
+ col_fplusminus.append(column.label)
629
+ if unmerged:
630
+ logging.warning(unexpected_column_warning)
631
+ elif column.type == "L":
632
+ logging.info(
633
+ f"Column with standard deviation associated to amplitude"
634
+ f" (type L, Friedel pairs) found: {column.label}"
635
+ )
636
+ col_sigfplusminus.append(column.label)
637
+ elif column.type == "F":
638
+ logging.info(
639
+ f"Column with amplitude (type F, no Friedel pairs)"
640
+ f" found: {column.label}"
641
+ )
642
+ col_fmean.append(column.label)
643
+ if unmerged:
644
+ logging.warning(unexpected_column_warning)
645
+
646
+ labin = None
647
+ if not prefer_amplitude:
648
+ if len(col_iplusminus) >= 2 and len(col_sigiplusminus) >= 2:
649
+ labin = (
650
+ f"{col_iplusminus[0]},{col_sigiplusminus[0]},"
651
+ f"{col_iplusminus[1]},{col_sigiplusminus[1]}"
652
+ )
653
+ elif col_imean and col_sigxmean:
654
+ labin = f"{col_imean[0]},{col_sigxmean[0]}"
655
+ if not labin:
656
+ if len(col_fplusminus) >= 2 and len(col_sigfplusminus) >= 2:
657
+ labin = (
658
+ f"{col_fplusminus[0]},{col_sigfplusminus[0]},"
659
+ f"{col_fplusminus[1]},{col_sigfplusminus[1]}"
660
+ )
661
+ elif col_fmean and col_sigxmean:
662
+ labin = f"{col_fmean[0]},{col_sigxmean[0]}"
663
+
664
+ if not labin:
665
+ raise RuntimeError(f"Neither intensities nor amplitudes found in {hklin}.")
666
+ elif (
667
+ (len(col_fplusminus) >= 2 and len(col_sigfplusminus) >= 2)
668
+ or (col_fmean and col_sigxmean)
669
+ ) and not (
670
+ (len(col_iplusminus) >= 2 and len(col_sigiplusminus) >= 2)
671
+ or (col_imean and col_sigxmean)
672
+ ):
673
+ logging.warning(
674
+ "The file contain only amplitudes but not intensities, however,"
675
+ " providing intensities is recommended."
676
+ )
677
+ logging.info(f"Using these columns for refinement: {labin}")
678
+
679
+ return labin, anom, col_free
680
+
681
+
682
+ def copy_cell_mtz(mtz_input, mtz_reference):
683
+ """
684
+ Copy unit cell parameters from a reference MTZ file to
685
+ an MTZ file using gemmi.
686
+
687
+ Args:
688
+ mtz_input (str): Path to input MTZ file
689
+ mtz_reference (str): Path to reference MTZ file
690
+ Returns:
691
+ str: Output MTZ file name
692
+ """
693
+ mtz = gemmi.read_mtz_file(mtz_input)
694
+ mtz_ref = gemmi.read_mtz_file(mtz_reference)
695
+ mtz.set_cell_for_all(mtz_ref.cell)
696
+ mtz_out_filepath = os.path.basename(mtz_input).replace(".mtz", "_cell.mtz")
697
+ mtz.write_to_file(mtz_out_filepath)
698
+ logging.info(
699
+ f"Copied unit cell parameters"
700
+ f" {mtz_ref.cell.a} {mtz_ref.cell.b} {mtz_ref.cell.c}"
701
+ f" {mtz_ref.cell.alpha} {mtz_ref.cell.beta} {mtz_ref.cell.gamma}"
702
+ f" from {mtz_reference} to {mtz_out_filepath}"
703
+ )
704
+ return mtz_out_filepath
705
+
706
+
707
+ def merge_in_groups(
708
+ unmerged,
709
+ n_bins,
710
+ prefix,
711
+ n_batches_per_group=60,
712
+ batches_edges=[],
713
+ merge_whole_file=False,
714
+ i_group_prefix=0,
715
+ ):
716
+
717
+ def merge_group(
718
+ df_groups,
719
+ i_group,
720
+ cell,
721
+ spacegroup,
722
+ binner,
723
+ n_expected,
724
+ wavelength=0,
725
+ n_groups=0,
726
+ anom=True,
727
+ prefix="",
728
+ i_group_prefix=0,
729
+ ):
730
+ intensities = gemmi.Intensities()
731
+ intensities.set_data(
732
+ cell,
733
+ spacegroup,
734
+ df_groups[i_group][["H", "K", "L"]].values,
735
+ df_groups[i_group]["I"].values,
736
+ df_groups[i_group]["SIGI"].values,
737
+ )
738
+ if anom:
739
+ intensities.prepare_for_merging(gemmi.DataType.Anomalous)
740
+ else:
741
+ intensities.prepare_for_merging(gemmi.DataType.Mean)
742
+ bin_stats = intensities.calculate_merging_stats(binner)
743
+ # Collect bin statistics into a list of dictionaries
744
+ bin_stats_list = []
745
+ for n, stats in enumerate(bin_stats):
746
+ bin_stats_list.append(
747
+ {
748
+ "bin": n + 1,
749
+ "dmax": binner.dmax_of_bin(n),
750
+ "dmin": binner.dmin_of_bin(n),
751
+ "CC1/2": stats.cc_half(),
752
+ "CC*": stats.cc_star(),
753
+ "Rmeas": stats.r_meas(),
754
+ "Rpim": stats.r_pim(),
755
+ }
756
+ )
757
+ # Add overall stats as a separate entry
758
+ overall_stats = intensities.calculate_merging_stats(None)
759
+ bin_stats_list.append(
760
+ {
761
+ "bin": "overall",
762
+ "dmax": intensities.resolution_range()[0],
763
+ "dmin": intensities.resolution_range()[1],
764
+ "CC1/2": overall_stats[0].cc_half(),
765
+ "CC*": overall_stats[0].cc_star(),
766
+ "Rmeas": overall_stats[0].r_meas(),
767
+ "Rpim": overall_stats[0].r_pim(),
768
+ }
769
+ )
770
+
771
+ if anom:
772
+ intensities.merge_in_place(gemmi.DataType.Anomalous)
773
+ else:
774
+ intensities.merge_in_place(gemmi.DataType.Mean)
775
+ # SIGI from merging: 1/sqrt(∑w), where w=1/sigma^2
776
+ mtz_group_merged = intensities.prepare_merged_mtz(with_nobs=True)
777
+ if wavelength:
778
+ mtz_group_merged.dataset(0).wavelength = wavelength
779
+ if n_groups:
780
+ g_with_leading_zeros = str(i_group_prefix + i_group + 1).zfill(
781
+ len(str(n_groups))
782
+ )
783
+ else:
784
+ g_with_leading_zeros = i_group_prefix + i_group + 1
785
+ # After merging, add <I> <I/sigI> n_unique n_obs completeness multiplicity
786
+ df = pandas.DataFrame(
787
+ data=mtz_group_merged.array, columns=mtz_group_merged.column_labels()
788
+ )
789
+ df["BIN"] = binner.get_bins(mtz_group_merged)
790
+ for b in range(binner.size):
791
+ df_bin = df[df["BIN"] == b]
792
+ bin_n_unique_expected = gemmi.count_reflections(
793
+ cell,
794
+ spacegroup,
795
+ binner.dmin_of_bin(b),
796
+ binner.dmax_of_bin(b),
797
+ unique=True,
798
+ )
799
+ bin_stats_list[b]["<I>"] = df_bin["IMEAN"].mean()
800
+ bin_stats_list[b]["<I/sigI>"] = (
801
+ df_bin["IMEAN"] / df_bin["SIGIMEAN"]
802
+ ).mean()
803
+ bin_stats_list[b]["n_unique"] = len(df_bin)
804
+ bin_stats_list[b]["completeness"] = len(df_bin) / bin_n_unique_expected
805
+ bin_stats_list[b]["n_obs"] = int(df_bin["NOBS"].sum())
806
+ bin_stats_list[b]["multiplicity"] = df_bin["NOBS"].mean()
807
+ n_unique_expected = gemmi.count_reflections(
808
+ cell,
809
+ spacegroup,
810
+ intensities.resolution_range()[1],
811
+ intensities.resolution_range()[0],
812
+ unique=True,
813
+ )
814
+ bin_stats_list[-1]["<I>"] = df["IMEAN"].mean()
815
+ bin_stats_list[-1]["<I/sigI>"] = (df["IMEAN"] / df["SIGIMEAN"]).mean()
816
+ bin_stats_list[-1]["n_unique"] = len(df)
817
+ # TODO: overall completeness is not correct
818
+ bin_stats_list[-1]["completeness"] = len(df) / n_unique_expected
819
+ bin_stats_list[-1]["n_obs"] = int(df["NOBS"].sum())
820
+ bin_stats_list[-1]["multiplicity"] = df["NOBS"].mean()
821
+ bin_stats_list[-1]["n_unique"] = len(intensities.miller_array)
822
+ stats_filename = (
823
+ f"{prefix}group{i_group_prefix + i_group + 1}_merging_stats.txt"
824
+ )
825
+ write_bin_stats(bin_stats_list, stats_filename)
826
+
827
+ ## n_expected = gemmi.count_reflections(m.cell, m.spacegroup, dmin, dmax)
828
+ completeness = len(intensities.miller_array) / n_expected
829
+ logging.info(
830
+ f"Merged group {i_group_prefix + i_group + 1} of batches: #reflections:"
831
+ f" {len(intensities.miller_array)}"
832
+ f" => completeness: {completeness:.3f}",
833
+ )
834
+
835
+ mtz_group_merged_filename = f"{prefix}group{g_with_leading_zeros}_I.mtz"
836
+ mtz_group_merged.write_to_file(mtz_group_merged_filename)
837
+
838
+ return mtz_group_merged_filename, bin_stats_list
839
+
840
+ if unmerged.lower().endswith(".hkl"):
841
+ xds_ascii = gemmi.read_xds_ascii(unmerged)
842
+ m = xds_ascii.to_mtz()
843
+ # Calculate resolution range from the unmerged file
844
+ d_array = m.cell.calculate_d_array(m.make_miller_array())
845
+ dmax = max(d_array)
846
+ dmin = min(d_array)
847
+ if unmerged.lower().endswith(".cif") or unmerged.lower().endswith(".ent"):
848
+ doc = gemmi.cif.read(unmerged)
849
+ rblocks = gemmi.as_refln_blocks(doc)
850
+ for rblock in rblocks:
851
+ if not rblock.is_merged():
852
+ Convert = gemmi.CifToMtz()
853
+ m = Convert.convert_block_to_mtz(rblock)
854
+ d_array = m.cell.calculate_d_array(m.make_miller_array())
855
+ dmax = max(d_array)
856
+ dmin = min(d_array)
857
+ break
858
+ else:
859
+ m = gemmi.read_mtz_file(unmerged)
860
+ dmax = m.resolution_low()
861
+ dmin = m.resolution_high()
862
+
863
+ logging.info(
864
+ f"Resolution limits: {dmax:.3f} - {dmin:.3f} A\n"
865
+ f"Space group: {m.spacegroup.hm} (No. {m.spacegroup.number})\n"
866
+ f"Unit cell: {m.cell.a:.3f} {m.cell.b:.3f} {m.cell.c:.3f}"
867
+ f" {m.cell.alpha:.3f} {m.cell.beta:.3f} {m.cell.gamma:.3f}"
868
+ )
869
+ # Scan the columns of the input unmerged MTZ file
870
+ # and check if Friedel pairs are present or not
871
+ anom = False
872
+ labin, anom, col_free = check_reflection_file_columns(m, unmerged=True)
873
+ # print(m.dataset(0).wavelength) == 0.0
874
+ # print(m.dataset(1).wavelength) OK
875
+ # print(m.datasets[0].wavelength) == 0.0
876
+ if m.datasets[-1].wavelength:
877
+ wavelength = m.datasets[-1].wavelength
878
+ logging.info(f"Wavelength from input file: {m.datasets[-1].wavelength}")
879
+ else:
880
+ wavelength = 0.0
881
+ logging.info("No wavelength found in input file.")
882
+ logging.info(
883
+ "Setting up resolution bins according to the file"
884
+ f" {unmerged} with resolution limits {dmax:.3f} - {dmin:.3f} A"
885
+ )
886
+ binner_master = gemmi.Binner()
887
+ binner_master.setup_from_1_d2(
888
+ n_bins, gemmi.Binner.Method.Dstar2, m.make_1_d2_array(), m.get_cell()
889
+ )
890
+ # n_expected = len(gemmi.make_miller_array(m.cell, m.spacegroup, 2.2, float('inf')))
891
+ n_expected = gemmi.count_reflections(m.cell, m.spacegroup, dmin, dmax)
892
+ logging.info(
893
+ f"Expected number of reflections for resolution range ({dmax:.3f} - {dmin:.3f}"
894
+ f" A), cell and symmetry from the input file {unmerged}: {str(n_expected)}"
895
+ )
896
+
897
+ logging.info(f"No. batches: {len(m.batches)}")
898
+ batch = m.batches[0]
899
+ # print(batch)
900
+ # print(batch.number) (0 + 1 = 1)
901
+ # print(batch.title)
902
+ # print(batch.axes)
903
+ # print(batch.dataset_id)
904
+ # print(list(batch.ints))
905
+ # print(list(batch.floats))
906
+ try:
907
+ if len(batch.floats) > 37 and batch.floats[36] and batch.floats[37]:
908
+ logging.info(
909
+ "Start/end phi of the first batch found:"
910
+ f" {batch.floats[36]}, {batch.floats[37]}"
911
+ )
912
+ else:
913
+ logging.info("Batch start/end phi of the first batch not found.")
914
+ except (IndexError, AttributeError) as e:
915
+ logging.info(f"Batch start and end of phi not found. Error: {e}")
916
+
917
+ # hkl = m.make_miller_array()
918
+ # intensity = m.column_with_label('I')
919
+ # sigma = m.column_with_label('SIGI')
920
+ # batch_col = m.column_with_label('BATCH')
921
+ # print(batch_col[0])
922
+ # print(batch.__dir__())
923
+
924
+ df_groups = []
925
+ df = pandas.DataFrame(data=m.array, columns=m.column_labels())
926
+ df = df.astype({name: "int32" for name in ["H", "K", "L"]})
927
+
928
+ # TODO: a function that converts any selection criteria in lists of batches.
929
+ # Note that batch numberring is 1, 2, ..., 2000
930
+ # but Python numberring is 0, 1, ..., 1999
931
+ if merge_whole_file:
932
+ batches_split = [0, len(m.batches)]
933
+ elif batches_edges:
934
+ batches_split = [0]
935
+ for i in range(len(batches_edges)):
936
+ batches_split.append(batches_split[-1] + batches_edges[i])
937
+ elif n_batches_per_group:
938
+ batches_split = list(range(0, len(m.batches), n_batches_per_group))
939
+ batches_split.append(len(m.batches))
940
+ logging.info(f"Batch edges for merging groups: {batches_split}")
941
+
942
+ mtz_groups = []
943
+ bin_stats_lists = []
944
+ for i_group in range(len(batches_split) - 1):
945
+ # print(batches_split[i_group], batches_split[i_group+1])
946
+ df_group = df.loc[
947
+ (df["BATCH"] >= batches_split[i_group])
948
+ & (df["BATCH"] < batches_split[i_group + 1])
949
+ ]
950
+ df_groups.append(df_group)
951
+ mtz_group, bin_stats_list = merge_group(
952
+ df_groups,
953
+ i_group,
954
+ m.cell,
955
+ m.spacegroup,
956
+ binner_master,
957
+ n_expected,
958
+ wavelength,
959
+ n_groups=len(batches_split),
960
+ anom=anom,
961
+ prefix=prefix,
962
+ i_group_prefix=i_group_prefix,
963
+ )
964
+ mtz_groups.append(mtz_group)
965
+ bin_stats_lists.append(bin_stats_list)
966
+ n_expected_list = [n_expected] * len(mtz_groups)
967
+ logging.info(f"Merged MTZ files: {mtz_groups}")
968
+ return mtz_groups, bin_stats_lists, n_expected_list, binner_master
969
+
970
+
971
+ def run_molrep(model, mtz):
972
+ import shutil
973
+
974
+ prefix_local = f"{os.path.splitext(os.path.basename(mtz))[0]}_molrep"
975
+ log_filename = prefix_local + ".log"
976
+ pdb_filename = prefix_local + ".pdb"
977
+ cmd = [
978
+ "molrep",
979
+ "-f",
980
+ mtz,
981
+ "-m",
982
+ model,
983
+ ]
984
+ logging.info("Running command: %s", shlex.join(cmd))
985
+ try:
986
+ with open(log_filename, "w") as log_file:
987
+ subprocess.run(cmd, check=True, stdout=log_file, stderr=subprocess.STDOUT)
988
+ except subprocess.CalledProcessError as e:
989
+ logging.error(f"Error occurred while running command: {e}")
990
+ shutil.copy2(
991
+ os.path.join(os.getcwd(), "molrep.pdb"), os.path.join(os.getcwd(), pdb_filename)
992
+ )
993
+ return pdb_filename
994
+
995
+
996
+ def dict_deep_merge(base: dict, update: dict) -> dict:
997
+ """
998
+ Recursively merge `update` into `base`.
999
+
1000
+ Merge rules:
1001
+ - If both values are dictionaries, merge them recursively.
1002
+ - If both values are lists, append items from `update` that are not
1003
+ already present in `base` while preserving order.
1004
+ - Otherwise, replace the value in ``base`` with the value from `update`.
1005
+
1006
+ Args:
1007
+ base: dict
1008
+ Dictionary to be updated in place.
1009
+ update: dict
1010
+ Dictionary containing additional or overriding values.
1011
+
1012
+ Returns:
1013
+ dict
1014
+ The merged dictionary (same object as `base`).
1015
+
1016
+ Examples
1017
+ --------
1018
+ >>> base = {"a": [1, 2], "b": {"x": 1}}
1019
+ >>> update = {"a": [2, 3], "b": {"y": 2}}
1020
+ >>> deep_merge(base, update)
1021
+ {'a': [1, 2, 3], 'b': {'x': 1, 'y': 2}}
1022
+ """
1023
+ for key, value in update.items():
1024
+ if key not in base:
1025
+ base[key] = value
1026
+ elif isinstance(base[key], dict) and isinstance(value, dict):
1027
+ dict_deep_merge(base[key], value)
1028
+ elif isinstance(base[key], list) and isinstance(value, list):
1029
+ # Append new items while preserving order
1030
+ seen = set(base[key])
1031
+ for item in value:
1032
+ if item not in seen:
1033
+ base[key].append(item)
1034
+ seen.add(item)
1035
+ else:
1036
+ base[key] = value
1037
+ return base
1038
+
1039
+
1040
+ def run_servalcat_fwt(mtz_groups_i, prefix="", n_proc=1):
1041
+ """
1042
+ Run `servalcat fw` to perform French Wilson conversion of intensities
1043
+ to structure factor amplitudes.
1044
+
1045
+ Args:
1046
+ mtz_groups_i (list): List of input MTZ files.
1047
+ prefix (str): Prefix for the output files.
1048
+
1049
+ Returns:
1050
+ list: List of output MTZ files, columns
1051
+ H K L F SIGF I SIG I + if Friedel pairs also F(+) SIGF(+) F(-) SIGF(-).
1052
+ servalcat fw drops NOBS, NOBS(+) and NOBS(-) columns.
1053
+ """
1054
+ logging.info(
1055
+ "Running servalcat fw to convert intensities to structure factor amplitudes..."
1056
+ )
1057
+ mtz_groups_fi = []
1058
+
1059
+ def run_fw_one(args):
1060
+ i_group, mtz_group_i = args
1061
+ group_fi_prefix = os.path.splitext(os.path.basename(mtz_group_i))[0] + "F"
1062
+ log_group_fi = f"{group_fi_prefix}.log"
1063
+ mtz_group_fi = f"{group_fi_prefix}.mtz"
1064
+ cmd = ["servalcat", "fw", "--hklin", mtz_group_i, "-o", group_fi_prefix]
1065
+ logging.info("Running command: %s", shlex.join(cmd))
1066
+ try:
1067
+ with open(log_group_fi, "w") as log_file:
1068
+ subprocess.run(
1069
+ cmd, check=True, stdout=log_file, stderr=subprocess.STDOUT
1070
+ )
1071
+ return mtz_group_fi
1072
+ except subprocess.CalledProcessError as e:
1073
+ logging.error(f"Error occurred while running command: {e}")
1074
+ return None
1075
+
1076
+ with concurrent.futures.ThreadPoolExecutor(max_workers=n_proc) as executor:
1077
+ results = list(executor.map(run_fw_one, enumerate(mtz_groups_i)))
1078
+ mtz_groups_fi.extend([r for r in results if r])
1079
+ return mtz_groups_fi
1080
+
1081
+
1082
+ def run_servalcat_refine(
1083
+ mtzs_fi,
1084
+ labins,
1085
+ models,
1086
+ mtzs_free=[],
1087
+ prefix="multixem_",
1088
+ source="xray",
1089
+ keyword_file="",
1090
+ config_yaml="",
1091
+ arguments=[],
1092
+ sigmaa=True,
1093
+ quick=False,
1094
+ n_proc=1,
1095
+ ):
1096
+ refined_mmcifs = []
1097
+ refined_mtzs = []
1098
+ refined_jsons = []
1099
+
1100
+ def refine_one(params):
1101
+ i_mtz, (mtz_fi, labin, mtz_free, model, prefix) = params
1102
+ local_refined_mmcifs = []
1103
+ local_refined_mtzs = []
1104
+ local_refined_jsons = []
1105
+
1106
+ if mtzs_free and "--labin_llweight" in arguments:
1107
+ prefix += f"llweight{i_mtz}_"
1108
+ if "--weight" in arguments:
1109
+ prefix += "unre_"
1110
+ prefix += "refine"
1111
+ log_filename = prefix + ".log"
1112
+
1113
+ cmd = [
1114
+ "servalcat",
1115
+ "refine_xtal_norefmac",
1116
+ "--hklin",
1117
+ mtz_fi,
1118
+ "--model",
1119
+ model,
1120
+ "-s",
1121
+ source,
1122
+ "--labin",
1123
+ labin,
1124
+ "--hout",
1125
+ "-o",
1126
+ prefix,
1127
+ ]
1128
+ if mtz_free:
1129
+ cmd.extend(["--hklin_free", mtz_free])
1130
+ if keyword_file:
1131
+ cmd.extend(["--keyword_file", keyword_file])
1132
+ if config_yaml:
1133
+ cmd.extend(["--config", config_yaml])
1134
+ if arguments:
1135
+ cmd.extend(arguments)
1136
+ if quick:
1137
+ cmd.extend(["--ncycle", "1"])
1138
+ logging.info("Running command: %s", shlex.join(cmd))
1139
+ try:
1140
+ with open(log_filename, "w") as log_file:
1141
+ process = subprocess.Popen(
1142
+ cmd, stdout=log_file, stderr=subprocess.PIPE, text=True, bufsize=1
1143
+ )
1144
+ for line in process.stderr:
1145
+ logging.error(line.rstrip())
1146
+ log_file.write(line)
1147
+ process.wait()
1148
+ if process.returncode != 0:
1149
+ raise subprocess.CalledProcessError(process.returncode, cmd)
1150
+ except subprocess.CalledProcessError as e:
1151
+ logging.error(f"Error occurred while running command: {e}")
1152
+ json_filename = prefix + "_stats.json"
1153
+ if not os.path.exists(json_filename):
1154
+ raise FileNotFoundError(f"Expected stats file not found: {json_filename}")
1155
+ with open(json_filename, "r") as stats_file:
1156
+ stats = json.load(stats_file)
1157
+ stats_line_list = [
1158
+ f"{stat} = {stats[-1]['data']['summary'][stat]:.4f}"
1159
+ for stat in stats[-1]["data"]["summary"]
1160
+ if stat != "-LL"
1161
+ ]
1162
+ logging.info(f"Finished: {prefix}.mmcif {', '.join(stats_line_list)}")
1163
+ if sigmaa:
1164
+ log_filename_sigmaa = prefix + "_sigmaa.log"
1165
+ cmd_sigmaa = [
1166
+ "servalcat",
1167
+ "sigmaa",
1168
+ "--hklin",
1169
+ mtz_fi,
1170
+ "--model",
1171
+ prefix + ".mmcif",
1172
+ "-s",
1173
+ source,
1174
+ "--labin",
1175
+ labin,
1176
+ "-o",
1177
+ prefix + "_sigmaa",
1178
+ ]
1179
+ if mtz_free:
1180
+ cmd_sigmaa.extend(["--hklin_free", mtz_free])
1181
+ # if arguments:
1182
+ # cmd_sigmaa.extend(arguments)
1183
+ logging.info("Running command: %s", shlex.join(cmd_sigmaa))
1184
+ try:
1185
+ with open(log_filename_sigmaa, "w") as log_file_sigmaa:
1186
+ subprocess.run(
1187
+ cmd_sigmaa,
1188
+ check=True,
1189
+ stdout=log_file_sigmaa,
1190
+ stderr=subprocess.STDOUT,
1191
+ )
1192
+ except subprocess.CalledProcessError as e:
1193
+ logging.error(f"Error occurred while running command: {e}")
1194
+ local_refined_mtzs.append(prefix + "_sigmaa.mtz")
1195
+ else:
1196
+ local_refined_mtzs.append(prefix + ".mtz")
1197
+ local_refined_mmcifs.append(prefix + ".mmcif")
1198
+ local_refined_jsons.append(prefix + "_stats.json")
1199
+ return local_refined_mmcifs[0], local_refined_mtzs[0], local_refined_jsons[0]
1200
+
1201
+ if len(mtzs_fi) == len(models) >= 2:
1202
+ models_list = models
1203
+ elif len(models) == len(mtzs_free) >= 2:
1204
+ models_list = models
1205
+ else:
1206
+ models_list = [models[0]] * max(len(mtzs_fi), len(mtzs_free))
1207
+
1208
+ if mtzs_free and len(mtzs_free) >= 2 and len(mtzs_fi) == 1:
1209
+ # refinement after bootstrapping
1210
+ params = zip(
1211
+ mtzs_fi * len(mtzs_free),
1212
+ labins * len(mtzs_free),
1213
+ mtzs_free,
1214
+ models_list,
1215
+ [prefix] * len(mtzs_free),
1216
+ )
1217
+ elif not mtzs_free:
1218
+ # refinement after merging, no free set provided
1219
+ if len(mtzs_fi) == 1 and "group" not in prefix:
1220
+ prefixes = [prefix] * len(mtzs_fi)
1221
+ else:
1222
+ prefixes = [f"{prefix}group{i_mtz + 1}_" for i_mtz in range(len(mtzs_fi))]
1223
+ params = zip(mtzs_fi, labins, [None] * len(mtzs_fi), models_list, prefixes)
1224
+ elif len(mtzs_free) == 1:
1225
+ # refinement after merging, single free set provided
1226
+ if len(mtzs_fi) == 1 and "group" not in prefix:
1227
+ prefixes = [prefix] * len(mtzs_fi)
1228
+ else:
1229
+ prefixes = [f"{prefix}group{i_mtz + 1}_" for i_mtz in range(len(mtzs_fi))]
1230
+ params = zip(
1231
+ mtzs_fi,
1232
+ labins,
1233
+ mtzs_free * len(mtzs_fi),
1234
+ models_list,
1235
+ prefixes,
1236
+ )
1237
+ else:
1238
+ # unexpected case, should not happen
1239
+ raise ValueError(
1240
+ "Unexpected case: both mtzs_fi and mtzs_free have more than one element."
1241
+ )
1242
+ with concurrent.futures.ThreadPoolExecutor(max_workers=n_proc) as executor:
1243
+ results = list(executor.map(refine_one, enumerate(params)))
1244
+ for mmcif, mtz, stats in results:
1245
+ refined_mmcifs.append(mmcif)
1246
+ refined_mtzs.append(mtz)
1247
+ refined_jsons.append(stats)
1248
+ return refined_mmcifs, refined_mtzs, refined_jsons
1249
+
1250
+
1251
+ def compare_mtzs_fi(
1252
+ mtzs_fi: list,
1253
+ binner: gemmi.Binner,
1254
+ labins: list = ["IMEAN,SIGIMEAN"],
1255
+ bin_stats_matrix: list = [],
1256
+ n_expected: list = [],
1257
+ ):
1258
+ """
1259
+ Compare multiple MTZ files containing reflection data (preferably
1260
+ reflection intensities) and calculate statistics for each pair of MTZ files.
1261
+ """
1262
+
1263
+ # noqa: E501
1264
+ def compare_mtz_fi_pair(
1265
+ mtz_fi1: str,
1266
+ mtz_fi2: str,
1267
+ binner: gemmi.Binner,
1268
+ labin: str = "IMEAN,SIGIMEAN",
1269
+ bin_stats_list1: list = [],
1270
+ bin_stats_list2: list = [],
1271
+ ):
1272
+ # f_col = "F"
1273
+ i_col = labin.split(",")[
1274
+ 0
1275
+ ] # usually IMEAN # can be just "I" after servalcat fw
1276
+ column_labels_dropna = [i_col, f"SIG{i_col}"] # or F?
1277
+ mtz1 = gemmi.read_mtz_file(mtz_fi1)
1278
+ mtz2 = gemmi.read_mtz_file(mtz_fi2)
1279
+ logging.info("")
1280
+ logging.info(
1281
+ f"Unit cell {str(mtz1.cell.a)} {str(mtz1.cell.b)} {str(mtz1.cell.c)}"
1282
+ f" {str(mtz1.cell.alpha)} {str(mtz1.cell.beta)} {str(mtz1.cell.gamma)}"
1283
+ f" in file {mtz_fi1}"
1284
+ )
1285
+ logging.info(
1286
+ f"Unit cell {str(mtz2.cell.a)} {str(mtz2.cell.b)} {str(mtz2.cell.c)}"
1287
+ f" {str(mtz2.cell.alpha)} {str(mtz2.cell.beta)} {str(mtz2.cell.gamma)}"
1288
+ f" in file {mtz_fi2}"
1289
+ )
1290
+ if mtz1.cell != mtz2.cell:
1291
+ logging.warning("Unit cell parameters are different.")
1292
+ mtz_df1 = pandas.DataFrame(data=mtz1.array, columns=mtz1.column_labels())
1293
+ mtz_df2 = pandas.DataFrame(data=mtz2.array, columns=mtz2.column_labels())
1294
+ mtz_df1 = mtz_df1.astype({name: "int32" for name in ["H", "K", "L"]})
1295
+ mtz_df2 = mtz_df2.astype({name: "int32" for name in ["H", "K", "L"]})
1296
+
1297
+ column_labels = mtz1.column_labels()
1298
+ column_labels.remove("H")
1299
+ column_labels.remove("K")
1300
+ column_labels.remove("L")
1301
+ # afterwards, rename to FP1, SIGFP1, ..., FP2, SIGFP2, ...
1302
+ # columns1 = [col + '1' for col in columns]
1303
+ column_labels_dict1 = {col: col + "1" for col in column_labels}
1304
+ # columns2 = [col + '2' for col in columns]
1305
+ column_labels_dict2 = {col: col + "2" for col in column_labels}
1306
+
1307
+ # mtz_df1 = mtz_df1[['H', 'K', 'L'] + columns] # Select only relevant columns
1308
+ mtz_df1 = mtz_df1.dropna(
1309
+ subset=column_labels_dropna
1310
+ ) # Select only reflections with F
1311
+ mtz_df1 = mtz_df1.rename(columns=column_labels_dict1) # Rename
1312
+ # print("")
1313
+ # print(mtz_df1.head(10))
1314
+ n_refl1 = len(mtz_df1)
1315
+ logging.info(f"No. unique reflections: {n_refl1} in file {mtz_fi1}")
1316
+
1317
+ # mtz_df2 = mtz_df2[['H', 'K', 'L'] + columns]
1318
+ mtz_df2 = mtz_df2.dropna(subset=column_labels_dropna)
1319
+ mtz_df2 = mtz_df2.rename(columns=column_labels_dict2)
1320
+ n_refl2 = len(mtz_df2)
1321
+ logging.info(f"No. unique reflections: {n_refl2} in file {mtz_fi2}")
1322
+
1323
+ # Extract common Miller indices (H, K, L)
1324
+ df = pandas.merge(mtz_df1, mtz_df2, on=["H", "K", "L"])
1325
+ n_refl = len(df)
1326
+ logging.info(
1327
+ f"No. unique reflections: {n_refl} in common;"
1328
+ f" ratios to the originals: {n_refl / n_refl1:.4f} {n_refl / n_refl2:.4f}"
1329
+ )
1330
+ # print("")
1331
+ # print(df.head(10))
1332
+ # print(df.describe())
1333
+ hkl_common_array = numpy.array(df[["H", "K", "L"]].values, numpy.int32)
1334
+ hkl_common_array = numpy.ascontiguousarray(hkl_common_array, dtype=numpy.int32)
1335
+ # print(len(hkl_common_array))
1336
+ # print(hkl_common_array.flags.c_contiguous)
1337
+ # print(hkl_common_array[:10])
1338
+ n_refl_list = [n_refl1, n_refl2, n_refl]
1339
+
1340
+ """# Scaling per resolution bins - at least 100 reflections per bin
1341
+ n_bins = int(n_refl / 200) # only starting point
1342
+ binner = gemmi.Binner()
1343
+ binner.setup(n_bins, gemmi.Binner.Method.Dstar2, hkl_common_array, mtz1.cell)
1344
+ bins_tmp = binner.get_bins(hkl_common_array)
1345
+ min_n_bins = min(Counter(bins_tmp).values())
1346
+ while min_n_bins <= 100:
1347
+ n_bins = n_bins - 1
1348
+ binner.setup(
1349
+ n_bins, gemmi.Binner.Method.Dstar2, hkl_common_array, mtz1.cell
1350
+ )
1351
+ bins_tmp = binner.get_bins(hkl_common_array)
1352
+ min_n_bins = min(Counter(bins_tmp).values())"""
1353
+ bins_tmp = binner.get_bins(hkl_common_array)
1354
+ min_n_bins = min(Counter(bins_tmp).values())
1355
+ if min_n_bins < 100:
1356
+ logging.warning(
1357
+ "Less than 100 reflections per bin"
1358
+ " - it is recommended to set up a lower number of bins."
1359
+ )
1360
+ df["BIN"] = bins_tmp
1361
+ # print("Binner min_n_bins:", min_n_bins)
1362
+ n_bins = len(set(bins_tmp)) # TODO how to use args.n_bins?
1363
+ bins_stats = []
1364
+ df2_scaled = pandas.DataFrame()
1365
+ # mtz_df2_scaled = mtz_df2[["H", "K", "L"]].copy()
1366
+ for b in range(n_bins):
1367
+ df_bin = df[df["BIN"] == b]
1368
+ # scale_delfofo = sum_hkl F1 * F2 / sum_hkl F2**2
1369
+ """
1370
+ scale_delfofo_numer = (df_bin[f_col + "1"] * df_bin[f_col + "2"]).sum()
1371
+ scale_delfofo_denomin = (df_bin[f_col + "2"] ** 2).sum()
1372
+ scale_delfofo = scale_delfofo_numer / scale_delfofo_denomin
1373
+ ccF_iso = numpy.corrcoef(
1374
+ df_bin[f_col + "1"], scale_delfofo * df_bin[f_col + "2"]
1375
+ )[0, 1]
1376
+ rF_iso_numer = (
1377
+ abs(df_bin[f_col + "1"] - scale_delfofo * df_bin[f_col + "2"])
1378
+ ).sum()
1379
+ rF_iso_denom = (
1380
+ abs(df_bin[f_col + "1"] + scale_delFfofo * df_bin[f_col + "2"])
1381
+ ).sum()
1382
+ rF_iso = 2 * rF_iso_numer / rF_iso_denom"""
1383
+ # DELFOFO
1384
+ """df.loc[df_bin.index, 'DELFOFO'] = \
1385
+ numpy.abs(df_bin[f_col + '1'] - scale_delfofo * df_bin[f_col + '2'])
1386
+ # df.loc[df_bin.index, 'PHDELFOFO'] = df_bin['PHFC1']
1387
+ df.loc[df_bin.index, 'DELFOFOSIG'] = \
1388
+ numpy.sign(df_bin[f_col + '1'] - scale_delfofo * df_bin[f_col + '2'])"""
1389
+ """# If FP1 - scale * FP2 < 0, then add/subtract 180deg to phase
1390
+ df_bin_noflip = df[(df['BIN'] == b) & (df['DELFOFOSIG'] != -1)]
1391
+ df_bin_flip_plus = \
1392
+ df[(df['BIN'] == b) & (df['DELFOFOSIG'] == -1) & (df['PHFC1'] <= 0)]
1393
+ df_bin_flip_minus = \
1394
+ df[(df['BIN'] == b) & (df['DELFOFOSIG'] == -1) & (df['PHFC1'] > 0)]
1395
+ # print(len(df_bin_noflip), len(df_bin_flip_plus), len(df_bin_flip_minus))
1396
+ df.loc[df_bin_noflip.index, 'PHDELFOFO'] = df_bin_noflip['PHFC1']
1397
+ df.loc[df_bin_flip_plus.index, 'PHDELFOFO'] = \
1398
+ df_bin_flip_plus['PHFC1'] + 180
1399
+ df.loc[df_bin_flip_minus.index, 'PHDELFOFO'] = \
1400
+ df_bin_flip_minus['PHFC1'] - 180"""
1401
+ scale_delioio = calc_scale_real(
1402
+ df_bin, i_col, b, binner.dmax_of_bin(b), binner.dmin_of_bin(b)
1403
+ )
1404
+ df_bin_scaled = df_bin.copy()
1405
+ df_bin_scaled[f"{i_col}2_scaled"] = (
1406
+ scale_delioio * df_bin_scaled[f"{i_col}2"]
1407
+ )
1408
+ df_bin_scaled[f"SIG{i_col}2_scaled"] = (
1409
+ scale_delioio * df_bin_scaled[f"SIG{i_col}2"]
1410
+ )
1411
+ df2_scaled = pandas.concat(
1412
+ [df2_scaled, df_bin_scaled]
1413
+ ) # , ignore_index=True)
1414
+ ccI_iso = numpy.corrcoef(
1415
+ df_bin[i_col + "1"], df_bin_scaled[i_col + "2_scaled"]
1416
+ )[0, 1]
1417
+ """
1418
+ rI_iso_numer = \
1419
+ (abs(df_bin[i_col + "1"] - scale_delioio * df_bin[i_col + "2"])).sum()
1420
+ rI_iso_denom = \
1421
+ (abs(df_bin[i_col + "1"] + scale_delioio * df_bin[i_col + "2"])).sum()
1422
+ rI_iso = 2 * rI_iso_numer / rI_iso_denom"""
1423
+
1424
+ bins_stats.append(
1425
+ {
1426
+ "bin": b + 1,
1427
+ "dmax": binner.dmax_of_bin(b),
1428
+ "dmin": binner.dmin_of_bin(b),
1429
+ "dmin_star2": 1 / (binner.dmin_of_bin(b) ** 2),
1430
+ "count": len(df_bin),
1431
+ # "scale_delfofo": scale_delfofo,
1432
+ # "ccF_iso": ccF_iso,
1433
+ # "rF_iso": rF_iso,
1434
+ "scale_delioio": scale_delioio,
1435
+ "ccI_iso": ccI_iso,
1436
+ # "rI_iso": rI_iso,
1437
+ }
1438
+ )
1439
+
1440
+ if (
1441
+ bin_stats_list1
1442
+ and bin_stats_list2
1443
+ and len(bin_stats_list1) == n_bins + 1
1444
+ and len(bin_stats_list2) == n_bins + 1
1445
+ ):
1446
+ # Add CC* from bin_stats_list1 and bin_stats_list2 if available
1447
+ # TODO: fix bug - CC*1 CC*2 CC12true are now available only for i==0 j==1
1448
+ for b in range(n_bins):
1449
+ bins_stats[b]["CC*1"] = bin_stats_list1[b]["CC*"]
1450
+ bins_stats[b]["CC*2"] = bin_stats_list2[b]["CC*"]
1451
+ bins_stats[b]["CC12true"] = bins_stats[b]["ccI_iso"] / (
1452
+ bins_stats[b]["CC*1"] * bins_stats[b]["CC*2"]
1453
+ )
1454
+
1455
+ bins_stats_df = pandas.DataFrame(bins_stats)
1456
+ # Calculate weighted average of cc over bins
1457
+ """ccF_iso_avg = (
1458
+ bins_stats_df["ccF_iso"] * bins_stats_df["count"]
1459
+ ).sum() / bins_stats_df["count"].sum()"""
1460
+ ccI_iso_avg = (
1461
+ bins_stats_df["ccI_iso"] * bins_stats_df["count"]
1462
+ ).sum() / bins_stats_df["count"].sum()
1463
+ # cc_iso_avg_list = [ccF_iso_avg, ccI_iso_avg]
1464
+ mtz_fi1_base = os.path.splitext(os.path.basename(mtz_fi1))[0]
1465
+ mtz_fi2_base = os.path.splitext(os.path.basename(mtz_fi2))[0]
1466
+
1467
+ mtz_df2 = mtz_df2.drop(columns=[f"{i_col}2", f"SIG{i_col}2"])
1468
+ df2_scaled = df2_scaled.reset_index()
1469
+ mtz_df2 = mtz_df2.merge(
1470
+ df2_scaled[["H", "K", "L", f"{i_col}2_scaled", f"SIG{i_col}2_scaled"]],
1471
+ on=["H", "K", "L"],
1472
+ how="left",
1473
+ )
1474
+ mtz_df2 = mtz_df2.rename(
1475
+ columns={
1476
+ f"{i_col}2_scaled": f"{i_col}",
1477
+ f"SIG{i_col}2_scaled": f"SIG{i_col}",
1478
+ }
1479
+ )
1480
+ write_mtz_from_df(
1481
+ mtz_df2,
1482
+ mtz2,
1483
+ columns={f"{i_col}": "J", f"SIG{i_col}": "Q"},
1484
+ filename=f"{mtz_fi2_base}_scaled_to_{mtz_fi1_base}.mtz",
1485
+ )
1486
+
1487
+ # Make a plot
1488
+ """def star2(x):
1489
+ # Vectorized 1/x^2, treating x==0 manually
1490
+ x = numpy.array(x, float)
1491
+ near_zero = numpy.isclose(x, 0)
1492
+ x[near_zero] = 9999
1493
+ x[~near_zero] = 1 / (x[~near_zero] ** 2)
1494
+ return x
1495
+ def star_sqrt(x):
1496
+ # Vectorized 1/sqrt(x), treating x<0 manually
1497
+ x = numpy.array(x, float)
1498
+ negative = numpy.less(x, 0)
1499
+ x[negative] = 9999
1500
+ x[~negative] = 1 / numpy.sqrt(x[~negative])
1501
+ return x"""
1502
+ plt.figure(figsize=(8, 6))
1503
+ plt.plot(
1504
+ bins_stats_df["dmin_star2"],
1505
+ bins_stats_df["ccI_iso"],
1506
+ marker="o",
1507
+ label="ccI_iso",
1508
+ )
1509
+ if "CC*1" in bins_stats_df.columns:
1510
+ plt.plot(
1511
+ bins_stats_df["dmin_star2"],
1512
+ bins_stats_df["CC*1"],
1513
+ marker="s",
1514
+ label="CC*1",
1515
+ )
1516
+ if "CC*2" in bins_stats_df.columns:
1517
+ plt.plot(
1518
+ bins_stats_df["dmin_star2"],
1519
+ bins_stats_df["CC*2"],
1520
+ marker="^",
1521
+ label="CC*2",
1522
+ )
1523
+ if "CC12true" in bins_stats_df.columns:
1524
+ plt.plot(
1525
+ bins_stats_df["dmin_star2"],
1526
+ bins_stats_df["CC12true"],
1527
+ marker="x",
1528
+ label="CC12true",
1529
+ )
1530
+ plt.xlabel("Resolution (Å⁻²)")
1531
+ # Show dmin labels on a secondary x-axis at the bottom
1532
+ ax = plt.gca()
1533
+ """secax = ax.secondary_xaxis(
1534
+ 'bottom',
1535
+ functions=(lambda x: 1 / numpy.sqrt(x), lambda d: 1 / (d ** 2))
1536
+ )
1537
+ secax = ax.secondary_xaxis('top', functions=(star_sqrt, star2))"""
1538
+ secax = ax.secondary_xaxis("top")
1539
+ secax.set_xlabel("Resolution (Å)")
1540
+ N = max(1, len(bins_stats_df) // 5) # Show only some labels to avoid overlap
1541
+ ticks = bins_stats_df["dmin_star2"]
1542
+ dmins = bins_stats_df["dmin"]
1543
+ labels = []
1544
+ for i, d in enumerate(dmins):
1545
+ if i == 0 or i == len(dmins) - 1: # Always show the first and last labels
1546
+ labels.append(f"{d:.2f}")
1547
+ elif i % N == 0:
1548
+ labels.append(f"{d:.2f}")
1549
+ else:
1550
+ labels.append("")
1551
+ secax.set_xticks(ticks)
1552
+ secax.set_xticklabels(labels)
1553
+ plt.ylabel("Correlation")
1554
+ plt.title("Correlation statistics per resolution bin")
1555
+ plt.legend()
1556
+ plt.tight_layout()
1557
+ plt.savefig(f"{mtz_fi1_base}_vs_{mtz_fi2_base}_cc_plot.png")
1558
+ plt.close()
1559
+ # Round float columns to 4 decimal places and save as fixed-width file
1560
+ stats_filename = f"{mtz_fi1_base}_vs_{mtz_fi2_base}_bin_stats.txt"
1561
+ write_bin_stats(bins_stats, stats_filename)
1562
+ return bins_stats, n_refl_list, ccI_iso_avg
1563
+
1564
+ n_refl_matrix = numpy.zeros((len(mtzs_fi), len(mtzs_fi)), dtype=int)
1565
+ ratio_refl_matrix = numpy.identity(len(mtzs_fi), dtype=float)
1566
+ # ccF_iso_matrix = numpy.identity(len(mtzs_fi), dtype=float)
1567
+ ccI_iso_matrix = numpy.identity(len(mtzs_fi), dtype=float)
1568
+ for i in range(len(mtzs_fi)):
1569
+ for j in range(i + 1, len(mtzs_fi)):
1570
+ # print(i, j)
1571
+ bins_stats, n_refl_list, ccI_iso_avg = compare_mtz_fi_pair(
1572
+ mtzs_fi[i],
1573
+ mtzs_fi[j],
1574
+ binner,
1575
+ labins[i],
1576
+ bin_stats_matrix[i][i],
1577
+ bin_stats_matrix[j][j],
1578
+ )
1579
+ bin_stats_matrix[i][j] = bins_stats
1580
+ bin_stats_matrix[j][i] = bin_stats_matrix[i][j]
1581
+ if i == 0:
1582
+ n_refl_matrix[j, j] = n_refl_list[1]
1583
+ if j == 1:
1584
+ n_refl_matrix[i, i] = n_refl_list[0]
1585
+ n_refl_matrix[i, j] = n_refl_list[2]
1586
+ n_refl_matrix[j, i] = n_refl_list[2]
1587
+ # No. reflections in common / No. reflections in the first file
1588
+ ratio_refl_matrix[i, j] = n_refl_list[2] / n_refl_list[0]
1589
+ # No. reflections in common / No. reflections in the second file
1590
+ ratio_refl_matrix[j, i] = n_refl_list[2] / n_refl_list[1]
1591
+ # ccF_iso_matrix[i, j] = cc_iso_avg_list[0]
1592
+ # ccF_iso_matrix[j, i] = cc_iso_avg_list[0]
1593
+ ccI_iso_matrix[i, j] = ccI_iso_avg
1594
+ ccI_iso_matrix[j, i] = ccI_iso_avg
1595
+ logging.info("\nNo. unique reflections:")
1596
+ logging.info(n_refl_matrix)
1597
+ if n_expected and len(n_expected) == len(mtzs_fi):
1598
+ completeness_matrix = n_refl_matrix / max(n_expected)
1599
+ logging.info("\nCompleteness:")
1600
+ logging.info(completeness_matrix)
1601
+ logging.info(
1602
+ "\nRatio of No. unique reflections in common and No. reflections in a data set:"
1603
+ )
1604
+ logging.info(ratio_refl_matrix)
1605
+ # print("Average CCFiso:")
1606
+ # print(ccF_iso_matrix)
1607
+ logging.info("\nAverage CCIiso:")
1608
+ logging.info(ccI_iso_matrix)
1609
+ logging.info("")
1610
+ # TODO: multiplicity
1611
+ return bin_stats_matrix, n_refl_matrix, ratio_refl_matrix
1612
+
1613
+
1614
+ def main():
1615
+ parser = create_parser()
1616
+ args = parser.parse_args()
1617
+ if hasattr(args, "func"):
1618
+ args.func(args)
1619
+
1620
+ # subcommand mean
1621
+ if args.command == "mean":
1622
+ setup_logging()
1623
+
1624
+ n_proc = min(os.cpu_count(), args.n_proc)
1625
+ if args.prefix:
1626
+ prefix = args.prefix
1627
+ else:
1628
+ prefix = args.file_name_template
1629
+ # Remove trailing _group{number}_ pattern to avoid duplication
1630
+ prefix = re.sub(r"_group\d+$", "", prefix)
1631
+ if prefix[-1] != "_":
1632
+ prefix += "_"
1633
+
1634
+ refined_jsons = sort_llweight_files(
1635
+ glob.glob(f"{args.file_name_template}_llweight*_refine_stats.json")
1636
+ )
1637
+ refined_json_ref_candidate = f"{args.file_name_template}_refine_stats.json"
1638
+ refined_json_ref = (
1639
+ refined_json_ref_candidate
1640
+ if os.path.isfile(refined_json_ref_candidate)
1641
+ else None
1642
+ )
1643
+ llweight_R1_outliers = []
1644
+ if refined_jsons:
1645
+ llweight_R1_outliers = bootstrap_analyse_stats(
1646
+ refined_jsons, refined_json_ref, idx=0, prefix=prefix
1647
+ )
1648
+
1649
+ refined_mmcif_ref_candidate = f"{args.file_name_template}_refine.mmcif"
1650
+ refined_mmcif_ref = (
1651
+ refined_mmcif_ref_candidate
1652
+ if os.path.isfile(refined_mmcif_ref_candidate)
1653
+ else None
1654
+ )
1655
+
1656
+ refined_mtz_ref_candidate = f"{args.file_name_template}_refine.mtz"
1657
+ refined_mtz_ref = (
1658
+ refined_mtz_ref_candidate
1659
+ if os.path.isfile(refined_mtz_ref_candidate)
1660
+ else None
1661
+ )
1662
+
1663
+ for f in [refined_json_ref, refined_mmcif_ref, refined_mtz_ref]:
1664
+ if f:
1665
+ logging.info(f"Reference file found: {f}")
1666
+
1667
+ refined_mmcifs = sort_llweight_files(
1668
+ glob.glob(f"{args.file_name_template}_llweight*_refine.mmcif")
1669
+ )
1670
+ # refined_mmcifs2 = sort_llweight_files(
1671
+ # glob.glob(f"{args.file_name_template}_llweight*_refine.cif")
1672
+ # )
1673
+ # refined_mmcifs = refined_mmcifs + refined_mmcifs2
1674
+ if llweight_R1_outliers:
1675
+ refined_mmcifs = filter_out_refined_files(
1676
+ refined_mmcifs, llweight_R1_outliers
1677
+ )
1678
+
1679
+ if refined_mmcifs:
1680
+ geometry_objects_ref = []
1681
+ if args.geometry_cids and refined_mmcif_ref:
1682
+ st_ref = gemmi.read_structure(refined_mmcif_ref)
1683
+ geometry_objects_ref = select_cids_for_geometry_analysis(
1684
+ args.geometry_cids
1685
+ )
1686
+ geometry_objects_ref = geometry_analysis_load(
1687
+ st_ref, geometry_objects_ref
1688
+ )
1689
+ bootstrap_analyse_structures(
1690
+ refined_mmcifs,
1691
+ refined_mmcif_ref,
1692
+ idx=0,
1693
+ prefix=prefix,
1694
+ skip_hydrogen=True,
1695
+ smcif=args.cif,
1696
+ geometry_cids_file=args.geometry_cids,
1697
+ geometry_objects_ref=geometry_objects_ref,
1698
+ no_origin_shift=args.no_origin_shift,
1699
+ )
1700
+ else:
1701
+ logging.warning(
1702
+ f"No refined mmCIF files found with a filename template"
1703
+ f" {args.file_name_template}_llweight*_refine.mmcif"
1704
+ )
1705
+ refined_mtzs = sort_llweight_files(
1706
+ glob.glob(f"{args.file_name_template}_llweight*_refine.mtz")
1707
+ )
1708
+ if llweight_R1_outliers:
1709
+ refined_mtzs = filter_out_refined_files(refined_mtzs, llweight_R1_outliers)
1710
+ if not args.calculate_mean_map:
1711
+ logging.info("Skipping mean map calculation.")
1712
+ return
1713
+ if refined_mtzs:
1714
+ if refined_mtz_ref:
1715
+ m = gemmi.read_mtz_file(refined_mtz_ref)
1716
+ binner = gemmi.Binner()
1717
+ binner.setup_from_1_d2(
1718
+ args.n_bins,
1719
+ gemmi.Binner.Method.Dstar2,
1720
+ m.make_1_d2_array(),
1721
+ m.get_cell(),
1722
+ )
1723
+ bootstrap_mean_map(
1724
+ refined_mtzs,
1725
+ idx=0,
1726
+ prefix=prefix,
1727
+ binner=binner,
1728
+ mtz_ref=refined_mtz_ref,
1729
+ n_proc=n_proc,
1730
+ )
1731
+ else:
1732
+ logging.warning(
1733
+ f"No refined MTZ files found with a filename template"
1734
+ f" {args.file_name_template}_llweight*_refine.mtz"
1735
+ )
1736
+ return
1737
+
1738
+ elif args.command not in ["pipeline", "bootstrap"] and args.command is not None:
1739
+ parser.print_help()
1740
+ return
1741
+
1742
+ # subcommand pipeline or bootstrap
1743
+ # elif args.command in ["pipeline", "bootstrap"] or args.command is None:
1744
+
1745
+ if args.prefix:
1746
+ prefix = args.prefix
1747
+ if prefix[-1] != "_":
1748
+ prefix += "_"
1749
+ working_dir_name = f"multixem_{prefix[:-1]}"
1750
+ else:
1751
+ prefix = "multixem_"
1752
+ working_dir_name = "multixem_project"
1753
+
1754
+ os.mkdir(working_dir_name)
1755
+ os.chdir(working_dir_name)
1756
+
1757
+ setup_logging()
1758
+
1759
+ logging.info(f"Command line: {shlex.join(sys.argv)}")
1760
+ logging.info(f"Running multixem version: {__version__}")
1761
+ args_dict = vars(args)
1762
+ logging.info("Parsed arguments:")
1763
+ for key, value in args_dict.items():
1764
+ logging.info(f" {key}: {value}")
1765
+ logging.info(f"Current working directory: {os.getcwd()}")
1766
+ logging.info(f"Prefix for the output files: {prefix}")
1767
+
1768
+ n_proc = min(os.cpu_count(), args.n_proc)
1769
+ # parse servalcat args preserving quoted groups
1770
+ servalcat_args = shlex.split(args.servalcat_args) if args.servalcat_args else []
1771
+
1772
+ mtzs_merged = []
1773
+ labins = []
1774
+ cols_free = []
1775
+ bin_stats_lists = []
1776
+ n_expected_list = []
1777
+ binner_master = None
1778
+
1779
+ if args.command == "pipeline" and args.hklin_unmerged:
1780
+ logging.info(f"Unmerged diffraction data files: {args.hklin_unmerged}")
1781
+ n_groups = 0
1782
+ mtz_groups_i = []
1783
+ for i, hklin_unmerged in enumerate(args.hklin_unmerged):
1784
+ logging.info("")
1785
+ logging.info(f"Unmerged diffraction data file: {hklin_unmerged}")
1786
+ if args.merge_whole_file:
1787
+ logging.info(
1788
+ "All the reflections in all the batches"
1789
+ " in this file will be merged."
1790
+ )
1791
+ _mtz_groups_i, _bin_stats_lists, _n_expected_list, _binner_master = (
1792
+ merge_in_groups(
1793
+ hklin_unmerged,
1794
+ args.n_bins,
1795
+ prefix,
1796
+ merge_whole_file=args.merge_whole_file,
1797
+ i_group_prefix=n_groups,
1798
+ )
1799
+ )
1800
+ elif len(args.n_batches) == 1:
1801
+ n_batches_per_group = args.n_batches[0]
1802
+ logging.info(
1803
+ f"Number of batches in merging group: {n_batches_per_group}"
1804
+ )
1805
+ _mtz_groups_i, _bin_stats_lists, _n_expected_list, _binner_master = (
1806
+ merge_in_groups(
1807
+ hklin_unmerged,
1808
+ args.n_bins,
1809
+ prefix,
1810
+ n_batches_per_group=n_batches_per_group,
1811
+ merge_whole_file=False,
1812
+ i_group_prefix=n_groups,
1813
+ )
1814
+ )
1815
+ else:
1816
+ _mtz_groups_i, _bin_stats_lists, _n_expected_list, _binner_master = (
1817
+ merge_in_groups(
1818
+ hklin_unmerged,
1819
+ args.n_bins,
1820
+ prefix,
1821
+ batches_edges=args.n_batches,
1822
+ merge_whole_file=False,
1823
+ i_group_prefix=n_groups,
1824
+ )
1825
+ )
1826
+ print(_mtz_groups_i)
1827
+ mtz_groups_i.extend(_mtz_groups_i)
1828
+ n_groups += len(_mtz_groups_i)
1829
+ n_groups_this_file = len(_mtz_groups_i)
1830
+ bin_stats_lists.extend(_bin_stats_lists)
1831
+ n_expected_list.extend(_n_expected_list)
1832
+ if i == 0:
1833
+ binner_master = _binner_master
1834
+ if args.amplitude:
1835
+ _mtzs_fwt = run_servalcat_fwt(_mtz_groups_i, prefix, n_proc)
1836
+ mtzs_merged.extend(_mtzs_fwt)
1837
+ labins.extend(["F,SIGF"] * n_groups_this_file)
1838
+ else:
1839
+ mtzs_merged.extend(_mtz_groups_i)
1840
+ labins.extend(["IMEAN,SIGIMEAN"] * n_groups_this_file)
1841
+ # TODO: free reflections if not given
1842
+ # TODO: check that input files have FI(R?)
1843
+ # TODO: mmCIF
1844
+ print(mtzs_merged)
1845
+ print(labins)
1846
+
1847
+ if args.hklin:
1848
+ logging.info(f"Merged diffraction data files: {args.hklin}")
1849
+ for i, hklin_i in enumerate(args.hklin):
1850
+ logging.info("")
1851
+ logging.info(f"Merged diffraction data file: {hklin_i}")
1852
+ if hklin_i.lower().endswith(".cif") or hklin_i.lower().endswith(".ent"):
1853
+ mtz = None
1854
+ doc = gemmi.cif.read(hklin_i)
1855
+ rblocks = gemmi.as_refln_blocks(doc)
1856
+ for rblock in rblocks:
1857
+ if rblock.is_merged():
1858
+ Convert = gemmi.CifToMtz()
1859
+ mtz = Convert.convert_block_to_mtz(rblock)
1860
+ break
1861
+ if mtz:
1862
+ mtz_filename = (
1863
+ f"{os.path.splitext(os.path.basename(hklin_i))[0]}.mtz"
1864
+ )
1865
+ mtz.write_to_file(mtz_filename)
1866
+ mtzs_merged.append(mtz_filename)
1867
+ elif servalcat_imported:
1868
+ mtz, _, _ = servalcat_utils.fileio.read_smcif_shelx(hklin_i)
1869
+ if mtz:
1870
+ mtz_filename = (
1871
+ f"{os.path.splitext(os.path.basename(hklin_i))[0]}.mtz"
1872
+ )
1873
+ mtz.write_to_file(mtz_filename)
1874
+ mtzs_merged.append(mtz_filename)
1875
+ else:
1876
+ raise RuntimeError(
1877
+ f"Could not recognise format of diffraction data file {hklin_i}"
1878
+ )
1879
+ elif hklin_i.lower().endswith(".hkl") or hklin_i.lower().endswith(".ent"):
1880
+ if not servalcat_imported:
1881
+ raise RuntimeError(
1882
+ f"Cannot read .hkl file {hklin_i}"
1883
+ " because servalcat is not installed."
1884
+ )
1885
+ mtz = servalcat_utils.fileio.read_smcif_hkl(hklin_i)
1886
+ mtz_filename = f"{os.path.splitext(os.path.basename(hklin_i))[0]}.mtz"
1887
+ mtz.write_to_file(mtz_filename)
1888
+ mtzs_merged.append(mtz_filename)
1889
+ else:
1890
+ mtz = gemmi.read_mtz_file(hklin_i)
1891
+ mtzs_merged.append(hklin_i)
1892
+ dmin = mtz.resolution_high()
1893
+ dmax = mtz.resolution_low()
1894
+ # Check for None or nan values and recalculate if necessary
1895
+ if dmax is None or dmin is None or numpy.isnan(dmax) or numpy.isnan(dmin):
1896
+ d_array = mtz.cell.calculate_d_array(mtz.make_miller_array())
1897
+ dmax = max(d_array)
1898
+ dmin = min(d_array)
1899
+ logging.info(f"Resolution limits: {dmax:.3f}" f" - {dmin:.3f} A")
1900
+ logging.info(
1901
+ f"Space group: {mtz.spacegroup.hm} (No. {mtz.spacegroup.number})"
1902
+ )
1903
+ logging.info(
1904
+ f"Unit cell: {mtz.cell.a:.3f} {mtz.cell.b:.3f} {mtz.cell.c:.3f}"
1905
+ f" {mtz.cell.alpha:.3f} {mtz.cell.beta:.3f} {mtz.cell.gamma:.3f}"
1906
+ )
1907
+ labin, anom_present, col_free = check_reflection_file_columns(
1908
+ mtz, unmerged=False, prefer_amplitude=args.amplitude
1909
+ ) # col_free may be reset later if args.hklin_free is given
1910
+ # elif i_present and not f_present: TODO FW
1911
+ labins.append(labin)
1912
+ cols_free.append(col_free)
1913
+ bin_stats_lists.append([])
1914
+ # TODO: check and fix n_expected
1915
+ n_expected = gemmi.count_reflections(mtz.cell, mtz.spacegroup, dmin, dmax)
1916
+ n_expected_list.append(n_expected)
1917
+ if not binner_master or dmin < 1 / numpy.sqrt(binner_master.max_1_d2):
1918
+ logging.info(
1919
+ "Setting up resolution bins according to the file"
1920
+ f" {hklin_i} with resolution limits {dmax:.3f}"
1921
+ f" - {dmin:.3f} A"
1922
+ )
1923
+ binner_master = gemmi.Binner()
1924
+ binner_master.setup_from_1_d2(
1925
+ args.n_bins,
1926
+ gemmi.Binner.Method.Dstar2,
1927
+ mtz.make_1_d2_array(),
1928
+ mtz.get_cell(),
1929
+ )
1930
+ logging.info("")
1931
+ n_mtzs_merged = len(mtzs_merged)
1932
+ assert n_mtzs_merged == len(labins)
1933
+
1934
+ # Avoid shared inner-list aliasing by creating separate lists
1935
+ bin_stats_matrix = [
1936
+ [[] for _ in range(n_mtzs_merged)] for _ in range(n_mtzs_merged)
1937
+ ]
1938
+ for i in range(n_mtzs_merged):
1939
+ if i < len(bin_stats_lists) and bin_stats_lists[i]:
1940
+ bin_stats_matrix[i][i] = bin_stats_lists[i]
1941
+
1942
+ # TODO what if dealing with amplitudes?
1943
+ if n_mtzs_merged >= 2 and not args.amplitude:
1944
+ bin_stats_matrix, n_refl_matrix, ratio_refl_matrix = compare_mtzs_fi(
1945
+ mtzs_merged, binner_master, labins, bin_stats_matrix, n_expected_list
1946
+ )
1947
+
1948
+ if args.hklin_free:
1949
+ logging.info(f"Free R flag data file: {args.hklin_free}")
1950
+ mtz_free = gemmi.read_mtz_file(args.hklin_free)
1951
+ mtzs_merged.append(args.hklin_free)
1952
+ dmin_free = mtz_free.resolution_high()
1953
+ dmax_free = mtz_free.resolution_low()
1954
+ # Check for None or nan values and recalculate if necessary
1955
+ if (
1956
+ dmax_free is None
1957
+ or dmin_free is None
1958
+ or numpy.isnan(dmax_free)
1959
+ or numpy.isnan(dmin_free)
1960
+ ):
1961
+ d_array_free = mtz_free.cell.calculate_d_array(mtz_free.make_miller_array())
1962
+ dmax_free = max(d_array_free)
1963
+ dmin_free = min(d_array_free)
1964
+ logging.info(f"Resolution limits: {dmax_free:.3f}" f" - {dmin_free:.3f} A")
1965
+ logging.info(
1966
+ f"Space group: {mtz_free.spacegroup.hm} (No. {mtz_free.spacegroup.number})"
1967
+ )
1968
+ logging.info(
1969
+ "Unit cell:"
1970
+ f" {mtz_free.cell.a:.3f} {mtz_free.cell.b:.3f} {mtz_free.cell.c:.3f}"
1971
+ f" {mtz_free.cell.alpha:.3f} {mtz_free.cell.beta:.3f}"
1972
+ f" {mtz_free.cell.gamma:.3f}"
1973
+ )
1974
+ # Overwrite cols_free
1975
+ _, _, col_free = check_reflection_file_columns(
1976
+ args.hklin_free, unmerged=False, search_col_free=True
1977
+ )
1978
+ cols_free = len(cols_free) * [col_free]
1979
+ logging.info("")
1980
+
1981
+ if args.command == "pipeline" and args.unify_cell:
1982
+ for i, mtz in enumerate(mtzs_merged):
1983
+ if i == 0:
1984
+ continue
1985
+ mtzs_merged[i] = copy_cell_mtz(mtzs_merged[i], mtzs_merged[0])
1986
+
1987
+ models = []
1988
+ if args.model:
1989
+ if args.command == "pipeline" and args.molrep:
1990
+ models_molrep = []
1991
+ for model, mtz_i in zip(args.model, mtzs_merged):
1992
+ logging.info(
1993
+ "Running MolRep to generate a model from the input structure."
1994
+ )
1995
+ model_molrep = run_molrep(model, mtz_i)
1996
+ models_molrep.append(model_molrep)
1997
+ models = models_molrep
1998
+ else:
1999
+ models = args.model
2000
+
2001
+ config_yaml = ""
2002
+ geometry_objects_ref = []
2003
+ if args.geometry_cids or args.servalcat_config:
2004
+ config_yaml_dict = {}
2005
+ if args.servalcat_config:
2006
+ with open(args.servalcat_config, "r") as f:
2007
+ config_yaml_dict = yaml.safe_load(f) or {}
2008
+ if args.geometry_cids:
2009
+ geometry_objects_ref = select_cids_for_geometry_analysis(
2010
+ args.geometry_cids
2011
+ )
2012
+ config_yaml_unrestrain_dict = unrestrain_yaml(geometry_objects_ref)
2013
+ config_yaml_dict = dict_deep_merge(
2014
+ config_yaml_dict, config_yaml_unrestrain_dict
2015
+ )
2016
+ config_yaml = "servalcat_config.yaml"
2017
+ with open(config_yaml, "w") as f:
2018
+ yaml.dump(config_yaml_dict, f, sort_keys=False)
2019
+
2020
+ refined_mmcifs, refined_mtzs, refined_jsons = run_servalcat_refine(
2021
+ mtzs_merged,
2022
+ labins,
2023
+ models,
2024
+ mtzs_free=[args.hklin_free],
2025
+ prefix=prefix,
2026
+ source=args.source,
2027
+ arguments=servalcat_args,
2028
+ config_yaml=config_yaml,
2029
+ sigmaa=(not args.amplitude),
2030
+ quick=args.quick,
2031
+ n_proc=n_proc,
2032
+ )
2033
+ write_coot_script(
2034
+ refined_mmcifs,
2035
+ maps=refined_mtzs,
2036
+ maps_labels=("FWT", "PHWT"),
2037
+ diffmaps=refined_mtzs,
2038
+ diffmaps_labels=("DELFWT", "PHDELWT"),
2039
+ )
2040
+ adp_analysis_histograms(refined_mmcifs, prefix)
2041
+ if len(refined_mmcifs) >= 2:
2042
+ compute_structure_differences(refined_mmcifs)
2043
+ bin_stats_matrix, diffmaps, diffmaps_fwt = compute_difference_maps(
2044
+ refined_mtzs, binner_master, bin_stats_matrix, args.amplitude
2045
+ )
2046
+ write_coot_script(
2047
+ refined_mmcifs,
2048
+ maps=[],
2049
+ maps_labels=(),
2050
+ diffmaps=diffmaps,
2051
+ diffmaps_labels=("DELFOFO", "PHDELFOFO"),
2052
+ )
2053
+ write_coot_script(
2054
+ refined_mmcifs,
2055
+ maps=[],
2056
+ maps_labels=(),
2057
+ diffmaps=diffmaps_fwt,
2058
+ diffmaps_labels=("DELFWTFWTSCall", "PHDELFWTFWTSCall"),
2059
+ )
2060
+ if args.command == "bootstrap" or (
2061
+ args.command == "pipeline" and args.bootstrap
2062
+ ):
2063
+ n_samples = (
2064
+ args.bootstrap
2065
+ if (args.command == "pipeline" and args.bootstrap)
2066
+ else args.n_samples
2067
+ )
2068
+ # TODO: some features assume only single model...
2069
+ for i_mtz, (mtz_in, labin, model, col_free) in enumerate(
2070
+ zip(mtzs_merged, labins, models, cols_free)
2071
+ ):
2072
+ if args.command == "bootstrap":
2073
+ idx = 0
2074
+ prefix_servalcat = prefix
2075
+ else:
2076
+ idx = i_mtz + 1
2077
+ prefix_servalcat = f"{prefix}group{idx}_"
2078
+ # restraints_file = ""
2079
+ if args.geometry_cids and geometry_objects_ref:
2080
+ st_ref = gemmi.read_structure(refined_mmcifs[i_mtz])
2081
+ geometry_objects_ref = geometry_analysis_load(
2082
+ st_ref, geometry_objects_ref
2083
+ )
2084
+ # restraints_file = unrestrain(geometry_objects_ref, model)
2085
+ mtzs_bootstrap = bootstrap_dataset(
2086
+ mtz_in,
2087
+ binner_master,
2088
+ seeds=range(1001, 1001 + n_samples),
2089
+ labin=labin,
2090
+ draw_factor=args.draw_factor,
2091
+ random_weights=args.random_weights,
2092
+ col_free=col_free,
2093
+ fraction_zero=args.fraction_zero,
2094
+ )
2095
+ if args.model_dir and args.models:
2096
+ input_model_s = args.models
2097
+ else:
2098
+ input_model_s = [model]
2099
+ if args.unre:
2100
+ (
2101
+ refined_mmcifs_bootstrap_unre,
2102
+ _,
2103
+ _,
2104
+ ) = run_servalcat_refine(
2105
+ [mtz_in],
2106
+ [labin],
2107
+ input_model_s,
2108
+ mtzs_free=mtzs_bootstrap,
2109
+ prefix=prefix_servalcat,
2110
+ source=args.source,
2111
+ keyword_file="",
2112
+ config_yaml=config_yaml,
2113
+ arguments=servalcat_args
2114
+ + ["--labin_llweight", "llweight"]
2115
+ + [
2116
+ "--weight",
2117
+ "2.0",
2118
+ "--hydrogen",
2119
+ "yes",
2120
+ "--ncycle",
2121
+ str(args.unre),
2122
+ ],
2123
+ sigmaa=False,
2124
+ quick=args.quick,
2125
+ n_proc=n_proc,
2126
+ )
2127
+ input_model_s = refined_mmcifs_bootstrap_unre
2128
+ (
2129
+ refined_mmcifs_bootstrap,
2130
+ refined_mtzs_bootstrap,
2131
+ refined_jsons_bootstrap,
2132
+ ) = run_servalcat_refine(
2133
+ [mtz_in],
2134
+ [labin],
2135
+ input_model_s,
2136
+ mtzs_free=mtzs_bootstrap,
2137
+ prefix=prefix_servalcat,
2138
+ source=args.source,
2139
+ # keyword_file=restraints_file,
2140
+ config_yaml=config_yaml,
2141
+ arguments=servalcat_args + ["--labin_llweight", "llweight"],
2142
+ sigmaa=False,
2143
+ quick=args.quick,
2144
+ n_proc=n_proc,
2145
+ )
2146
+ llweight_R1_outliers = bootstrap_analyse_stats(
2147
+ refined_jsons_bootstrap,
2148
+ refined_jsons[i_mtz],
2149
+ idx=idx,
2150
+ prefix=prefix,
2151
+ )
2152
+
2153
+ if llweight_R1_outliers is not None:
2154
+ # Filter out the outliers
2155
+ refined_mmcifs_bootstrap = filter_out_refined_files(
2156
+ refined_mmcifs_bootstrap, llweight_R1_outliers
2157
+ )
2158
+ refined_mtzs_bootstrap = filter_out_refined_files(
2159
+ refined_mtzs_bootstrap, llweight_R1_outliers
2160
+ )
2161
+
2162
+ if os.path.splitext(model)[1] == ".cif":
2163
+ bootstrap_analyse_structures(
2164
+ refined_mmcifs_bootstrap,
2165
+ refined_mmcifs[i_mtz],
2166
+ idx=idx,
2167
+ prefix=prefix,
2168
+ skip_hydrogen=True,
2169
+ smcif=model,
2170
+ geometry_cids_file=args.geometry_cids,
2171
+ geometry_objects_ref=geometry_objects_ref,
2172
+ no_origin_shift=args.no_origin_shift,
2173
+ )
2174
+ else:
2175
+ bootstrap_analyse_structures(
2176
+ refined_mmcifs_bootstrap,
2177
+ refined_mmcifs[i_mtz],
2178
+ idx=idx,
2179
+ prefix=prefix,
2180
+ skip_hydrogen=True,
2181
+ smcif="",
2182
+ geometry_cids_file=args.geometry_cids,
2183
+ geometry_objects_ref=geometry_objects_ref,
2184
+ no_origin_shift=args.no_origin_shift,
2185
+ )
2186
+ bootstrap_mean_map(
2187
+ refined_mtzs_bootstrap,
2188
+ idx=idx,
2189
+ prefix=prefix,
2190
+ binner=binner_master,
2191
+ mtz_ref=refined_mtzs[i_mtz],
2192
+ n_proc=n_proc,
2193
+ )