multixem 0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multixem/__init__.py +6 -0
- multixem/analyse_refinement.py +656 -0
- multixem/bootstrap_dataset.py +310 -0
- multixem/bootstrap_maps.py +558 -0
- multixem/bootstrap_statistics.py +783 -0
- multixem/bootstrap_structures.py +1569 -0
- multixem/run.py +2193 -0
- multixem/tools.py +521 -0
- multixem-0.3.dist-info/METADATA +175 -0
- multixem-0.3.dist-info/RECORD +14 -0
- multixem-0.3.dist-info/WHEEL +5 -0
- multixem-0.3.dist-info/entry_points.txt +2 -0
- multixem-0.3.dist-info/licenses/LICENSE +373 -0
- multixem-0.3.dist-info/top_level.txt +1 -0
multixem/run.py
ADDED
|
@@ -0,0 +1,2193 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
import argparse
|
|
5
|
+
import subprocess
|
|
6
|
+
import numpy
|
|
7
|
+
import pandas
|
|
8
|
+
import matplotlib.pyplot as plt
|
|
9
|
+
import matplotlib
|
|
10
|
+
import logging
|
|
11
|
+
import json
|
|
12
|
+
import shlex
|
|
13
|
+
import glob
|
|
14
|
+
import re
|
|
15
|
+
import yaml
|
|
16
|
+
from collections import Counter
|
|
17
|
+
import concurrent.futures
|
|
18
|
+
|
|
19
|
+
# servalcat 0.4.145 must be imported before gemmi
|
|
20
|
+
try:
|
|
21
|
+
from servalcat import utils as servalcat_utils
|
|
22
|
+
|
|
23
|
+
servalcat_imported = True
|
|
24
|
+
except ModuleNotFoundError:
|
|
25
|
+
servalcat_imported = False
|
|
26
|
+
import gemmi
|
|
27
|
+
from . import __version__
|
|
28
|
+
from .tools import write_bin_stats, calc_scale_real, write_mtz_from_df
|
|
29
|
+
from .analyse_refinement import (
|
|
30
|
+
adp_analysis_histograms,
|
|
31
|
+
compute_difference_maps,
|
|
32
|
+
compute_structure_differences,
|
|
33
|
+
write_coot_script,
|
|
34
|
+
)
|
|
35
|
+
from .bootstrap_dataset import bootstrap_dataset
|
|
36
|
+
from .bootstrap_statistics import bootstrap_analyse_stats
|
|
37
|
+
from .bootstrap_structures import (
|
|
38
|
+
# unrestrain,
|
|
39
|
+
unrestrain_yaml,
|
|
40
|
+
geometry_analysis_load,
|
|
41
|
+
select_cids_for_geometry_analysis,
|
|
42
|
+
bootstrap_analyse_structures,
|
|
43
|
+
)
|
|
44
|
+
from .bootstrap_maps import bootstrap_mean_map
|
|
45
|
+
|
|
46
|
+
matplotlib.use("Agg")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def setup_logging():
|
|
50
|
+
"""
|
|
51
|
+
Set up logging to output to both console and file.
|
|
52
|
+
"""
|
|
53
|
+
# Create logger
|
|
54
|
+
logger = logging.getLogger()
|
|
55
|
+
logger.setLevel(logging.INFO)
|
|
56
|
+
|
|
57
|
+
# Remove any existing handlers
|
|
58
|
+
for handler in logger.handlers[:]:
|
|
59
|
+
logger.removeHandler(handler)
|
|
60
|
+
|
|
61
|
+
# Create formatters - simple format for INFO, detailed for others
|
|
62
|
+
class CustomFormatter(logging.Formatter):
|
|
63
|
+
def format(self, record):
|
|
64
|
+
if record.levelno in [logging.INFO, logging.DEBUG]:
|
|
65
|
+
return record.getMessage()
|
|
66
|
+
else:
|
|
67
|
+
formatted_time = self.formatTime(record, datefmt="%Y-%m-%d %H:%M:%S")
|
|
68
|
+
return f"{formatted_time} - {record.levelname} - {record.getMessage()}"
|
|
69
|
+
|
|
70
|
+
formatter = CustomFormatter()
|
|
71
|
+
|
|
72
|
+
# Console handler
|
|
73
|
+
console_handler = logging.StreamHandler(sys.stdout)
|
|
74
|
+
console_handler.setLevel(logging.INFO)
|
|
75
|
+
console_handler.setFormatter(formatter)
|
|
76
|
+
logger.addHandler(console_handler)
|
|
77
|
+
|
|
78
|
+
# File handler - capture everything including DEBUG
|
|
79
|
+
file_handler = logging.FileHandler("multixem.log", mode="a")
|
|
80
|
+
file_handler.setLevel(logging.DEBUG)
|
|
81
|
+
file_handler.setFormatter(formatter)
|
|
82
|
+
logger.addHandler(file_handler)
|
|
83
|
+
|
|
84
|
+
# Capture warnings and route them through logging
|
|
85
|
+
logging.captureWarnings(True)
|
|
86
|
+
warnings_logger = logging.getLogger("py.warnings")
|
|
87
|
+
warnings_logger.setLevel(logging.WARNING)
|
|
88
|
+
|
|
89
|
+
# Also capture uncaught exceptions
|
|
90
|
+
def handle_exception(exc_type, exc_value, exc_traceback):
|
|
91
|
+
if issubclass(exc_type, KeyboardInterrupt):
|
|
92
|
+
# Allow KeyboardInterrupt to be handled normally
|
|
93
|
+
sys.__excepthook__(exc_type, exc_value, exc_traceback)
|
|
94
|
+
return
|
|
95
|
+
# Log uncaught exceptions with full traceback
|
|
96
|
+
import traceback
|
|
97
|
+
|
|
98
|
+
tb_str = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback))
|
|
99
|
+
logger.error(f"Uncaught exception:\n{tb_str}")
|
|
100
|
+
|
|
101
|
+
sys.excepthook = handle_exception
|
|
102
|
+
|
|
103
|
+
return logger
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def sort_llweight_files(file_list):
|
|
107
|
+
"""
|
|
108
|
+
Sort files containing '_llweight<number>_' by the numerical value.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
file_list: List of file paths containing '_llweight<number>_'
|
|
112
|
+
|
|
113
|
+
Returns:
|
|
114
|
+
Sorted list of file paths
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
def extract_llweight_number(filepath):
|
|
118
|
+
match = re.search(r"_llweight(\d+)_", filepath)
|
|
119
|
+
if match:
|
|
120
|
+
return int(match.group(1))
|
|
121
|
+
return -1 # Files without llweight number will be sorted first
|
|
122
|
+
|
|
123
|
+
return sorted(file_list, key=extract_llweight_number)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def filter_out_refined_files(mmcif_files, llweights):
|
|
127
|
+
"""
|
|
128
|
+
Filter out refined mmcif files corresponding to outliers in R1 values.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
mmcif_files: List of refined mmcif file paths.
|
|
132
|
+
llweights: List of llweight numbers to exclude.
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
Filtered list of mmcif file paths.
|
|
136
|
+
"""
|
|
137
|
+
if not llweights:
|
|
138
|
+
return mmcif_files
|
|
139
|
+
filtered_files = []
|
|
140
|
+
for file in mmcif_files:
|
|
141
|
+
match = re.search(r"_llweight(\d+)_", file)
|
|
142
|
+
if match:
|
|
143
|
+
llweight_num = int(match.group(1))
|
|
144
|
+
if llweight_num not in llweights:
|
|
145
|
+
filtered_files.append(file)
|
|
146
|
+
else:
|
|
147
|
+
logging.warning(
|
|
148
|
+
f"Could not get llweight number from {file}, including it by default."
|
|
149
|
+
)
|
|
150
|
+
filtered_files.append(file)
|
|
151
|
+
return filtered_files
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def create_parser():
|
|
155
|
+
"""
|
|
156
|
+
Create the argument parser for the command-line interface.
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
argparse.ArgumentParser: The argument parser object.
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
def positive_int(value):
|
|
163
|
+
if isinstance(value, list):
|
|
164
|
+
positive_int(v for v in value)
|
|
165
|
+
else:
|
|
166
|
+
try:
|
|
167
|
+
ivalue = int(value)
|
|
168
|
+
except ValueError:
|
|
169
|
+
raise argparse.ArgumentTypeError(f"{value} is not an integer.")
|
|
170
|
+
if ivalue <= 0:
|
|
171
|
+
raise argparse.ArgumentTypeError(f"{value} is not a positive integer.")
|
|
172
|
+
return ivalue
|
|
173
|
+
|
|
174
|
+
def positive_float(value):
|
|
175
|
+
try:
|
|
176
|
+
fvalue = float(value)
|
|
177
|
+
except ValueError:
|
|
178
|
+
raise argparse.ArgumentTypeError(f"{value} is not a float.")
|
|
179
|
+
if fvalue <= 0:
|
|
180
|
+
raise argparse.ArgumentTypeError(f"{value} is not a positive float.")
|
|
181
|
+
return fvalue
|
|
182
|
+
|
|
183
|
+
def existing_file(path):
|
|
184
|
+
abs_norm_path = os.path.abspath(os.path.normpath(path))
|
|
185
|
+
if not os.path.isfile(abs_norm_path):
|
|
186
|
+
raise argparse.ArgumentTypeError(f"File does not exist: {abs_norm_path}")
|
|
187
|
+
return abs_norm_path
|
|
188
|
+
|
|
189
|
+
def existing_directory(path):
|
|
190
|
+
abs_norm_path = os.path.abspath(os.path.normpath(path))
|
|
191
|
+
if not os.path.isdir(abs_norm_path):
|
|
192
|
+
raise argparse.ArgumentTypeError(
|
|
193
|
+
f"Directory does not exist: {abs_norm_path}"
|
|
194
|
+
)
|
|
195
|
+
return abs_norm_path
|
|
196
|
+
|
|
197
|
+
class ArgumentDefaultsHelpFormatterCustom(argparse.ArgumentDefaultsHelpFormatter):
|
|
198
|
+
def _get_help_string(self, action):
|
|
199
|
+
help_str = action.help
|
|
200
|
+
# Only show default if it's not None and not suppressed
|
|
201
|
+
if action.default is not None and action.default != argparse.SUPPRESS:
|
|
202
|
+
if "%(default)" not in help_str:
|
|
203
|
+
help_str += f" (default: {action.default})"
|
|
204
|
+
return help_str
|
|
205
|
+
|
|
206
|
+
parser = argparse.ArgumentParser(
|
|
207
|
+
prog="multixem",
|
|
208
|
+
description="Refinement pipeline for multiple data sets in structure biology.",
|
|
209
|
+
formatter_class=ArgumentDefaultsHelpFormatterCustom,
|
|
210
|
+
)
|
|
211
|
+
parser.add_argument(
|
|
212
|
+
"-v",
|
|
213
|
+
"--version",
|
|
214
|
+
action="version",
|
|
215
|
+
version=__version__,
|
|
216
|
+
help="show version and exit",
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
subparsers = parser.add_subparsers(dest="command", help="Available commands")
|
|
220
|
+
|
|
221
|
+
# Parent parser for common options
|
|
222
|
+
common_parent = argparse.ArgumentParser(add_help=False)
|
|
223
|
+
common_parent.add_argument(
|
|
224
|
+
"--n_bins",
|
|
225
|
+
type=positive_int,
|
|
226
|
+
default=20,
|
|
227
|
+
help="Number of resolution bins. Must be a positive integer.",
|
|
228
|
+
)
|
|
229
|
+
common_parent.add_argument(
|
|
230
|
+
"-p", "--prefix", type=str, help="Prefix for the output files."
|
|
231
|
+
)
|
|
232
|
+
common_parent.add_argument(
|
|
233
|
+
"--n_proc",
|
|
234
|
+
type=positive_int,
|
|
235
|
+
default=4,
|
|
236
|
+
help="Number of processes to use for parallelisation."
|
|
237
|
+
+ " Must be a positive integer.",
|
|
238
|
+
)
|
|
239
|
+
common_parent.add_argument(
|
|
240
|
+
"--geometry_cids",
|
|
241
|
+
type=existing_file,
|
|
242
|
+
help=(
|
|
243
|
+
"Input file with atomic CIDs defining bonds, angles or torsions to be"
|
|
244
|
+
" analysed while performing bootstrap."
|
|
245
|
+
),
|
|
246
|
+
)
|
|
247
|
+
common_parent.add_argument(
|
|
248
|
+
"--no_origin_shift",
|
|
249
|
+
action="store_true",
|
|
250
|
+
default=False,
|
|
251
|
+
help=(
|
|
252
|
+
"In some point groups, origin is not fixed (floating origin problem)."
|
|
253
|
+
" The program automatically detects this and applies a shift to"
|
|
254
|
+
" account for it. Use this option to disable this automatic correction."
|
|
255
|
+
),
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
common_refinement_parent = argparse.ArgumentParser(add_help=False)
|
|
259
|
+
# TODO does not work for single command line argument
|
|
260
|
+
common_refinement_parent.add_argument(
|
|
261
|
+
"--servalcat_args",
|
|
262
|
+
type=str,
|
|
263
|
+
default=[],
|
|
264
|
+
help="Command line arguments for Servalcat (quoted)."
|
|
265
|
+
+ " Do not use options -s, --source, --config and --hout here.",
|
|
266
|
+
)
|
|
267
|
+
common_refinement_parent.add_argument(
|
|
268
|
+
"--model",
|
|
269
|
+
type=existing_file,
|
|
270
|
+
nargs="+",
|
|
271
|
+
help="Input atomic structure model file(s).",
|
|
272
|
+
required=False,
|
|
273
|
+
)
|
|
274
|
+
# TODO more files
|
|
275
|
+
common_refinement_parent.add_argument(
|
|
276
|
+
"--hklin",
|
|
277
|
+
type=existing_file,
|
|
278
|
+
nargs="+",
|
|
279
|
+
help="Input merged diffraction data file(s).",
|
|
280
|
+
)
|
|
281
|
+
common_refinement_parent.add_argument(
|
|
282
|
+
"--hklin_free", type=existing_file, help="Input MTZ file for test flags."
|
|
283
|
+
)
|
|
284
|
+
common_refinement_parent.add_argument(
|
|
285
|
+
"--amplitude",
|
|
286
|
+
action="store_true",
|
|
287
|
+
help="Use amplitude rather than intensities (not recommended).",
|
|
288
|
+
)
|
|
289
|
+
common_refinement_parent.add_argument(
|
|
290
|
+
"--model_dir",
|
|
291
|
+
type=existing_directory,
|
|
292
|
+
help=(
|
|
293
|
+
"Directory containing multiple input atomic structure model files"
|
|
294
|
+
" for bootstrap."
|
|
295
|
+
),
|
|
296
|
+
)
|
|
297
|
+
common_refinement_parent.add_argument(
|
|
298
|
+
"--unre",
|
|
299
|
+
type=positive_int,
|
|
300
|
+
default=0,
|
|
301
|
+
help="Number of cycles for prior unrestrained refinement during bootstrap.",
|
|
302
|
+
)
|
|
303
|
+
common_refinement_parent.add_argument(
|
|
304
|
+
"--quick",
|
|
305
|
+
action="store_true",
|
|
306
|
+
help="Quick run (only for development).",
|
|
307
|
+
)
|
|
308
|
+
# TODO: if input has Friedel pairs but a user wants to merge them
|
|
309
|
+
common_refinement_parent.add_argument(
|
|
310
|
+
"-s",
|
|
311
|
+
"--source",
|
|
312
|
+
type=str,
|
|
313
|
+
default="xray",
|
|
314
|
+
choices=["xray", "electron", "neutron"],
|
|
315
|
+
help="Radiation source, xray or electron or neutron.",
|
|
316
|
+
)
|
|
317
|
+
common_refinement_parent.add_argument(
|
|
318
|
+
"--servalcat_config",
|
|
319
|
+
type=existing_file,
|
|
320
|
+
help=(
|
|
321
|
+
"YAML configuration file for Servalcat"
|
|
322
|
+
" (e.g. to specify occupancy refinement)"
|
|
323
|
+
),
|
|
324
|
+
)
|
|
325
|
+
common_refinement_parent.add_argument(
|
|
326
|
+
"--draw_factor",
|
|
327
|
+
type=positive_float,
|
|
328
|
+
default=1.0,
|
|
329
|
+
help=(
|
|
330
|
+
"Factor for a number of draws in bootstrap resampling."
|
|
331
|
+
" (the default 1.0 means the number of draws is equal"
|
|
332
|
+
" to the number of reflections in each bin)."
|
|
333
|
+
),
|
|
334
|
+
)
|
|
335
|
+
common_refinement_parent.add_argument(
|
|
336
|
+
"--random_weights",
|
|
337
|
+
action="store_true",
|
|
338
|
+
help=(
|
|
339
|
+
"Use fractional-random-weight bootstrap method, i.e. assign random weights"
|
|
340
|
+
" to reflection likelihood terms in the target log-likelihood function."
|
|
341
|
+
" A fraction of reflection will be assigned zero weights"
|
|
342
|
+
" based on the provided free flag in --hklin_free or --hklin,"
|
|
343
|
+
" or randomly if no free flag is provided, controlled by --fraction_zero."
|
|
344
|
+
),
|
|
345
|
+
)
|
|
346
|
+
common_refinement_parent.add_argument(
|
|
347
|
+
"--fraction_zero",
|
|
348
|
+
type=float,
|
|
349
|
+
default=0.0,
|
|
350
|
+
help=(
|
|
351
|
+
"If random weighting is used, this specifies the fraction of"
|
|
352
|
+
" zero weights to include. Must be in [0.0, 1.0)."
|
|
353
|
+
" If a value of 0.0 is set (default), zero weights will be assigned"
|
|
354
|
+
" according to the free flags in --hklin_free or --hklin."
|
|
355
|
+
" Recommended value: 0.05."
|
|
356
|
+
),
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
# Main pipeline subcommand (default behavior)
|
|
360
|
+
pipeline_parser = subparsers.add_parser(
|
|
361
|
+
"pipeline",
|
|
362
|
+
parents=[common_parent, common_refinement_parent],
|
|
363
|
+
help="Run the main refinement pipeline",
|
|
364
|
+
formatter_class=ArgumentDefaultsHelpFormatterCustom,
|
|
365
|
+
)
|
|
366
|
+
pipeline_parser.add_argument(
|
|
367
|
+
"-u",
|
|
368
|
+
"--hklin_unmerged",
|
|
369
|
+
type=existing_file,
|
|
370
|
+
nargs="+",
|
|
371
|
+
help="Input unmerged diffraction data file(s).",
|
|
372
|
+
)
|
|
373
|
+
pipeline_parser.add_argument(
|
|
374
|
+
"--n_batches",
|
|
375
|
+
type=positive_int,
|
|
376
|
+
nargs="+",
|
|
377
|
+
default=0, # default values set up in validate_args()
|
|
378
|
+
help="Number of batches per merging group, or list of batch edges"
|
|
379
|
+
+ " where to split the data."
|
|
380
|
+
+ " Must be a positive integer or space-separated list of positive integers.",
|
|
381
|
+
)
|
|
382
|
+
pipeline_parser.add_argument(
|
|
383
|
+
"--merge_whole_file",
|
|
384
|
+
action="store_true",
|
|
385
|
+
help="Merge all the batches in the input unmerged file(s).",
|
|
386
|
+
)
|
|
387
|
+
pipeline_parser.add_argument(
|
|
388
|
+
"--molrep",
|
|
389
|
+
action="store_true",
|
|
390
|
+
help="Run MolRep for molecular replacement before structure refinement.",
|
|
391
|
+
)
|
|
392
|
+
pipeline_parser.add_argument(
|
|
393
|
+
"--unify_cell",
|
|
394
|
+
action="store_true",
|
|
395
|
+
help=(
|
|
396
|
+
"Set the same unit cell parameters for all datasets, "
|
|
397
|
+
"based on the first dataset."
|
|
398
|
+
),
|
|
399
|
+
)
|
|
400
|
+
pipeline_parser.add_argument(
|
|
401
|
+
"--bootstrap",
|
|
402
|
+
type=positive_int,
|
|
403
|
+
default=0,
|
|
404
|
+
help=(
|
|
405
|
+
"No. of bootstrap resampled sub data sets to be created and"
|
|
406
|
+
" used for refinement. Must be a positive integer higher than 1."
|
|
407
|
+
),
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
# mean subcommand - mean and std for statistics, model and maps
|
|
411
|
+
mean_parser = subparsers.add_parser(
|
|
412
|
+
"mean",
|
|
413
|
+
parents=[common_parent],
|
|
414
|
+
help="Calculate mean maps from bootstrapped refinement results",
|
|
415
|
+
formatter_class=ArgumentDefaultsHelpFormatterCustom,
|
|
416
|
+
)
|
|
417
|
+
mean_parser.add_argument(
|
|
418
|
+
"file_name_template",
|
|
419
|
+
type=str,
|
|
420
|
+
help=(
|
|
421
|
+
"Template name for input files, e.g. put `dataset`"
|
|
422
|
+
" for `dataset_llweight*_refine.mtz` and `dataset_llweight*_refine.mmcif`."
|
|
423
|
+
),
|
|
424
|
+
)
|
|
425
|
+
mean_parser.add_argument(
|
|
426
|
+
"--calculate_mean_map",
|
|
427
|
+
type=bool,
|
|
428
|
+
default=False,
|
|
429
|
+
help="Calculate mean maps.",
|
|
430
|
+
)
|
|
431
|
+
mean_parser.add_argument(
|
|
432
|
+
"--cif",
|
|
433
|
+
type=existing_file,
|
|
434
|
+
help="Path to a small molecule CIF file.",
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
# Bootstrap subcommand (optional new command)
|
|
438
|
+
bootstrap_parser = subparsers.add_parser(
|
|
439
|
+
"bootstrap",
|
|
440
|
+
parents=[common_parent, common_refinement_parent],
|
|
441
|
+
help="Run multiple refinement against bootstrap sub data sets",
|
|
442
|
+
formatter_class=ArgumentDefaultsHelpFormatterCustom,
|
|
443
|
+
)
|
|
444
|
+
bootstrap_parser.add_argument(
|
|
445
|
+
"n_samples",
|
|
446
|
+
type=positive_int,
|
|
447
|
+
default=5000,
|
|
448
|
+
help=(
|
|
449
|
+
"No. of bootstrap resampled sub data sets to be created and"
|
|
450
|
+
" used for refinement. Must be a positive integer higher than 1."
|
|
451
|
+
),
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
def validate_common(args):
|
|
455
|
+
if args.n_proc <= 0:
|
|
456
|
+
parser.error("--n_proc must be positive")
|
|
457
|
+
|
|
458
|
+
def validate_model_dir(args, n_required):
|
|
459
|
+
if not args.model_dir:
|
|
460
|
+
return
|
|
461
|
+
model_files = []
|
|
462
|
+
for pattern in ["*.pdb", "*.cif", "*.mmcif"]:
|
|
463
|
+
model_files.extend(glob.glob(os.path.join(args.model_dir, pattern)))
|
|
464
|
+
|
|
465
|
+
if len(model_files) < n_required:
|
|
466
|
+
parser.error(
|
|
467
|
+
f"--model_dir contains {len(model_files)} structure model files, "
|
|
468
|
+
f"but at least {n_required} are required."
|
|
469
|
+
)
|
|
470
|
+
args.models = sorted(model_files)[:n_required]
|
|
471
|
+
|
|
472
|
+
def validate_pipeline(args):
|
|
473
|
+
validate_common(args)
|
|
474
|
+
if args.n_batches and not args.hklin_unmerged:
|
|
475
|
+
parser.error("--n_batches requires --hklin_unmerged to be provided.")
|
|
476
|
+
if args.merge_whole_file and not args.hklin_unmerged:
|
|
477
|
+
parser.error("--merge_whole_file requires --hklin_unmerged to be provided.")
|
|
478
|
+
if args.hklin_unmerged and not args.n_batches and not args.merge_whole_file:
|
|
479
|
+
args.n_batches = [60] # Default
|
|
480
|
+
if args.model and len(args.model) > 1:
|
|
481
|
+
if len(args.model or []) < (
|
|
482
|
+
len(args.hklin or []) + len(args.hklin_unmerged or [])
|
|
483
|
+
):
|
|
484
|
+
parser.error(
|
|
485
|
+
"Just a single model can be provided for multiple data sets, "
|
|
486
|
+
"or the number of models must match the number of data sets."
|
|
487
|
+
)
|
|
488
|
+
if args.bootstrap and args.bootstrap < 2:
|
|
489
|
+
parser.error("--bootstrap must be at least 2.")
|
|
490
|
+
if args.geometry_cids and not args.bootstrap:
|
|
491
|
+
parser.error("--geometry_cids requires --bootstrap to be provided.")
|
|
492
|
+
if args.unre and not args.bootstrap:
|
|
493
|
+
parser.error("--unre requires --bootstrap to be provided.")
|
|
494
|
+
if args.bootstrap and args.model_dir:
|
|
495
|
+
validate_model_dir(args, args.bootstrap)
|
|
496
|
+
if args.random_weights and (
|
|
497
|
+
args.fraction_zero < 0.0 or args.fraction_zero >= 1.0
|
|
498
|
+
):
|
|
499
|
+
parser.error("--fraction_zero must be in the range [0.0, 1.0).")
|
|
500
|
+
|
|
501
|
+
def validate_mean(args):
|
|
502
|
+
validate_common(args)
|
|
503
|
+
|
|
504
|
+
def validate_bootstrap(args):
|
|
505
|
+
validate_common(args)
|
|
506
|
+
# TODO
|
|
507
|
+
if not args.hklin:
|
|
508
|
+
parser.error("--hklin is required for bootstrap.")
|
|
509
|
+
if args.n_samples < 2:
|
|
510
|
+
parser.error("Number of samples must be at least 2.")
|
|
511
|
+
if args.model_dir:
|
|
512
|
+
model_files = glob.glob(os.path.join(args.model_dir, "*.pdb"))
|
|
513
|
+
model_files += glob.glob(os.path.join(args.model_dir, "*.cif"))
|
|
514
|
+
model_files += glob.glob(os.path.join(args.model_dir, "*.mmcif"))
|
|
515
|
+
if len(model_files) < args.n_samples:
|
|
516
|
+
parser.error(
|
|
517
|
+
f"--model_dir contains {len(model_files)} model files, "
|
|
518
|
+
f"but at least {args.n_samples} are required."
|
|
519
|
+
)
|
|
520
|
+
args.models = sorted(model_files)[: args.n_samples]
|
|
521
|
+
if args.model_dir:
|
|
522
|
+
validate_model_dir(args, args.n_samples)
|
|
523
|
+
if args.random_weights and (
|
|
524
|
+
args.fraction_zero < 0.0 or args.fraction_zero >= 1.0
|
|
525
|
+
):
|
|
526
|
+
parser.error("--fraction_zero must be in the range [0.0, 1.0).")
|
|
527
|
+
|
|
528
|
+
pipeline_parser.set_defaults(func=validate_pipeline)
|
|
529
|
+
mean_parser.set_defaults(func=validate_mean)
|
|
530
|
+
bootstrap_parser.set_defaults(func=validate_bootstrap)
|
|
531
|
+
|
|
532
|
+
# Parse and validate immediately
|
|
533
|
+
original_parse_args = parser.parse_args
|
|
534
|
+
|
|
535
|
+
def parse_and_validate(cmd_args=None):
|
|
536
|
+
args = original_parse_args(cmd_args)
|
|
537
|
+
if hasattr(args, "func"):
|
|
538
|
+
args.func(args)
|
|
539
|
+
return args
|
|
540
|
+
|
|
541
|
+
parser.parse_args = parse_and_validate
|
|
542
|
+
|
|
543
|
+
return parser
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def check_reflection_file_columns(
|
|
547
|
+
hklin: str,
|
|
548
|
+
unmerged: bool = False,
|
|
549
|
+
prefer_amplitude: bool = False,
|
|
550
|
+
search_col_free: bool = False,
|
|
551
|
+
):
|
|
552
|
+
"""
|
|
553
|
+
Check the input reflection file for the presence of intensities,
|
|
554
|
+
amplitudes and Friedel pairs. Find the column labels and decide which to use.
|
|
555
|
+
|
|
556
|
+
Args:
|
|
557
|
+
hklin (str): gemmi.Mtz object or path to the input reflection file.
|
|
558
|
+
unmerged (bool): Whether the input file is unmerged data.
|
|
559
|
+
prefer_amplitude (bool): If both intensities and amplitudes are found,
|
|
560
|
+
prefer amplitudes over intensities.
|
|
561
|
+
search_col_free (bool): Whether to search only for a free reflection column
|
|
562
|
+
|
|
563
|
+
Returns:
|
|
564
|
+
tuple: A tuple containing three boolean values:
|
|
565
|
+
- labin (str): Column labels for --labin option in Servalcat.
|
|
566
|
+
- anom: True if Friedel pairs are present.
|
|
567
|
+
"""
|
|
568
|
+
if not isinstance(hklin, gemmi.Mtz):
|
|
569
|
+
m = gemmi.read_mtz_file(hklin)
|
|
570
|
+
else:
|
|
571
|
+
m = hklin
|
|
572
|
+
# TODO: CIF
|
|
573
|
+
# dmax = m.resolution_low()
|
|
574
|
+
# dmin = m.resolution_high()
|
|
575
|
+
unexpected_column_warning = (
|
|
576
|
+
"This is quite unusual for unmerged data file, are you sure about the file?"
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
anom = False
|
|
580
|
+
|
|
581
|
+
col_iplusminus = []
|
|
582
|
+
col_sigiplusminus = []
|
|
583
|
+
col_imean = []
|
|
584
|
+
col_fplusminus = []
|
|
585
|
+
col_sigfplusminus = []
|
|
586
|
+
col_fmean = []
|
|
587
|
+
col_sigxmean = []
|
|
588
|
+
col_free = ""
|
|
589
|
+
|
|
590
|
+
for column in m.columns:
|
|
591
|
+
if not col_free and column.type == "I":
|
|
592
|
+
if "free" in column.label.lower() and "flag" in column.label.lower():
|
|
593
|
+
col_free = column.label
|
|
594
|
+
logging.info(f"Column with free R flag (type I) found: {column.label}")
|
|
595
|
+
if search_col_free:
|
|
596
|
+
logging.info("Free R flag column found successfully.")
|
|
597
|
+
return None, None, col_free
|
|
598
|
+
elif column.type == "K":
|
|
599
|
+
logging.info(
|
|
600
|
+
f"Column with intensity (type K, Friedel pairs) found: {column.label}"
|
|
601
|
+
)
|
|
602
|
+
logging.info("Friedel pairs will be kept separately.")
|
|
603
|
+
anom = True
|
|
604
|
+
col_iplusminus.append(column.label)
|
|
605
|
+
elif column.type == "M":
|
|
606
|
+
logging.info(
|
|
607
|
+
f"Column with standard deviation associated to intensity column"
|
|
608
|
+
f" (type M, Friedel pairs) found: {column.label}"
|
|
609
|
+
)
|
|
610
|
+
col_sigiplusminus.append(column.label)
|
|
611
|
+
elif column.type == "J":
|
|
612
|
+
logging.info(
|
|
613
|
+
f"Column with intensity (type J, no Friedel pairs)"
|
|
614
|
+
f" found: {column.label}"
|
|
615
|
+
)
|
|
616
|
+
col_imean.append(column.label)
|
|
617
|
+
elif column.type == "Q":
|
|
618
|
+
logging.info(
|
|
619
|
+
f"Column with standard deviation associated to intensity/amplitude"
|
|
620
|
+
f" column (type Q, no Friedel pairs) found: {column.label}"
|
|
621
|
+
)
|
|
622
|
+
col_sigxmean.append(column.label)
|
|
623
|
+
elif column.type == "G":
|
|
624
|
+
logging.info(
|
|
625
|
+
f"Column with amplitude (type G, Friedel pairs) found: {column.label}"
|
|
626
|
+
)
|
|
627
|
+
anom = True
|
|
628
|
+
col_fplusminus.append(column.label)
|
|
629
|
+
if unmerged:
|
|
630
|
+
logging.warning(unexpected_column_warning)
|
|
631
|
+
elif column.type == "L":
|
|
632
|
+
logging.info(
|
|
633
|
+
f"Column with standard deviation associated to amplitude"
|
|
634
|
+
f" (type L, Friedel pairs) found: {column.label}"
|
|
635
|
+
)
|
|
636
|
+
col_sigfplusminus.append(column.label)
|
|
637
|
+
elif column.type == "F":
|
|
638
|
+
logging.info(
|
|
639
|
+
f"Column with amplitude (type F, no Friedel pairs)"
|
|
640
|
+
f" found: {column.label}"
|
|
641
|
+
)
|
|
642
|
+
col_fmean.append(column.label)
|
|
643
|
+
if unmerged:
|
|
644
|
+
logging.warning(unexpected_column_warning)
|
|
645
|
+
|
|
646
|
+
labin = None
|
|
647
|
+
if not prefer_amplitude:
|
|
648
|
+
if len(col_iplusminus) >= 2 and len(col_sigiplusminus) >= 2:
|
|
649
|
+
labin = (
|
|
650
|
+
f"{col_iplusminus[0]},{col_sigiplusminus[0]},"
|
|
651
|
+
f"{col_iplusminus[1]},{col_sigiplusminus[1]}"
|
|
652
|
+
)
|
|
653
|
+
elif col_imean and col_sigxmean:
|
|
654
|
+
labin = f"{col_imean[0]},{col_sigxmean[0]}"
|
|
655
|
+
if not labin:
|
|
656
|
+
if len(col_fplusminus) >= 2 and len(col_sigfplusminus) >= 2:
|
|
657
|
+
labin = (
|
|
658
|
+
f"{col_fplusminus[0]},{col_sigfplusminus[0]},"
|
|
659
|
+
f"{col_fplusminus[1]},{col_sigfplusminus[1]}"
|
|
660
|
+
)
|
|
661
|
+
elif col_fmean and col_sigxmean:
|
|
662
|
+
labin = f"{col_fmean[0]},{col_sigxmean[0]}"
|
|
663
|
+
|
|
664
|
+
if not labin:
|
|
665
|
+
raise RuntimeError(f"Neither intensities nor amplitudes found in {hklin}.")
|
|
666
|
+
elif (
|
|
667
|
+
(len(col_fplusminus) >= 2 and len(col_sigfplusminus) >= 2)
|
|
668
|
+
or (col_fmean and col_sigxmean)
|
|
669
|
+
) and not (
|
|
670
|
+
(len(col_iplusminus) >= 2 and len(col_sigiplusminus) >= 2)
|
|
671
|
+
or (col_imean and col_sigxmean)
|
|
672
|
+
):
|
|
673
|
+
logging.warning(
|
|
674
|
+
"The file contain only amplitudes but not intensities, however,"
|
|
675
|
+
" providing intensities is recommended."
|
|
676
|
+
)
|
|
677
|
+
logging.info(f"Using these columns for refinement: {labin}")
|
|
678
|
+
|
|
679
|
+
return labin, anom, col_free
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def copy_cell_mtz(mtz_input, mtz_reference):
|
|
683
|
+
"""
|
|
684
|
+
Copy unit cell parameters from a reference MTZ file to
|
|
685
|
+
an MTZ file using gemmi.
|
|
686
|
+
|
|
687
|
+
Args:
|
|
688
|
+
mtz_input (str): Path to input MTZ file
|
|
689
|
+
mtz_reference (str): Path to reference MTZ file
|
|
690
|
+
Returns:
|
|
691
|
+
str: Output MTZ file name
|
|
692
|
+
"""
|
|
693
|
+
mtz = gemmi.read_mtz_file(mtz_input)
|
|
694
|
+
mtz_ref = gemmi.read_mtz_file(mtz_reference)
|
|
695
|
+
mtz.set_cell_for_all(mtz_ref.cell)
|
|
696
|
+
mtz_out_filepath = os.path.basename(mtz_input).replace(".mtz", "_cell.mtz")
|
|
697
|
+
mtz.write_to_file(mtz_out_filepath)
|
|
698
|
+
logging.info(
|
|
699
|
+
f"Copied unit cell parameters"
|
|
700
|
+
f" {mtz_ref.cell.a} {mtz_ref.cell.b} {mtz_ref.cell.c}"
|
|
701
|
+
f" {mtz_ref.cell.alpha} {mtz_ref.cell.beta} {mtz_ref.cell.gamma}"
|
|
702
|
+
f" from {mtz_reference} to {mtz_out_filepath}"
|
|
703
|
+
)
|
|
704
|
+
return mtz_out_filepath
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def merge_in_groups(
|
|
708
|
+
unmerged,
|
|
709
|
+
n_bins,
|
|
710
|
+
prefix,
|
|
711
|
+
n_batches_per_group=60,
|
|
712
|
+
batches_edges=[],
|
|
713
|
+
merge_whole_file=False,
|
|
714
|
+
i_group_prefix=0,
|
|
715
|
+
):
|
|
716
|
+
|
|
717
|
+
def merge_group(
|
|
718
|
+
df_groups,
|
|
719
|
+
i_group,
|
|
720
|
+
cell,
|
|
721
|
+
spacegroup,
|
|
722
|
+
binner,
|
|
723
|
+
n_expected,
|
|
724
|
+
wavelength=0,
|
|
725
|
+
n_groups=0,
|
|
726
|
+
anom=True,
|
|
727
|
+
prefix="",
|
|
728
|
+
i_group_prefix=0,
|
|
729
|
+
):
|
|
730
|
+
intensities = gemmi.Intensities()
|
|
731
|
+
intensities.set_data(
|
|
732
|
+
cell,
|
|
733
|
+
spacegroup,
|
|
734
|
+
df_groups[i_group][["H", "K", "L"]].values,
|
|
735
|
+
df_groups[i_group]["I"].values,
|
|
736
|
+
df_groups[i_group]["SIGI"].values,
|
|
737
|
+
)
|
|
738
|
+
if anom:
|
|
739
|
+
intensities.prepare_for_merging(gemmi.DataType.Anomalous)
|
|
740
|
+
else:
|
|
741
|
+
intensities.prepare_for_merging(gemmi.DataType.Mean)
|
|
742
|
+
bin_stats = intensities.calculate_merging_stats(binner)
|
|
743
|
+
# Collect bin statistics into a list of dictionaries
|
|
744
|
+
bin_stats_list = []
|
|
745
|
+
for n, stats in enumerate(bin_stats):
|
|
746
|
+
bin_stats_list.append(
|
|
747
|
+
{
|
|
748
|
+
"bin": n + 1,
|
|
749
|
+
"dmax": binner.dmax_of_bin(n),
|
|
750
|
+
"dmin": binner.dmin_of_bin(n),
|
|
751
|
+
"CC1/2": stats.cc_half(),
|
|
752
|
+
"CC*": stats.cc_star(),
|
|
753
|
+
"Rmeas": stats.r_meas(),
|
|
754
|
+
"Rpim": stats.r_pim(),
|
|
755
|
+
}
|
|
756
|
+
)
|
|
757
|
+
# Add overall stats as a separate entry
|
|
758
|
+
overall_stats = intensities.calculate_merging_stats(None)
|
|
759
|
+
bin_stats_list.append(
|
|
760
|
+
{
|
|
761
|
+
"bin": "overall",
|
|
762
|
+
"dmax": intensities.resolution_range()[0],
|
|
763
|
+
"dmin": intensities.resolution_range()[1],
|
|
764
|
+
"CC1/2": overall_stats[0].cc_half(),
|
|
765
|
+
"CC*": overall_stats[0].cc_star(),
|
|
766
|
+
"Rmeas": overall_stats[0].r_meas(),
|
|
767
|
+
"Rpim": overall_stats[0].r_pim(),
|
|
768
|
+
}
|
|
769
|
+
)
|
|
770
|
+
|
|
771
|
+
if anom:
|
|
772
|
+
intensities.merge_in_place(gemmi.DataType.Anomalous)
|
|
773
|
+
else:
|
|
774
|
+
intensities.merge_in_place(gemmi.DataType.Mean)
|
|
775
|
+
# SIGI from merging: 1/sqrt(∑w), where w=1/sigma^2
|
|
776
|
+
mtz_group_merged = intensities.prepare_merged_mtz(with_nobs=True)
|
|
777
|
+
if wavelength:
|
|
778
|
+
mtz_group_merged.dataset(0).wavelength = wavelength
|
|
779
|
+
if n_groups:
|
|
780
|
+
g_with_leading_zeros = str(i_group_prefix + i_group + 1).zfill(
|
|
781
|
+
len(str(n_groups))
|
|
782
|
+
)
|
|
783
|
+
else:
|
|
784
|
+
g_with_leading_zeros = i_group_prefix + i_group + 1
|
|
785
|
+
# After merging, add <I> <I/sigI> n_unique n_obs completeness multiplicity
|
|
786
|
+
df = pandas.DataFrame(
|
|
787
|
+
data=mtz_group_merged.array, columns=mtz_group_merged.column_labels()
|
|
788
|
+
)
|
|
789
|
+
df["BIN"] = binner.get_bins(mtz_group_merged)
|
|
790
|
+
for b in range(binner.size):
|
|
791
|
+
df_bin = df[df["BIN"] == b]
|
|
792
|
+
bin_n_unique_expected = gemmi.count_reflections(
|
|
793
|
+
cell,
|
|
794
|
+
spacegroup,
|
|
795
|
+
binner.dmin_of_bin(b),
|
|
796
|
+
binner.dmax_of_bin(b),
|
|
797
|
+
unique=True,
|
|
798
|
+
)
|
|
799
|
+
bin_stats_list[b]["<I>"] = df_bin["IMEAN"].mean()
|
|
800
|
+
bin_stats_list[b]["<I/sigI>"] = (
|
|
801
|
+
df_bin["IMEAN"] / df_bin["SIGIMEAN"]
|
|
802
|
+
).mean()
|
|
803
|
+
bin_stats_list[b]["n_unique"] = len(df_bin)
|
|
804
|
+
bin_stats_list[b]["completeness"] = len(df_bin) / bin_n_unique_expected
|
|
805
|
+
bin_stats_list[b]["n_obs"] = int(df_bin["NOBS"].sum())
|
|
806
|
+
bin_stats_list[b]["multiplicity"] = df_bin["NOBS"].mean()
|
|
807
|
+
n_unique_expected = gemmi.count_reflections(
|
|
808
|
+
cell,
|
|
809
|
+
spacegroup,
|
|
810
|
+
intensities.resolution_range()[1],
|
|
811
|
+
intensities.resolution_range()[0],
|
|
812
|
+
unique=True,
|
|
813
|
+
)
|
|
814
|
+
bin_stats_list[-1]["<I>"] = df["IMEAN"].mean()
|
|
815
|
+
bin_stats_list[-1]["<I/sigI>"] = (df["IMEAN"] / df["SIGIMEAN"]).mean()
|
|
816
|
+
bin_stats_list[-1]["n_unique"] = len(df)
|
|
817
|
+
# TODO: overall completeness is not correct
|
|
818
|
+
bin_stats_list[-1]["completeness"] = len(df) / n_unique_expected
|
|
819
|
+
bin_stats_list[-1]["n_obs"] = int(df["NOBS"].sum())
|
|
820
|
+
bin_stats_list[-1]["multiplicity"] = df["NOBS"].mean()
|
|
821
|
+
bin_stats_list[-1]["n_unique"] = len(intensities.miller_array)
|
|
822
|
+
stats_filename = (
|
|
823
|
+
f"{prefix}group{i_group_prefix + i_group + 1}_merging_stats.txt"
|
|
824
|
+
)
|
|
825
|
+
write_bin_stats(bin_stats_list, stats_filename)
|
|
826
|
+
|
|
827
|
+
## n_expected = gemmi.count_reflections(m.cell, m.spacegroup, dmin, dmax)
|
|
828
|
+
completeness = len(intensities.miller_array) / n_expected
|
|
829
|
+
logging.info(
|
|
830
|
+
f"Merged group {i_group_prefix + i_group + 1} of batches: #reflections:"
|
|
831
|
+
f" {len(intensities.miller_array)}"
|
|
832
|
+
f" => completeness: {completeness:.3f}",
|
|
833
|
+
)
|
|
834
|
+
|
|
835
|
+
mtz_group_merged_filename = f"{prefix}group{g_with_leading_zeros}_I.mtz"
|
|
836
|
+
mtz_group_merged.write_to_file(mtz_group_merged_filename)
|
|
837
|
+
|
|
838
|
+
return mtz_group_merged_filename, bin_stats_list
|
|
839
|
+
|
|
840
|
+
if unmerged.lower().endswith(".hkl"):
|
|
841
|
+
xds_ascii = gemmi.read_xds_ascii(unmerged)
|
|
842
|
+
m = xds_ascii.to_mtz()
|
|
843
|
+
# Calculate resolution range from the unmerged file
|
|
844
|
+
d_array = m.cell.calculate_d_array(m.make_miller_array())
|
|
845
|
+
dmax = max(d_array)
|
|
846
|
+
dmin = min(d_array)
|
|
847
|
+
if unmerged.lower().endswith(".cif") or unmerged.lower().endswith(".ent"):
|
|
848
|
+
doc = gemmi.cif.read(unmerged)
|
|
849
|
+
rblocks = gemmi.as_refln_blocks(doc)
|
|
850
|
+
for rblock in rblocks:
|
|
851
|
+
if not rblock.is_merged():
|
|
852
|
+
Convert = gemmi.CifToMtz()
|
|
853
|
+
m = Convert.convert_block_to_mtz(rblock)
|
|
854
|
+
d_array = m.cell.calculate_d_array(m.make_miller_array())
|
|
855
|
+
dmax = max(d_array)
|
|
856
|
+
dmin = min(d_array)
|
|
857
|
+
break
|
|
858
|
+
else:
|
|
859
|
+
m = gemmi.read_mtz_file(unmerged)
|
|
860
|
+
dmax = m.resolution_low()
|
|
861
|
+
dmin = m.resolution_high()
|
|
862
|
+
|
|
863
|
+
logging.info(
|
|
864
|
+
f"Resolution limits: {dmax:.3f} - {dmin:.3f} A\n"
|
|
865
|
+
f"Space group: {m.spacegroup.hm} (No. {m.spacegroup.number})\n"
|
|
866
|
+
f"Unit cell: {m.cell.a:.3f} {m.cell.b:.3f} {m.cell.c:.3f}"
|
|
867
|
+
f" {m.cell.alpha:.3f} {m.cell.beta:.3f} {m.cell.gamma:.3f}"
|
|
868
|
+
)
|
|
869
|
+
# Scan the columns of the input unmerged MTZ file
|
|
870
|
+
# and check if Friedel pairs are present or not
|
|
871
|
+
anom = False
|
|
872
|
+
labin, anom, col_free = check_reflection_file_columns(m, unmerged=True)
|
|
873
|
+
# print(m.dataset(0).wavelength) == 0.0
|
|
874
|
+
# print(m.dataset(1).wavelength) OK
|
|
875
|
+
# print(m.datasets[0].wavelength) == 0.0
|
|
876
|
+
if m.datasets[-1].wavelength:
|
|
877
|
+
wavelength = m.datasets[-1].wavelength
|
|
878
|
+
logging.info(f"Wavelength from input file: {m.datasets[-1].wavelength}")
|
|
879
|
+
else:
|
|
880
|
+
wavelength = 0.0
|
|
881
|
+
logging.info("No wavelength found in input file.")
|
|
882
|
+
logging.info(
|
|
883
|
+
"Setting up resolution bins according to the file"
|
|
884
|
+
f" {unmerged} with resolution limits {dmax:.3f} - {dmin:.3f} A"
|
|
885
|
+
)
|
|
886
|
+
binner_master = gemmi.Binner()
|
|
887
|
+
binner_master.setup_from_1_d2(
|
|
888
|
+
n_bins, gemmi.Binner.Method.Dstar2, m.make_1_d2_array(), m.get_cell()
|
|
889
|
+
)
|
|
890
|
+
# n_expected = len(gemmi.make_miller_array(m.cell, m.spacegroup, 2.2, float('inf')))
|
|
891
|
+
n_expected = gemmi.count_reflections(m.cell, m.spacegroup, dmin, dmax)
|
|
892
|
+
logging.info(
|
|
893
|
+
f"Expected number of reflections for resolution range ({dmax:.3f} - {dmin:.3f}"
|
|
894
|
+
f" A), cell and symmetry from the input file {unmerged}: {str(n_expected)}"
|
|
895
|
+
)
|
|
896
|
+
|
|
897
|
+
logging.info(f"No. batches: {len(m.batches)}")
|
|
898
|
+
batch = m.batches[0]
|
|
899
|
+
# print(batch)
|
|
900
|
+
# print(batch.number) (0 + 1 = 1)
|
|
901
|
+
# print(batch.title)
|
|
902
|
+
# print(batch.axes)
|
|
903
|
+
# print(batch.dataset_id)
|
|
904
|
+
# print(list(batch.ints))
|
|
905
|
+
# print(list(batch.floats))
|
|
906
|
+
try:
|
|
907
|
+
if len(batch.floats) > 37 and batch.floats[36] and batch.floats[37]:
|
|
908
|
+
logging.info(
|
|
909
|
+
"Start/end phi of the first batch found:"
|
|
910
|
+
f" {batch.floats[36]}, {batch.floats[37]}"
|
|
911
|
+
)
|
|
912
|
+
else:
|
|
913
|
+
logging.info("Batch start/end phi of the first batch not found.")
|
|
914
|
+
except (IndexError, AttributeError) as e:
|
|
915
|
+
logging.info(f"Batch start and end of phi not found. Error: {e}")
|
|
916
|
+
|
|
917
|
+
# hkl = m.make_miller_array()
|
|
918
|
+
# intensity = m.column_with_label('I')
|
|
919
|
+
# sigma = m.column_with_label('SIGI')
|
|
920
|
+
# batch_col = m.column_with_label('BATCH')
|
|
921
|
+
# print(batch_col[0])
|
|
922
|
+
# print(batch.__dir__())
|
|
923
|
+
|
|
924
|
+
df_groups = []
|
|
925
|
+
df = pandas.DataFrame(data=m.array, columns=m.column_labels())
|
|
926
|
+
df = df.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
927
|
+
|
|
928
|
+
# TODO: a function that converts any selection criteria in lists of batches.
|
|
929
|
+
# Note that batch numberring is 1, 2, ..., 2000
|
|
930
|
+
# but Python numberring is 0, 1, ..., 1999
|
|
931
|
+
if merge_whole_file:
|
|
932
|
+
batches_split = [0, len(m.batches)]
|
|
933
|
+
elif batches_edges:
|
|
934
|
+
batches_split = [0]
|
|
935
|
+
for i in range(len(batches_edges)):
|
|
936
|
+
batches_split.append(batches_split[-1] + batches_edges[i])
|
|
937
|
+
elif n_batches_per_group:
|
|
938
|
+
batches_split = list(range(0, len(m.batches), n_batches_per_group))
|
|
939
|
+
batches_split.append(len(m.batches))
|
|
940
|
+
logging.info(f"Batch edges for merging groups: {batches_split}")
|
|
941
|
+
|
|
942
|
+
mtz_groups = []
|
|
943
|
+
bin_stats_lists = []
|
|
944
|
+
for i_group in range(len(batches_split) - 1):
|
|
945
|
+
# print(batches_split[i_group], batches_split[i_group+1])
|
|
946
|
+
df_group = df.loc[
|
|
947
|
+
(df["BATCH"] >= batches_split[i_group])
|
|
948
|
+
& (df["BATCH"] < batches_split[i_group + 1])
|
|
949
|
+
]
|
|
950
|
+
df_groups.append(df_group)
|
|
951
|
+
mtz_group, bin_stats_list = merge_group(
|
|
952
|
+
df_groups,
|
|
953
|
+
i_group,
|
|
954
|
+
m.cell,
|
|
955
|
+
m.spacegroup,
|
|
956
|
+
binner_master,
|
|
957
|
+
n_expected,
|
|
958
|
+
wavelength,
|
|
959
|
+
n_groups=len(batches_split),
|
|
960
|
+
anom=anom,
|
|
961
|
+
prefix=prefix,
|
|
962
|
+
i_group_prefix=i_group_prefix,
|
|
963
|
+
)
|
|
964
|
+
mtz_groups.append(mtz_group)
|
|
965
|
+
bin_stats_lists.append(bin_stats_list)
|
|
966
|
+
n_expected_list = [n_expected] * len(mtz_groups)
|
|
967
|
+
logging.info(f"Merged MTZ files: {mtz_groups}")
|
|
968
|
+
return mtz_groups, bin_stats_lists, n_expected_list, binner_master
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def run_molrep(model, mtz):
|
|
972
|
+
import shutil
|
|
973
|
+
|
|
974
|
+
prefix_local = f"{os.path.splitext(os.path.basename(mtz))[0]}_molrep"
|
|
975
|
+
log_filename = prefix_local + ".log"
|
|
976
|
+
pdb_filename = prefix_local + ".pdb"
|
|
977
|
+
cmd = [
|
|
978
|
+
"molrep",
|
|
979
|
+
"-f",
|
|
980
|
+
mtz,
|
|
981
|
+
"-m",
|
|
982
|
+
model,
|
|
983
|
+
]
|
|
984
|
+
logging.info("Running command: %s", shlex.join(cmd))
|
|
985
|
+
try:
|
|
986
|
+
with open(log_filename, "w") as log_file:
|
|
987
|
+
subprocess.run(cmd, check=True, stdout=log_file, stderr=subprocess.STDOUT)
|
|
988
|
+
except subprocess.CalledProcessError as e:
|
|
989
|
+
logging.error(f"Error occurred while running command: {e}")
|
|
990
|
+
shutil.copy2(
|
|
991
|
+
os.path.join(os.getcwd(), "molrep.pdb"), os.path.join(os.getcwd(), pdb_filename)
|
|
992
|
+
)
|
|
993
|
+
return pdb_filename
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
def dict_deep_merge(base: dict, update: dict) -> dict:
|
|
997
|
+
"""
|
|
998
|
+
Recursively merge `update` into `base`.
|
|
999
|
+
|
|
1000
|
+
Merge rules:
|
|
1001
|
+
- If both values are dictionaries, merge them recursively.
|
|
1002
|
+
- If both values are lists, append items from `update` that are not
|
|
1003
|
+
already present in `base` while preserving order.
|
|
1004
|
+
- Otherwise, replace the value in ``base`` with the value from `update`.
|
|
1005
|
+
|
|
1006
|
+
Args:
|
|
1007
|
+
base: dict
|
|
1008
|
+
Dictionary to be updated in place.
|
|
1009
|
+
update: dict
|
|
1010
|
+
Dictionary containing additional or overriding values.
|
|
1011
|
+
|
|
1012
|
+
Returns:
|
|
1013
|
+
dict
|
|
1014
|
+
The merged dictionary (same object as `base`).
|
|
1015
|
+
|
|
1016
|
+
Examples
|
|
1017
|
+
--------
|
|
1018
|
+
>>> base = {"a": [1, 2], "b": {"x": 1}}
|
|
1019
|
+
>>> update = {"a": [2, 3], "b": {"y": 2}}
|
|
1020
|
+
>>> deep_merge(base, update)
|
|
1021
|
+
{'a': [1, 2, 3], 'b': {'x': 1, 'y': 2}}
|
|
1022
|
+
"""
|
|
1023
|
+
for key, value in update.items():
|
|
1024
|
+
if key not in base:
|
|
1025
|
+
base[key] = value
|
|
1026
|
+
elif isinstance(base[key], dict) and isinstance(value, dict):
|
|
1027
|
+
dict_deep_merge(base[key], value)
|
|
1028
|
+
elif isinstance(base[key], list) and isinstance(value, list):
|
|
1029
|
+
# Append new items while preserving order
|
|
1030
|
+
seen = set(base[key])
|
|
1031
|
+
for item in value:
|
|
1032
|
+
if item not in seen:
|
|
1033
|
+
base[key].append(item)
|
|
1034
|
+
seen.add(item)
|
|
1035
|
+
else:
|
|
1036
|
+
base[key] = value
|
|
1037
|
+
return base
|
|
1038
|
+
|
|
1039
|
+
|
|
1040
|
+
def run_servalcat_fwt(mtz_groups_i, prefix="", n_proc=1):
|
|
1041
|
+
"""
|
|
1042
|
+
Run `servalcat fw` to perform French Wilson conversion of intensities
|
|
1043
|
+
to structure factor amplitudes.
|
|
1044
|
+
|
|
1045
|
+
Args:
|
|
1046
|
+
mtz_groups_i (list): List of input MTZ files.
|
|
1047
|
+
prefix (str): Prefix for the output files.
|
|
1048
|
+
|
|
1049
|
+
Returns:
|
|
1050
|
+
list: List of output MTZ files, columns
|
|
1051
|
+
H K L F SIGF I SIG I + if Friedel pairs also F(+) SIGF(+) F(-) SIGF(-).
|
|
1052
|
+
servalcat fw drops NOBS, NOBS(+) and NOBS(-) columns.
|
|
1053
|
+
"""
|
|
1054
|
+
logging.info(
|
|
1055
|
+
"Running servalcat fw to convert intensities to structure factor amplitudes..."
|
|
1056
|
+
)
|
|
1057
|
+
mtz_groups_fi = []
|
|
1058
|
+
|
|
1059
|
+
def run_fw_one(args):
|
|
1060
|
+
i_group, mtz_group_i = args
|
|
1061
|
+
group_fi_prefix = os.path.splitext(os.path.basename(mtz_group_i))[0] + "F"
|
|
1062
|
+
log_group_fi = f"{group_fi_prefix}.log"
|
|
1063
|
+
mtz_group_fi = f"{group_fi_prefix}.mtz"
|
|
1064
|
+
cmd = ["servalcat", "fw", "--hklin", mtz_group_i, "-o", group_fi_prefix]
|
|
1065
|
+
logging.info("Running command: %s", shlex.join(cmd))
|
|
1066
|
+
try:
|
|
1067
|
+
with open(log_group_fi, "w") as log_file:
|
|
1068
|
+
subprocess.run(
|
|
1069
|
+
cmd, check=True, stdout=log_file, stderr=subprocess.STDOUT
|
|
1070
|
+
)
|
|
1071
|
+
return mtz_group_fi
|
|
1072
|
+
except subprocess.CalledProcessError as e:
|
|
1073
|
+
logging.error(f"Error occurred while running command: {e}")
|
|
1074
|
+
return None
|
|
1075
|
+
|
|
1076
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=n_proc) as executor:
|
|
1077
|
+
results = list(executor.map(run_fw_one, enumerate(mtz_groups_i)))
|
|
1078
|
+
mtz_groups_fi.extend([r for r in results if r])
|
|
1079
|
+
return mtz_groups_fi
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
def run_servalcat_refine(
|
|
1083
|
+
mtzs_fi,
|
|
1084
|
+
labins,
|
|
1085
|
+
models,
|
|
1086
|
+
mtzs_free=[],
|
|
1087
|
+
prefix="multixem_",
|
|
1088
|
+
source="xray",
|
|
1089
|
+
keyword_file="",
|
|
1090
|
+
config_yaml="",
|
|
1091
|
+
arguments=[],
|
|
1092
|
+
sigmaa=True,
|
|
1093
|
+
quick=False,
|
|
1094
|
+
n_proc=1,
|
|
1095
|
+
):
|
|
1096
|
+
refined_mmcifs = []
|
|
1097
|
+
refined_mtzs = []
|
|
1098
|
+
refined_jsons = []
|
|
1099
|
+
|
|
1100
|
+
def refine_one(params):
|
|
1101
|
+
i_mtz, (mtz_fi, labin, mtz_free, model, prefix) = params
|
|
1102
|
+
local_refined_mmcifs = []
|
|
1103
|
+
local_refined_mtzs = []
|
|
1104
|
+
local_refined_jsons = []
|
|
1105
|
+
|
|
1106
|
+
if mtzs_free and "--labin_llweight" in arguments:
|
|
1107
|
+
prefix += f"llweight{i_mtz}_"
|
|
1108
|
+
if "--weight" in arguments:
|
|
1109
|
+
prefix += "unre_"
|
|
1110
|
+
prefix += "refine"
|
|
1111
|
+
log_filename = prefix + ".log"
|
|
1112
|
+
|
|
1113
|
+
cmd = [
|
|
1114
|
+
"servalcat",
|
|
1115
|
+
"refine_xtal_norefmac",
|
|
1116
|
+
"--hklin",
|
|
1117
|
+
mtz_fi,
|
|
1118
|
+
"--model",
|
|
1119
|
+
model,
|
|
1120
|
+
"-s",
|
|
1121
|
+
source,
|
|
1122
|
+
"--labin",
|
|
1123
|
+
labin,
|
|
1124
|
+
"--hout",
|
|
1125
|
+
"-o",
|
|
1126
|
+
prefix,
|
|
1127
|
+
]
|
|
1128
|
+
if mtz_free:
|
|
1129
|
+
cmd.extend(["--hklin_free", mtz_free])
|
|
1130
|
+
if keyword_file:
|
|
1131
|
+
cmd.extend(["--keyword_file", keyword_file])
|
|
1132
|
+
if config_yaml:
|
|
1133
|
+
cmd.extend(["--config", config_yaml])
|
|
1134
|
+
if arguments:
|
|
1135
|
+
cmd.extend(arguments)
|
|
1136
|
+
if quick:
|
|
1137
|
+
cmd.extend(["--ncycle", "1"])
|
|
1138
|
+
logging.info("Running command: %s", shlex.join(cmd))
|
|
1139
|
+
try:
|
|
1140
|
+
with open(log_filename, "w") as log_file:
|
|
1141
|
+
process = subprocess.Popen(
|
|
1142
|
+
cmd, stdout=log_file, stderr=subprocess.PIPE, text=True, bufsize=1
|
|
1143
|
+
)
|
|
1144
|
+
for line in process.stderr:
|
|
1145
|
+
logging.error(line.rstrip())
|
|
1146
|
+
log_file.write(line)
|
|
1147
|
+
process.wait()
|
|
1148
|
+
if process.returncode != 0:
|
|
1149
|
+
raise subprocess.CalledProcessError(process.returncode, cmd)
|
|
1150
|
+
except subprocess.CalledProcessError as e:
|
|
1151
|
+
logging.error(f"Error occurred while running command: {e}")
|
|
1152
|
+
json_filename = prefix + "_stats.json"
|
|
1153
|
+
if not os.path.exists(json_filename):
|
|
1154
|
+
raise FileNotFoundError(f"Expected stats file not found: {json_filename}")
|
|
1155
|
+
with open(json_filename, "r") as stats_file:
|
|
1156
|
+
stats = json.load(stats_file)
|
|
1157
|
+
stats_line_list = [
|
|
1158
|
+
f"{stat} = {stats[-1]['data']['summary'][stat]:.4f}"
|
|
1159
|
+
for stat in stats[-1]["data"]["summary"]
|
|
1160
|
+
if stat != "-LL"
|
|
1161
|
+
]
|
|
1162
|
+
logging.info(f"Finished: {prefix}.mmcif {', '.join(stats_line_list)}")
|
|
1163
|
+
if sigmaa:
|
|
1164
|
+
log_filename_sigmaa = prefix + "_sigmaa.log"
|
|
1165
|
+
cmd_sigmaa = [
|
|
1166
|
+
"servalcat",
|
|
1167
|
+
"sigmaa",
|
|
1168
|
+
"--hklin",
|
|
1169
|
+
mtz_fi,
|
|
1170
|
+
"--model",
|
|
1171
|
+
prefix + ".mmcif",
|
|
1172
|
+
"-s",
|
|
1173
|
+
source,
|
|
1174
|
+
"--labin",
|
|
1175
|
+
labin,
|
|
1176
|
+
"-o",
|
|
1177
|
+
prefix + "_sigmaa",
|
|
1178
|
+
]
|
|
1179
|
+
if mtz_free:
|
|
1180
|
+
cmd_sigmaa.extend(["--hklin_free", mtz_free])
|
|
1181
|
+
# if arguments:
|
|
1182
|
+
# cmd_sigmaa.extend(arguments)
|
|
1183
|
+
logging.info("Running command: %s", shlex.join(cmd_sigmaa))
|
|
1184
|
+
try:
|
|
1185
|
+
with open(log_filename_sigmaa, "w") as log_file_sigmaa:
|
|
1186
|
+
subprocess.run(
|
|
1187
|
+
cmd_sigmaa,
|
|
1188
|
+
check=True,
|
|
1189
|
+
stdout=log_file_sigmaa,
|
|
1190
|
+
stderr=subprocess.STDOUT,
|
|
1191
|
+
)
|
|
1192
|
+
except subprocess.CalledProcessError as e:
|
|
1193
|
+
logging.error(f"Error occurred while running command: {e}")
|
|
1194
|
+
local_refined_mtzs.append(prefix + "_sigmaa.mtz")
|
|
1195
|
+
else:
|
|
1196
|
+
local_refined_mtzs.append(prefix + ".mtz")
|
|
1197
|
+
local_refined_mmcifs.append(prefix + ".mmcif")
|
|
1198
|
+
local_refined_jsons.append(prefix + "_stats.json")
|
|
1199
|
+
return local_refined_mmcifs[0], local_refined_mtzs[0], local_refined_jsons[0]
|
|
1200
|
+
|
|
1201
|
+
if len(mtzs_fi) == len(models) >= 2:
|
|
1202
|
+
models_list = models
|
|
1203
|
+
elif len(models) == len(mtzs_free) >= 2:
|
|
1204
|
+
models_list = models
|
|
1205
|
+
else:
|
|
1206
|
+
models_list = [models[0]] * max(len(mtzs_fi), len(mtzs_free))
|
|
1207
|
+
|
|
1208
|
+
if mtzs_free and len(mtzs_free) >= 2 and len(mtzs_fi) == 1:
|
|
1209
|
+
# refinement after bootstrapping
|
|
1210
|
+
params = zip(
|
|
1211
|
+
mtzs_fi * len(mtzs_free),
|
|
1212
|
+
labins * len(mtzs_free),
|
|
1213
|
+
mtzs_free,
|
|
1214
|
+
models_list,
|
|
1215
|
+
[prefix] * len(mtzs_free),
|
|
1216
|
+
)
|
|
1217
|
+
elif not mtzs_free:
|
|
1218
|
+
# refinement after merging, no free set provided
|
|
1219
|
+
if len(mtzs_fi) == 1 and "group" not in prefix:
|
|
1220
|
+
prefixes = [prefix] * len(mtzs_fi)
|
|
1221
|
+
else:
|
|
1222
|
+
prefixes = [f"{prefix}group{i_mtz + 1}_" for i_mtz in range(len(mtzs_fi))]
|
|
1223
|
+
params = zip(mtzs_fi, labins, [None] * len(mtzs_fi), models_list, prefixes)
|
|
1224
|
+
elif len(mtzs_free) == 1:
|
|
1225
|
+
# refinement after merging, single free set provided
|
|
1226
|
+
if len(mtzs_fi) == 1 and "group" not in prefix:
|
|
1227
|
+
prefixes = [prefix] * len(mtzs_fi)
|
|
1228
|
+
else:
|
|
1229
|
+
prefixes = [f"{prefix}group{i_mtz + 1}_" for i_mtz in range(len(mtzs_fi))]
|
|
1230
|
+
params = zip(
|
|
1231
|
+
mtzs_fi,
|
|
1232
|
+
labins,
|
|
1233
|
+
mtzs_free * len(mtzs_fi),
|
|
1234
|
+
models_list,
|
|
1235
|
+
prefixes,
|
|
1236
|
+
)
|
|
1237
|
+
else:
|
|
1238
|
+
# unexpected case, should not happen
|
|
1239
|
+
raise ValueError(
|
|
1240
|
+
"Unexpected case: both mtzs_fi and mtzs_free have more than one element."
|
|
1241
|
+
)
|
|
1242
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=n_proc) as executor:
|
|
1243
|
+
results = list(executor.map(refine_one, enumerate(params)))
|
|
1244
|
+
for mmcif, mtz, stats in results:
|
|
1245
|
+
refined_mmcifs.append(mmcif)
|
|
1246
|
+
refined_mtzs.append(mtz)
|
|
1247
|
+
refined_jsons.append(stats)
|
|
1248
|
+
return refined_mmcifs, refined_mtzs, refined_jsons
|
|
1249
|
+
|
|
1250
|
+
|
|
1251
|
+
def compare_mtzs_fi(
|
|
1252
|
+
mtzs_fi: list,
|
|
1253
|
+
binner: gemmi.Binner,
|
|
1254
|
+
labins: list = ["IMEAN,SIGIMEAN"],
|
|
1255
|
+
bin_stats_matrix: list = [],
|
|
1256
|
+
n_expected: list = [],
|
|
1257
|
+
):
|
|
1258
|
+
"""
|
|
1259
|
+
Compare multiple MTZ files containing reflection data (preferably
|
|
1260
|
+
reflection intensities) and calculate statistics for each pair of MTZ files.
|
|
1261
|
+
"""
|
|
1262
|
+
|
|
1263
|
+
# noqa: E501
|
|
1264
|
+
def compare_mtz_fi_pair(
|
|
1265
|
+
mtz_fi1: str,
|
|
1266
|
+
mtz_fi2: str,
|
|
1267
|
+
binner: gemmi.Binner,
|
|
1268
|
+
labin: str = "IMEAN,SIGIMEAN",
|
|
1269
|
+
bin_stats_list1: list = [],
|
|
1270
|
+
bin_stats_list2: list = [],
|
|
1271
|
+
):
|
|
1272
|
+
# f_col = "F"
|
|
1273
|
+
i_col = labin.split(",")[
|
|
1274
|
+
0
|
|
1275
|
+
] # usually IMEAN # can be just "I" after servalcat fw
|
|
1276
|
+
column_labels_dropna = [i_col, f"SIG{i_col}"] # or F?
|
|
1277
|
+
mtz1 = gemmi.read_mtz_file(mtz_fi1)
|
|
1278
|
+
mtz2 = gemmi.read_mtz_file(mtz_fi2)
|
|
1279
|
+
logging.info("")
|
|
1280
|
+
logging.info(
|
|
1281
|
+
f"Unit cell {str(mtz1.cell.a)} {str(mtz1.cell.b)} {str(mtz1.cell.c)}"
|
|
1282
|
+
f" {str(mtz1.cell.alpha)} {str(mtz1.cell.beta)} {str(mtz1.cell.gamma)}"
|
|
1283
|
+
f" in file {mtz_fi1}"
|
|
1284
|
+
)
|
|
1285
|
+
logging.info(
|
|
1286
|
+
f"Unit cell {str(mtz2.cell.a)} {str(mtz2.cell.b)} {str(mtz2.cell.c)}"
|
|
1287
|
+
f" {str(mtz2.cell.alpha)} {str(mtz2.cell.beta)} {str(mtz2.cell.gamma)}"
|
|
1288
|
+
f" in file {mtz_fi2}"
|
|
1289
|
+
)
|
|
1290
|
+
if mtz1.cell != mtz2.cell:
|
|
1291
|
+
logging.warning("Unit cell parameters are different.")
|
|
1292
|
+
mtz_df1 = pandas.DataFrame(data=mtz1.array, columns=mtz1.column_labels())
|
|
1293
|
+
mtz_df2 = pandas.DataFrame(data=mtz2.array, columns=mtz2.column_labels())
|
|
1294
|
+
mtz_df1 = mtz_df1.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
1295
|
+
mtz_df2 = mtz_df2.astype({name: "int32" for name in ["H", "K", "L"]})
|
|
1296
|
+
|
|
1297
|
+
column_labels = mtz1.column_labels()
|
|
1298
|
+
column_labels.remove("H")
|
|
1299
|
+
column_labels.remove("K")
|
|
1300
|
+
column_labels.remove("L")
|
|
1301
|
+
# afterwards, rename to FP1, SIGFP1, ..., FP2, SIGFP2, ...
|
|
1302
|
+
# columns1 = [col + '1' for col in columns]
|
|
1303
|
+
column_labels_dict1 = {col: col + "1" for col in column_labels}
|
|
1304
|
+
# columns2 = [col + '2' for col in columns]
|
|
1305
|
+
column_labels_dict2 = {col: col + "2" for col in column_labels}
|
|
1306
|
+
|
|
1307
|
+
# mtz_df1 = mtz_df1[['H', 'K', 'L'] + columns] # Select only relevant columns
|
|
1308
|
+
mtz_df1 = mtz_df1.dropna(
|
|
1309
|
+
subset=column_labels_dropna
|
|
1310
|
+
) # Select only reflections with F
|
|
1311
|
+
mtz_df1 = mtz_df1.rename(columns=column_labels_dict1) # Rename
|
|
1312
|
+
# print("")
|
|
1313
|
+
# print(mtz_df1.head(10))
|
|
1314
|
+
n_refl1 = len(mtz_df1)
|
|
1315
|
+
logging.info(f"No. unique reflections: {n_refl1} in file {mtz_fi1}")
|
|
1316
|
+
|
|
1317
|
+
# mtz_df2 = mtz_df2[['H', 'K', 'L'] + columns]
|
|
1318
|
+
mtz_df2 = mtz_df2.dropna(subset=column_labels_dropna)
|
|
1319
|
+
mtz_df2 = mtz_df2.rename(columns=column_labels_dict2)
|
|
1320
|
+
n_refl2 = len(mtz_df2)
|
|
1321
|
+
logging.info(f"No. unique reflections: {n_refl2} in file {mtz_fi2}")
|
|
1322
|
+
|
|
1323
|
+
# Extract common Miller indices (H, K, L)
|
|
1324
|
+
df = pandas.merge(mtz_df1, mtz_df2, on=["H", "K", "L"])
|
|
1325
|
+
n_refl = len(df)
|
|
1326
|
+
logging.info(
|
|
1327
|
+
f"No. unique reflections: {n_refl} in common;"
|
|
1328
|
+
f" ratios to the originals: {n_refl / n_refl1:.4f} {n_refl / n_refl2:.4f}"
|
|
1329
|
+
)
|
|
1330
|
+
# print("")
|
|
1331
|
+
# print(df.head(10))
|
|
1332
|
+
# print(df.describe())
|
|
1333
|
+
hkl_common_array = numpy.array(df[["H", "K", "L"]].values, numpy.int32)
|
|
1334
|
+
hkl_common_array = numpy.ascontiguousarray(hkl_common_array, dtype=numpy.int32)
|
|
1335
|
+
# print(len(hkl_common_array))
|
|
1336
|
+
# print(hkl_common_array.flags.c_contiguous)
|
|
1337
|
+
# print(hkl_common_array[:10])
|
|
1338
|
+
n_refl_list = [n_refl1, n_refl2, n_refl]
|
|
1339
|
+
|
|
1340
|
+
"""# Scaling per resolution bins - at least 100 reflections per bin
|
|
1341
|
+
n_bins = int(n_refl / 200) # only starting point
|
|
1342
|
+
binner = gemmi.Binner()
|
|
1343
|
+
binner.setup(n_bins, gemmi.Binner.Method.Dstar2, hkl_common_array, mtz1.cell)
|
|
1344
|
+
bins_tmp = binner.get_bins(hkl_common_array)
|
|
1345
|
+
min_n_bins = min(Counter(bins_tmp).values())
|
|
1346
|
+
while min_n_bins <= 100:
|
|
1347
|
+
n_bins = n_bins - 1
|
|
1348
|
+
binner.setup(
|
|
1349
|
+
n_bins, gemmi.Binner.Method.Dstar2, hkl_common_array, mtz1.cell
|
|
1350
|
+
)
|
|
1351
|
+
bins_tmp = binner.get_bins(hkl_common_array)
|
|
1352
|
+
min_n_bins = min(Counter(bins_tmp).values())"""
|
|
1353
|
+
bins_tmp = binner.get_bins(hkl_common_array)
|
|
1354
|
+
min_n_bins = min(Counter(bins_tmp).values())
|
|
1355
|
+
if min_n_bins < 100:
|
|
1356
|
+
logging.warning(
|
|
1357
|
+
"Less than 100 reflections per bin"
|
|
1358
|
+
" - it is recommended to set up a lower number of bins."
|
|
1359
|
+
)
|
|
1360
|
+
df["BIN"] = bins_tmp
|
|
1361
|
+
# print("Binner min_n_bins:", min_n_bins)
|
|
1362
|
+
n_bins = len(set(bins_tmp)) # TODO how to use args.n_bins?
|
|
1363
|
+
bins_stats = []
|
|
1364
|
+
df2_scaled = pandas.DataFrame()
|
|
1365
|
+
# mtz_df2_scaled = mtz_df2[["H", "K", "L"]].copy()
|
|
1366
|
+
for b in range(n_bins):
|
|
1367
|
+
df_bin = df[df["BIN"] == b]
|
|
1368
|
+
# scale_delfofo = sum_hkl F1 * F2 / sum_hkl F2**2
|
|
1369
|
+
"""
|
|
1370
|
+
scale_delfofo_numer = (df_bin[f_col + "1"] * df_bin[f_col + "2"]).sum()
|
|
1371
|
+
scale_delfofo_denomin = (df_bin[f_col + "2"] ** 2).sum()
|
|
1372
|
+
scale_delfofo = scale_delfofo_numer / scale_delfofo_denomin
|
|
1373
|
+
ccF_iso = numpy.corrcoef(
|
|
1374
|
+
df_bin[f_col + "1"], scale_delfofo * df_bin[f_col + "2"]
|
|
1375
|
+
)[0, 1]
|
|
1376
|
+
rF_iso_numer = (
|
|
1377
|
+
abs(df_bin[f_col + "1"] - scale_delfofo * df_bin[f_col + "2"])
|
|
1378
|
+
).sum()
|
|
1379
|
+
rF_iso_denom = (
|
|
1380
|
+
abs(df_bin[f_col + "1"] + scale_delFfofo * df_bin[f_col + "2"])
|
|
1381
|
+
).sum()
|
|
1382
|
+
rF_iso = 2 * rF_iso_numer / rF_iso_denom"""
|
|
1383
|
+
# DELFOFO
|
|
1384
|
+
"""df.loc[df_bin.index, 'DELFOFO'] = \
|
|
1385
|
+
numpy.abs(df_bin[f_col + '1'] - scale_delfofo * df_bin[f_col + '2'])
|
|
1386
|
+
# df.loc[df_bin.index, 'PHDELFOFO'] = df_bin['PHFC1']
|
|
1387
|
+
df.loc[df_bin.index, 'DELFOFOSIG'] = \
|
|
1388
|
+
numpy.sign(df_bin[f_col + '1'] - scale_delfofo * df_bin[f_col + '2'])"""
|
|
1389
|
+
"""# If FP1 - scale * FP2 < 0, then add/subtract 180deg to phase
|
|
1390
|
+
df_bin_noflip = df[(df['BIN'] == b) & (df['DELFOFOSIG'] != -1)]
|
|
1391
|
+
df_bin_flip_plus = \
|
|
1392
|
+
df[(df['BIN'] == b) & (df['DELFOFOSIG'] == -1) & (df['PHFC1'] <= 0)]
|
|
1393
|
+
df_bin_flip_minus = \
|
|
1394
|
+
df[(df['BIN'] == b) & (df['DELFOFOSIG'] == -1) & (df['PHFC1'] > 0)]
|
|
1395
|
+
# print(len(df_bin_noflip), len(df_bin_flip_plus), len(df_bin_flip_minus))
|
|
1396
|
+
df.loc[df_bin_noflip.index, 'PHDELFOFO'] = df_bin_noflip['PHFC1']
|
|
1397
|
+
df.loc[df_bin_flip_plus.index, 'PHDELFOFO'] = \
|
|
1398
|
+
df_bin_flip_plus['PHFC1'] + 180
|
|
1399
|
+
df.loc[df_bin_flip_minus.index, 'PHDELFOFO'] = \
|
|
1400
|
+
df_bin_flip_minus['PHFC1'] - 180"""
|
|
1401
|
+
scale_delioio = calc_scale_real(
|
|
1402
|
+
df_bin, i_col, b, binner.dmax_of_bin(b), binner.dmin_of_bin(b)
|
|
1403
|
+
)
|
|
1404
|
+
df_bin_scaled = df_bin.copy()
|
|
1405
|
+
df_bin_scaled[f"{i_col}2_scaled"] = (
|
|
1406
|
+
scale_delioio * df_bin_scaled[f"{i_col}2"]
|
|
1407
|
+
)
|
|
1408
|
+
df_bin_scaled[f"SIG{i_col}2_scaled"] = (
|
|
1409
|
+
scale_delioio * df_bin_scaled[f"SIG{i_col}2"]
|
|
1410
|
+
)
|
|
1411
|
+
df2_scaled = pandas.concat(
|
|
1412
|
+
[df2_scaled, df_bin_scaled]
|
|
1413
|
+
) # , ignore_index=True)
|
|
1414
|
+
ccI_iso = numpy.corrcoef(
|
|
1415
|
+
df_bin[i_col + "1"], df_bin_scaled[i_col + "2_scaled"]
|
|
1416
|
+
)[0, 1]
|
|
1417
|
+
"""
|
|
1418
|
+
rI_iso_numer = \
|
|
1419
|
+
(abs(df_bin[i_col + "1"] - scale_delioio * df_bin[i_col + "2"])).sum()
|
|
1420
|
+
rI_iso_denom = \
|
|
1421
|
+
(abs(df_bin[i_col + "1"] + scale_delioio * df_bin[i_col + "2"])).sum()
|
|
1422
|
+
rI_iso = 2 * rI_iso_numer / rI_iso_denom"""
|
|
1423
|
+
|
|
1424
|
+
bins_stats.append(
|
|
1425
|
+
{
|
|
1426
|
+
"bin": b + 1,
|
|
1427
|
+
"dmax": binner.dmax_of_bin(b),
|
|
1428
|
+
"dmin": binner.dmin_of_bin(b),
|
|
1429
|
+
"dmin_star2": 1 / (binner.dmin_of_bin(b) ** 2),
|
|
1430
|
+
"count": len(df_bin),
|
|
1431
|
+
# "scale_delfofo": scale_delfofo,
|
|
1432
|
+
# "ccF_iso": ccF_iso,
|
|
1433
|
+
# "rF_iso": rF_iso,
|
|
1434
|
+
"scale_delioio": scale_delioio,
|
|
1435
|
+
"ccI_iso": ccI_iso,
|
|
1436
|
+
# "rI_iso": rI_iso,
|
|
1437
|
+
}
|
|
1438
|
+
)
|
|
1439
|
+
|
|
1440
|
+
if (
|
|
1441
|
+
bin_stats_list1
|
|
1442
|
+
and bin_stats_list2
|
|
1443
|
+
and len(bin_stats_list1) == n_bins + 1
|
|
1444
|
+
and len(bin_stats_list2) == n_bins + 1
|
|
1445
|
+
):
|
|
1446
|
+
# Add CC* from bin_stats_list1 and bin_stats_list2 if available
|
|
1447
|
+
# TODO: fix bug - CC*1 CC*2 CC12true are now available only for i==0 j==1
|
|
1448
|
+
for b in range(n_bins):
|
|
1449
|
+
bins_stats[b]["CC*1"] = bin_stats_list1[b]["CC*"]
|
|
1450
|
+
bins_stats[b]["CC*2"] = bin_stats_list2[b]["CC*"]
|
|
1451
|
+
bins_stats[b]["CC12true"] = bins_stats[b]["ccI_iso"] / (
|
|
1452
|
+
bins_stats[b]["CC*1"] * bins_stats[b]["CC*2"]
|
|
1453
|
+
)
|
|
1454
|
+
|
|
1455
|
+
bins_stats_df = pandas.DataFrame(bins_stats)
|
|
1456
|
+
# Calculate weighted average of cc over bins
|
|
1457
|
+
"""ccF_iso_avg = (
|
|
1458
|
+
bins_stats_df["ccF_iso"] * bins_stats_df["count"]
|
|
1459
|
+
).sum() / bins_stats_df["count"].sum()"""
|
|
1460
|
+
ccI_iso_avg = (
|
|
1461
|
+
bins_stats_df["ccI_iso"] * bins_stats_df["count"]
|
|
1462
|
+
).sum() / bins_stats_df["count"].sum()
|
|
1463
|
+
# cc_iso_avg_list = [ccF_iso_avg, ccI_iso_avg]
|
|
1464
|
+
mtz_fi1_base = os.path.splitext(os.path.basename(mtz_fi1))[0]
|
|
1465
|
+
mtz_fi2_base = os.path.splitext(os.path.basename(mtz_fi2))[0]
|
|
1466
|
+
|
|
1467
|
+
mtz_df2 = mtz_df2.drop(columns=[f"{i_col}2", f"SIG{i_col}2"])
|
|
1468
|
+
df2_scaled = df2_scaled.reset_index()
|
|
1469
|
+
mtz_df2 = mtz_df2.merge(
|
|
1470
|
+
df2_scaled[["H", "K", "L", f"{i_col}2_scaled", f"SIG{i_col}2_scaled"]],
|
|
1471
|
+
on=["H", "K", "L"],
|
|
1472
|
+
how="left",
|
|
1473
|
+
)
|
|
1474
|
+
mtz_df2 = mtz_df2.rename(
|
|
1475
|
+
columns={
|
|
1476
|
+
f"{i_col}2_scaled": f"{i_col}",
|
|
1477
|
+
f"SIG{i_col}2_scaled": f"SIG{i_col}",
|
|
1478
|
+
}
|
|
1479
|
+
)
|
|
1480
|
+
write_mtz_from_df(
|
|
1481
|
+
mtz_df2,
|
|
1482
|
+
mtz2,
|
|
1483
|
+
columns={f"{i_col}": "J", f"SIG{i_col}": "Q"},
|
|
1484
|
+
filename=f"{mtz_fi2_base}_scaled_to_{mtz_fi1_base}.mtz",
|
|
1485
|
+
)
|
|
1486
|
+
|
|
1487
|
+
# Make a plot
|
|
1488
|
+
"""def star2(x):
|
|
1489
|
+
# Vectorized 1/x^2, treating x==0 manually
|
|
1490
|
+
x = numpy.array(x, float)
|
|
1491
|
+
near_zero = numpy.isclose(x, 0)
|
|
1492
|
+
x[near_zero] = 9999
|
|
1493
|
+
x[~near_zero] = 1 / (x[~near_zero] ** 2)
|
|
1494
|
+
return x
|
|
1495
|
+
def star_sqrt(x):
|
|
1496
|
+
# Vectorized 1/sqrt(x), treating x<0 manually
|
|
1497
|
+
x = numpy.array(x, float)
|
|
1498
|
+
negative = numpy.less(x, 0)
|
|
1499
|
+
x[negative] = 9999
|
|
1500
|
+
x[~negative] = 1 / numpy.sqrt(x[~negative])
|
|
1501
|
+
return x"""
|
|
1502
|
+
plt.figure(figsize=(8, 6))
|
|
1503
|
+
plt.plot(
|
|
1504
|
+
bins_stats_df["dmin_star2"],
|
|
1505
|
+
bins_stats_df["ccI_iso"],
|
|
1506
|
+
marker="o",
|
|
1507
|
+
label="ccI_iso",
|
|
1508
|
+
)
|
|
1509
|
+
if "CC*1" in bins_stats_df.columns:
|
|
1510
|
+
plt.plot(
|
|
1511
|
+
bins_stats_df["dmin_star2"],
|
|
1512
|
+
bins_stats_df["CC*1"],
|
|
1513
|
+
marker="s",
|
|
1514
|
+
label="CC*1",
|
|
1515
|
+
)
|
|
1516
|
+
if "CC*2" in bins_stats_df.columns:
|
|
1517
|
+
plt.plot(
|
|
1518
|
+
bins_stats_df["dmin_star2"],
|
|
1519
|
+
bins_stats_df["CC*2"],
|
|
1520
|
+
marker="^",
|
|
1521
|
+
label="CC*2",
|
|
1522
|
+
)
|
|
1523
|
+
if "CC12true" in bins_stats_df.columns:
|
|
1524
|
+
plt.plot(
|
|
1525
|
+
bins_stats_df["dmin_star2"],
|
|
1526
|
+
bins_stats_df["CC12true"],
|
|
1527
|
+
marker="x",
|
|
1528
|
+
label="CC12true",
|
|
1529
|
+
)
|
|
1530
|
+
plt.xlabel("Resolution (Å⁻²)")
|
|
1531
|
+
# Show dmin labels on a secondary x-axis at the bottom
|
|
1532
|
+
ax = plt.gca()
|
|
1533
|
+
"""secax = ax.secondary_xaxis(
|
|
1534
|
+
'bottom',
|
|
1535
|
+
functions=(lambda x: 1 / numpy.sqrt(x), lambda d: 1 / (d ** 2))
|
|
1536
|
+
)
|
|
1537
|
+
secax = ax.secondary_xaxis('top', functions=(star_sqrt, star2))"""
|
|
1538
|
+
secax = ax.secondary_xaxis("top")
|
|
1539
|
+
secax.set_xlabel("Resolution (Å)")
|
|
1540
|
+
N = max(1, len(bins_stats_df) // 5) # Show only some labels to avoid overlap
|
|
1541
|
+
ticks = bins_stats_df["dmin_star2"]
|
|
1542
|
+
dmins = bins_stats_df["dmin"]
|
|
1543
|
+
labels = []
|
|
1544
|
+
for i, d in enumerate(dmins):
|
|
1545
|
+
if i == 0 or i == len(dmins) - 1: # Always show the first and last labels
|
|
1546
|
+
labels.append(f"{d:.2f}")
|
|
1547
|
+
elif i % N == 0:
|
|
1548
|
+
labels.append(f"{d:.2f}")
|
|
1549
|
+
else:
|
|
1550
|
+
labels.append("")
|
|
1551
|
+
secax.set_xticks(ticks)
|
|
1552
|
+
secax.set_xticklabels(labels)
|
|
1553
|
+
plt.ylabel("Correlation")
|
|
1554
|
+
plt.title("Correlation statistics per resolution bin")
|
|
1555
|
+
plt.legend()
|
|
1556
|
+
plt.tight_layout()
|
|
1557
|
+
plt.savefig(f"{mtz_fi1_base}_vs_{mtz_fi2_base}_cc_plot.png")
|
|
1558
|
+
plt.close()
|
|
1559
|
+
# Round float columns to 4 decimal places and save as fixed-width file
|
|
1560
|
+
stats_filename = f"{mtz_fi1_base}_vs_{mtz_fi2_base}_bin_stats.txt"
|
|
1561
|
+
write_bin_stats(bins_stats, stats_filename)
|
|
1562
|
+
return bins_stats, n_refl_list, ccI_iso_avg
|
|
1563
|
+
|
|
1564
|
+
n_refl_matrix = numpy.zeros((len(mtzs_fi), len(mtzs_fi)), dtype=int)
|
|
1565
|
+
ratio_refl_matrix = numpy.identity(len(mtzs_fi), dtype=float)
|
|
1566
|
+
# ccF_iso_matrix = numpy.identity(len(mtzs_fi), dtype=float)
|
|
1567
|
+
ccI_iso_matrix = numpy.identity(len(mtzs_fi), dtype=float)
|
|
1568
|
+
for i in range(len(mtzs_fi)):
|
|
1569
|
+
for j in range(i + 1, len(mtzs_fi)):
|
|
1570
|
+
# print(i, j)
|
|
1571
|
+
bins_stats, n_refl_list, ccI_iso_avg = compare_mtz_fi_pair(
|
|
1572
|
+
mtzs_fi[i],
|
|
1573
|
+
mtzs_fi[j],
|
|
1574
|
+
binner,
|
|
1575
|
+
labins[i],
|
|
1576
|
+
bin_stats_matrix[i][i],
|
|
1577
|
+
bin_stats_matrix[j][j],
|
|
1578
|
+
)
|
|
1579
|
+
bin_stats_matrix[i][j] = bins_stats
|
|
1580
|
+
bin_stats_matrix[j][i] = bin_stats_matrix[i][j]
|
|
1581
|
+
if i == 0:
|
|
1582
|
+
n_refl_matrix[j, j] = n_refl_list[1]
|
|
1583
|
+
if j == 1:
|
|
1584
|
+
n_refl_matrix[i, i] = n_refl_list[0]
|
|
1585
|
+
n_refl_matrix[i, j] = n_refl_list[2]
|
|
1586
|
+
n_refl_matrix[j, i] = n_refl_list[2]
|
|
1587
|
+
# No. reflections in common / No. reflections in the first file
|
|
1588
|
+
ratio_refl_matrix[i, j] = n_refl_list[2] / n_refl_list[0]
|
|
1589
|
+
# No. reflections in common / No. reflections in the second file
|
|
1590
|
+
ratio_refl_matrix[j, i] = n_refl_list[2] / n_refl_list[1]
|
|
1591
|
+
# ccF_iso_matrix[i, j] = cc_iso_avg_list[0]
|
|
1592
|
+
# ccF_iso_matrix[j, i] = cc_iso_avg_list[0]
|
|
1593
|
+
ccI_iso_matrix[i, j] = ccI_iso_avg
|
|
1594
|
+
ccI_iso_matrix[j, i] = ccI_iso_avg
|
|
1595
|
+
logging.info("\nNo. unique reflections:")
|
|
1596
|
+
logging.info(n_refl_matrix)
|
|
1597
|
+
if n_expected and len(n_expected) == len(mtzs_fi):
|
|
1598
|
+
completeness_matrix = n_refl_matrix / max(n_expected)
|
|
1599
|
+
logging.info("\nCompleteness:")
|
|
1600
|
+
logging.info(completeness_matrix)
|
|
1601
|
+
logging.info(
|
|
1602
|
+
"\nRatio of No. unique reflections in common and No. reflections in a data set:"
|
|
1603
|
+
)
|
|
1604
|
+
logging.info(ratio_refl_matrix)
|
|
1605
|
+
# print("Average CCFiso:")
|
|
1606
|
+
# print(ccF_iso_matrix)
|
|
1607
|
+
logging.info("\nAverage CCIiso:")
|
|
1608
|
+
logging.info(ccI_iso_matrix)
|
|
1609
|
+
logging.info("")
|
|
1610
|
+
# TODO: multiplicity
|
|
1611
|
+
return bin_stats_matrix, n_refl_matrix, ratio_refl_matrix
|
|
1612
|
+
|
|
1613
|
+
|
|
1614
|
+
def main():
|
|
1615
|
+
parser = create_parser()
|
|
1616
|
+
args = parser.parse_args()
|
|
1617
|
+
if hasattr(args, "func"):
|
|
1618
|
+
args.func(args)
|
|
1619
|
+
|
|
1620
|
+
# subcommand mean
|
|
1621
|
+
if args.command == "mean":
|
|
1622
|
+
setup_logging()
|
|
1623
|
+
|
|
1624
|
+
n_proc = min(os.cpu_count(), args.n_proc)
|
|
1625
|
+
if args.prefix:
|
|
1626
|
+
prefix = args.prefix
|
|
1627
|
+
else:
|
|
1628
|
+
prefix = args.file_name_template
|
|
1629
|
+
# Remove trailing _group{number}_ pattern to avoid duplication
|
|
1630
|
+
prefix = re.sub(r"_group\d+$", "", prefix)
|
|
1631
|
+
if prefix[-1] != "_":
|
|
1632
|
+
prefix += "_"
|
|
1633
|
+
|
|
1634
|
+
refined_jsons = sort_llweight_files(
|
|
1635
|
+
glob.glob(f"{args.file_name_template}_llweight*_refine_stats.json")
|
|
1636
|
+
)
|
|
1637
|
+
refined_json_ref_candidate = f"{args.file_name_template}_refine_stats.json"
|
|
1638
|
+
refined_json_ref = (
|
|
1639
|
+
refined_json_ref_candidate
|
|
1640
|
+
if os.path.isfile(refined_json_ref_candidate)
|
|
1641
|
+
else None
|
|
1642
|
+
)
|
|
1643
|
+
llweight_R1_outliers = []
|
|
1644
|
+
if refined_jsons:
|
|
1645
|
+
llweight_R1_outliers = bootstrap_analyse_stats(
|
|
1646
|
+
refined_jsons, refined_json_ref, idx=0, prefix=prefix
|
|
1647
|
+
)
|
|
1648
|
+
|
|
1649
|
+
refined_mmcif_ref_candidate = f"{args.file_name_template}_refine.mmcif"
|
|
1650
|
+
refined_mmcif_ref = (
|
|
1651
|
+
refined_mmcif_ref_candidate
|
|
1652
|
+
if os.path.isfile(refined_mmcif_ref_candidate)
|
|
1653
|
+
else None
|
|
1654
|
+
)
|
|
1655
|
+
|
|
1656
|
+
refined_mtz_ref_candidate = f"{args.file_name_template}_refine.mtz"
|
|
1657
|
+
refined_mtz_ref = (
|
|
1658
|
+
refined_mtz_ref_candidate
|
|
1659
|
+
if os.path.isfile(refined_mtz_ref_candidate)
|
|
1660
|
+
else None
|
|
1661
|
+
)
|
|
1662
|
+
|
|
1663
|
+
for f in [refined_json_ref, refined_mmcif_ref, refined_mtz_ref]:
|
|
1664
|
+
if f:
|
|
1665
|
+
logging.info(f"Reference file found: {f}")
|
|
1666
|
+
|
|
1667
|
+
refined_mmcifs = sort_llweight_files(
|
|
1668
|
+
glob.glob(f"{args.file_name_template}_llweight*_refine.mmcif")
|
|
1669
|
+
)
|
|
1670
|
+
# refined_mmcifs2 = sort_llweight_files(
|
|
1671
|
+
# glob.glob(f"{args.file_name_template}_llweight*_refine.cif")
|
|
1672
|
+
# )
|
|
1673
|
+
# refined_mmcifs = refined_mmcifs + refined_mmcifs2
|
|
1674
|
+
if llweight_R1_outliers:
|
|
1675
|
+
refined_mmcifs = filter_out_refined_files(
|
|
1676
|
+
refined_mmcifs, llweight_R1_outliers
|
|
1677
|
+
)
|
|
1678
|
+
|
|
1679
|
+
if refined_mmcifs:
|
|
1680
|
+
geometry_objects_ref = []
|
|
1681
|
+
if args.geometry_cids and refined_mmcif_ref:
|
|
1682
|
+
st_ref = gemmi.read_structure(refined_mmcif_ref)
|
|
1683
|
+
geometry_objects_ref = select_cids_for_geometry_analysis(
|
|
1684
|
+
args.geometry_cids
|
|
1685
|
+
)
|
|
1686
|
+
geometry_objects_ref = geometry_analysis_load(
|
|
1687
|
+
st_ref, geometry_objects_ref
|
|
1688
|
+
)
|
|
1689
|
+
bootstrap_analyse_structures(
|
|
1690
|
+
refined_mmcifs,
|
|
1691
|
+
refined_mmcif_ref,
|
|
1692
|
+
idx=0,
|
|
1693
|
+
prefix=prefix,
|
|
1694
|
+
skip_hydrogen=True,
|
|
1695
|
+
smcif=args.cif,
|
|
1696
|
+
geometry_cids_file=args.geometry_cids,
|
|
1697
|
+
geometry_objects_ref=geometry_objects_ref,
|
|
1698
|
+
no_origin_shift=args.no_origin_shift,
|
|
1699
|
+
)
|
|
1700
|
+
else:
|
|
1701
|
+
logging.warning(
|
|
1702
|
+
f"No refined mmCIF files found with a filename template"
|
|
1703
|
+
f" {args.file_name_template}_llweight*_refine.mmcif"
|
|
1704
|
+
)
|
|
1705
|
+
refined_mtzs = sort_llweight_files(
|
|
1706
|
+
glob.glob(f"{args.file_name_template}_llweight*_refine.mtz")
|
|
1707
|
+
)
|
|
1708
|
+
if llweight_R1_outliers:
|
|
1709
|
+
refined_mtzs = filter_out_refined_files(refined_mtzs, llweight_R1_outliers)
|
|
1710
|
+
if not args.calculate_mean_map:
|
|
1711
|
+
logging.info("Skipping mean map calculation.")
|
|
1712
|
+
return
|
|
1713
|
+
if refined_mtzs:
|
|
1714
|
+
if refined_mtz_ref:
|
|
1715
|
+
m = gemmi.read_mtz_file(refined_mtz_ref)
|
|
1716
|
+
binner = gemmi.Binner()
|
|
1717
|
+
binner.setup_from_1_d2(
|
|
1718
|
+
args.n_bins,
|
|
1719
|
+
gemmi.Binner.Method.Dstar2,
|
|
1720
|
+
m.make_1_d2_array(),
|
|
1721
|
+
m.get_cell(),
|
|
1722
|
+
)
|
|
1723
|
+
bootstrap_mean_map(
|
|
1724
|
+
refined_mtzs,
|
|
1725
|
+
idx=0,
|
|
1726
|
+
prefix=prefix,
|
|
1727
|
+
binner=binner,
|
|
1728
|
+
mtz_ref=refined_mtz_ref,
|
|
1729
|
+
n_proc=n_proc,
|
|
1730
|
+
)
|
|
1731
|
+
else:
|
|
1732
|
+
logging.warning(
|
|
1733
|
+
f"No refined MTZ files found with a filename template"
|
|
1734
|
+
f" {args.file_name_template}_llweight*_refine.mtz"
|
|
1735
|
+
)
|
|
1736
|
+
return
|
|
1737
|
+
|
|
1738
|
+
elif args.command not in ["pipeline", "bootstrap"] and args.command is not None:
|
|
1739
|
+
parser.print_help()
|
|
1740
|
+
return
|
|
1741
|
+
|
|
1742
|
+
# subcommand pipeline or bootstrap
|
|
1743
|
+
# elif args.command in ["pipeline", "bootstrap"] or args.command is None:
|
|
1744
|
+
|
|
1745
|
+
if args.prefix:
|
|
1746
|
+
prefix = args.prefix
|
|
1747
|
+
if prefix[-1] != "_":
|
|
1748
|
+
prefix += "_"
|
|
1749
|
+
working_dir_name = f"multixem_{prefix[:-1]}"
|
|
1750
|
+
else:
|
|
1751
|
+
prefix = "multixem_"
|
|
1752
|
+
working_dir_name = "multixem_project"
|
|
1753
|
+
|
|
1754
|
+
os.mkdir(working_dir_name)
|
|
1755
|
+
os.chdir(working_dir_name)
|
|
1756
|
+
|
|
1757
|
+
setup_logging()
|
|
1758
|
+
|
|
1759
|
+
logging.info(f"Command line: {shlex.join(sys.argv)}")
|
|
1760
|
+
logging.info(f"Running multixem version: {__version__}")
|
|
1761
|
+
args_dict = vars(args)
|
|
1762
|
+
logging.info("Parsed arguments:")
|
|
1763
|
+
for key, value in args_dict.items():
|
|
1764
|
+
logging.info(f" {key}: {value}")
|
|
1765
|
+
logging.info(f"Current working directory: {os.getcwd()}")
|
|
1766
|
+
logging.info(f"Prefix for the output files: {prefix}")
|
|
1767
|
+
|
|
1768
|
+
n_proc = min(os.cpu_count(), args.n_proc)
|
|
1769
|
+
# parse servalcat args preserving quoted groups
|
|
1770
|
+
servalcat_args = shlex.split(args.servalcat_args) if args.servalcat_args else []
|
|
1771
|
+
|
|
1772
|
+
mtzs_merged = []
|
|
1773
|
+
labins = []
|
|
1774
|
+
cols_free = []
|
|
1775
|
+
bin_stats_lists = []
|
|
1776
|
+
n_expected_list = []
|
|
1777
|
+
binner_master = None
|
|
1778
|
+
|
|
1779
|
+
if args.command == "pipeline" and args.hklin_unmerged:
|
|
1780
|
+
logging.info(f"Unmerged diffraction data files: {args.hklin_unmerged}")
|
|
1781
|
+
n_groups = 0
|
|
1782
|
+
mtz_groups_i = []
|
|
1783
|
+
for i, hklin_unmerged in enumerate(args.hklin_unmerged):
|
|
1784
|
+
logging.info("")
|
|
1785
|
+
logging.info(f"Unmerged diffraction data file: {hklin_unmerged}")
|
|
1786
|
+
if args.merge_whole_file:
|
|
1787
|
+
logging.info(
|
|
1788
|
+
"All the reflections in all the batches"
|
|
1789
|
+
" in this file will be merged."
|
|
1790
|
+
)
|
|
1791
|
+
_mtz_groups_i, _bin_stats_lists, _n_expected_list, _binner_master = (
|
|
1792
|
+
merge_in_groups(
|
|
1793
|
+
hklin_unmerged,
|
|
1794
|
+
args.n_bins,
|
|
1795
|
+
prefix,
|
|
1796
|
+
merge_whole_file=args.merge_whole_file,
|
|
1797
|
+
i_group_prefix=n_groups,
|
|
1798
|
+
)
|
|
1799
|
+
)
|
|
1800
|
+
elif len(args.n_batches) == 1:
|
|
1801
|
+
n_batches_per_group = args.n_batches[0]
|
|
1802
|
+
logging.info(
|
|
1803
|
+
f"Number of batches in merging group: {n_batches_per_group}"
|
|
1804
|
+
)
|
|
1805
|
+
_mtz_groups_i, _bin_stats_lists, _n_expected_list, _binner_master = (
|
|
1806
|
+
merge_in_groups(
|
|
1807
|
+
hklin_unmerged,
|
|
1808
|
+
args.n_bins,
|
|
1809
|
+
prefix,
|
|
1810
|
+
n_batches_per_group=n_batches_per_group,
|
|
1811
|
+
merge_whole_file=False,
|
|
1812
|
+
i_group_prefix=n_groups,
|
|
1813
|
+
)
|
|
1814
|
+
)
|
|
1815
|
+
else:
|
|
1816
|
+
_mtz_groups_i, _bin_stats_lists, _n_expected_list, _binner_master = (
|
|
1817
|
+
merge_in_groups(
|
|
1818
|
+
hklin_unmerged,
|
|
1819
|
+
args.n_bins,
|
|
1820
|
+
prefix,
|
|
1821
|
+
batches_edges=args.n_batches,
|
|
1822
|
+
merge_whole_file=False,
|
|
1823
|
+
i_group_prefix=n_groups,
|
|
1824
|
+
)
|
|
1825
|
+
)
|
|
1826
|
+
print(_mtz_groups_i)
|
|
1827
|
+
mtz_groups_i.extend(_mtz_groups_i)
|
|
1828
|
+
n_groups += len(_mtz_groups_i)
|
|
1829
|
+
n_groups_this_file = len(_mtz_groups_i)
|
|
1830
|
+
bin_stats_lists.extend(_bin_stats_lists)
|
|
1831
|
+
n_expected_list.extend(_n_expected_list)
|
|
1832
|
+
if i == 0:
|
|
1833
|
+
binner_master = _binner_master
|
|
1834
|
+
if args.amplitude:
|
|
1835
|
+
_mtzs_fwt = run_servalcat_fwt(_mtz_groups_i, prefix, n_proc)
|
|
1836
|
+
mtzs_merged.extend(_mtzs_fwt)
|
|
1837
|
+
labins.extend(["F,SIGF"] * n_groups_this_file)
|
|
1838
|
+
else:
|
|
1839
|
+
mtzs_merged.extend(_mtz_groups_i)
|
|
1840
|
+
labins.extend(["IMEAN,SIGIMEAN"] * n_groups_this_file)
|
|
1841
|
+
# TODO: free reflections if not given
|
|
1842
|
+
# TODO: check that input files have FI(R?)
|
|
1843
|
+
# TODO: mmCIF
|
|
1844
|
+
print(mtzs_merged)
|
|
1845
|
+
print(labins)
|
|
1846
|
+
|
|
1847
|
+
if args.hklin:
|
|
1848
|
+
logging.info(f"Merged diffraction data files: {args.hklin}")
|
|
1849
|
+
for i, hklin_i in enumerate(args.hklin):
|
|
1850
|
+
logging.info("")
|
|
1851
|
+
logging.info(f"Merged diffraction data file: {hklin_i}")
|
|
1852
|
+
if hklin_i.lower().endswith(".cif") or hklin_i.lower().endswith(".ent"):
|
|
1853
|
+
mtz = None
|
|
1854
|
+
doc = gemmi.cif.read(hklin_i)
|
|
1855
|
+
rblocks = gemmi.as_refln_blocks(doc)
|
|
1856
|
+
for rblock in rblocks:
|
|
1857
|
+
if rblock.is_merged():
|
|
1858
|
+
Convert = gemmi.CifToMtz()
|
|
1859
|
+
mtz = Convert.convert_block_to_mtz(rblock)
|
|
1860
|
+
break
|
|
1861
|
+
if mtz:
|
|
1862
|
+
mtz_filename = (
|
|
1863
|
+
f"{os.path.splitext(os.path.basename(hklin_i))[0]}.mtz"
|
|
1864
|
+
)
|
|
1865
|
+
mtz.write_to_file(mtz_filename)
|
|
1866
|
+
mtzs_merged.append(mtz_filename)
|
|
1867
|
+
elif servalcat_imported:
|
|
1868
|
+
mtz, _, _ = servalcat_utils.fileio.read_smcif_shelx(hklin_i)
|
|
1869
|
+
if mtz:
|
|
1870
|
+
mtz_filename = (
|
|
1871
|
+
f"{os.path.splitext(os.path.basename(hklin_i))[0]}.mtz"
|
|
1872
|
+
)
|
|
1873
|
+
mtz.write_to_file(mtz_filename)
|
|
1874
|
+
mtzs_merged.append(mtz_filename)
|
|
1875
|
+
else:
|
|
1876
|
+
raise RuntimeError(
|
|
1877
|
+
f"Could not recognise format of diffraction data file {hklin_i}"
|
|
1878
|
+
)
|
|
1879
|
+
elif hklin_i.lower().endswith(".hkl") or hklin_i.lower().endswith(".ent"):
|
|
1880
|
+
if not servalcat_imported:
|
|
1881
|
+
raise RuntimeError(
|
|
1882
|
+
f"Cannot read .hkl file {hklin_i}"
|
|
1883
|
+
" because servalcat is not installed."
|
|
1884
|
+
)
|
|
1885
|
+
mtz = servalcat_utils.fileio.read_smcif_hkl(hklin_i)
|
|
1886
|
+
mtz_filename = f"{os.path.splitext(os.path.basename(hklin_i))[0]}.mtz"
|
|
1887
|
+
mtz.write_to_file(mtz_filename)
|
|
1888
|
+
mtzs_merged.append(mtz_filename)
|
|
1889
|
+
else:
|
|
1890
|
+
mtz = gemmi.read_mtz_file(hklin_i)
|
|
1891
|
+
mtzs_merged.append(hklin_i)
|
|
1892
|
+
dmin = mtz.resolution_high()
|
|
1893
|
+
dmax = mtz.resolution_low()
|
|
1894
|
+
# Check for None or nan values and recalculate if necessary
|
|
1895
|
+
if dmax is None or dmin is None or numpy.isnan(dmax) or numpy.isnan(dmin):
|
|
1896
|
+
d_array = mtz.cell.calculate_d_array(mtz.make_miller_array())
|
|
1897
|
+
dmax = max(d_array)
|
|
1898
|
+
dmin = min(d_array)
|
|
1899
|
+
logging.info(f"Resolution limits: {dmax:.3f}" f" - {dmin:.3f} A")
|
|
1900
|
+
logging.info(
|
|
1901
|
+
f"Space group: {mtz.spacegroup.hm} (No. {mtz.spacegroup.number})"
|
|
1902
|
+
)
|
|
1903
|
+
logging.info(
|
|
1904
|
+
f"Unit cell: {mtz.cell.a:.3f} {mtz.cell.b:.3f} {mtz.cell.c:.3f}"
|
|
1905
|
+
f" {mtz.cell.alpha:.3f} {mtz.cell.beta:.3f} {mtz.cell.gamma:.3f}"
|
|
1906
|
+
)
|
|
1907
|
+
labin, anom_present, col_free = check_reflection_file_columns(
|
|
1908
|
+
mtz, unmerged=False, prefer_amplitude=args.amplitude
|
|
1909
|
+
) # col_free may be reset later if args.hklin_free is given
|
|
1910
|
+
# elif i_present and not f_present: TODO FW
|
|
1911
|
+
labins.append(labin)
|
|
1912
|
+
cols_free.append(col_free)
|
|
1913
|
+
bin_stats_lists.append([])
|
|
1914
|
+
# TODO: check and fix n_expected
|
|
1915
|
+
n_expected = gemmi.count_reflections(mtz.cell, mtz.spacegroup, dmin, dmax)
|
|
1916
|
+
n_expected_list.append(n_expected)
|
|
1917
|
+
if not binner_master or dmin < 1 / numpy.sqrt(binner_master.max_1_d2):
|
|
1918
|
+
logging.info(
|
|
1919
|
+
"Setting up resolution bins according to the file"
|
|
1920
|
+
f" {hklin_i} with resolution limits {dmax:.3f}"
|
|
1921
|
+
f" - {dmin:.3f} A"
|
|
1922
|
+
)
|
|
1923
|
+
binner_master = gemmi.Binner()
|
|
1924
|
+
binner_master.setup_from_1_d2(
|
|
1925
|
+
args.n_bins,
|
|
1926
|
+
gemmi.Binner.Method.Dstar2,
|
|
1927
|
+
mtz.make_1_d2_array(),
|
|
1928
|
+
mtz.get_cell(),
|
|
1929
|
+
)
|
|
1930
|
+
logging.info("")
|
|
1931
|
+
n_mtzs_merged = len(mtzs_merged)
|
|
1932
|
+
assert n_mtzs_merged == len(labins)
|
|
1933
|
+
|
|
1934
|
+
# Avoid shared inner-list aliasing by creating separate lists
|
|
1935
|
+
bin_stats_matrix = [
|
|
1936
|
+
[[] for _ in range(n_mtzs_merged)] for _ in range(n_mtzs_merged)
|
|
1937
|
+
]
|
|
1938
|
+
for i in range(n_mtzs_merged):
|
|
1939
|
+
if i < len(bin_stats_lists) and bin_stats_lists[i]:
|
|
1940
|
+
bin_stats_matrix[i][i] = bin_stats_lists[i]
|
|
1941
|
+
|
|
1942
|
+
# TODO what if dealing with amplitudes?
|
|
1943
|
+
if n_mtzs_merged >= 2 and not args.amplitude:
|
|
1944
|
+
bin_stats_matrix, n_refl_matrix, ratio_refl_matrix = compare_mtzs_fi(
|
|
1945
|
+
mtzs_merged, binner_master, labins, bin_stats_matrix, n_expected_list
|
|
1946
|
+
)
|
|
1947
|
+
|
|
1948
|
+
if args.hklin_free:
|
|
1949
|
+
logging.info(f"Free R flag data file: {args.hklin_free}")
|
|
1950
|
+
mtz_free = gemmi.read_mtz_file(args.hklin_free)
|
|
1951
|
+
mtzs_merged.append(args.hklin_free)
|
|
1952
|
+
dmin_free = mtz_free.resolution_high()
|
|
1953
|
+
dmax_free = mtz_free.resolution_low()
|
|
1954
|
+
# Check for None or nan values and recalculate if necessary
|
|
1955
|
+
if (
|
|
1956
|
+
dmax_free is None
|
|
1957
|
+
or dmin_free is None
|
|
1958
|
+
or numpy.isnan(dmax_free)
|
|
1959
|
+
or numpy.isnan(dmin_free)
|
|
1960
|
+
):
|
|
1961
|
+
d_array_free = mtz_free.cell.calculate_d_array(mtz_free.make_miller_array())
|
|
1962
|
+
dmax_free = max(d_array_free)
|
|
1963
|
+
dmin_free = min(d_array_free)
|
|
1964
|
+
logging.info(f"Resolution limits: {dmax_free:.3f}" f" - {dmin_free:.3f} A")
|
|
1965
|
+
logging.info(
|
|
1966
|
+
f"Space group: {mtz_free.spacegroup.hm} (No. {mtz_free.spacegroup.number})"
|
|
1967
|
+
)
|
|
1968
|
+
logging.info(
|
|
1969
|
+
"Unit cell:"
|
|
1970
|
+
f" {mtz_free.cell.a:.3f} {mtz_free.cell.b:.3f} {mtz_free.cell.c:.3f}"
|
|
1971
|
+
f" {mtz_free.cell.alpha:.3f} {mtz_free.cell.beta:.3f}"
|
|
1972
|
+
f" {mtz_free.cell.gamma:.3f}"
|
|
1973
|
+
)
|
|
1974
|
+
# Overwrite cols_free
|
|
1975
|
+
_, _, col_free = check_reflection_file_columns(
|
|
1976
|
+
args.hklin_free, unmerged=False, search_col_free=True
|
|
1977
|
+
)
|
|
1978
|
+
cols_free = len(cols_free) * [col_free]
|
|
1979
|
+
logging.info("")
|
|
1980
|
+
|
|
1981
|
+
if args.command == "pipeline" and args.unify_cell:
|
|
1982
|
+
for i, mtz in enumerate(mtzs_merged):
|
|
1983
|
+
if i == 0:
|
|
1984
|
+
continue
|
|
1985
|
+
mtzs_merged[i] = copy_cell_mtz(mtzs_merged[i], mtzs_merged[0])
|
|
1986
|
+
|
|
1987
|
+
models = []
|
|
1988
|
+
if args.model:
|
|
1989
|
+
if args.command == "pipeline" and args.molrep:
|
|
1990
|
+
models_molrep = []
|
|
1991
|
+
for model, mtz_i in zip(args.model, mtzs_merged):
|
|
1992
|
+
logging.info(
|
|
1993
|
+
"Running MolRep to generate a model from the input structure."
|
|
1994
|
+
)
|
|
1995
|
+
model_molrep = run_molrep(model, mtz_i)
|
|
1996
|
+
models_molrep.append(model_molrep)
|
|
1997
|
+
models = models_molrep
|
|
1998
|
+
else:
|
|
1999
|
+
models = args.model
|
|
2000
|
+
|
|
2001
|
+
config_yaml = ""
|
|
2002
|
+
geometry_objects_ref = []
|
|
2003
|
+
if args.geometry_cids or args.servalcat_config:
|
|
2004
|
+
config_yaml_dict = {}
|
|
2005
|
+
if args.servalcat_config:
|
|
2006
|
+
with open(args.servalcat_config, "r") as f:
|
|
2007
|
+
config_yaml_dict = yaml.safe_load(f) or {}
|
|
2008
|
+
if args.geometry_cids:
|
|
2009
|
+
geometry_objects_ref = select_cids_for_geometry_analysis(
|
|
2010
|
+
args.geometry_cids
|
|
2011
|
+
)
|
|
2012
|
+
config_yaml_unrestrain_dict = unrestrain_yaml(geometry_objects_ref)
|
|
2013
|
+
config_yaml_dict = dict_deep_merge(
|
|
2014
|
+
config_yaml_dict, config_yaml_unrestrain_dict
|
|
2015
|
+
)
|
|
2016
|
+
config_yaml = "servalcat_config.yaml"
|
|
2017
|
+
with open(config_yaml, "w") as f:
|
|
2018
|
+
yaml.dump(config_yaml_dict, f, sort_keys=False)
|
|
2019
|
+
|
|
2020
|
+
refined_mmcifs, refined_mtzs, refined_jsons = run_servalcat_refine(
|
|
2021
|
+
mtzs_merged,
|
|
2022
|
+
labins,
|
|
2023
|
+
models,
|
|
2024
|
+
mtzs_free=[args.hklin_free],
|
|
2025
|
+
prefix=prefix,
|
|
2026
|
+
source=args.source,
|
|
2027
|
+
arguments=servalcat_args,
|
|
2028
|
+
config_yaml=config_yaml,
|
|
2029
|
+
sigmaa=(not args.amplitude),
|
|
2030
|
+
quick=args.quick,
|
|
2031
|
+
n_proc=n_proc,
|
|
2032
|
+
)
|
|
2033
|
+
write_coot_script(
|
|
2034
|
+
refined_mmcifs,
|
|
2035
|
+
maps=refined_mtzs,
|
|
2036
|
+
maps_labels=("FWT", "PHWT"),
|
|
2037
|
+
diffmaps=refined_mtzs,
|
|
2038
|
+
diffmaps_labels=("DELFWT", "PHDELWT"),
|
|
2039
|
+
)
|
|
2040
|
+
adp_analysis_histograms(refined_mmcifs, prefix)
|
|
2041
|
+
if len(refined_mmcifs) >= 2:
|
|
2042
|
+
compute_structure_differences(refined_mmcifs)
|
|
2043
|
+
bin_stats_matrix, diffmaps, diffmaps_fwt = compute_difference_maps(
|
|
2044
|
+
refined_mtzs, binner_master, bin_stats_matrix, args.amplitude
|
|
2045
|
+
)
|
|
2046
|
+
write_coot_script(
|
|
2047
|
+
refined_mmcifs,
|
|
2048
|
+
maps=[],
|
|
2049
|
+
maps_labels=(),
|
|
2050
|
+
diffmaps=diffmaps,
|
|
2051
|
+
diffmaps_labels=("DELFOFO", "PHDELFOFO"),
|
|
2052
|
+
)
|
|
2053
|
+
write_coot_script(
|
|
2054
|
+
refined_mmcifs,
|
|
2055
|
+
maps=[],
|
|
2056
|
+
maps_labels=(),
|
|
2057
|
+
diffmaps=diffmaps_fwt,
|
|
2058
|
+
diffmaps_labels=("DELFWTFWTSCall", "PHDELFWTFWTSCall"),
|
|
2059
|
+
)
|
|
2060
|
+
if args.command == "bootstrap" or (
|
|
2061
|
+
args.command == "pipeline" and args.bootstrap
|
|
2062
|
+
):
|
|
2063
|
+
n_samples = (
|
|
2064
|
+
args.bootstrap
|
|
2065
|
+
if (args.command == "pipeline" and args.bootstrap)
|
|
2066
|
+
else args.n_samples
|
|
2067
|
+
)
|
|
2068
|
+
# TODO: some features assume only single model...
|
|
2069
|
+
for i_mtz, (mtz_in, labin, model, col_free) in enumerate(
|
|
2070
|
+
zip(mtzs_merged, labins, models, cols_free)
|
|
2071
|
+
):
|
|
2072
|
+
if args.command == "bootstrap":
|
|
2073
|
+
idx = 0
|
|
2074
|
+
prefix_servalcat = prefix
|
|
2075
|
+
else:
|
|
2076
|
+
idx = i_mtz + 1
|
|
2077
|
+
prefix_servalcat = f"{prefix}group{idx}_"
|
|
2078
|
+
# restraints_file = ""
|
|
2079
|
+
if args.geometry_cids and geometry_objects_ref:
|
|
2080
|
+
st_ref = gemmi.read_structure(refined_mmcifs[i_mtz])
|
|
2081
|
+
geometry_objects_ref = geometry_analysis_load(
|
|
2082
|
+
st_ref, geometry_objects_ref
|
|
2083
|
+
)
|
|
2084
|
+
# restraints_file = unrestrain(geometry_objects_ref, model)
|
|
2085
|
+
mtzs_bootstrap = bootstrap_dataset(
|
|
2086
|
+
mtz_in,
|
|
2087
|
+
binner_master,
|
|
2088
|
+
seeds=range(1001, 1001 + n_samples),
|
|
2089
|
+
labin=labin,
|
|
2090
|
+
draw_factor=args.draw_factor,
|
|
2091
|
+
random_weights=args.random_weights,
|
|
2092
|
+
col_free=col_free,
|
|
2093
|
+
fraction_zero=args.fraction_zero,
|
|
2094
|
+
)
|
|
2095
|
+
if args.model_dir and args.models:
|
|
2096
|
+
input_model_s = args.models
|
|
2097
|
+
else:
|
|
2098
|
+
input_model_s = [model]
|
|
2099
|
+
if args.unre:
|
|
2100
|
+
(
|
|
2101
|
+
refined_mmcifs_bootstrap_unre,
|
|
2102
|
+
_,
|
|
2103
|
+
_,
|
|
2104
|
+
) = run_servalcat_refine(
|
|
2105
|
+
[mtz_in],
|
|
2106
|
+
[labin],
|
|
2107
|
+
input_model_s,
|
|
2108
|
+
mtzs_free=mtzs_bootstrap,
|
|
2109
|
+
prefix=prefix_servalcat,
|
|
2110
|
+
source=args.source,
|
|
2111
|
+
keyword_file="",
|
|
2112
|
+
config_yaml=config_yaml,
|
|
2113
|
+
arguments=servalcat_args
|
|
2114
|
+
+ ["--labin_llweight", "llweight"]
|
|
2115
|
+
+ [
|
|
2116
|
+
"--weight",
|
|
2117
|
+
"2.0",
|
|
2118
|
+
"--hydrogen",
|
|
2119
|
+
"yes",
|
|
2120
|
+
"--ncycle",
|
|
2121
|
+
str(args.unre),
|
|
2122
|
+
],
|
|
2123
|
+
sigmaa=False,
|
|
2124
|
+
quick=args.quick,
|
|
2125
|
+
n_proc=n_proc,
|
|
2126
|
+
)
|
|
2127
|
+
input_model_s = refined_mmcifs_bootstrap_unre
|
|
2128
|
+
(
|
|
2129
|
+
refined_mmcifs_bootstrap,
|
|
2130
|
+
refined_mtzs_bootstrap,
|
|
2131
|
+
refined_jsons_bootstrap,
|
|
2132
|
+
) = run_servalcat_refine(
|
|
2133
|
+
[mtz_in],
|
|
2134
|
+
[labin],
|
|
2135
|
+
input_model_s,
|
|
2136
|
+
mtzs_free=mtzs_bootstrap,
|
|
2137
|
+
prefix=prefix_servalcat,
|
|
2138
|
+
source=args.source,
|
|
2139
|
+
# keyword_file=restraints_file,
|
|
2140
|
+
config_yaml=config_yaml,
|
|
2141
|
+
arguments=servalcat_args + ["--labin_llweight", "llweight"],
|
|
2142
|
+
sigmaa=False,
|
|
2143
|
+
quick=args.quick,
|
|
2144
|
+
n_proc=n_proc,
|
|
2145
|
+
)
|
|
2146
|
+
llweight_R1_outliers = bootstrap_analyse_stats(
|
|
2147
|
+
refined_jsons_bootstrap,
|
|
2148
|
+
refined_jsons[i_mtz],
|
|
2149
|
+
idx=idx,
|
|
2150
|
+
prefix=prefix,
|
|
2151
|
+
)
|
|
2152
|
+
|
|
2153
|
+
if llweight_R1_outliers is not None:
|
|
2154
|
+
# Filter out the outliers
|
|
2155
|
+
refined_mmcifs_bootstrap = filter_out_refined_files(
|
|
2156
|
+
refined_mmcifs_bootstrap, llweight_R1_outliers
|
|
2157
|
+
)
|
|
2158
|
+
refined_mtzs_bootstrap = filter_out_refined_files(
|
|
2159
|
+
refined_mtzs_bootstrap, llweight_R1_outliers
|
|
2160
|
+
)
|
|
2161
|
+
|
|
2162
|
+
if os.path.splitext(model)[1] == ".cif":
|
|
2163
|
+
bootstrap_analyse_structures(
|
|
2164
|
+
refined_mmcifs_bootstrap,
|
|
2165
|
+
refined_mmcifs[i_mtz],
|
|
2166
|
+
idx=idx,
|
|
2167
|
+
prefix=prefix,
|
|
2168
|
+
skip_hydrogen=True,
|
|
2169
|
+
smcif=model,
|
|
2170
|
+
geometry_cids_file=args.geometry_cids,
|
|
2171
|
+
geometry_objects_ref=geometry_objects_ref,
|
|
2172
|
+
no_origin_shift=args.no_origin_shift,
|
|
2173
|
+
)
|
|
2174
|
+
else:
|
|
2175
|
+
bootstrap_analyse_structures(
|
|
2176
|
+
refined_mmcifs_bootstrap,
|
|
2177
|
+
refined_mmcifs[i_mtz],
|
|
2178
|
+
idx=idx,
|
|
2179
|
+
prefix=prefix,
|
|
2180
|
+
skip_hydrogen=True,
|
|
2181
|
+
smcif="",
|
|
2182
|
+
geometry_cids_file=args.geometry_cids,
|
|
2183
|
+
geometry_objects_ref=geometry_objects_ref,
|
|
2184
|
+
no_origin_shift=args.no_origin_shift,
|
|
2185
|
+
)
|
|
2186
|
+
bootstrap_mean_map(
|
|
2187
|
+
refined_mtzs_bootstrap,
|
|
2188
|
+
idx=idx,
|
|
2189
|
+
prefix=prefix,
|
|
2190
|
+
binner=binner_master,
|
|
2191
|
+
mtz_ref=refined_mtzs[i_mtz],
|
|
2192
|
+
n_proc=n_proc,
|
|
2193
|
+
)
|