servalcat 0.4.88__cp313-cp313-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of servalcat might be problematic. Click here for more details.

Files changed (45) hide show
  1. servalcat/__init__.py +10 -0
  2. servalcat/__main__.py +120 -0
  3. servalcat/ext.cp313-win_amd64.pyd +0 -0
  4. servalcat/refine/__init__.py +0 -0
  5. servalcat/refine/cgsolve.py +100 -0
  6. servalcat/refine/refine.py +823 -0
  7. servalcat/refine/refine_geom.py +220 -0
  8. servalcat/refine/refine_spa.py +345 -0
  9. servalcat/refine/refine_xtal.py +268 -0
  10. servalcat/refine/spa.py +136 -0
  11. servalcat/refine/xtal.py +273 -0
  12. servalcat/refmac/__init__.py +0 -0
  13. servalcat/refmac/exte.py +182 -0
  14. servalcat/refmac/refmac_keywords.py +639 -0
  15. servalcat/refmac/refmac_wrapper.py +403 -0
  16. servalcat/spa/__init__.py +0 -0
  17. servalcat/spa/fofc.py +473 -0
  18. servalcat/spa/fsc.py +387 -0
  19. servalcat/spa/localcc.py +188 -0
  20. servalcat/spa/realspcc_from_var.py +128 -0
  21. servalcat/spa/run_refmac.py +972 -0
  22. servalcat/spa/shift_maps.py +293 -0
  23. servalcat/spa/shiftback.py +137 -0
  24. servalcat/spa/translate.py +129 -0
  25. servalcat/utils/__init__.py +35 -0
  26. servalcat/utils/commands.py +1397 -0
  27. servalcat/utils/fileio.py +737 -0
  28. servalcat/utils/generate_operators.py +296 -0
  29. servalcat/utils/hkl.py +712 -0
  30. servalcat/utils/logger.py +116 -0
  31. servalcat/utils/maps.py +345 -0
  32. servalcat/utils/model.py +782 -0
  33. servalcat/utils/refmac.py +760 -0
  34. servalcat/utils/restraints.py +782 -0
  35. servalcat/utils/symmetry.py +295 -0
  36. servalcat/xtal/__init__.py +0 -0
  37. servalcat/xtal/french_wilson.py +256 -0
  38. servalcat/xtal/run_refmac_small.py +240 -0
  39. servalcat/xtal/sigmaa.py +1622 -0
  40. servalcat/xtal/twin.py +115 -0
  41. servalcat-0.4.88.dist-info/METADATA +55 -0
  42. servalcat-0.4.88.dist-info/RECORD +45 -0
  43. servalcat-0.4.88.dist-info/WHEEL +5 -0
  44. servalcat-0.4.88.dist-info/entry_points.txt +4 -0
  45. servalcat-0.4.88.dist-info/licenses/LICENSE +373 -0
@@ -0,0 +1,972 @@
1
+ """
2
+ Author: "Keitaro Yamashita, Garib N. Murshudov"
3
+ MRC Laboratory of Molecular Biology
4
+
5
+ This software is released under the
6
+ Mozilla Public License, version 2.0; see LICENSE.
7
+ """
8
+ from __future__ import absolute_import, division, print_function, generators
9
+ import gemmi
10
+ import numpy
11
+ import json
12
+ import os
13
+ import io
14
+ import shutil
15
+ import argparse
16
+ from servalcat.utils import logger
17
+ from servalcat import utils
18
+ from servalcat import spa # this is not a right style
19
+ from servalcat.spa import shift_maps
20
+ from servalcat.refmac.refmac_wrapper import prepare_crd
21
+
22
+ def add_arguments(parser):
23
+ parser.description = 'Run REFMAC5 for SPA'
24
+
25
+ parser.add_argument('--exe', default="refmac5",
26
+ help='refmac5 binary (default: %(default)s)')
27
+ parser.add_argument("--monlib",
28
+ help="Monomer library path. Default: $CLIBD_MON")
29
+ # sfcalc options
30
+ group = parser.add_mutually_exclusive_group(required=True)
31
+ group.add_argument("--halfmaps", nargs=2, help="Input half map files")
32
+ group.add_argument("--map", help="Use this only if you really do not have half maps.")
33
+ parser.add_argument('--mask',
34
+ help='Mask file')
35
+ parser.add_argument('--model', required=True,
36
+ help='Input atomic model file')
37
+ parser.add_argument('--mask_radius',
38
+ type=float, default=3,
39
+ help='')
40
+ parser.add_argument('--mask_soft_edge',
41
+ type=float, default=0,
42
+ help='Add soft edge to model mask. Should use with --no_sharpen_before_mask?')
43
+ parser.add_argument('--padding',
44
+ type=float,
45
+ help='Default: 2*mask_radius')
46
+ parser.add_argument('--no_mask', action='store_true')
47
+ parser.add_argument('--invert_mask', action='store_true', help='not for refinement.')
48
+ parser.add_argument('--pixel_size', type=float,
49
+ help='Override pixel size (A)')
50
+ parser.add_argument('--resolution',
51
+ type=float,
52
+ help='')
53
+ parser.add_argument('--no_trim',
54
+ action='store_true',
55
+ help='Keep original box (not recommended)')
56
+ parser.add_argument('--blur',
57
+ type=float, default=0,
58
+ help='Sharpening or blurring B')
59
+ utils.symmetry.add_symmetry_args(parser) # add --pg etc
60
+ parser.add_argument('--contacting_only', action="store_true", help="Filter out non-contacting NCS")
61
+ parser.add_argument('--ignore_symmetry',
62
+ help='Ignore symmetry information (MTRIX/_struct_ncs_oper) in the model file')
63
+ parser.add_argument('--find_links', action='store_true',
64
+ help='Automatically add links')
65
+ parser.add_argument("--b_before_mask", type=float,
66
+ help="sharpening B value for sharpen-mask-unsharpen procedure. By default it is determined automatically.")
67
+ parser.add_argument('--no_sharpen_before_mask', action='store_true',
68
+ help='By default half maps are sharpened before masking by std of signal and unsharpened after masking. This option disables it.')
69
+ parser.add_argument('--no_fix_microheterogeneity', action='store_true',
70
+ help='By default it will fix microheterogeneity for Refmac')
71
+ parser.add_argument('--no_fix_resi9999', action='store_true',
72
+ help='By default it will split chain if max residue number > 9999 which is not supported by Refmac')
73
+ parser.add_argument('--no_check_ncs_overlaps', action='store_true',
74
+ help='Disable model overlap (e.g. expanded model is used with --pg) test')
75
+ parser.add_argument('--no_check_ncs_map', action='store_true',
76
+ help='Disable map NCS consistency test')
77
+ parser.add_argument('--no_check_mask_with_model', action='store_true',
78
+ help='Disable mask test using model')
79
+ parser.add_argument("--prepare_only", action='store_true',
80
+ help="Stop before refinement")
81
+ parser.add_argument("--no_refmacat", action='store_true',
82
+ help="By default uses gemmi for crd/rst file preparation (do not use makecif)")
83
+ # run_refmac options
84
+ # TODO use group! like refmac options
85
+ parser.add_argument('--ligand', nargs="*", action="append",
86
+ help="restraint dictionary cif file(s)")
87
+ parser.add_argument('--bfactor', type=float,
88
+ help="reset all atomic B values to specified value")
89
+ parser.add_argument('--ncsr', default="local", choices=["local", "global"],
90
+ help="local or global NCS restrained (default: %(default)s)")
91
+ parser.add_argument('--ncycle', type=int, default=10,
92
+ help="number of cycles in Refmac (default: %(default)d)")
93
+ parser.add_argument('--tlscycle', type=int, default=0,
94
+ help="number of TLS cycles in Refmac (default: %(default)d)")
95
+ parser.add_argument('--tlsin',
96
+ help="TLS parameter input for Refmac")
97
+ parser.add_argument('--hydrogen', default="all", choices=["all", "yes", "no"],
98
+ help="all: add riding hydrogen atoms, yes: use hydrogen atoms if present, no: remove hydrogen atoms in input. "
99
+ "Default: %(default)s")
100
+ parser.add_argument('--jellybody', action='store_true',
101
+ help="Use jelly body restraints")
102
+ parser.add_argument('--jellybody_params', nargs=2, type=float,
103
+ metavar=("sigma", "dmax"), default=[0.01, 4.2],
104
+ help="Jelly body sigma and dmax (default: %(default)s)")
105
+ parser.add_argument('--hout', action='store_true', help="write hydrogen atoms in the output model")
106
+ group = parser.add_mutually_exclusive_group()
107
+ group.add_argument('--weight_auto_scale', type=float,
108
+ help="'weight auto' scale value. automatically determined from resolution and mask/box volume ratio if unspecified")
109
+ parser.add_argument('--keywords', nargs='+', action="append",
110
+ help="refmac keyword(s)")
111
+ parser.add_argument('--keyword_file', nargs='+', action="append",
112
+ help="refmac keyword file(s)")
113
+ parser.add_argument('--external_restraints_json')
114
+ parser.add_argument('--show_refmac_log', action='store_true',
115
+ help="show all Refmac log instead of summary")
116
+ parser.add_argument('--output_prefix', default="refined",
117
+ help='output file name prefix (default: %(default)s)')
118
+ parser.add_argument('--cross_validation', action='store_true',
119
+ help='Run cross validation')
120
+ parser.add_argument('--cross_validation_method', default="shake", choices=["throughout", "shake"],
121
+ help="shake: randomize a model refined against a full map and then refine it against a half map, "
122
+ "throughout: use only a half map for refinement (another half map is used for error estimation) "
123
+ "Default: %(default)s")
124
+ parser.add_argument('--shake_radius', default=0.3,
125
+ help='Shake rmsd in case of --cross_validation_method=shake (default: %(default).1f)')
126
+ group = parser.add_mutually_exclusive_group()
127
+ group.add_argument('--mask_for_fofc', help="Mask file for Fo-Fc map calculation")
128
+ group.add_argument('--mask_radius_for_fofc', type=float, help="Mask radius for Fo-Fc map calculation")
129
+ parser.add_argument('--trim_fofc_mtz', action="store_true", help="diffmap.mtz will have smaller cell (if --mask_for_fofc is given)")
130
+ parser.add_argument("--fsc_resolution", type=float,
131
+ help="High resolution limit for FSC calculation. Default: Nyquist")
132
+
133
+ # add_arguments()
134
+
135
+ def parse_args(arg_list):
136
+ parser = argparse.ArgumentParser()
137
+ add_arguments(parser)
138
+ return parser.parse_args(arg_list)
139
+ # parse_args()
140
+
141
+ def calc_fsc(st, output_prefix, maps, d_min, mask, mask_radius, soft_edge, b_before_mask, no_sharpen_before_mask, make_hydrogen, monlib,
142
+ blur=0, d_min_fsc=None, cross_validation=False, cross_validation_method=None, st_sr=None):
143
+ # st_sr: shaken-and-refined st in case of cross_validation_method=="shake"
144
+ if cross_validation:
145
+ assert len(maps) == 2
146
+ assert cross_validation_method in ("shake", "throughout")
147
+ if cross_validation and cross_validation_method == "shake":
148
+ assert st_sr is not None
149
+ else:
150
+ assert st_sr is None
151
+
152
+ logger.writeln("Calculating map-model FSC..")
153
+
154
+ if d_min_fsc is None:
155
+ d_min_fsc = utils.maps.nyquist_resolution(maps[0][0])
156
+ logger.writeln(" --fsc_resolution is not specified. Using Nyquist resolution: {:.2f}".format(d_min_fsc))
157
+
158
+ st = st.clone()
159
+ if st_sr is not None: st_sr = st_sr.clone()
160
+
161
+ if make_hydrogen == "all":
162
+ utils.restraints.add_hydrogens(st, monlib)
163
+ if st_sr is not None: utils.restraints.add_hydrogens(st_sr, monlib)
164
+
165
+ if mask is not None or mask_radius is not None:
166
+ if mask is None:
167
+ assert maps[0][0].unit_cell == st.cell
168
+ mask = utils.maps.mask_from_model(st, mask_radius, soft_edge=soft_edge, grid=maps[0][0])
169
+ if no_sharpen_before_mask or len(maps) < 2:
170
+ for ma in maps: ma[0].array[:] *= mask
171
+ else:
172
+ # It seems we need different B for different resolution limit
173
+ if b_before_mask is None: b_before_mask = determine_b_before_mask(st, maps, maps[0][1], mask, d_min_fsc)
174
+ maps = utils.maps.sharpen_mask_unsharpen(maps, mask, d_min_fsc, b=b_before_mask)
175
+
176
+ hkldata = utils.maps.mask_and_fft_maps(maps, d_min_fsc)
177
+ hkldata.df["FC"] = utils.model.calc_fc_fft(st, d_min_fsc - 1e-6, monlib=monlib, source="electron",
178
+ miller_array=hkldata.miller_array())
179
+ # XXX didn't apply mask to FC!!
180
+ labs_fc = ["FC"]
181
+
182
+ if st_sr is not None:
183
+ hkldata.df["FC_sr"] = utils.model.calc_fc_fft(st_sr, d_min_fsc - 1e-6, monlib=monlib, source="electron",
184
+ miller_array=hkldata.miller_array())
185
+ labs_fc.append("FC_sr")
186
+
187
+ if blur != 0:
188
+ logger.writeln(" Unblurring Fc with B={} for FSC calculation".format(blur))
189
+ unblur = numpy.exp(blur/hkldata.d_spacings().to_numpy()**2/4.)
190
+ for lab in labs_fc:
191
+ hkldata.df[lab] *= unblur
192
+
193
+ hkldata.setup_relion_binning()
194
+ stats = spa.fsc.calc_fsc_all(hkldata, labs_fc=labs_fc, lab_f="FP",
195
+ labs_half=["F_map1", "F_map2"] if len(maps)==2 else None)
196
+
197
+ hkldata2 = hkldata.copy(d_min=d_min) # for FSCaverage at resolution for refinement # XXX more efficient way
198
+ hkldata2.setup_relion_binning()
199
+ stats2 = spa.fsc.calc_fsc_all(hkldata2, labs_fc=labs_fc, lab_f="FP",
200
+ labs_half=["F_map1", "F_map2"] if len(maps)==2 else None)
201
+
202
+ if "fsc_half" in stats:
203
+ with numpy.errstate(invalid="ignore"): # XXX negative fsc results in nan!
204
+ stats.loc[:,"fsc_full_sqrt"] = numpy.sqrt(2*stats.fsc_half/(1+stats.fsc_half))
205
+
206
+ logger.writeln(stats.to_string()+"\n")
207
+
208
+ # remove and rename columns
209
+ for s in (stats, stats2):
210
+ s.rename(columns=dict(fsc_FC_full="fsc_model", Rcmplx_FC_full="Rcmplx"), inplace=True)
211
+ if cross_validation:
212
+ if cross_validation_method == "shake":
213
+ s.drop(columns=["fsc_FC_half1", "fsc_FC_half2", "fsc_FC_sr_full", "Rcmplx_FC_sr_full"], inplace=True)
214
+ s.rename(columns=dict(fsc_FC_sr_half1="fsc_model_half1",
215
+ fsc_FC_sr_half2="fsc_model_half2"), inplace=True)
216
+ else:
217
+ s.rename(columns=dict(fsc_FC_half1="fsc_model_half1",
218
+ fsc_FC_half2="fsc_model_half2"), inplace=True)
219
+ else:
220
+ s.drop(columns=[x for x in s if x.startswith("fsc_FC") and x.endswith(("half1","half2"))], inplace=True)
221
+
222
+ # FSCaverages
223
+ fscavg_text = "Map-model FSCaverages (at {:.2f} A):\n".format(d_min)
224
+ fscavg_text += " FSCaverage(full) = {: .4f}\n".format(spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model))
225
+ if cross_validation:
226
+ fscavg_text += "Cross-validated map-model FSCaverages:\n"
227
+ fscavg_text += " FSCaverage(half1)= {: .4f}\n".format(spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half1))
228
+ fscavg_text += " FSCaverage(half2)= {: .4f}\n".format(spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half2))
229
+
230
+ # for loggraph
231
+ fsc_logfile = "{}_fsc.log".format(output_prefix)
232
+ with open(fsc_logfile, "w") as ofs:
233
+ columns = "1/resol^2 ncoef ln(Mn(|Fo|^2)) ln(Mn(|Fc|^2)) FSC(full,model) FSC_half FSC_full_sqrt FSC(half1,model) FSC(half2,model) Rcmplx(full,model)".split()
234
+
235
+ ofs.write("$TABLE: Map-model FSC after refinement:\n")
236
+ if len(maps) == 2:
237
+ if cross_validation: fsc_cols = [5,6,7,8,9]
238
+ else: fsc_cols = [5,6,7]
239
+ else: fsc_cols = [5]
240
+ fsc_cols.append(10)
241
+ if len(maps) == 2: ofs.write("$GRAPHS: FSC :A:1,5,6,7:\n")
242
+ else: ofs.write("$GRAPHS: FSC :A:1,5:\n")
243
+ if cross_validation: ofs.write(": cross-validated FSC :A:1,8,9:\n".format(",".join(map(str,fsc_cols))))
244
+ ofs.write(": Rcmplx :A:1,{}:\n".format(4+len(fsc_cols)))
245
+ ofs.write(": ln(Mn(|F|^2)) :A:1,3,4:\n")
246
+ ofs.write(": number of Fourier coeffs :A:1,2:\n")
247
+ ofs.write("$$ {}$$\n".format(" ".join(columns[:4]+[columns[i-1] for i in fsc_cols])))
248
+ ofs.write("$$\n")
249
+
250
+ plot_columns = ["d_min", "ncoeffs", "power_FP", "power_FC", "fsc_model"]
251
+ if len(maps) == 2:
252
+ plot_columns.extend(["fsc_half", "fsc_full_sqrt"])
253
+ if cross_validation:
254
+ plot_columns.extend(["fsc_model_half1", "fsc_model_half2"])
255
+ plot_columns.append("Rcmplx")
256
+ with numpy.errstate(divide="ignore"):
257
+ log_format = lambda x: "{:.3f}".format(numpy.log(x))
258
+ ofs.write(stats.to_string(header=False, index=False, index_names=False, columns=plot_columns,
259
+ formatters=dict(d_min=lambda x: "{:.4f}".format(1/x**2),
260
+ power_FP=log_format, power_FC=log_format)))
261
+ ofs.write("\n")
262
+ ofs.write("$$\n\n")
263
+ ofs.write(fscavg_text)
264
+
265
+ logger.write(fscavg_text)
266
+ logger.writeln("Run loggraph {} to see plots.".format(fsc_logfile))
267
+
268
+ # write json
269
+ json.dump(stats.to_dict("records"),
270
+ open("{}_fsc.json".format(output_prefix), "w"),
271
+ indent=True)
272
+
273
+ return fscavg_text
274
+ # calc_fsc()
275
+
276
+ def calc_fofc(st, st_expanded, maps, monlib, model_format, args, diffmap_prefix="diffmap"):
277
+ logger.writeln("Starting Fo-Fc calculation..")
278
+ if not args.halfmaps: logger.writeln(" with limited functionality because half maps were not provided")
279
+ logger.writeln(" model: {}".format(args.output_prefix+model_format))
280
+
281
+ # for Fo-Fc in case of helical reconstruction, expand model more
282
+ # XXX should we do it for FSC calculation also? Probably we should not do sharpen-unsharpen procedure for FSC calc either.
283
+ if args.twist is not None:
284
+ logger.writeln("Generating all helical copies in the box")
285
+ st_expanded = st.clone()
286
+ utils.symmetry.update_ncs_from_args(args, st_expanded, map_and_start=maps[0], filter_contacting=False)
287
+ utils.model.expand_ncs(st_expanded)
288
+ utils.fileio.write_model(st_expanded, args.output_prefix+"_expanded_all", pdb=True, cif=True)
289
+
290
+ if args.mask_for_fofc:
291
+ logger.writeln(" mask: {}".format(args.mask_for_fofc))
292
+ mask = utils.fileio.read_ccp4_map(args.mask_for_fofc)[0]
293
+ elif args.mask_radius_for_fofc:
294
+ logger.writeln(" mask: using refined model with radius of {} A".format(args.mask_radius_for_fofc))
295
+ mask = utils.maps.mask_from_model(st_expanded, args.mask_radius_for_fofc, grid=maps[0][0]) # use soft edge?
296
+ else:
297
+ logger.writeln(" mask: not used")
298
+ mask = None
299
+
300
+ hkldata, map_labs, stats_str = spa.fofc.calc_fofc(st_expanded, args.resolution, maps, mask=mask, monlib=monlib,
301
+ half1_only=(args.cross_validation and args.cross_validation_method == "throughout"),
302
+ sharpening_b=None if args.halfmaps else 0.) # assume already sharpened if fullmap is given
303
+ spa.fofc.write_files(hkldata, map_labs, maps[0][1], stats_str,
304
+ mask=mask, output_prefix=diffmap_prefix,
305
+ trim_map=mask is not None, trim_mtz=args.trim_fofc_mtz)
306
+
307
+ # Create Coot script
308
+ spa.fofc.write_coot_script("{}_coot.py".format(args.output_prefix),
309
+ model_file="{}.pdb".format(args.output_prefix), # as Coot is not good at mmcif file..
310
+ mtz_file="{}.mtz".format(diffmap_prefix),
311
+ contour_fo=None if mask is None else 1.2,
312
+ contour_fofc=None if mask is None else 3.0,
313
+ ncs_ops=st.ncs)
314
+
315
+ # Create ChimeraX script
316
+ spa.fofc.write_chimerax_script(cxc_out="{}_chimerax.cxc".format(args.output_prefix),
317
+ model_file="{}.mmcif".format(args.output_prefix), # ChimeraX handles mmcif just fine
318
+ fo_mrc_file="{}_normalized_fo.mrc".format(diffmap_prefix),
319
+ fofc_mrc_file="{}_normalized_fofc.mrc".format(diffmap_prefix))
320
+ # calc_fofc()
321
+
322
+ def write_final_summary(st, refmac_summary, fscavg_text, output_prefix, is_mask_given):
323
+ if len(refmac_summary["cycles"]) > 1 and "actual_weight" in refmac_summary["cycles"][-2]:
324
+ final_weight = refmac_summary["cycles"][-2]["actual_weight"]
325
+ else:
326
+ final_weight = "???"
327
+
328
+ adpstats_txt = ""
329
+ adp_stats = utils.model.adp_stats_per_chain(st[0])
330
+ max_chain_len = max([len(x[0]) for x in adp_stats])
331
+ max_num_len = max([len(str(x[1])) for x in adp_stats])
332
+ for chain, natoms, qs in adp_stats:
333
+ adpstats_txt += " Chain {0:{1}s}".format(chain, max_chain_len) if chain!="*" else " {0:{1}s}".format("All", max_chain_len+6)
334
+ adpstats_txt += " ({0:{1}d} atoms) min={2:5.1f} median={3:5.1f} max={4:5.1f} A^2\n".format(natoms, max_num_len, qs[0],qs[2],qs[4])
335
+
336
+ if is_mask_given:
337
+ map_peaks_str = """\
338
+ List Fo-Fc map peaks in the ASU:
339
+ servalcat util map_peaks --map diffmap_normalized_fofc.mrc --model {prefix}.pdb --abs_level 4.0 \
340
+ """.format(prefix=output_prefix)
341
+ else:
342
+ map_peaks_str = "WARNING: --mask_for_fofc was not given, so the Fo-Fc map was not normalized."
343
+
344
+ logger.writeln("""
345
+ =============================================================================
346
+ * Final Summary *
347
+
348
+ Rmsd from ideal
349
+ bond lengths: {rmsbond} A
350
+ bond angles: {rmsangle} deg
351
+
352
+ {fscavgs}
353
+ Run loggraph {fsclog} to see plots
354
+
355
+ ADP statistics
356
+ {adpstats}
357
+
358
+ Weight used: {final_weight}
359
+ If you want to change the weight, give larger (looser restraints)
360
+ or smaller (tighter) value to --weight_auto_scale=.
361
+
362
+ Open refined model and diffmap.mtz with COOT:
363
+ coot --script {prefix}_coot.py
364
+
365
+ Open refined model, map and difference map with ChimeraX/ISOLDE:
366
+ chimerax {prefix}_chimerax.cxc
367
+
368
+ {map_peaks_msg}
369
+ =============================================================================
370
+ """.format(rmsbond=refmac_summary["cycles"][-1].get("rms_bond", "???"),
371
+ rmsangle=refmac_summary["cycles"][-1].get("rms_angle", "???"),
372
+ fscavgs=fscavg_text.rstrip(),
373
+ fsclog="{}_fsc.log".format(output_prefix),
374
+ adpstats=adpstats_txt.rstrip(),
375
+ final_weight=final_weight,
376
+ prefix=output_prefix,
377
+ map_peaks_msg=map_peaks_str))
378
+ # write_final_summary()
379
+
380
+ def lab_f_suffix(blur):
381
+ if blur is None or blur == 0.:
382
+ return ""
383
+ elif blur > 0:
384
+ return "Blur_{:.2f}".format(blur)
385
+ else:
386
+ return "Sharp_{:.2f}".format(-blur)
387
+ # lab_f_suffix()
388
+
389
+ def write_map_mtz(hkldata, mtz_out, map_labs, sig_lab=None, blur=0):
390
+ nblur = 2 if blur != 0 else 1
391
+ mean_f = hkldata.df[map_labs].abs().mean().min()
392
+ data_labs = map_labs + ([sig_lab] if sig_lab else [])
393
+
394
+ if mean_f < 1:
395
+ scale = 10. / mean_f
396
+ logger.writeln("Mean(|F|)= {:.2e} may be too small for Refmac. Applying scale= {:.1f}".format(mean_f, scale))
397
+ for lab in data_labs:
398
+ hkldata.df[lab] *= scale
399
+
400
+ mtz_labs = data_labs + []
401
+ mtz_types = {}
402
+ if sig_lab: mtz_types[sig_lab] = "Q"
403
+
404
+ if blur != 0:
405
+ temp = hkldata.debye_waller_factors(b_iso=blur)
406
+ for lab in data_labs:
407
+ data = hkldata.df[lab]
408
+ newlab = lab + lab_f_suffix(blur)
409
+ if numpy.iscomplexobj(data): data = numpy.abs(data)
410
+ hkldata.df[newlab] = data * temp
411
+ mtz_labs.append(newlab)
412
+ mtz_types[newlab] = "F" if lab != sig_lab else "Q"
413
+
414
+ hkldata.write_mtz(mtz_out, labs=mtz_labs, types=mtz_types,
415
+ phase_label_decorator=lambda x: "P"+x[1:])
416
+ # write_map_mtz()
417
+
418
+ def determine_b_before_mask(st, maps, grid_start, mask, resolution):
419
+ logger.writeln("Determining b_before_mask..")
420
+ # work in masked map for the speed
421
+ new_cell, new_shape, starts, shifts = shift_maps.determine_shape_and_shift(mask=mask,
422
+ grid_start=grid_start,
423
+ padding=5,
424
+ mask_cutoff=0.5,
425
+ noncentered=True,
426
+ noncubic=True,
427
+ json_out=None)
428
+ st = st.clone()
429
+ st.cell = new_cell
430
+ st.spacegroup_hm = "P 1"
431
+ for cra in st[0].all():
432
+ cra.atom.pos += shifts
433
+ cra.atom.b_iso = 0
434
+ cra.atom.aniso = gemmi.SMat33f(0,0,0,0,0,0)
435
+
436
+ newmaps = []
437
+ for i in range(len(maps)): # Update maps
438
+ g = gemmi.FloatGrid(maps[i][0].array*mask,
439
+ maps[i][0].unit_cell, maps[i][0].spacegroup)
440
+
441
+ suba = g.get_subarray(starts, new_shape)
442
+ new_grid = gemmi.FloatGrid(suba, new_cell, st.find_spacegroup())
443
+ newmaps.append([new_grid]+maps[i][1:])
444
+
445
+ hkldata = utils.maps.mask_and_fft_maps(newmaps, resolution)
446
+ hkldata.df["FC"] = utils.model.calc_fc_fft(st, resolution - 1e-6, source="electron",
447
+ miller_array=hkldata.miller_array())
448
+ k, b = hkldata.scale_k_and_b("FC", "FP")
449
+ return -b
450
+ # determine_b_before_mask()
451
+
452
+ def process_input(st, maps, resolution, monlib, mask_in, args,
453
+ shifted_model_prefix="shifted",
454
+ output_masked_prefix="masked_fs",
455
+ output_mtz_prefix="starting_map",
456
+ use_gemmi_prep=False, no_refmac_fix=False,
457
+ use_refmac=True, find_links=False):
458
+ ret = {} # instructions for refinement
459
+ maps = utils.maps.copy_maps(maps) # not to modify maps
460
+
461
+ grid_start = maps[0][1]
462
+ unit_cell = maps[0][0].unit_cell
463
+ spacegroup = gemmi.SpaceGroup(1)
464
+ start_xyz = numpy.array(maps[0][0].get_position(*grid_start).tolist())
465
+ A = numpy.array(unit_cell.orthogonalization_matrix.tolist())
466
+ center = numpy.sum(A, axis=1) / 2 #+ start_xyz
467
+
468
+ # Create mask
469
+ if mask_in:
470
+ logger.writeln("Input mask file: {}".format(mask_in))
471
+ mask = utils.fileio.read_ccp4_map(mask_in)[0]
472
+ else:
473
+ mask = None
474
+
475
+ st.cell = unit_cell
476
+ st.spacegroup_hm = "P 1"
477
+ if use_refmac:
478
+ ret["model_format"] = ".mmcif" if st.input_format == gemmi.CoorFormat.Mmcif else ".pdb"
479
+ max_seq_num = max([max(res.seqid.num for res in chain) for model in st for chain in model])
480
+ if max_seq_num > 9999 and ret["model_format"] == ".pdb":
481
+ logger.writeln("Max residue number ({}) exceeds 9999. Will use mmcif format".format(max_seq_num))
482
+ ret["model_format"] = ".mmcif"
483
+
484
+ if len(st.ncs) > 0 and args.ignore_symmetry:
485
+ logger.writeln("Removing symmetry information from model.")
486
+ st.ncs.clear()
487
+ utils.symmetry.update_ncs_from_args(args, st, map_and_start=maps[0], filter_contacting=args.contacting_only)
488
+ st_expanded = st.clone()
489
+ if not all(op.given for op in st.ncs):
490
+ # symmetry is not exact in helical reconstructions
491
+ cc_cutoff = 0.9 if args.twist is None else 0.7
492
+ if not args.no_check_ncs_overlaps and utils.model.check_symmetry_related_model_duplication(st):
493
+ raise SystemExit("\nError: Too many symmetry-related contacts detected.\n"
494
+ "Please provide an asymmetric unit model along with symmetry operators.")
495
+ if not args.no_check_ncs_map and utils.maps.check_symmetry_related_map_values(st, maps[0][0], cc_cutoff=cc_cutoff):
496
+ raise SystemExit("\nError: Map correlation is too small. Please ensure your map follows the model's symmetry")
497
+ args.keywords.extend(utils.symmetry.ncs_ops_for_refmac(st.ncs))
498
+ utils.model.expand_ncs(st_expanded)
499
+ logger.writeln(" Saving expanded model: input_model_expanded.*")
500
+ utils.fileio.write_model(st_expanded, "input_model_expanded", pdb=True, cif=True)
501
+
502
+ if mask is not None and not args.no_check_mask_with_model:
503
+ if not utils.maps.test_mask_with_model(mask, st_expanded):
504
+ raise SystemExit("\nError: Model is out of mask.\n"
505
+ "Please check your --model and --mask. You can disable this test with --no_check_mask_with_model.")
506
+ if args.mask_for_fofc:
507
+ masktmp = utils.fileio.read_ccp4_map(args.mask_for_fofc)[0]
508
+ if masktmp.shape != maps[0][0].shape:
509
+ raise SystemExit("\nError: mask from --mask_for_fofc has a different shape from input map(s)")
510
+ if not args.no_check_mask_with_model and not utils.maps.test_mask_with_model(masktmp, st):
511
+ raise SystemExit("\nError: Model is out of mask.\n"
512
+ "Please check your --model and --mask_for_fofc. You can disable this test with --no_check_mask_with_model.")
513
+ del masktmp
514
+
515
+ if mask is None and args.mask_radius:
516
+ logger.writeln("Creating mask..")
517
+ mask = utils.maps.mask_from_model(st_expanded, args.mask_radius, soft_edge=args.mask_soft_edge, grid=maps[0][0])
518
+ #utils.maps.write_ccp4_map("mask_from_model.ccp4", mask)
519
+
520
+ if use_refmac:
521
+ logger.writeln(" Saving input model with unit cell information")
522
+ utils.fileio.write_model(st, "starting_model", pdb=True, cif=True)
523
+ ret["model_file"] = "starting_model" + ret["model_format"]
524
+
525
+ if mask is not None:
526
+ if args.invert_mask:
527
+ logger.writeln("Inverting mask..")
528
+ mask_max, mask_min = numpy.max(mask), numpy.min(mask)
529
+ logger.writeln(" mask_max, mask_min= {}, {}".format(mask_max, mask_min))
530
+ mask = mask_max + mask_min - mask
531
+
532
+ # Mask maps
533
+ if args.no_sharpen_before_mask or len(maps) < 2:
534
+ logger.writeln("Applying mask..")
535
+ for ma in maps: ma[0].array[:] *= mask
536
+ else:
537
+ logger.writeln("Sharpen-mask-unsharpen..")
538
+ b_before_mask = args.b_before_mask
539
+ if b_before_mask is None: b_before_mask = determine_b_before_mask(st, maps, grid_start, mask, resolution)
540
+ maps = utils.maps.sharpen_mask_unsharpen(maps, mask, resolution, b=b_before_mask)
541
+
542
+ if not args.no_trim:
543
+ logger.writeln(" Shifting maps and/or model..")
544
+ if args.padding is None: args.padding = args.mask_radius * 2
545
+ new_cell, new_shape, starts, shifts = shift_maps.determine_shape_and_shift(mask=mask,
546
+ grid_start=grid_start,
547
+ padding=args.padding,
548
+ mask_cutoff=0.5,
549
+ noncentered=True,
550
+ noncubic=True,
551
+ json_out=None)
552
+ ret["shifts"] = shifts
553
+ vol_mask = numpy.count_nonzero(mask.array>0.5)
554
+ vol_map = new_shape[0] * new_shape[1] * new_shape[2]
555
+ ret["vol_ratio"] = vol_mask / vol_map
556
+ logger.writeln(" Vol_mask/Vol_map= {:.2e}".format(ret["vol_ratio"]))
557
+
558
+ # Model may be built out of the box (with 'unit cell' translation symmetry)
559
+ # It is only valid with original unit cell, but no longer valid with the new cell
560
+ # It still would not work if model is built over multiple 'cells'.
561
+ extra_shift = utils.model.translate_into_box(st,
562
+ origin=gemmi.Position(*start_xyz),
563
+ apply_shift=False)
564
+ if numpy.linalg.norm(extra_shift) > 0:
565
+ logger.writeln("Input model is out of the box. Required shift= {}".format(extra_shift))
566
+ ret["shifts"] += gemmi.Position(*extra_shift)
567
+ logger.writeln("Shift for model has been adjusted: {}".format(numpy.array(ret["shifts"].tolist())))
568
+
569
+ st.cell = new_cell
570
+ st.spacegroup_hm = "P 1"
571
+ if use_refmac:
572
+ logger.writeln(" Saving model in trimmed map..")
573
+ utils.fileio.write_model(st, shifted_model_prefix, pdb=True, cif=True)
574
+ ret["model_file"] = shifted_model_prefix + ret["model_format"]
575
+
576
+ logger.writeln(" Trimming maps..")
577
+ for i in range(len(maps)): # Update maps
578
+ suba = maps[i][0].get_subarray(starts, new_shape)
579
+ new_grid = gemmi.FloatGrid(suba, new_cell, spacegroup)
580
+ maps[i][0] = new_grid
581
+
582
+ st.setup_cell_images()
583
+ utils.restraints.find_and_fix_links(st, monlib, add_found=find_links,
584
+ # link via ncsc is not supported as of Refmac5.8.0411
585
+ find_symmetry_related=not use_refmac)
586
+ # workaround for Refmac
587
+ # TODO need to check external restraints
588
+ if use_refmac:
589
+ if use_gemmi_prep:
590
+ h_change = {"all":gemmi.HydrogenChange.ReAddButWater,
591
+ "yes":gemmi.HydrogenChange.NoChange,
592
+ "no":gemmi.HydrogenChange.Remove}[args.hydrogen]
593
+ topo, metal_kws = utils.restraints.prepare_topology(st, monlib, h_change=h_change, raise_error=False)
594
+ args.keywords = metal_kws + args.keywords
595
+ elif not no_refmac_fix:
596
+ topo = gemmi.prepare_topology(st, monlib, warnings=io.StringIO(), ignore_unknown_links=True)
597
+ else:
598
+ topo = None # not used
599
+ if not no_refmac_fix:
600
+ ret["refmac_fixes"] = utils.refmac.FixForRefmac()
601
+ ret["refmac_fixes"].fix_before_topology(st, topo,
602
+ fix_microheterogeneity=not args.no_fix_microheterogeneity and not use_gemmi_prep,
603
+ fix_resimax=not args.no_fix_resi9999,
604
+ fix_nonpolymer=False)
605
+ chain_id_len_max = max([len(x) for x in utils.model.all_chain_ids(st)])
606
+ if chain_id_len_max > 1 and ret["model_format"] == ".pdb":
607
+ logger.writeln("Long chain ID (length: {}) detected. Will use mmcif format".format(chain_id_len_max))
608
+ ret["model_format"] = ".mmcif"
609
+ if not no_refmac_fix and ret["model_format"] == ".mmcif" and not use_gemmi_prep:
610
+ ret["refmac_fixes"].fix_nonpolymer(st)
611
+
612
+ if use_refmac and use_gemmi_prep:
613
+ # TODO: make cispept, make link, remove unknown link id
614
+ # TODO: cross validation?
615
+ crdout = os.path.splitext(ret["model_file"])[0] + ".crd"
616
+ ret["model_file"] = crdout
617
+ ret["model_format"] = ".mmcif"
618
+ args.keywords.append("make cr prepared")
619
+ gemmi.setup_for_crd(st)
620
+ doc = gemmi.prepare_refmac_crd(st, topo, monlib, h_change)
621
+ doc.write_file(crdout, style=gemmi.cif.Style.NoBlankLines)
622
+ logger.writeln("crd file written: {}".format(crdout))
623
+
624
+ hkldata = utils.maps.mask_and_fft_maps(maps, resolution, None, with_000=False)
625
+ hkldata.setup_relion_binning()
626
+ if len(maps) == 2:
627
+ map_labs = ["Fmap1", "Fmap2", "Fout"]
628
+ ret["lab_f_half1"] = "Fmap1" + lab_f_suffix(args.blur)
629
+ # TODO Add SIGF in case of half maps, when refmac is ready
630
+ ret["lab_phi_half1"] = "Pmap1"
631
+ ret["lab_f_half2"] = "Fmap2" + lab_f_suffix(args.blur)
632
+ ret["lab_phi_half2"] = "Pmap2"
633
+ utils.maps.calc_noise_var_from_halfmaps(hkldata)
634
+ d_eff_full = hkldata.d_eff("FSCfull")
635
+ logger.writeln("Effective resolution from FSCfull= {:.2f}".format(d_eff_full))
636
+ ret["d_eff"] = d_eff_full
637
+ else:
638
+ map_labs = ["Fout"]
639
+ sig_lab = None
640
+
641
+ if use_refmac:
642
+ if args.no_mask:
643
+ logger.writeln("Saving unmasked maps as mtz file..")
644
+ mtzout = output_mtz_prefix+".mtz"
645
+ else:
646
+ logger.writeln(" Saving masked maps as mtz file..")
647
+ mtzout = output_masked_prefix+"_obs.mtz"
648
+
649
+ hkldata.df.rename(columns=dict(F_map1="Fmap1", F_map2="Fmap2", FP="Fout"), inplace=True)
650
+ if "shifts" in ret:
651
+ for lab in map_labs: # apply phase shift
652
+ logger.writeln(" applying phase shift for {} with translation {}".format(lab, -ret["shifts"]))
653
+ hkldata.translate(lab, -ret["shifts"])
654
+
655
+ write_map_mtz(hkldata, mtzout, map_labs=map_labs, blur=args.blur)
656
+ ret["mtz_file"] = mtzout
657
+ ret["lab_f"] = "Fout" + lab_f_suffix(args.blur)
658
+ ret["lab_phi"] = "Pout"
659
+ else:
660
+ fac = hkldata.debye_waller_factors(b_iso=args.blur)
661
+ if "shifts" in ret: fac *= hkldata.translation_factor(-ret["shifts"])
662
+ for lab in ("F_map1", "F_map2", "FP"):
663
+ if lab in hkldata.df: hkldata.df[lab] *= fac
664
+ return hkldata, ret
665
+ # process_input()
666
+
667
+ def check_args(args):
668
+ if not os.path.exists(args.model):
669
+ raise SystemExit("Error: --model {} does not exist.".format(args.model))
670
+
671
+ if args.cross_validation and not args.halfmaps:
672
+ raise SystemExit("Error: half maps are needed when --cross_validation is given")
673
+
674
+ if args.mask_for_fofc and not os.path.exists(args.mask_for_fofc):
675
+ raise SystemExit("Error: --mask_for_fofc {} does not exist".format(args.mask_for_fofc))
676
+
677
+ if args.mask_for_fofc and args.mask_radius_for_fofc:
678
+ raise SystemExit("Error: you cannot specify both --mask_for_fofc and --mask_radius_for_fofc")
679
+
680
+ if args.trim_fofc_mtz and not (args.mask_for_fofc or args.mask_radius_for_fofc):
681
+ raise SystemExit("Error: --trim_fofc_mtz is specified but --mask_for_fofc is not given")
682
+
683
+ if args.ligand: args.ligand = sum(args.ligand, [])
684
+
685
+ if args.keywords:
686
+ args.keywords = sum(args.keywords, [])
687
+ else:
688
+ args.keywords = []
689
+
690
+ if args.keyword_file:
691
+ args.keyword_file = sum(args.keyword_file, [])
692
+ for f in args.keyword_file:
693
+ if not os.path.exists(f):
694
+ raise SystemExit(f"Error: keyword file was not found: {f}")
695
+ logger.writeln("Keyword file: {}".format(f))
696
+ else:
697
+ args.keyword_file = []
698
+
699
+ if (args.twist, args.rise).count(None) == 1:
700
+ raise SystemExit("ERROR: give both helical parameters --twist and --rise")
701
+ if args.twist is not None:
702
+ logger.writeln("INFO: setting --contacting_only because helical symmetry is given")
703
+ args.contacting_only = True
704
+ if args.no_mask:
705
+ args.mask_radius = None
706
+ if not args.no_trim:
707
+ logger.writeln("WARNING: setting --no_trim because --no_mask is given")
708
+ args.no_trim = True
709
+ if args.mask:
710
+ logger.writeln("WARNING: Your --mask is ignored because --no_mask is given")
711
+ args.mask = None
712
+
713
+ #if args.mask_soft_edge > 0:
714
+ # logger.writeln("INFO: --mask_soft_edge={} is given. Turning off sharpen_before_mask.".format(args.mask_soft_edge))
715
+ # args.no_sharpen_before_mask = True
716
+
717
+ if args.resolution is None and args.model and utils.fileio.splitext(args.model)[1].endswith("cif"):
718
+ doc = gemmi.cif.read(args.model)
719
+ if len(doc) != 1:
720
+ raise SystemExit("cannot find resolution from cif. Give --resolution")
721
+ block = doc.sole_block()
722
+ reso_str = block.find_value("_em_3d_reconstruction.resolution")
723
+ try:
724
+ args.resolution = float(reso_str)
725
+ except:
726
+ raise SystemExit("ERROR: _em_3d_reconstruction.resolution is invalid. Give --resolution")
727
+ logger.writeln("WARNING: --resolution not given. Using _em_3d_reconstruction.resolution = {}".format(reso_str))
728
+
729
+ if args.resolution is None:
730
+ raise SystemExit("ERROR: --resolution is needed.")
731
+ # check_args()
732
+
733
+ def main(args):
734
+ check_args(args)
735
+ use_gemmi_prep = False
736
+ if not args.prepare_only:
737
+ refmac_ver = utils.refmac.check_version(args.exe)
738
+ if not refmac_ver:
739
+ raise SystemExit("Error: Check Refmac installation or use --exe to give the location.")
740
+ if not args.no_refmacat and refmac_ver >= (5, 8, 404):
741
+ logger.writeln(" will use gemmi to prepare restraints")
742
+ use_gemmi_prep = True
743
+ else:
744
+ logger.writeln(" will use makecif to prepare restraints")
745
+
746
+ logger.writeln("Input model: {}".format(args.model))
747
+ st = utils.fileio.read_structure(args.model)
748
+ if len(st) > 1:
749
+ logger.writeln(" Removing models 2-{}".format(len(st)))
750
+ for i in reversed(range(1, len(st))):
751
+ del st[i]
752
+
753
+ try:
754
+ monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand,
755
+ stop_for_unknowns=True)
756
+ except RuntimeError as e:
757
+ raise SystemExit("Error: {}".format(e))
758
+
759
+ utils.model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
760
+ try:
761
+ utils.restraints.prepare_topology(st.clone(), monlib, h_change=gemmi.HydrogenChange.NoChange,
762
+ check_hydrogen=(args.hydrogen=="yes"))
763
+ except RuntimeError as e:
764
+ raise SystemExit("Error: {}".format(e))
765
+
766
+ if args.halfmaps:
767
+ maps = utils.fileio.read_halfmaps(args.halfmaps, pixel_size=args.pixel_size)
768
+ else:
769
+ maps = [utils.fileio.read_ccp4_map(args.map, pixel_size=args.pixel_size)]
770
+
771
+ utils.model.remove_charge([st])
772
+ shifted_model_prefix = "shifted"
773
+ _, file_info = process_input(st, maps, resolution=args.resolution - 1e-6, monlib=monlib,
774
+ mask_in=args.mask, args=args,
775
+ shifted_model_prefix=shifted_model_prefix,
776
+ use_gemmi_prep=use_gemmi_prep,
777
+ find_links=args.find_links)
778
+ if args.prepare_only:
779
+ logger.writeln("\n--prepare_only is given. Stopping.")
780
+ return
781
+
782
+ args.mtz = file_info["mtz_file"]
783
+ if args.halfmaps: # FIXME if no_mask?
784
+ args.mtz_half = [file_info["mtz_file"], file_info["mtz_file"]]
785
+ args.lab_phi = file_info["lab_phi"] #"Pout0"
786
+ args.lab_f = file_info["lab_f"]
787
+ args.lab_sigf = None
788
+ args.model = file_info["model_file"] # refmac xyzin
789
+ model_format = file_info["model_format"]
790
+
791
+ if args.cross_validation and args.cross_validation_method == "throughout":
792
+ args.lab_f = file_info["lab_f_half1"]
793
+ args.lab_phi = file_info["lab_phi_half1"]
794
+ # XXX args.lab_sigf?
795
+
796
+ chain_id_lens = [len(x) for x in utils.model.all_chain_ids(st)]
797
+ keep_chain_ids = (chain_id_lens and max(chain_id_lens) == 1) # always kept unless one-letter chain IDs
798
+
799
+ # FIXME if mtz is given and sfcalc() not ran?
800
+
801
+ if not args.no_trim:
802
+ refmac_prefix = "{}_{}".format(shifted_model_prefix, args.output_prefix)
803
+ else:
804
+ refmac_prefix = args.output_prefix # XXX this should be different name (nomask etc?)
805
+
806
+ # Weight auto scale
807
+ if args.weight_auto_scale is None:
808
+ reso = file_info["d_eff"] if "d_eff" in file_info else args.resolution
809
+ if "vol_ratio" in file_info:
810
+ if "d_eff" in file_info:
811
+ rlmc = (-9.503541, 3.129882, 15.439744)
812
+ else:
813
+ rlmc = (-8.329418, 3.032409, 14.381907)
814
+ logger.writeln("Estimating weight auto scale using resolution and volume ratio")
815
+ ws = rlmc[0] + reso*rlmc[1] +file_info["vol_ratio"]*rlmc[2]
816
+ else:
817
+ if "d_eff" in file_info:
818
+ rlmc = (-5.903807, 2.870723)
819
+ else:
820
+ rlmc = (-4.891140, 2.746791)
821
+ logger.writeln("Estimating weight auto scale using resolution")
822
+ ws = rlmc[0] + args.resolution*rlmc[1]
823
+ args.weight_auto_scale = max(0.2, min(18.0, ws))
824
+ logger.writeln(" Will use weight auto {:.2f}".format(args.weight_auto_scale))
825
+
826
+ # Run Refmac
827
+ refmac = utils.refmac.Refmac(prefix=refmac_prefix, args=args, global_mode="spa",
828
+ keep_chain_ids=keep_chain_ids)
829
+ refmac.set_libin(args.ligand)
830
+ try:
831
+ refmac_summary = refmac.run_refmac()
832
+ except RuntimeError as e:
833
+ raise SystemExit("Error: {}".format(e))
834
+
835
+ # Modify output
836
+ st, cif_ref = utils.fileio.read_structure_from_pdb_and_mmcif(refmac_prefix+model_format)
837
+ utils.model.setup_entities(st, clear=True, overwrite_entity_type=True, force_subchain_names=True)
838
+
839
+ if not args.no_trim:
840
+ st.cell = maps[0][0].unit_cell
841
+ st.setup_cell_images()
842
+ if "refmac_fixes" in file_info:
843
+ file_info["refmac_fixes"].modify_back(st)
844
+ utils.model.adp_analysis(st)
845
+ utils.fileio.write_model(st, prefix=args.output_prefix,
846
+ pdb=True, cif=True, cif_ref=cif_ref)
847
+
848
+ # Take care of TLS out
849
+ if not args.no_trim: # if no_trim, there is nothing to do.
850
+ tlsout = refmac.tlsout()
851
+ if os.path.exists(tlsout):
852
+ logger.writeln("Copying tlsout")
853
+ shutil.copyfile(refmac.tlsout(), args.output_prefix+".tls")
854
+
855
+ # Expand sym here
856
+ st_expanded = st.clone()
857
+ if not all(op.given for op in st.ncs):
858
+ utils.model.expand_ncs(st_expanded)
859
+ utils.fileio.write_model(st_expanded, args.output_prefix+"_expanded", pdb=True, cif=True,
860
+ cif_ref=cif_ref)
861
+
862
+ if args.cross_validation and args.cross_validation_method == "shake":
863
+ logger.writeln("Cross validation is requested.")
864
+ refmac_prefix_shaken = refmac_prefix+"_shaken_refined"
865
+ logger.writeln("Starting refinement using half map 1 (model is shaken first)")
866
+ logger.writeln("In this refinement, hydrogen is removed regardless of --hydrogen option")
867
+ if use_gemmi_prep:
868
+ xyzin = refmac_prefix + ".crd"
869
+ st_tmp = utils.fileio.read_structure(refmac_prefix+model_format)
870
+ utils.model.setup_entities(st_tmp, clear=True, overwrite_entity_type=True, force_subchain_names=True)
871
+ prepare_crd(st_tmp,
872
+ crdout=xyzin, ligand=[refmac_prefix+model_format],
873
+ make={"hydr":"n"},
874
+ fix_long_resnames=False) # we do not need output file - do we?
875
+ else:
876
+ xyzin = refmac_prefix + model_format
877
+ refmac_hm1 = refmac.copy(hklin=args.mtz_half[0],
878
+ xyzin=xyzin,
879
+ prefix=refmac_prefix_shaken,
880
+ shake=args.shake_radius,
881
+ jellybody=False, # makes no sense to use jelly body after shaking
882
+ hydrogen="no") # should not use hydrogen after shaking
883
+ if args.jellybody: logger.writeln(" Turning off jelly body")
884
+ if "lab_f_half1" in file_info:
885
+ refmac_hm1.lab_f = file_info["lab_f_half1"]
886
+ refmac_hm1.lab_phi = file_info["lab_phi_half1"]
887
+ # SIGMA?
888
+
889
+ try:
890
+ refmac_hm1.run_refmac()
891
+ except RuntimeError as e:
892
+ raise SystemExit("Error: {}".format(e))
893
+
894
+ if args.hydrogen != "no": # does not work properly when 'yes' - we would need to keep hydrogen in input
895
+ logger.writeln("Cross validation: 2nd run with hydrogen")
896
+ if use_gemmi_prep:
897
+ xyzin = refmac_prefix_shaken + ".crd"
898
+ st_tmp = utils.fileio.read_structure(refmac_prefix_shaken+model_format)
899
+ utils.model.setup_entities(st_tmp, clear=True, overwrite_entity_type=True, force_subchain_names=True)
900
+ prepare_crd(st_tmp,
901
+ crdout=xyzin, ligand=[refmac_prefix+model_format],
902
+ make={"hydr":"a"},
903
+ fix_long_resnames=False) # we do not need output file - do we?
904
+ else:
905
+ xyzin = refmac_prefix_shaken + model_format
906
+ refmac_prefix_shaken = refmac_prefix+"_shaken_refined2"
907
+ refmac_hm1_2 = refmac_hm1.copy(xyzin=xyzin,
908
+ prefix=refmac_prefix_shaken,
909
+ shake=None,
910
+ hydrogen="all")
911
+ try:
912
+ refmac_hm1_2.run_refmac()
913
+ except RuntimeError as e:
914
+ raise SystemExit("Error: {}".format(e))
915
+
916
+ # Modify output
917
+ st_sr, cif_ref_sr = utils.fileio.read_structure_from_pdb_and_mmcif(refmac_prefix_shaken+model_format)
918
+ utils.model.setup_entities(st_sr, clear=True, overwrite_entity_type=True, force_subchain_names=True)
919
+ if not args.no_trim:
920
+ st_sr.cell = maps[0][0].unit_cell
921
+ st_sr.setup_cell_images()
922
+ if "refmac_fixes" in file_info:
923
+ file_info["refmac_fixes"].modify_back(st_sr)
924
+
925
+ utils.fileio.write_model(st_sr, prefix=args.output_prefix+"_shaken_refined",
926
+ pdb=True, cif=True, cif_ref=cif_ref_sr)
927
+
928
+ # Expand sym here
929
+ st_sr_expanded = st_sr.clone()
930
+ if not all(op.given for op in st_sr.ncs):
931
+ utils.model.expand_ncs(st_sr_expanded)
932
+ utils.fileio.write_model(st_sr_expanded, args.output_prefix+"_shaken_refined_expanded",
933
+ pdb=True, cif=True, cif_ref=cif_ref_sr)
934
+ if args.twist is not None: # as requested by a user
935
+ st_sr_expanded_all = st_sr.clone()
936
+ utils.symmetry.update_ncs_from_args(args, st_sr_expanded_all, map_and_start=maps[0], filter_contacting=False)
937
+ utils.model.expand_ncs(st_sr_expanded_all)
938
+ utils.fileio.write_model(st_sr_expanded_all, args.output_prefix+"_shaken_refined_expanded_all", pdb=True, cif=True,
939
+ cif_ref=cif_ref)
940
+ else:
941
+ st_sr_expanded = None
942
+
943
+ if args.mask:
944
+ mask = utils.fileio.read_ccp4_map(args.mask)[0]
945
+ else:
946
+ mask = None
947
+
948
+ # Calc FSC
949
+ fscavg_text = calc_fsc(st_expanded, args.output_prefix, maps,
950
+ args.resolution, mask=mask, mask_radius=args.mask_radius if not args.no_mask else None,
951
+ soft_edge=args.mask_soft_edge,
952
+ b_before_mask=args.b_before_mask,
953
+ no_sharpen_before_mask=args.no_sharpen_before_mask,
954
+ make_hydrogen=args.hydrogen,
955
+ monlib=monlib, cross_validation=args.cross_validation,
956
+ blur=args.blur,
957
+ d_min_fsc=args.fsc_resolution,
958
+ cross_validation_method=args.cross_validation_method, st_sr=st_sr_expanded)
959
+
960
+ # Calc Fo-Fc (and updated) maps
961
+ calc_fofc(st, st_expanded, maps, monlib, model_format, args)
962
+
963
+ # Final summary
964
+ write_final_summary(st, refmac_summary, fscavg_text, args.output_prefix,
965
+ args.mask_for_fofc or args.mask_radius_for_fofc)
966
+ # main()
967
+
968
+ if __name__ == "__main__":
969
+ import sys
970
+ args = parse_args(sys.argv[1:])
971
+ main(args)
972
+