servalcat 0.4.99__cp39-cp39-macosx_10_14_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of servalcat might be problematic. Click here for more details.

Files changed (45) hide show
  1. servalcat/__init__.py +10 -0
  2. servalcat/__main__.py +120 -0
  3. servalcat/ext.cpython-39-darwin.so +0 -0
  4. servalcat/refine/__init__.py +0 -0
  5. servalcat/refine/cgsolve.py +100 -0
  6. servalcat/refine/refine.py +906 -0
  7. servalcat/refine/refine_geom.py +233 -0
  8. servalcat/refine/refine_spa.py +366 -0
  9. servalcat/refine/refine_xtal.py +281 -0
  10. servalcat/refine/spa.py +144 -0
  11. servalcat/refine/xtal.py +276 -0
  12. servalcat/refmac/__init__.py +0 -0
  13. servalcat/refmac/exte.py +182 -0
  14. servalcat/refmac/refmac_keywords.py +639 -0
  15. servalcat/refmac/refmac_wrapper.py +395 -0
  16. servalcat/spa/__init__.py +0 -0
  17. servalcat/spa/fofc.py +479 -0
  18. servalcat/spa/fsc.py +385 -0
  19. servalcat/spa/localcc.py +188 -0
  20. servalcat/spa/realspcc_from_var.py +128 -0
  21. servalcat/spa/run_refmac.py +977 -0
  22. servalcat/spa/shift_maps.py +293 -0
  23. servalcat/spa/shiftback.py +137 -0
  24. servalcat/spa/translate.py +129 -0
  25. servalcat/utils/__init__.py +35 -0
  26. servalcat/utils/commands.py +1547 -0
  27. servalcat/utils/fileio.py +744 -0
  28. servalcat/utils/generate_operators.py +296 -0
  29. servalcat/utils/hkl.py +714 -0
  30. servalcat/utils/logger.py +140 -0
  31. servalcat/utils/maps.py +345 -0
  32. servalcat/utils/model.py +782 -0
  33. servalcat/utils/refmac.py +760 -0
  34. servalcat/utils/restraints.py +781 -0
  35. servalcat/utils/symmetry.py +295 -0
  36. servalcat/xtal/__init__.py +0 -0
  37. servalcat/xtal/french_wilson.py +258 -0
  38. servalcat/xtal/run_refmac_small.py +240 -0
  39. servalcat/xtal/sigmaa.py +1644 -0
  40. servalcat/xtal/twin.py +121 -0
  41. servalcat-0.4.99.dist-info/METADATA +55 -0
  42. servalcat-0.4.99.dist-info/RECORD +45 -0
  43. servalcat-0.4.99.dist-info/WHEEL +5 -0
  44. servalcat-0.4.99.dist-info/entry_points.txt +4 -0
  45. servalcat-0.4.99.dist-info/licenses/LICENSE +373 -0
@@ -0,0 +1,977 @@
1
+ """
2
+ Author: "Keitaro Yamashita, Garib N. Murshudov"
3
+ MRC Laboratory of Molecular Biology
4
+
5
+ This software is released under the
6
+ Mozilla Public License, version 2.0; see LICENSE.
7
+ """
8
+ from __future__ import absolute_import, division, print_function, generators
9
+ import gemmi
10
+ import numpy
11
+ import json
12
+ import os
13
+ import shutil
14
+ import argparse
15
+ from servalcat.utils import logger
16
+ from servalcat import utils
17
+ from servalcat import spa # this is not a right style
18
+ from servalcat.spa import shift_maps
19
+ from servalcat.refmac.refmac_wrapper import prepare_crd
20
+
21
+ def add_arguments(parser):
22
+ parser.description = 'Run REFMAC5 for SPA'
23
+
24
+ parser.add_argument('--exe', default="refmac5",
25
+ help='refmac5 binary (default: %(default)s)')
26
+ parser.add_argument("--monlib",
27
+ help="Monomer library path. Default: $CLIBD_MON")
28
+ # sfcalc options
29
+ group = parser.add_mutually_exclusive_group(required=True)
30
+ group.add_argument("--halfmaps", nargs=2, help="Input half map files")
31
+ group.add_argument("--map", help="Use this only if you really do not have half maps.")
32
+ parser.add_argument('--mask',
33
+ help='Mask file')
34
+ parser.add_argument('--model', required=True,
35
+ help='Input atomic model file')
36
+ parser.add_argument('--mask_radius',
37
+ type=float, default=3,
38
+ help='')
39
+ parser.add_argument('--mask_soft_edge',
40
+ type=float, default=0,
41
+ help='Add soft edge to model mask. Should use with --no_sharpen_before_mask?')
42
+ parser.add_argument('--padding',
43
+ type=float,
44
+ help='Default: 2*mask_radius')
45
+ parser.add_argument('--no_mask', action='store_true')
46
+ parser.add_argument('--invert_mask', action='store_true', help='not for refinement.')
47
+ parser.add_argument('--pixel_size', type=float,
48
+ help='Override pixel size (A)')
49
+ parser.add_argument('--resolution',
50
+ type=float,
51
+ help='')
52
+ parser.add_argument('--no_trim',
53
+ action='store_true',
54
+ help='Keep original box (not recommended)')
55
+ parser.add_argument('--blur',
56
+ type=float, default=0,
57
+ help='Sharpening or blurring B')
58
+ utils.symmetry.add_symmetry_args(parser) # add --pg etc
59
+ parser.add_argument('--contacting_only', action="store_true", help="Filter out non-contacting NCS")
60
+ parser.add_argument('--ignore_symmetry', action='store_true',
61
+ help='Ignore symmetry information (MTRIX/_struct_ncs_oper) in the model file')
62
+ parser.add_argument('--find_links', action='store_true',
63
+ help='Automatically add links')
64
+ parser.add_argument("--b_before_mask", type=float,
65
+ help="sharpening B value for sharpen-mask-unsharpen procedure. By default it is determined automatically.")
66
+ parser.add_argument('--no_sharpen_before_mask', action='store_true',
67
+ help='By default half maps are sharpened before masking by std of signal and unsharpened after masking. This option disables it.')
68
+ parser.add_argument('--no_fix_microheterogeneity', action='store_true',
69
+ help='By default it will fix microheterogeneity for Refmac')
70
+ parser.add_argument('--no_fix_resi9999', action='store_true',
71
+ help='By default it will split chain if max residue number > 9999 which is not supported by Refmac')
72
+ parser.add_argument('--no_check_ncs_overlaps', action='store_true',
73
+ help='Disable model overlap (e.g. expanded model is used with --pg) test')
74
+ parser.add_argument('--no_check_ncs_map', action='store_true',
75
+ help='Disable map NCS consistency test')
76
+ parser.add_argument('--no_check_mask_with_model', action='store_true',
77
+ help='Disable mask test using model')
78
+ parser.add_argument("--prepare_only", action='store_true',
79
+ help="Stop before refinement")
80
+ parser.add_argument("--no_refmacat", action='store_true',
81
+ help="By default uses gemmi for crd/rst file preparation (do not use makecif)")
82
+ # run_refmac options
83
+ # TODO use group! like refmac options
84
+ parser.add_argument('--ligand', nargs="*", action="append",
85
+ help="restraint dictionary cif file(s)")
86
+ parser.add_argument('--bfactor', type=float,
87
+ help="reset all atomic B values to specified value")
88
+ parser.add_argument('--ncsr', default="local", choices=["local", "global"],
89
+ help="local or global NCS restrained (default: %(default)s)")
90
+ parser.add_argument('--ncycle', type=int, default=10,
91
+ help="number of cycles in Refmac (default: %(default)d)")
92
+ parser.add_argument('--tlscycle', type=int, default=0,
93
+ help="number of TLS cycles in Refmac (default: %(default)d)")
94
+ parser.add_argument('--tlsin',
95
+ help="TLS parameter input for Refmac")
96
+ parser.add_argument('--hydrogen', default="all", choices=["all", "yes", "no"],
97
+ help="all: add riding hydrogen atoms, yes: use hydrogen atoms if present, no: remove hydrogen atoms in input. "
98
+ "Default: %(default)s")
99
+ parser.add_argument('--jellybody', action='store_true',
100
+ help="Use jelly body restraints")
101
+ parser.add_argument('--jellybody_params', nargs=2, type=float,
102
+ metavar=("sigma", "dmax"), default=[0.01, 4.2],
103
+ help="Jelly body sigma and dmax (default: %(default)s)")
104
+ parser.add_argument('--hout', action='store_true', help="write hydrogen atoms in the output model")
105
+ group = parser.add_mutually_exclusive_group()
106
+ group.add_argument('--weight_auto_scale', type=float,
107
+ help="'weight auto' scale value. automatically determined from resolution and mask/box volume ratio if unspecified")
108
+ parser.add_argument('--keywords', nargs='+', action="append",
109
+ help="refmac keyword(s)")
110
+ parser.add_argument('--keyword_file', nargs='+', action="append",
111
+ help="refmac keyword file(s)")
112
+ parser.add_argument('--external_restraints_json')
113
+ parser.add_argument('--show_refmac_log', action='store_true',
114
+ help="show all Refmac log instead of summary")
115
+ parser.add_argument('--output_prefix', default="refined",
116
+ help='output file name prefix (default: %(default)s)')
117
+ parser.add_argument('--cross_validation', action='store_true',
118
+ help='Run cross validation')
119
+ parser.add_argument('--cross_validation_method', default="shake", choices=["throughout", "shake"],
120
+ help="shake: randomize a model refined against a full map and then refine it against a half map, "
121
+ "throughout: use only a half map for refinement (another half map is used for error estimation) "
122
+ "Default: %(default)s")
123
+ parser.add_argument('--shake_radius', default=0.3,
124
+ help='Shake rmsd in case of --cross_validation_method=shake (default: %(default).1f)')
125
+ group = parser.add_mutually_exclusive_group()
126
+ group.add_argument('--mask_for_fofc', help="Mask file for Fo-Fc map calculation")
127
+ group.add_argument('--mask_radius_for_fofc', type=float, help="Mask radius for Fo-Fc map calculation")
128
+ parser.add_argument('--trim_fofc_mtz', action="store_true", help="diffmap.mtz will have smaller cell (if --mask_for_fofc is given)")
129
+ parser.add_argument("--fsc_resolution", type=float,
130
+ help="High resolution limit for FSC calculation. Default: Nyquist")
131
+
132
+ # add_arguments()
133
+
134
+ def parse_args(arg_list):
135
+ parser = argparse.ArgumentParser()
136
+ add_arguments(parser)
137
+ return parser.parse_args(arg_list)
138
+ # parse_args()
139
+
140
+ def calc_fsc(st, output_prefix, maps, d_min, mask, mask_radius, soft_edge, b_before_mask, no_sharpen_before_mask, make_hydrogen, monlib,
141
+ blur=0, d_min_fsc=None, cross_validation=False, cross_validation_method=None, st_sr=None):
142
+ # st_sr: shaken-and-refined st in case of cross_validation_method=="shake"
143
+ if cross_validation:
144
+ assert len(maps) == 2
145
+ assert cross_validation_method in ("shake", "throughout")
146
+ if cross_validation and cross_validation_method == "shake":
147
+ assert st_sr is not None
148
+ else:
149
+ assert st_sr is None
150
+
151
+ logger.writeln("Calculating map-model FSC..")
152
+ ret = {"summary": {}}
153
+
154
+ if d_min_fsc is None:
155
+ d_min_fsc = utils.maps.nyquist_resolution(maps[0][0])
156
+ logger.writeln(" --fsc_resolution is not specified. Using Nyquist resolution: {:.2f}".format(d_min_fsc))
157
+
158
+ st = st.clone()
159
+ if st_sr is not None: st_sr = st_sr.clone()
160
+
161
+ if make_hydrogen == "all":
162
+ utils.restraints.add_hydrogens(st, monlib)
163
+ if st_sr is not None: utils.restraints.add_hydrogens(st_sr, monlib)
164
+
165
+ if mask is not None or mask_radius is not None:
166
+ if mask is None:
167
+ assert maps[0][0].unit_cell == st.cell
168
+ mask = utils.maps.mask_from_model(st, mask_radius, soft_edge=soft_edge, grid=maps[0][0])
169
+ if no_sharpen_before_mask or len(maps) < 2:
170
+ for ma in maps: ma[0].array[:] *= mask
171
+ else:
172
+ # It seems we need different B for different resolution limit
173
+ if b_before_mask is None: b_before_mask = determine_b_before_mask(st, maps, maps[0][1], mask, d_min_fsc)
174
+ maps = utils.maps.sharpen_mask_unsharpen(maps, mask, d_min_fsc, b=b_before_mask)
175
+
176
+ hkldata = utils.maps.mask_and_fft_maps(maps, d_min_fsc)
177
+ hkldata.df["FC"] = utils.model.calc_fc_fft(st, d_min_fsc - 1e-6, monlib=monlib, source="electron",
178
+ miller_array=hkldata.miller_array())
179
+ # XXX didn't apply mask to FC!!
180
+ labs_fc = ["FC"]
181
+
182
+ if st_sr is not None:
183
+ hkldata.df["FC_sr"] = utils.model.calc_fc_fft(st_sr, d_min_fsc - 1e-6, monlib=monlib, source="electron",
184
+ miller_array=hkldata.miller_array())
185
+ labs_fc.append("FC_sr")
186
+
187
+ if blur != 0:
188
+ logger.writeln(" Unblurring Fc with B={} for FSC calculation".format(blur))
189
+ unblur = numpy.exp(blur/hkldata.d_spacings().to_numpy()**2/4.)
190
+ for lab in labs_fc:
191
+ hkldata.df[lab] *= unblur
192
+
193
+ hkldata.setup_relion_binning()
194
+ stats = spa.fsc.calc_fsc_all(hkldata, labs_fc=labs_fc, lab_f="FP",
195
+ labs_half=["F_map1", "F_map2"] if len(maps)==2 else None)
196
+
197
+ hkldata2 = hkldata.copy(d_min=d_min) # for FSCaverage at resolution for refinement # XXX more efficient way
198
+ hkldata2.setup_relion_binning()
199
+ stats2 = spa.fsc.calc_fsc_all(hkldata2, labs_fc=labs_fc, lab_f="FP",
200
+ labs_half=["F_map1", "F_map2"] if len(maps)==2 else None)
201
+
202
+ if "fsc_half" in stats:
203
+ with numpy.errstate(invalid="ignore"): # XXX negative fsc results in nan!
204
+ stats.loc[:,"fsc_full_sqrt"] = numpy.sqrt(2*stats.fsc_half/(1+stats.fsc_half))
205
+
206
+ logger.writeln(stats.to_string()+"\n")
207
+
208
+ # remove and rename columns
209
+ for s in (stats, stats2):
210
+ s.rename(columns=dict(fsc_FC_full="fsc_model", Rcmplx_FC_full="Rcmplx"), inplace=True)
211
+ if cross_validation:
212
+ if cross_validation_method == "shake":
213
+ s.drop(columns=["fsc_FC_half1", "fsc_FC_half2", "fsc_FC_sr_full", "Rcmplx_FC_sr_full"], inplace=True)
214
+ s.rename(columns=dict(fsc_FC_sr_half1="fsc_model_half1",
215
+ fsc_FC_sr_half2="fsc_model_half2"), inplace=True)
216
+ else:
217
+ s.rename(columns=dict(fsc_FC_half1="fsc_model_half1",
218
+ fsc_FC_half2="fsc_model_half2"), inplace=True)
219
+ else:
220
+ s.drop(columns=[x for x in s if x.startswith("fsc_FC") and x.endswith(("half1","half2"))], inplace=True)
221
+
222
+ # FSCaverages
223
+ ret["summary"]["d_min"] = d_min
224
+ ret["summary"]["FSCaverage"] = spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model)
225
+ if cross_validation:
226
+ ret["summary"]["FSCaverage_half1"] = spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half1)
227
+ ret["summary"]["FSCaverage_half2"] = spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half2)
228
+ fscavg_text = "Map-model FSCaverages (at {:.2f} A):\n".format(d_min)
229
+ fscavg_text += " FSCaverage(full) = {: .4f}\n".format(ret["summary"]["FSCaverage"])
230
+ if cross_validation:
231
+ fscavg_text += "Cross-validated map-model FSCaverages:\n"
232
+ fscavg_text += " FSCaverage(half1)= {: .4f}\n".format(ret["summary"]["FSCaverage_half1"])
233
+ fscavg_text += " FSCaverage(half2)= {: .4f}\n".format(ret["summary"]["FSCaverage_half2"])
234
+
235
+ # for loggraph
236
+ fsc_logfile = "{}_fsc.log".format(output_prefix)
237
+ with open(fsc_logfile, "w") as ofs:
238
+ columns = "1/resol^2 ncoef ln(Mn(|Fo|^2)) ln(Mn(|Fc|^2)) FSC(full,model) FSC_half FSC_full_sqrt FSC(half1,model) FSC(half2,model) Rcmplx(full,model)".split()
239
+
240
+ ofs.write("$TABLE: Map-model FSC after refinement:\n")
241
+ if len(maps) == 2:
242
+ if cross_validation: fsc_cols = [5,6,7,8,9]
243
+ else: fsc_cols = [5,6,7]
244
+ else: fsc_cols = [5]
245
+ fsc_cols.append(10)
246
+ if len(maps) == 2: ofs.write("$GRAPHS: FSC :A:1,5,6,7:\n")
247
+ else: ofs.write("$GRAPHS: FSC :A:1,5:\n")
248
+ if cross_validation: ofs.write(": cross-validated FSC :A:1,8,9:\n".format(",".join(map(str,fsc_cols))))
249
+ ofs.write(": Rcmplx :A:1,{}:\n".format(4+len(fsc_cols)))
250
+ ofs.write(": ln(Mn(|F|^2)) :A:1,3,4:\n")
251
+ ofs.write(": number of Fourier coeffs :A:1,2:\n")
252
+ ofs.write("$$ {}$$\n".format(" ".join(columns[:4]+[columns[i-1] for i in fsc_cols])))
253
+ ofs.write("$$\n")
254
+
255
+ plot_columns = ["d_min", "ncoeffs", "power_FP", "power_FC", "fsc_model"]
256
+ if len(maps) == 2:
257
+ plot_columns.extend(["fsc_half", "fsc_full_sqrt"])
258
+ if cross_validation:
259
+ plot_columns.extend(["fsc_model_half1", "fsc_model_half2"])
260
+ plot_columns.append("Rcmplx")
261
+ with numpy.errstate(divide="ignore"):
262
+ log_format = lambda x: "{:.3f}".format(numpy.log(x))
263
+ ofs.write(stats.to_string(header=False, index=False, index_names=False, columns=plot_columns,
264
+ formatters=dict(d_min=lambda x: "{:.4f}".format(1/x**2),
265
+ power_FP=log_format, power_FC=log_format)))
266
+ ofs.write("\n")
267
+ ofs.write("$$\n\n")
268
+ ofs.write(fscavg_text)
269
+
270
+ logger.write(fscavg_text)
271
+ logger.writeln("Run loggraph {} to see plots.".format(fsc_logfile))
272
+
273
+ # write json
274
+ json.dump(stats.to_dict("records"),
275
+ open("{}_fsc.json".format(output_prefix), "w"),
276
+ indent=True)
277
+ ret["binned"] = stats2.to_dict(orient="records")
278
+ return fscavg_text, ret
279
+ # calc_fsc()
280
+
281
+ def calc_fofc(st, st_expanded, maps, monlib, model_format, args, diffmap_prefix="diffmap"):
282
+ logger.writeln("Starting Fo-Fc calculation..")
283
+ if not args.halfmaps: logger.writeln(" with limited functionality because half maps were not provided")
284
+ logger.writeln(" model: {}".format(args.output_prefix+model_format))
285
+
286
+ # for Fo-Fc in case of helical reconstruction, expand model more
287
+ # XXX should we do it for FSC calculation also? Probably we should not do sharpen-unsharpen procedure for FSC calc either.
288
+ if args.twist is not None:
289
+ logger.writeln("Generating all helical copies in the box")
290
+ st_expanded = st.clone()
291
+ utils.symmetry.update_ncs_from_args(args, st_expanded, map_and_start=maps[0], filter_contacting=False)
292
+ utils.model.expand_ncs(st_expanded)
293
+ utils.fileio.write_model(st_expanded, args.output_prefix+"_expanded_all", pdb=True, cif=True)
294
+
295
+ if args.mask_for_fofc:
296
+ logger.writeln(" mask: {}".format(args.mask_for_fofc))
297
+ mask = utils.fileio.read_ccp4_map(args.mask_for_fofc)[0]
298
+ elif args.mask_radius_for_fofc:
299
+ logger.writeln(" mask: using refined model with radius of {} A".format(args.mask_radius_for_fofc))
300
+ mask = utils.maps.mask_from_model(st_expanded, args.mask_radius_for_fofc, grid=maps[0][0]) # use soft edge?
301
+ else:
302
+ logger.writeln(" mask: not used")
303
+ mask = None
304
+
305
+ hkldata, map_labs, stats_str = spa.fofc.calc_fofc(st_expanded, args.resolution, maps, mask=mask, monlib=monlib,
306
+ half1_only=(args.cross_validation and args.cross_validation_method == "throughout"),
307
+ sharpening_b=None if args.halfmaps else 0.) # assume already sharpened if fullmap is given
308
+ spa.fofc.write_files(hkldata, map_labs, maps[0][1], stats_str,
309
+ mask=mask, output_prefix=diffmap_prefix,
310
+ trim_map=mask is not None, trim_mtz=args.trim_fofc_mtz)
311
+
312
+ # Create Coot script
313
+ spa.fofc.write_coot_script("{}_coot.py".format(args.output_prefix),
314
+ model_file="{}.pdb".format(args.output_prefix), # as Coot is not good at mmcif file..
315
+ mtz_file="{}.mtz".format(diffmap_prefix),
316
+ contour_fo=None if mask is None else 1.2,
317
+ contour_fofc=None if mask is None else 3.0,
318
+ ncs_ops=st.ncs)
319
+
320
+ # Create ChimeraX script
321
+ spa.fofc.write_chimerax_script(cxc_out="{}_chimerax.cxc".format(args.output_prefix),
322
+ model_file="{}.mmcif".format(args.output_prefix), # ChimeraX handles mmcif just fine
323
+ fo_mrc_file="{}_normalized_fo.mrc".format(diffmap_prefix),
324
+ fofc_mrc_file="{}_normalized_fofc.mrc".format(diffmap_prefix))
325
+ # calc_fofc()
326
+
327
+ def write_final_summary(st, refmac_summary, fscavg_text, output_prefix, is_mask_given):
328
+ if len(refmac_summary["cycles"]) > 1 and "actual_weight" in refmac_summary["cycles"][-2]:
329
+ final_weight = refmac_summary["cycles"][-2]["actual_weight"]
330
+ else:
331
+ final_weight = "???"
332
+
333
+ adpstats_txt = ""
334
+ adp_stats = utils.model.adp_stats_per_chain(st[0])
335
+ max_chain_len = max([len(x[0]) for x in adp_stats])
336
+ max_num_len = max([len(str(x[1])) for x in adp_stats])
337
+ for chain, natoms, qs in adp_stats:
338
+ adpstats_txt += " Chain {0:{1}s}".format(chain, max_chain_len) if chain!="*" else " {0:{1}s}".format("All", max_chain_len+6)
339
+ adpstats_txt += " ({0:{1}d} atoms) min={2:5.1f} median={3:5.1f} max={4:5.1f} A^2\n".format(natoms, max_num_len, qs[0],qs[2],qs[4])
340
+
341
+ if is_mask_given:
342
+ map_peaks_str = """\
343
+ List Fo-Fc map peaks in the ASU:
344
+ servalcat util map_peaks --map diffmap_normalized_fofc.mrc --model {prefix}.pdb --abs_level 4.0 \
345
+ """.format(prefix=output_prefix)
346
+ else:
347
+ map_peaks_str = "WARNING: --mask_for_fofc was not given, so the Fo-Fc map was not normalized."
348
+
349
+ logger.writeln("""
350
+ =============================================================================
351
+ * Final Summary *
352
+
353
+ Rmsd from ideal
354
+ bond lengths: {rmsbond} A
355
+ bond angles: {rmsangle} deg
356
+
357
+ {fscavgs}
358
+ Run loggraph {fsclog} to see plots
359
+
360
+ ADP statistics
361
+ {adpstats}
362
+
363
+ Weight used: {final_weight}
364
+ If you want to change the weight, give larger (looser restraints)
365
+ or smaller (tighter) value to --weight_auto_scale=.
366
+
367
+ Open refined model and diffmap.mtz with COOT:
368
+ coot --script {prefix}_coot.py
369
+
370
+ Open refined model, map and difference map with ChimeraX/ISOLDE:
371
+ chimerax {prefix}_chimerax.cxc
372
+
373
+ {map_peaks_msg}
374
+ =============================================================================
375
+ """.format(rmsbond=refmac_summary["cycles"][-1].get("rms_bond", "???"),
376
+ rmsangle=refmac_summary["cycles"][-1].get("rms_angle", "???"),
377
+ fscavgs=fscavg_text.rstrip(),
378
+ fsclog="{}_fsc.log".format(output_prefix),
379
+ adpstats=adpstats_txt.rstrip(),
380
+ final_weight=final_weight,
381
+ prefix=output_prefix,
382
+ map_peaks_msg=map_peaks_str))
383
+ # write_final_summary()
384
+
385
+ def lab_f_suffix(blur):
386
+ if blur is None or blur == 0.:
387
+ return ""
388
+ elif blur > 0:
389
+ return "Blur_{:.2f}".format(blur)
390
+ else:
391
+ return "Sharp_{:.2f}".format(-blur)
392
+ # lab_f_suffix()
393
+
394
+ def write_map_mtz(hkldata, mtz_out, map_labs, sig_lab=None, blur=0):
395
+ nblur = 2 if blur != 0 else 1
396
+ mean_f = hkldata.df[map_labs].abs().mean().min()
397
+ data_labs = map_labs + ([sig_lab] if sig_lab else [])
398
+
399
+ if mean_f < 1:
400
+ scale = 10. / mean_f
401
+ logger.writeln("Mean(|F|)= {:.2e} may be too small for Refmac. Applying scale= {:.1f}".format(mean_f, scale))
402
+ for lab in data_labs:
403
+ hkldata.df[lab] *= scale
404
+
405
+ mtz_labs = data_labs + []
406
+ mtz_types = {}
407
+ if sig_lab: mtz_types[sig_lab] = "Q"
408
+
409
+ if blur != 0:
410
+ temp = hkldata.debye_waller_factors(b_iso=blur)
411
+ for lab in data_labs:
412
+ data = hkldata.df[lab]
413
+ newlab = lab + lab_f_suffix(blur)
414
+ if numpy.iscomplexobj(data): data = numpy.abs(data)
415
+ hkldata.df[newlab] = data * temp
416
+ mtz_labs.append(newlab)
417
+ mtz_types[newlab] = "F" if lab != sig_lab else "Q"
418
+
419
+ hkldata.write_mtz(mtz_out, labs=mtz_labs, types=mtz_types,
420
+ phase_label_decorator=lambda x: "P"+x[1:])
421
+ # write_map_mtz()
422
+
423
+ def determine_b_before_mask(st, maps, grid_start, mask, resolution):
424
+ logger.writeln("Determining b_before_mask..")
425
+ # work in masked map for the speed
426
+ new_cell, new_shape, starts, shifts = shift_maps.determine_shape_and_shift(mask=mask,
427
+ grid_start=grid_start,
428
+ padding=5,
429
+ mask_cutoff=0.5,
430
+ noncentered=True,
431
+ noncubic=True,
432
+ json_out=None)
433
+ st = st.clone()
434
+ st.cell = new_cell
435
+ st.spacegroup_hm = "P 1"
436
+ for cra in st[0].all():
437
+ cra.atom.pos += shifts
438
+ cra.atom.b_iso = 0
439
+ cra.atom.aniso = gemmi.SMat33f(0,0,0,0,0,0)
440
+
441
+ newmaps = []
442
+ for i in range(len(maps)): # Update maps
443
+ g = gemmi.FloatGrid(maps[i][0].array*mask,
444
+ maps[i][0].unit_cell, maps[i][0].spacegroup)
445
+
446
+ suba = g.get_subarray(starts, new_shape)
447
+ new_grid = gemmi.FloatGrid(suba, new_cell, st.find_spacegroup())
448
+ newmaps.append([new_grid]+maps[i][1:])
449
+
450
+ hkldata = utils.maps.mask_and_fft_maps(newmaps, resolution)
451
+ hkldata.df["FC"] = utils.model.calc_fc_fft(st, resolution - 1e-6, source="electron",
452
+ miller_array=hkldata.miller_array())
453
+ k, b = hkldata.scale_k_and_b("FC", "FP")
454
+ return -b
455
+ # determine_b_before_mask()
456
+
457
+ def process_input(st, maps, resolution, monlib, mask_in, args,
458
+ shifted_model_prefix="shifted",
459
+ output_masked_prefix="masked_fs",
460
+ output_mtz_prefix="starting_map",
461
+ use_gemmi_prep=False, no_refmac_fix=False,
462
+ use_refmac=True, find_links=False):
463
+ ret = {} # instructions for refinement
464
+ maps = utils.maps.copy_maps(maps) # not to modify maps
465
+
466
+ grid_start = maps[0][1]
467
+ unit_cell = maps[0][0].unit_cell
468
+ spacegroup = gemmi.SpaceGroup(1)
469
+ start_xyz = numpy.array(maps[0][0].get_position(*grid_start).tolist())
470
+ A = unit_cell.orth.mat.array
471
+ center = numpy.sum(A, axis=1) / 2 #+ start_xyz
472
+
473
+ # Create mask
474
+ if mask_in:
475
+ logger.writeln("Input mask file: {}".format(mask_in))
476
+ mask = utils.fileio.read_ccp4_map(mask_in)[0]
477
+ else:
478
+ mask = None
479
+
480
+ st.cell = unit_cell
481
+ st.spacegroup_hm = "P 1"
482
+ if use_refmac:
483
+ ret["model_format"] = ".mmcif" if st.input_format == gemmi.CoorFormat.Mmcif else ".pdb"
484
+ max_seq_num = max([max(res.seqid.num for res in chain) for model in st for chain in model])
485
+ if max_seq_num > 9999 and ret["model_format"] == ".pdb":
486
+ logger.writeln("Max residue number ({}) exceeds 9999. Will use mmcif format".format(max_seq_num))
487
+ ret["model_format"] = ".mmcif"
488
+
489
+ if len(st.ncs) > 0 and args.ignore_symmetry:
490
+ logger.writeln("Removing symmetry information from model.")
491
+ st.ncs.clear()
492
+ utils.symmetry.update_ncs_from_args(args, st, map_and_start=maps[0], filter_contacting=args.contacting_only)
493
+ st_expanded = st.clone()
494
+ if not all(op.given for op in st.ncs):
495
+ # symmetry is not exact in helical reconstructions
496
+ cc_cutoff = 0.9 if args.twist is None else 0.7
497
+ if not args.no_check_ncs_overlaps and utils.model.check_symmetry_related_model_duplication(st):
498
+ raise SystemExit("\nError: Too many symmetry-related contacts detected.\n"
499
+ "Please provide an asymmetric unit model along with symmetry operators.")
500
+ if not args.no_check_ncs_map and utils.maps.check_symmetry_related_map_values(st, maps[0][0], cc_cutoff=cc_cutoff):
501
+ raise SystemExit("\nError: Map correlation is too small. Please ensure your map follows the model's symmetry")
502
+ args.keywords.extend(utils.symmetry.ncs_ops_for_refmac(st.ncs))
503
+ utils.model.expand_ncs(st_expanded)
504
+ logger.writeln(" Saving expanded model: input_model_expanded.*")
505
+ utils.fileio.write_model(st_expanded, "input_model_expanded", pdb=True, cif=True)
506
+
507
+ if mask is not None and not args.no_check_mask_with_model:
508
+ if not utils.maps.test_mask_with_model(mask, st_expanded):
509
+ raise SystemExit("\nError: Model is out of mask.\n"
510
+ "Please check your --model and --mask. You can disable this test with --no_check_mask_with_model.")
511
+ if args.mask_for_fofc:
512
+ masktmp = utils.fileio.read_ccp4_map(args.mask_for_fofc)[0]
513
+ if masktmp.shape != maps[0][0].shape:
514
+ raise SystemExit("\nError: mask from --mask_for_fofc has a different shape from input map(s)")
515
+ if not args.no_check_mask_with_model and not utils.maps.test_mask_with_model(masktmp, st):
516
+ raise SystemExit("\nError: Model is out of mask.\n"
517
+ "Please check your --model and --mask_for_fofc. You can disable this test with --no_check_mask_with_model.")
518
+ del masktmp
519
+
520
+ if mask is None and args.mask_radius:
521
+ logger.writeln("Creating mask..")
522
+ mask = utils.maps.mask_from_model(st_expanded, args.mask_radius, soft_edge=args.mask_soft_edge, grid=maps[0][0])
523
+ #utils.maps.write_ccp4_map("mask_from_model.ccp4", mask)
524
+
525
+ if use_refmac:
526
+ logger.writeln(" Saving input model with unit cell information")
527
+ utils.fileio.write_model(st, "starting_model", pdb=True, cif=True)
528
+ ret["model_file"] = "starting_model" + ret["model_format"]
529
+
530
+ if mask is not None:
531
+ if args.invert_mask:
532
+ logger.writeln("Inverting mask..")
533
+ mask_max, mask_min = numpy.max(mask), numpy.min(mask)
534
+ logger.writeln(" mask_max, mask_min= {}, {}".format(mask_max, mask_min))
535
+ mask = mask_max + mask_min - mask
536
+
537
+ # Mask maps
538
+ if args.no_sharpen_before_mask or len(maps) < 2:
539
+ logger.writeln("Applying mask..")
540
+ for ma in maps: ma[0].array[:] *= mask
541
+ else:
542
+ logger.writeln("Sharpen-mask-unsharpen..")
543
+ b_before_mask = args.b_before_mask
544
+ if b_before_mask is None: b_before_mask = determine_b_before_mask(st, maps, grid_start, mask, resolution)
545
+ maps = utils.maps.sharpen_mask_unsharpen(maps, mask, resolution, b=b_before_mask)
546
+
547
+ if not args.no_trim:
548
+ logger.writeln(" Shifting maps and/or model..")
549
+ if args.padding is None: args.padding = args.mask_radius * 2
550
+ new_cell, new_shape, starts, shifts = shift_maps.determine_shape_and_shift(mask=mask,
551
+ grid_start=grid_start,
552
+ padding=args.padding,
553
+ mask_cutoff=0.5,
554
+ noncentered=True,
555
+ noncubic=True,
556
+ json_out=None)
557
+ ret["shifts"] = shifts
558
+ vol_mask = numpy.count_nonzero(mask.array>0.5)
559
+ vol_map = new_shape[0] * new_shape[1] * new_shape[2]
560
+ ret["vol_ratio"] = vol_mask / vol_map
561
+ logger.writeln(" Vol_mask/Vol_map= {:.2e}".format(ret["vol_ratio"]))
562
+
563
+ # Model may be built out of the box (with 'unit cell' translation symmetry)
564
+ # It is only valid with original unit cell, but no longer valid with the new cell
565
+ # It still would not work if model is built over multiple 'cells'.
566
+ extra_shift = utils.model.translate_into_box(st,
567
+ origin=gemmi.Position(*start_xyz),
568
+ apply_shift=False)
569
+ if numpy.linalg.norm(extra_shift) > 0:
570
+ logger.writeln("Input model is out of the box. Required shift= {}".format(extra_shift))
571
+ ret["shifts"] += gemmi.Position(*extra_shift)
572
+ logger.writeln("Shift for model has been adjusted: {}".format(numpy.array(ret["shifts"].tolist())))
573
+
574
+ st.cell = new_cell
575
+ st.spacegroup_hm = "P 1"
576
+ if use_refmac:
577
+ logger.writeln(" Saving model in trimmed map..")
578
+ utils.fileio.write_model(st, shifted_model_prefix, pdb=True, cif=True)
579
+ ret["model_file"] = shifted_model_prefix + ret["model_format"]
580
+
581
+ logger.writeln(" Trimming maps..")
582
+ for i in range(len(maps)): # Update maps
583
+ suba = maps[i][0].get_subarray(starts, new_shape)
584
+ new_grid = gemmi.FloatGrid(suba, new_cell, spacegroup)
585
+ maps[i][0] = new_grid
586
+
587
+ st.setup_cell_images()
588
+ utils.restraints.find_and_fix_links(st, monlib, add_found=find_links,
589
+ # link via ncsc is not supported as of Refmac5.8.0411
590
+ find_symmetry_related=not use_refmac)
591
+ # workaround for Refmac
592
+ # TODO need to check external restraints
593
+ if use_refmac:
594
+ if use_gemmi_prep:
595
+ h_change = {"all":gemmi.HydrogenChange.ReAddButWater,
596
+ "yes":gemmi.HydrogenChange.NoChange,
597
+ "no":gemmi.HydrogenChange.Remove}[args.hydrogen]
598
+ topo, metal_kws = utils.restraints.prepare_topology(st, monlib, h_change=h_change, raise_error=False)
599
+ args.keywords = metal_kws + args.keywords
600
+ elif not no_refmac_fix:
601
+ topo = gemmi.prepare_topology(st, monlib, warnings=logger.silent(), ignore_unknown_links=True)
602
+ else:
603
+ topo = None # not used
604
+ if not no_refmac_fix:
605
+ ret["refmac_fixes"] = utils.refmac.FixForRefmac()
606
+ ret["refmac_fixes"].fix_before_topology(st, topo,
607
+ fix_microheterogeneity=not args.no_fix_microheterogeneity and not use_gemmi_prep,
608
+ fix_resimax=not args.no_fix_resi9999,
609
+ fix_nonpolymer=False)
610
+ chain_id_len_max = max([len(x) for x in utils.model.all_chain_ids(st)])
611
+ if chain_id_len_max > 1 and ret["model_format"] == ".pdb":
612
+ logger.writeln("Long chain ID (length: {}) detected. Will use mmcif format".format(chain_id_len_max))
613
+ ret["model_format"] = ".mmcif"
614
+ if not no_refmac_fix and ret["model_format"] == ".mmcif" and not use_gemmi_prep:
615
+ ret["refmac_fixes"].fix_nonpolymer(st)
616
+
617
+ if use_refmac and use_gemmi_prep:
618
+ # TODO: make cispept, make link, remove unknown link id
619
+ # TODO: cross validation?
620
+ crdout = os.path.splitext(ret["model_file"])[0] + ".crd"
621
+ ret["model_file"] = crdout
622
+ ret["model_format"] = ".mmcif"
623
+ args.keywords.append("make cr prepared")
624
+ gemmi.setup_for_crd(st)
625
+ doc = gemmi.prepare_refmac_crd(st, topo, monlib, h_change)
626
+ doc.write_file(crdout, options=gemmi.cif.Style.NoBlankLines)
627
+ logger.writeln("crd file written: {}".format(crdout))
628
+
629
+ hkldata = utils.maps.mask_and_fft_maps(maps, resolution, None, with_000=False)
630
+ hkldata.setup_relion_binning()
631
+ if len(maps) == 2:
632
+ map_labs = ["Fmap1", "Fmap2", "Fout"]
633
+ ret["lab_f_half1"] = "Fmap1" + lab_f_suffix(args.blur)
634
+ # TODO Add SIGF in case of half maps, when refmac is ready
635
+ ret["lab_phi_half1"] = "Pmap1"
636
+ ret["lab_f_half2"] = "Fmap2" + lab_f_suffix(args.blur)
637
+ ret["lab_phi_half2"] = "Pmap2"
638
+ utils.maps.calc_noise_var_from_halfmaps(hkldata)
639
+ d_eff_full = hkldata.d_eff("FSCfull")
640
+ logger.writeln("Effective resolution from FSCfull= {:.2f}".format(d_eff_full))
641
+ ret["d_eff"] = d_eff_full
642
+ else:
643
+ map_labs = ["Fout"]
644
+ sig_lab = None
645
+
646
+ if use_refmac:
647
+ if args.no_mask:
648
+ logger.writeln("Saving unmasked maps as mtz file..")
649
+ mtzout = output_mtz_prefix+".mtz"
650
+ else:
651
+ logger.writeln(" Saving masked maps as mtz file..")
652
+ mtzout = output_masked_prefix+"_obs.mtz"
653
+
654
+ hkldata.df.rename(columns=dict(F_map1="Fmap1", F_map2="Fmap2", FP="Fout"), inplace=True)
655
+ if "shifts" in ret:
656
+ for lab in map_labs: # apply phase shift
657
+ logger.writeln(" applying phase shift for {} with translation {}".format(lab, -ret["shifts"]))
658
+ hkldata.translate(lab, -ret["shifts"])
659
+
660
+ write_map_mtz(hkldata, mtzout, map_labs=map_labs, blur=args.blur)
661
+ ret["mtz_file"] = mtzout
662
+ ret["lab_f"] = "Fout" + lab_f_suffix(args.blur)
663
+ ret["lab_phi"] = "Pout"
664
+ else:
665
+ fac = hkldata.debye_waller_factors(b_iso=args.blur)
666
+ if "shifts" in ret: fac *= hkldata.translation_factor(-ret["shifts"])
667
+ for lab in ("F_map1", "F_map2", "FP"):
668
+ if lab in hkldata.df: hkldata.df[lab] *= fac
669
+ return hkldata, ret
670
+ # process_input()
671
+
672
+ def check_args(args):
673
+ if not os.path.exists(args.model):
674
+ raise SystemExit("Error: --model {} does not exist.".format(args.model))
675
+
676
+ if args.cross_validation and not args.halfmaps:
677
+ raise SystemExit("Error: half maps are needed when --cross_validation is given")
678
+
679
+ if args.mask_for_fofc and not os.path.exists(args.mask_for_fofc):
680
+ raise SystemExit("Error: --mask_for_fofc {} does not exist".format(args.mask_for_fofc))
681
+
682
+ if args.mask_for_fofc and args.mask_radius_for_fofc:
683
+ raise SystemExit("Error: you cannot specify both --mask_for_fofc and --mask_radius_for_fofc")
684
+
685
+ if args.trim_fofc_mtz and not (args.mask_for_fofc or args.mask_radius_for_fofc):
686
+ raise SystemExit("Error: --trim_fofc_mtz is specified but --mask_for_fofc is not given")
687
+
688
+ if args.ligand: args.ligand = sum(args.ligand, [])
689
+
690
+ if args.keywords:
691
+ args.keywords = sum(args.keywords, [])
692
+ else:
693
+ args.keywords = []
694
+
695
+ if args.keyword_file:
696
+ args.keyword_file = sum(args.keyword_file, [])
697
+ for f in args.keyword_file:
698
+ if not os.path.exists(f):
699
+ raise SystemExit(f"Error: keyword file was not found: {f}")
700
+ logger.writeln("Keyword file: {}".format(f))
701
+ else:
702
+ args.keyword_file = []
703
+
704
+ if (args.twist, args.rise).count(None) == 1:
705
+ raise SystemExit("ERROR: give both helical parameters --twist and --rise")
706
+ if args.twist is not None:
707
+ logger.writeln("INFO: setting --contacting_only because helical symmetry is given")
708
+ args.contacting_only = True
709
+ if args.no_mask:
710
+ args.mask_radius = None
711
+ if not args.no_trim:
712
+ logger.writeln("WARNING: setting --no_trim because --no_mask is given")
713
+ args.no_trim = True
714
+ if args.mask:
715
+ logger.writeln("WARNING: Your --mask is ignored because --no_mask is given")
716
+ args.mask = None
717
+
718
+ #if args.mask_soft_edge > 0:
719
+ # logger.writeln("INFO: --mask_soft_edge={} is given. Turning off sharpen_before_mask.".format(args.mask_soft_edge))
720
+ # args.no_sharpen_before_mask = True
721
+
722
+ if args.resolution is None and args.model and utils.fileio.splitext(args.model)[1].endswith("cif"):
723
+ doc = gemmi.cif.read(args.model)
724
+ if len(doc) != 1:
725
+ raise SystemExit("cannot find resolution from cif. Give --resolution")
726
+ block = doc.sole_block()
727
+ reso_str = block.find_value("_em_3d_reconstruction.resolution")
728
+ try:
729
+ args.resolution = float(reso_str)
730
+ except:
731
+ raise SystemExit("ERROR: _em_3d_reconstruction.resolution is invalid. Give --resolution")
732
+ logger.writeln("WARNING: --resolution not given. Using _em_3d_reconstruction.resolution = {}".format(reso_str))
733
+
734
+ if args.resolution is None:
735
+ raise SystemExit("ERROR: --resolution is needed.")
736
+ # check_args()
737
+
738
+ def main(args):
739
+ check_args(args)
740
+ use_gemmi_prep = False
741
+ if not args.prepare_only:
742
+ refmac_ver = utils.refmac.check_version(args.exe)
743
+ if not refmac_ver:
744
+ raise SystemExit("Error: Check Refmac installation or use --exe to give the location.")
745
+ if not args.no_refmacat and refmac_ver >= (5, 8, 404):
746
+ logger.writeln(" will use gemmi to prepare restraints")
747
+ use_gemmi_prep = True
748
+ else:
749
+ logger.writeln(" will use makecif to prepare restraints")
750
+
751
+ logger.writeln("Input model: {}".format(args.model))
752
+ st = utils.fileio.read_structure(args.model)
753
+ if len(st) > 1:
754
+ logger.writeln(" Removing models 2-{}".format(len(st)))
755
+ for i in reversed(range(1, len(st))):
756
+ del st[i]
757
+
758
+ try:
759
+ monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand,
760
+ stop_for_unknowns=True)
761
+ except RuntimeError as e:
762
+ raise SystemExit("Error: {}".format(e))
763
+
764
+ utils.model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
765
+ try:
766
+ utils.restraints.prepare_topology(st.clone(), monlib, h_change=gemmi.HydrogenChange.NoChange,
767
+ check_hydrogen=(args.hydrogen=="yes"))
768
+ except RuntimeError as e:
769
+ raise SystemExit("Error: {}".format(e))
770
+
771
+ if args.halfmaps:
772
+ maps = utils.fileio.read_halfmaps(args.halfmaps, pixel_size=args.pixel_size)
773
+ else:
774
+ maps = [utils.fileio.read_ccp4_map(args.map, pixel_size=args.pixel_size)]
775
+
776
+ utils.model.remove_charge([st])
777
+ shifted_model_prefix = "shifted"
778
+ _, file_info = process_input(st, maps, resolution=args.resolution - 1e-6, monlib=monlib,
779
+ mask_in=args.mask, args=args,
780
+ shifted_model_prefix=shifted_model_prefix,
781
+ use_gemmi_prep=use_gemmi_prep,
782
+ find_links=args.find_links)
783
+ if args.prepare_only:
784
+ logger.writeln("\n--prepare_only is given. Stopping.")
785
+ return
786
+
787
+ args.mtz = file_info["mtz_file"]
788
+ if args.halfmaps: # FIXME if no_mask?
789
+ args.mtz_half = [file_info["mtz_file"], file_info["mtz_file"]]
790
+ args.lab_phi = file_info["lab_phi"] #"Pout0"
791
+ args.lab_f = file_info["lab_f"]
792
+ args.lab_sigf = None
793
+ args.model = file_info["model_file"] # refmac xyzin
794
+ model_format = file_info["model_format"]
795
+
796
+ if args.cross_validation and args.cross_validation_method == "throughout":
797
+ args.lab_f = file_info["lab_f_half1"]
798
+ args.lab_phi = file_info["lab_phi_half1"]
799
+ # XXX args.lab_sigf?
800
+
801
+ chain_id_lens = [len(x) for x in utils.model.all_chain_ids(st)]
802
+ keep_chain_ids = (chain_id_lens and max(chain_id_lens) == 1) # always kept unless one-letter chain IDs
803
+
804
+ # FIXME if mtz is given and sfcalc() not ran?
805
+
806
+ if not args.no_trim:
807
+ refmac_prefix = "{}_{}".format(shifted_model_prefix, args.output_prefix)
808
+ else:
809
+ refmac_prefix = args.output_prefix # XXX this should be different name (nomask etc?)
810
+
811
+ # Weight auto scale
812
+ if args.weight_auto_scale is None:
813
+ reso = file_info["d_eff"] if "d_eff" in file_info else args.resolution
814
+ if "vol_ratio" in file_info:
815
+ if "d_eff" in file_info:
816
+ rlmc = (-9.503541, 3.129882, 15.439744)
817
+ else:
818
+ rlmc = (-8.329418, 3.032409, 14.381907)
819
+ logger.writeln("Estimating weight auto scale using resolution and volume ratio")
820
+ ws = rlmc[0] + reso*rlmc[1] +file_info["vol_ratio"]*rlmc[2]
821
+ else:
822
+ if "d_eff" in file_info:
823
+ rlmc = (-5.903807, 2.870723)
824
+ else:
825
+ rlmc = (-4.891140, 2.746791)
826
+ logger.writeln("Estimating weight auto scale using resolution")
827
+ ws = rlmc[0] + args.resolution*rlmc[1]
828
+ args.weight_auto_scale = max(0.2, min(18.0, ws))
829
+ logger.writeln(" Will use weight auto {:.2f}".format(args.weight_auto_scale))
830
+
831
+ # Run Refmac
832
+ refmac = utils.refmac.Refmac(prefix=refmac_prefix, args=args, global_mode="spa",
833
+ keep_chain_ids=keep_chain_ids)
834
+ refmac.set_libin(args.ligand)
835
+ try:
836
+ refmac_summary = refmac.run_refmac()
837
+ except RuntimeError as e:
838
+ raise SystemExit("Error: {}".format(e))
839
+
840
+ # Modify output
841
+ st, cif_ref = utils.fileio.read_structure_from_pdb_and_mmcif(refmac_prefix+model_format)
842
+ utils.model.setup_entities(st, clear=True, overwrite_entity_type=True, force_subchain_names=True)
843
+
844
+ if not args.no_trim:
845
+ st.cell = maps[0][0].unit_cell
846
+ st.setup_cell_images()
847
+ if "refmac_fixes" in file_info:
848
+ file_info["refmac_fixes"].modify_back(st)
849
+ utils.model.adp_analysis(st)
850
+ utils.fileio.write_model(st, prefix=args.output_prefix,
851
+ pdb=True, cif=True, cif_ref=cif_ref)
852
+
853
+ # Take care of TLS out
854
+ if not args.no_trim: # if no_trim, there is nothing to do.
855
+ tlsout = refmac.tlsout()
856
+ if os.path.exists(tlsout):
857
+ logger.writeln("Copying tlsout")
858
+ shutil.copyfile(refmac.tlsout(), args.output_prefix+".tls")
859
+
860
+ # Expand sym here
861
+ st_expanded = st.clone()
862
+ if not all(op.given for op in st.ncs):
863
+ utils.model.expand_ncs(st_expanded)
864
+ utils.fileio.write_model(st_expanded, args.output_prefix+"_expanded", pdb=True, cif=True,
865
+ cif_ref=cif_ref)
866
+
867
+ if args.cross_validation and args.cross_validation_method == "shake":
868
+ logger.writeln("Cross validation is requested.")
869
+ refmac_prefix_shaken = refmac_prefix+"_shaken_refined"
870
+ logger.writeln("Starting refinement using half map 1 (model is shaken first)")
871
+ logger.writeln("In this refinement, hydrogen is removed regardless of --hydrogen option")
872
+ if use_gemmi_prep:
873
+ xyzin = refmac_prefix + ".crd"
874
+ st_tmp = utils.fileio.read_structure(refmac_prefix+model_format)
875
+ utils.model.setup_entities(st_tmp, clear=True, overwrite_entity_type=True, force_subchain_names=True)
876
+ prepare_crd(st_tmp,
877
+ crdout=xyzin, ligand=[refmac_prefix+model_format],
878
+ make={"hydr":"n"},
879
+ fix_long_resnames=False) # we do not need output file - do we?
880
+ else:
881
+ xyzin = refmac_prefix + model_format
882
+ refmac_hm1 = refmac.copy(hklin=args.mtz_half[0],
883
+ xyzin=xyzin,
884
+ prefix=refmac_prefix_shaken,
885
+ shake=args.shake_radius,
886
+ jellybody=False, # makes no sense to use jelly body after shaking
887
+ hydrogen="no") # should not use hydrogen after shaking
888
+ if args.jellybody: logger.writeln(" Turning off jelly body")
889
+ if "lab_f_half1" in file_info:
890
+ refmac_hm1.lab_f = file_info["lab_f_half1"]
891
+ refmac_hm1.lab_phi = file_info["lab_phi_half1"]
892
+ # SIGMA?
893
+
894
+ try:
895
+ refmac_hm1.run_refmac()
896
+ except RuntimeError as e:
897
+ raise SystemExit("Error: {}".format(e))
898
+
899
+ if args.hydrogen != "no": # does not work properly when 'yes' - we would need to keep hydrogen in input
900
+ logger.writeln("Cross validation: 2nd run with hydrogen")
901
+ if use_gemmi_prep:
902
+ xyzin = refmac_prefix_shaken + ".crd"
903
+ st_tmp = utils.fileio.read_structure(refmac_prefix_shaken+model_format)
904
+ utils.model.setup_entities(st_tmp, clear=True, overwrite_entity_type=True, force_subchain_names=True)
905
+ prepare_crd(st_tmp,
906
+ crdout=xyzin, ligand=[refmac_prefix+model_format],
907
+ make={"hydr":"a"},
908
+ fix_long_resnames=False) # we do not need output file - do we?
909
+ else:
910
+ xyzin = refmac_prefix_shaken + model_format
911
+ refmac_prefix_shaken = refmac_prefix+"_shaken_refined2"
912
+ refmac_hm1_2 = refmac_hm1.copy(xyzin=xyzin,
913
+ prefix=refmac_prefix_shaken,
914
+ shake=None,
915
+ hydrogen="all")
916
+ try:
917
+ refmac_hm1_2.run_refmac()
918
+ except RuntimeError as e:
919
+ raise SystemExit("Error: {}".format(e))
920
+
921
+ # Modify output
922
+ st_sr, cif_ref_sr = utils.fileio.read_structure_from_pdb_and_mmcif(refmac_prefix_shaken+model_format)
923
+ utils.model.setup_entities(st_sr, clear=True, overwrite_entity_type=True, force_subchain_names=True)
924
+ if not args.no_trim:
925
+ st_sr.cell = maps[0][0].unit_cell
926
+ st_sr.setup_cell_images()
927
+ if "refmac_fixes" in file_info:
928
+ file_info["refmac_fixes"].modify_back(st_sr)
929
+
930
+ utils.fileio.write_model(st_sr, prefix=args.output_prefix+"_shaken_refined",
931
+ pdb=True, cif=True, cif_ref=cif_ref_sr)
932
+
933
+ # Expand sym here
934
+ st_sr_expanded = st_sr.clone()
935
+ if not all(op.given for op in st_sr.ncs):
936
+ utils.model.expand_ncs(st_sr_expanded)
937
+ utils.fileio.write_model(st_sr_expanded, args.output_prefix+"_shaken_refined_expanded",
938
+ pdb=True, cif=True, cif_ref=cif_ref_sr)
939
+ if args.twist is not None: # as requested by a user
940
+ st_sr_expanded_all = st_sr.clone()
941
+ utils.symmetry.update_ncs_from_args(args, st_sr_expanded_all, map_and_start=maps[0], filter_contacting=False)
942
+ utils.model.expand_ncs(st_sr_expanded_all)
943
+ utils.fileio.write_model(st_sr_expanded_all, args.output_prefix+"_shaken_refined_expanded_all", pdb=True, cif=True,
944
+ cif_ref=cif_ref)
945
+ else:
946
+ st_sr_expanded = None
947
+
948
+ if args.mask:
949
+ mask = utils.fileio.read_ccp4_map(args.mask)[0]
950
+ else:
951
+ mask = None
952
+
953
+ # Calc FSC
954
+ fscavg_text = calc_fsc(st_expanded, args.output_prefix, maps,
955
+ args.resolution, mask=mask, mask_radius=args.mask_radius if not args.no_mask else None,
956
+ soft_edge=args.mask_soft_edge,
957
+ b_before_mask=args.b_before_mask,
958
+ no_sharpen_before_mask=args.no_sharpen_before_mask,
959
+ make_hydrogen=args.hydrogen,
960
+ monlib=monlib, cross_validation=args.cross_validation,
961
+ blur=args.blur,
962
+ d_min_fsc=args.fsc_resolution,
963
+ cross_validation_method=args.cross_validation_method, st_sr=st_sr_expanded)[0]
964
+
965
+ # Calc Fo-Fc (and updated) maps
966
+ calc_fofc(st, st_expanded, maps, monlib, model_format, args)
967
+
968
+ # Final summary
969
+ write_final_summary(st, refmac_summary, fscavg_text, args.output_prefix,
970
+ args.mask_for_fofc or args.mask_radius_for_fofc)
971
+ # main()
972
+
973
+ if __name__ == "__main__":
974
+ import sys
975
+ args = parse_args(sys.argv[1:])
976
+ main(args)
977
+