servalcat 0.4.88__cp312-cp312-macosx_11_0_arm64.whl → 0.4.100__cp312-cp312-macosx_11_0_arm64.whl

servalcat/refine/spa.py

@@ -38,8 +38,14 @@ class LL_SPA:
         self.lab_obs = lab_obs
         self.st = st
         self.monlib = monlib
-        self.d_min = hkldata.d_min_max()[0]
+        self.d_min_max = hkldata.d_min_max()
         self.ll = None
+        self.b_aniso = None
+
+    def refine_id(self):
+        if self.source == "electron":
+            return "ELECTRON MICROSCOPY"
+        return "NON-EM SPA" # does not happen, I guess
 
     def update_ml_params(self):
         # FIXME make sure D > 0
@@ -52,7 +58,7 @@ class LL_SPA:
         else:
             st = self.st
 
-        self.hkldata.df["FC"] = utils.model.calc_fc_fft(st, self.d_min - 1e-6,
+        self.hkldata.df["FC"] = utils.model.calc_fc_fft(st, self.d_min_max[0] - 1e-6,
                                                         monlib=self.monlib,
                                                         source=self.source,
                                                         mott_bethe=self.mott_bethe,
@@ -92,15 +98,17 @@ class LL_SPA:
         stats = fsc.calc_fsc_all(self.hkldata, labs_fc=["FC"], lab_f=self.lab_obs)
         fsca = fsc.fsc_average(stats.ncoeffs, stats.fsc_FC_full)
         logger.writeln("FSCaverage = {:.4f}".format(fsca))
+        ret = {"summary": {"FSCaverage": fsca, "-LL": self.calc_target()}}
         # XXX in fsc object, _full is misleading - it's not full in cross validation mode
         if "D" in self.hkldata.binned_df and "S" in self.hkldata.binned_df:
             stats[["D", "S"]] = self.hkldata.binned_df[["D", "S"]]
-        return {"bin_stats": stats, "summary": {"FSCaverage": fsca, "-LL": self.calc_target()}}
+        ret["bin_stats"] = stats
+        return ret
 
     def calc_grad(self, atom_pos, refine_xyz, adp_mode, refine_occ, refine_h, specs):
         dll_dab = numpy.empty_like(self.hkldata.df[self.lab_obs])
         d2ll_dab2 = numpy.zeros(len(self.hkldata.df.index))
-        blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min / 3) # TODO need more work
+        blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min_max[0] / 3) # TODO need more work
         logger.writeln("blur for deriv= {:.2f}".format(blur))
         for i_bin, idxes in self.hkldata.binned():
             D = self.hkldata.binned_df.D[i_bin]

servalcat/refine/xtal.py

@@ -30,7 +30,7 @@ class LL_Xtal:
         self.free = free
         self.st = st
         self.monlib = monlib
-        self.d_min = hkldata.d_min_max()[0]
+        self.d_min_max = hkldata.d_min_max()
         self.fc_labs = ["FC0"]
         self.use_solvent = use_solvent
         if use_solvent:
@@ -45,7 +45,7 @@ class LL_Xtal:
         self.ll = None
         self.scaling = sigmaa.LsqScale()
         if twin:
-            self.twin_data = find_twin_domains_from_data(self.hkldata)
+            self.twin_data, _ = find_twin_domains_from_data(self.hkldata)
         else:
             self.twin_data = None
         if self.twin_data:
@@ -54,6 +54,9 @@ class LL_Xtal:
         logger.writeln("will use {} reflections for parameter estimation".format(self.use_in_est))
         logger.writeln("will use {} reflections for refinement".format(self.use_in_target))
 
+    def refine_id(self):
+        return {"xray": "X-RAY", "electron": "ELECTRON", "neutron": "NEUTRON"}.get(self.source, "") + " DIFFRACTION"
+
     def update_ml_params(self):
         self.b_aniso = sigmaa.determine_ml_params(self.hkldata, self.is_int, self.fc_labs, self.D_labs, self.b_aniso,
                                                   self.centric_and_selections, use=self.use_in_est,
@@ -63,7 +66,7 @@ class LL_Xtal:
         #                             self.centric_and_selections)
     def update_fc(self):
         sigmaa.update_fc(st_list=[self.st], fc_labs=self.fc_labs,
-                         d_min=self.d_min, monlib=self.monlib,
+                         d_min=self.d_min_max[0], monlib=self.monlib,
                          source=self.source, mott_bethe=self.mott_bethe,
                          hkldata=self.hkldata, twin_data=self.twin_data)
 
@@ -78,20 +81,20 @@ class LL_Xtal:
                                            for sel in self.centric_and_selections[i_bin]])
             mask = numpy.empty(len(self.hkldata.df.index)) * numpy.nan
             mask[idxes] = 1 / self.hkldata.debye_waller_factors(b_cart=self.b_aniso)[idxes]**2
-            self.twin_data.est_f_true(self.hkldata.df.I * mask,
-                                      self.hkldata.df.SIGI * mask)
+            self.twin_data.est_f_true(self.hkldata.df.I.to_numpy() * mask,
+                                      self.hkldata.df.SIGI.to_numpy() * mask)
         
     def overall_scale(self, min_b=0.1):
         miller_array = self.twin_data.asu if self.twin_data else self.hkldata.miller_array()
         if self.use_solvent:
-            Fmask = sigmaa.calc_Fmask(self.st, self.d_min, miller_array)
+            Fmask = sigmaa.calc_Fmask(self.st, self.d_min_max[0], miller_array)
             if self.twin_data:
                 fc_sum = self.twin_data.f_calc[:,:-1].sum(axis=1)
             else:
                 fc_sum = self.hkldata.df[self.fc_labs[:-1]].sum(axis=1).to_numpy()
             fc_list = [fc_sum, Fmask]
         else:
-            if twin_data:
+            if self.twin_data:
                 fc_list = [self.twin_data.f_calc.sum(axis=1)]
             else:
                 fc_list = [self.hkldata.df[self.fc_labs].sum(axis=1).to_numpy()]
@@ -177,7 +180,7 @@ class LL_Xtal:
         return ret
 
     def calc_grad(self, atom_pos, refine_xyz, adp_mode, refine_occ, refine_h, specs=None):
-        blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min / 3) # TODO need more work
+        blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min_max[0] / 3) # TODO need more work
         logger.writeln("blur for deriv= {:.2f}".format(blur))
         if self.twin_data:
             dll_dab, d2ll_dab2 = self.twin_data.ll_der_fc0()
@@ -206,8 +209,8 @@ class LL_Xtal:
                         to = Io[cidxes] / sigIo[cidxes] - sigIo[cidxes] / (c+1) / k_ani[cidxes]**2 / S / epsilon
                         tf = k_ani[cidxes] * Fc_abs / numpy.sqrt(sigIo[cidxes])
                         sig1 = k_ani[cidxes]**2 * epsilon * S / sigIo[cidxes]
-                        k_num = 0.5 if c == 0 else 0. # acentric:0.5, centric: 0.
-                        r = ext.integ_J_ratio(k_num, k_num - 0.5, True, to, tf, sig1, c+1,
+                        k_num = numpy.repeat(0.5 if c == 0 else 0., to.size) # acentric:0.5, centric: 0.
+                        r = ext.integ_J_ratio(k_num, k_num - 0.5, True, to, tf, sig1, numpy.repeat(c+1, to.size),
                                               integr.exp2_threshold, integr.h, integr.N, integr.ewmax)
                         r *= numpy.sqrt(sigIo[cidxes]) / k_ani[cidxes]
                         g = (2-c) * (Fc_abs - r) / epsilon / S  * Ds[:,0]
@@ -232,11 +235,11 @@ class LL_Xtal:
                             m = numpy.tanh(X)
                             g = (Fc_abs - m * Fo) / Sigma * Ds[:,0]
                             dll_dab[cidxes] = g * expip
-                            d2ll_dab2[cidxes] = (1. / Sigma - (Fo / (Sigma * numpy.cosh(X)))**2) * Ds[:,0]**2
+                            d2ll_dab2[cidxes] = (1. / Sigma - (Fo / Sigma)**2 * (1. - m**2)) * Ds[:,0]**2
             dll_dab *= self.hkldata.debye_waller_factors(b_iso=-blur)
 
         if self.mott_bethe:
-            d2 = 1 / self.twin_data.s2_array if self.twin_data else self.hkldata.d_spacings()**2
+            d2 = numpy.reciprocal(self.twin_data.s2_array) if self.twin_data else self.hkldata.d_spacings()**2
             dll_dab *= d2 * gemmi.mott_bethe_const()
             d2ll_dab2 *= gemmi.mott_bethe_const()**2
 

servalcat/refmac/refmac_wrapper.py

@@ -11,7 +11,6 @@ import numpy
 import json
 import os
 import sys
-import io
 import tempfile
 import subprocess
 import argparse
@@ -126,8 +125,7 @@ def prepare_crd(st, crdout, ligand, make, monlib_path=None, h_pos="elec",
     max_seq_num = max([max(res.seqid.num for res in chain) for model in st for chain in model])
     if max_seq_num > 9999:
         logger.writeln("Max residue number ({}) exceeds 9999. Needs workaround.".format(max_seq_num))
-        sio = io.StringIO()
-        topo = gemmi.prepare_topology(st, monlib, warnings=sio, ignore_unknown_links=True)
+        topo = gemmi.prepare_topology(st, monlib, warnings=logger.silent(), ignore_unknown_links=True)
         refmac_fixes.fix_before_topology(st, topo, 
                                          fix_microheterogeneity=False,
                                          fix_resimax=True,
@@ -135,8 +133,7 @@ def prepare_crd(st, crdout, ligand, make, monlib_path=None, h_pos="elec",
 
     if unre:
         # Refmac5 does not seem to do anything to hydrogen when unre regardless of "make hydr"
-        sio = io.StringIO()
-        topo = gemmi.prepare_topology(st, monlib, warnings=sio, ignore_unknown_links=True)
+        topo = gemmi.prepare_topology(st, monlib, warnings=logger.silent(), ignore_unknown_links=True)
         metal_kws = []
     else:
         if make.get("hydr") == "a": logger.writeln("(re)generating hydrogen atoms")
@@ -194,7 +191,7 @@ def prepare_crd(st, crdout, ligand, make, monlib_path=None, h_pos="elec",
         if st.name.lower() in block_names:
             st.name = st.name + str(i)
     doc = gemmi.prepare_refmac_crd(st, topo, monlib, h_change)
-    doc.write_file(crdout, style=gemmi.cif.Style.NoBlankLines)
+    doc.write_file(crdout, options=gemmi.cif.Style.NoBlankLines)
     logger.writeln("crd file written: {}".format(crdout))
     return refmac_fixes, [x+"\n" for x in metal_kws]
 # prepare_crd()
@@ -276,11 +273,6 @@ def modify_output(pdbout, cifout, fixes, hout, cispeps, keep_original_output=Fal
         logger.writeln("This structure cannot be saved as an official PDB format. Using hybrid-36. Header part may be inaccurate.")
     if not hout:
         st.remove_hydrogens() # remove hydrogen from pdb, while kept in mmcif
-    # Use short name in pdb
-    st.shorten_ccd_codes()
-    if st.shortened_ccd_codes:
-        msg = " ".join("{}->{}".format(o,n) for o,n in st.shortened_ccd_codes)
-        logger.writeln("Using shortened residue names in the output pdb file: " + msg)
     os.rename(pdbout, pdbout + suffix)
     utils.fileio.write_pdb(st, pdbout)
     if not keep_original_output:

servalcat/spa/fofc.py

@@ -44,9 +44,11 @@ def add_arguments(parser):
     parser.add_argument("--monlib",
                         help="Monomer library path. Default: $CLIBD_MON")
     parser.add_argument("--omit_proton", action='store_true',
-                        help="Omit proton from model in map calculation")
+                        #help="Omit hydrogen proton (leaving electrons) from model in map calculation")
+                        help=argparse.SUPPRESS)
     parser.add_argument("--omit_h_electron", action='store_true',
-                        help="Omit hydrogen electrons from model in map calculation")
+                        #help="Omit hydrogen electrons (leaving protons) from model in map calculation")
+                        help=argparse.SUPPRESS)
     parser.add_argument("--source", choices=["electron", "xray", "neutron"], default="electron")
     parser.add_argument('-o','--output_prefix', default="diffmap",
                         help='output file name prefix (default: %(default)s)')
@@ -464,7 +466,11 @@ def main(args):
     logger.writeln("coot --script " + py_out)
     if mask is not None:
         logger.writeln("\nWant to list Fo-Fc map peaks? Try:")
-        logger.writeln("servalcat util map_peaks --map {}_normalized_fofc.mrc --model {} --abs_level 4.0".format(args.output_prefix, args.model))
+        if omit_h_electron:
+            logger.writeln("servalcat util map_peaks --map {}_normalized_fofc_flipsign.mrc --model {} --abs_level 4.0".format(args.output_prefix, args.model))
+        else:
+            logger.writeln("servalcat util map_peaks --map {}_normalized_fofc.mrc --model {} --abs_level 4.0".format(args.output_prefix, args.model))
+
 # main()
 
 if __name__ == "__main__":

servalcat/spa/fsc.py

@@ -20,13 +20,14 @@ def add_arguments(parser):
 
     parser.add_argument('--model',
                         help="")
-    parser.add_argument('--map',
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument('--map',
                         help='Input map file(s)')
-    parser.add_argument('--mtz',
+    group.add_argument("--halfmaps",  nargs=2)
+    group.add_argument('--mtz',
                         help='Input mtz file.')
     parser.add_argument('--labin', nargs=2,
                         help='label (F and PHI) for mtz')
-    parser.add_argument("--halfmaps",  nargs=2)
     parser.add_argument('--pixel_size', type=float,
                         help='Override pixel size (A)')
     parser.add_argument('--mask', help='Mask file')
@@ -68,13 +69,10 @@ def write_loggraph(stats, labs_fc, log_out):
     model_labs1 = [l for l in stats if any(l.startswith("fsc_"+fc) for fc in labs_fc)]
     model_labs2 = [l for l in stats if any(l.startswith(("cc_"+fc, "mcos_"+fc)) for fc in labs_fc)]
     power_labs = [l for l in stats if l.startswith("power_")]
-    half_labs1 = ["fsc_half_unmasked", "fsc_half_masked", "fsc_half_masked_rand", "fsc_half_masked_corrected"]
-    half_labs2 = ["cc_half", "mcos_half"]
-    if not all(l in stats for l in half_labs1):
-        if "fsc_half" in stats:
-            half_labs1 = ["fsc_half"]
-        else:
-            half_labs1 = []
+    half_labs1 = [l for l in ("fsc_half_unmasked", "fsc_half_masked", "fsc_half_masked_rand", "fsc_half_masked_corrected") if l in stats]
+    half_labs2 = [l for l in ("cc_half", "mcos_half") if l in stats]
+    if not half_labs1 and "fsc_half" in stats:
+        half_labs1 = ["fsc_half"]
 
     stats2 = stats.copy()
     stats2.insert(0, "bin", stats.index)

servalcat/spa/run_refmac.py

@@ -10,7 +10,6 @@ import gemmi
 import numpy
 import json
 import os
-import io
 import shutil
 import argparse
 from servalcat.utils import logger
@@ -58,7 +57,7 @@ def add_arguments(parser):
                         help='Sharpening or blurring B')
     utils.symmetry.add_symmetry_args(parser) # add --pg etc
     parser.add_argument('--contacting_only', action="store_true", help="Filter out non-contacting NCS")
-    parser.add_argument('--ignore_symmetry',
+    parser.add_argument('--ignore_symmetry', action='store_true',
                         help='Ignore symmetry information (MTRIX/_struct_ncs_oper) in the model file')
     parser.add_argument('--find_links', action='store_true', 
                         help='Automatically add links')
@@ -150,6 +149,7 @@ def calc_fsc(st, output_prefix, maps, d_min, mask, mask_radius, soft_edge, b_bef
         assert st_sr is None
     
     logger.writeln("Calculating map-model FSC..")
+    ret = {"summary": {}}
 
     if d_min_fsc is None:
         d_min_fsc = utils.maps.nyquist_resolution(maps[0][0])
@@ -220,12 +220,17 @@ def calc_fsc(st, output_prefix, maps, d_min, mask, mask_radius, soft_edge, b_bef
             s.drop(columns=[x for x in s if x.startswith("fsc_FC") and x.endswith(("half1","half2"))], inplace=True)
 
     # FSCaverages
+    ret["summary"]["d_min"] = d_min
+    ret["summary"]["FSCaverage"] = spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model)
+    if cross_validation:
+        ret["summary"]["FSCaverage_half1"] = spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half1)
+        ret["summary"]["FSCaverage_half2"] = spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half2)
     fscavg_text  = "Map-model FSCaverages (at {:.2f} A):\n".format(d_min)
-    fscavg_text += " FSCaverage(full) = {: .4f}\n".format(spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model))
+    fscavg_text += " FSCaverage(full) = {: .4f}\n".format(ret["summary"]["FSCaverage"])
     if cross_validation:
         fscavg_text += "Cross-validated map-model FSCaverages:\n"
-        fscavg_text += " FSCaverage(half1)= {: .4f}\n".format(spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half1))
-        fscavg_text += " FSCaverage(half2)= {: .4f}\n".format(spa.fsc.fsc_average(stats2.ncoeffs, stats2.fsc_model_half2))
+        fscavg_text += " FSCaverage(half1)= {: .4f}\n".format(ret["summary"]["FSCaverage_half1"])
+        fscavg_text += " FSCaverage(half2)= {: .4f}\n".format(ret["summary"]["FSCaverage_half2"])
     
     # for loggraph
     fsc_logfile = "{}_fsc.log".format(output_prefix)
@@ -269,8 +274,8 @@ def calc_fsc(st, output_prefix, maps, d_min, mask, mask_radius, soft_edge, b_bef
     json.dump(stats.to_dict("records"),
               open("{}_fsc.json".format(output_prefix), "w"),
               indent=True)
-
-    return fscavg_text
+    ret["binned"] = stats2.to_dict(orient="records")
+    return fscavg_text, ret
 # calc_fsc()
 
 def calc_fofc(st, st_expanded, maps, monlib, model_format, args, diffmap_prefix="diffmap"):
@@ -462,7 +467,7 @@ def process_input(st, maps, resolution, monlib, mask_in, args,
     unit_cell = maps[0][0].unit_cell
     spacegroup = gemmi.SpaceGroup(1)
     start_xyz = numpy.array(maps[0][0].get_position(*grid_start).tolist())
-    A = numpy.array(unit_cell.orthogonalization_matrix.tolist())
+    A = unit_cell.orth.mat.array
     center = numpy.sum(A, axis=1) / 2 #+ start_xyz
 
     # Create mask
@@ -593,7 +598,7 @@ def process_input(st, maps, resolution, monlib, mask_in, args,
             topo, metal_kws = utils.restraints.prepare_topology(st, monlib, h_change=h_change, raise_error=False)
             args.keywords = metal_kws + args.keywords
         elif not no_refmac_fix:
-            topo = gemmi.prepare_topology(st, monlib, warnings=io.StringIO(), ignore_unknown_links=True)
+            topo = gemmi.prepare_topology(st, monlib, warnings=logger.silent(), ignore_unknown_links=True)
         else:
             topo = None # not used
         if not no_refmac_fix:
@@ -618,7 +623,7 @@ def process_input(st, maps, resolution, monlib, mask_in, args,
         args.keywords.append("make cr prepared")
         gemmi.setup_for_crd(st)
         doc = gemmi.prepare_refmac_crd(st, topo, monlib, h_change)
-        doc.write_file(crdout, style=gemmi.cif.Style.NoBlankLines)
+        doc.write_file(crdout, options=gemmi.cif.Style.NoBlankLines)
         logger.writeln("crd file written: {}".format(crdout))
 
     hkldata = utils.maps.mask_and_fft_maps(maps, resolution, None, with_000=False)
@@ -955,7 +960,7 @@ def main(args):
                            monlib=monlib, cross_validation=args.cross_validation,
                            blur=args.blur,
                            d_min_fsc=args.fsc_resolution,
-                           cross_validation_method=args.cross_validation_method, st_sr=st_sr_expanded)
+                           cross_validation_method=args.cross_validation_method, st_sr=st_sr_expanded)[0]
 
     # Calc Fo-Fc (and updated) maps
     calc_fofc(st, st_expanded, maps, monlib, model_format, args)

servalcat/spa/translate.py

@@ -60,9 +60,9 @@ def find_peak(tf_map, ini_pos):
 
     x = tf_map.unit_cell.fractionalize(ini_pos)
     logger.writeln("       x0: [{}, {}, {}]".format(*x.tolist()))
-    logger.writeln("       f0: {}".format(-tf_map.tricubic_interpolation(x)))
+    logger.writeln("       f0: {}".format(-tf_map.interpolate_value(x, order=3)))
     
-    res = scipy.optimize.minimize(fun=lambda x:-tf_map.tricubic_interpolation(gemmi.Fractional(*x)),
+    res = scipy.optimize.minimize(fun=lambda x:-tf_map.interpolate_value(gemmi.Fractional(*x), order=3),
                                   x0=x.tolist(),
                                   jac=lambda x:-numpy.array(tf_map.tricubic_interpolation_der(gemmi.Fractional(*x))[1:]))
     logger.writeln(str(res))

servalcat/utils/commands.py

@@ -185,6 +185,17 @@ def add_arguments(p):
     parser.add_argument('-o', '--output_prefix', 
                         help="default: taken from input file")
 
+    # conf
+    parser = subparsers.add_parser("conf", description = 'Compare conformations')
+    parser.add_argument('models', nargs="+")
+    parser.add_argument("--min_diff", type=float, default=60.)
+    parser.add_argument('--ligand', nargs="*", action="append")
+    parser.add_argument("--monlib",
+                        help="Monomer library path. Default: $CLIBD_MON")
+    parser.add_argument("--same_chain", action='store_true', help="Only between same chains (more than one file)")
+    parser.add_argument('-o', '--output_prefix', default="conf",
+                        help="")
+
     # adp
     parser = subparsers.add_parser("adp", description = 'ADP analysis')
     parser.add_argument('model')
@@ -281,6 +292,9 @@ def add_arguments(p):
     parser = subparsers.add_parser("seq", description = 'Print/align model sequence')
     parser.add_argument("--model", required=True)
     parser.add_argument('--seq', nargs="*", action="append", help="Sequence file(s)")
+    parser.add_argument('--scoring', nargs=6, type=int, default=(1, 0, -1, -1, 0, -1),
+                        metavar=("match", "mismatch", "gapo", "gape", "good_gapo", "bad_gapo"),
+                        help="scoring function. default: %(default)s")
 
     # dnarna
     parser = subparsers.add_parser("dnarna", description = 'DNA to RNA or RNA to DNA model conversion')
@@ -919,6 +933,135 @@ def geometry(args):
         fileio.write_model(st, file_name="{}_per_atom_score{}".format(args.output_prefix, model_format))
 # geometry()
 
+def compare_conf(args):
+    def angle_abs_diff(a, b, full=360.):
+        # from gemmi/math.hpp
+        d = abs(a - b)
+        if d > full:
+            d -= numpy.floor(d / full) * full
+        return min(d, full - d)
+    # angle_abs_diff()
+    
+    if args.ligand: args.ligand = sum(args.ligand, [])
+    st = None
+    for i, f in enumerate(args.models):
+        tmp = fileio.read_structure(f)
+        if len(args.models) > 1:
+            for chain in tmp[0]:
+                chain.name = f"{i+1}_{chain.name}"
+        if i == 0:
+            st = tmp
+        else:
+            for chain in tmp[0]:
+                st[0].add_chain(chain)
+    try:
+        monlib = restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand, 
+                                                 stop_for_unknowns=True)
+    except RuntimeError as e:
+        raise SystemExit(f"Error: {e}")
+
+    model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
+    try:
+        topo, _ = restraints.prepare_topology(st, monlib, h_change=gemmi.HydrogenChange.NoChange,
+                                              check_hydrogen=False)
+    except RuntimeError as e:
+        raise SystemExit(f"Error: {e}")
+    ncslist = restraints.prepare_ncs_restraints(st)
+    lookup = {x.atom: x for x in st[0].all()}
+    ptypes = {x.name: x.polymer_type for x in st.entities}
+    resn_lookup = {(chain.name, res.seqid): res.name for chain in st[0] for res in chain}
+    confs = {}
+    for t in topo.torsions:
+        cra = lookup[t.atoms[0]]
+        ptype = ptypes[cra.residue.entity_id]
+        is_peptide = ptype in (gemmi.PolymerType.PeptideL, gemmi.PolymerType.PeptideD)
+        is_peptide_tors = t.restr.label.startswith("chi") or t.restr.label in ("omega", "phi", "psi")
+        is_na = ptype in (gemmi.PolymerType.Dna, gemmi.PolymerType.Rna, gemmi.PolymerType.DnaRnaHybrid)
+        is_na_tors = t.restr.label in ("C2e-chi", "alpha", "beta", "gamma", "C2e-nyu0", "epsilon", "zeta")
+        if (is_peptide and is_peptide_tors) or (is_na and is_na_tors):
+            confs.setdefault(cra.chain.name, {}).setdefault(cra.residue.seqid, {})[t.restr.label] = numpy.rad2deg(t.calculate())
+    fulls = {("ARG", "chi5"): 180., ("TYR", "chi2"): 180., ("PHE", "chi2"): 180., ("ASP", "chi2"): 180., ("GLU", "chi3"): 180.}
+    ret = []
+    for_coot = []
+    for ncs in ncslist.ncss:
+        c1, c2 = ncs.chains
+        if args.same_chain and len(args.models) > 1 and c1[c1.index("_"):] != c2[c2.index("_"):]:
+            continue
+        for s1, s2 in ncs.seqids:
+            if c1 in confs and s1 in confs[c1] and c2 in confs and s2 in confs[c2]:
+                conf1, conf2 = confs[c1][s1], confs[c2][s2]
+                resn = resn_lookup[(c1, s1)]
+                for t in conf1:
+                    if t in conf2:
+                        d = angle_abs_diff(conf1[t], conf2[t], fulls.get((resn, t), 360.))
+                        ret.append((c1, s1, c2, s2, resn, t, conf1[t], conf2[t], d))
+                        if d > args.min_diff:
+                            for_coot.append((c1, s1.num, c2, s2.num, resn, t, d))
+    df = pandas.DataFrame(ret, columns=["chain_1", "seq_1", "chain_2", "seq_2", "resn", "label", "conf_1", "conf_2", "diff"])
+    df.sort_values("diff", ascending=False, inplace=True)
+    logger.writeln(f"\nList of torsion angle differences (>{args.min_diff})")
+    logger.writeln(df[df["diff"] > args.min_diff].to_string(index=False))
+
+    for_coot.sort(key=lambda x:-x[-1])
+    coot_out = args.output_prefix + "_coot.py"
+    with open(coot_out, "w") as ofs:
+        # https://python-gtk-3-tutorial.readthedocs.io/en/latest/treeview.html
+        ofs.write("""\
+from __future__ import absolute_import, division, print_function
+import re
+import gtk
+class coot_serval_conf_list:
+  def __init__(self):
+    window = gtk.Window(gtk.WINDOW_TOPLEVEL)
+    window.set_title("Different conformations (Servalcat)")
+    window.set_default_size(600, 600)
+    scrolled_win = gtk.ScrolledWindow()
+    scrolled_win.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_ALWAYS)
+    vbox = gtk.VBox(False, 2)
+    self.liststore = gtk.ListStore(str, int, str, int, str, str, float)
+    self.filter = self.liststore.filter_new()
+    self.treeview = gtk.TreeView(model=self.filter)
+    for i, column_title in enumerate(["chain_1", "seq_1", "chain_2", "seq_2", "resn", "label", "diff"]):
+      renderer = gtk.CellRendererText()
+      column = gtk.TreeViewColumn(column_title, renderer, text=i)
+      self.treeview.append_column(column)
+    self.data = {}
+    self.add_data()
+    scrolled_win.add_with_viewport(self.treeview)  # add?
+    vbox.pack_start(scrolled_win, True, True, 0)
+    window.add(vbox)
+    window.show_all()
+    self.treeview.connect("row-activated", self.on_row_activated)
+
+  def on_row_activated(self, treeview, path, column):
+    assert len(path) == 1
+    col_idx = [i for i, c in enumerate(treeview.get_columns()) if column == c][0]
+    row = self.liststore[path[0]]
+    if col_idx < 2:
+      chain, resi = row[0], row[1]
+    elif col_idx < 4:
+      chain, resi = row[2], row[3]
+    else:
+      return
+    if re.search("^[0-9]+_[0-9A-Za-z]", chain):
+      chain = chain[chain.index("_")+1:]
+    imol = active_atom_spec()[1][0]
+    for name in (" CA ", " C1'"):
+      a = get_atom(imol, chain, resi, "", name)
+      if a:
+        set_rotation_center(*a[2])
+        break
+
+  def add_data(self):
+    for i, d in enumerate(self.data):
+      self.liststore.append(d)
+
+gui = coot_serval_conf_list()
+""".format(for_coot))
+    logger.writeln("\nRun:")
+    logger.writeln(f"coot --script {coot_out}")
+# compare_conf()
+
 def adp_stats(args):
     if not args.output_prefix: args.output_prefix = fileio.splitext(os.path.basename(args.model))[0] + "_adp"
     st = fileio.read_structure(args.model)
@@ -1242,6 +1385,9 @@ def seq(args):
         for sf in args.seq:
             seqs.extend(fileio.read_sequence_file(sf))
         
+    sc = gemmi.AlignmentScoring()
+    sc.match, sc.mismatch, sc.gapo, sc.gape, sc.good_gapo, sc.bad_gapo = args.scoring
+    
     st = fileio.read_structure(args.model) # TODO option to (or not to) expand NCS
     model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)    
     for chain in st[0]:
@@ -1257,17 +1403,20 @@ def seq(args):
                     gemmi.PolymerType.Rna: gemmi.ResidueKind.RNA}.get(p_type, gemmi.ResidueKind.AA)
             s = [gemmi.expand_one_letter(x, kind) for x in seq]
             if None in s: continue
-            results.append([name, gemmi.align_sequence_to_polymer(s, p, p_type), seq])
+            #als = [gemmi.align_sequence_to_polymer(s, p, p_type, gemmi.AlignmentScoring(x)) for x in ("s", "p")]
+            #results.append([name, max(als, key=lambda x: x.match_count), seq])
+            results.append([name, gemmi.align_sequence_to_polymer(s, p, p_type, sc), seq])
 
         if results:
             logger.writeln("Chain: {}".format(chain.name))
             logger.writeln(" polymer type: {}".format(str(p_type).replace("PolymerType.", "")))
-            name, al, s1 = max(results, key=lambda x: x[1].score)
+            name, al, s1 = max(results, key=lambda x: (x[1].match_count, x[1].score))
             logger.writeln(" match: {}".format(name))
+            logger.writeln(" aligned: {}".format(al.match_count))
             logger.writeln(" score: {}".format(al.score))
             p1, p2 = al.add_gaps(s1, 1), al.add_gaps(p_seq, 2)
-            unkseq = [x.start() for x in re.finditer("\-", p1)]
-            mismatches = [x.start() for x in re.finditer("\.", al.match_string)]
+            unkseq = [x.start() for x in re.finditer(r"\-", p1)]
+            mismatches = [x.start() for x in re.finditer(r"\.", al.match_string)]
             if mismatches or unkseq:
                 idxes = {x.start(): i for i, x in enumerate(re.finditer("[^-]", p2))}
                 seqnums = [str(x.seqid) for x in p]
@@ -1371,6 +1520,7 @@ def main(args):
                  merge_models=merge_models,
                  merge_dicts=merge_dicts,
                  geom=geometry,
+                 conf=compare_conf,
                  adp=adp_stats,
                  power=show_power,
                  fcalc=fcalc,

servalcat/utils/fileio.py

@@ -17,7 +17,6 @@ import re
 import subprocess
 import gemmi
 import numpy
-import numpy.lib.recfunctions
 import gzip
 
 def splitext(path):
@@ -83,10 +82,12 @@ def write_mmcif(st, cif_out, cif_ref=None):
         groups.scale = True
         groups.assembly = True
         groups.entity = True
+        groups.entity_poly = True
         groups.entity_poly_seq = True
         groups.cis = True
         groups.conn = True
         groups.software = True
+        groups.auth_all = True
         # FIXME is this all? 
         try:
             doc = read_cif_safe(cif_ref)
@@ -107,23 +108,27 @@ def write_mmcif(st, cif_out, cif_ref=None):
         block.find_mmcif_category("_atom_sites.").erase()
         st_new.update_mmcif_block(block, groups)
         if "_entry.id" in st_new.info: st_new.info["_entry.id"] = st_new.info["_entry.id"][:78]
-        doc.write_file(cif_out, style=gemmi.cif.Style.Aligned)
+        doc.write_file(cif_out, options=gemmi.cif.Style.Aligned)
     else:
         st_new.name = st_new.name[:78] # this will become _entry.id
         if "_entry.id" in st_new.info: st_new.info["_entry.id"] = st_new.info["_entry.id"][:78]
-        groups = gemmi.MmcifOutputGroups(True)
+        groups = gemmi.MmcifOutputGroups(True, auth_all=True)
         doc = gemmi.cif.Document()
         block = doc.add_new_block("new")
         st_new.update_mmcif_block(block, groups)
-        doc.write_file(cif_out, style=gemmi.cif.Style.Aligned)
+        doc.write_file(cif_out, options=gemmi.cif.Style.Aligned)
 # write_mmcif()
 
 def write_pdb(st, pdb_out):
     logger.writeln("Writing PDB file: {}".format(pdb_out))
+    st = st.clone()
     chain_id_lens = [len(x) for x in model.all_chain_ids(st)]
     if chain_id_lens and max(chain_id_lens) > 2:
-        st = st.clone()
         st.shorten_chain_names()
+    st.shorten_ccd_codes()
+    if st.shortened_ccd_codes:
+        msg = " ".join("{}->{}".format(o,n) for o,n in st.shortened_ccd_codes)
+        logger.writeln(" Using shortened residue names in the output pdb file: " + msg)
     st.write_pdb(pdb_out, use_linkr=True)
 # write_pdb()
 
@@ -339,6 +344,8 @@ def read_structure(xyz_in, assign_het_flags=True, merge_chain_parts=True):
                 if st is None:
                     logger.writeln("Reading chemical component file: {}".format(xyz_in))
                     st = gemmi.make_structure_from_chemcomp_block(block)
+                    for i in range(len(st)-1):
+                        del st[1]
     elif spext[1].lower() in (".ins", ".res"):
         logger.writeln("Reading SHELX ins/res file: {}".format(xyz_in))
         st = model.cx_to_mx(read_shelx_ins(ins_in=xyz_in)[0])
@@ -456,7 +463,7 @@ def merge_ligand_cif(cifs_in, cif_out):
             if b.name not in list_names:
                 doc.add_copied_block(b)
 
-    doc.write_file(cif_out, style=gemmi.cif.Style.Aligned)
+    doc.write_file(cif_out, options=gemmi.cif.Style.Aligned)
 # merge_ligand_cif()
 
 def read_shelx_ins(ins_in=None, lines_in=None, ignore_q_peaks=True): # TODO support gz?
@@ -571,7 +578,7 @@ def read_shelx_ins(ins_in=None, lines_in=None, ignore_q_peaks=True): # TODO supp
         symms.extend([x*gemmi.Op("-x,-y,-z") for x in symms])
 
     ss.symops = [op.triplet() for op in set(symms)]
-    ss.set_spacegroup("s")
+    ss.determine_and_set_spacegroup("s")
     # in case of non-regular setting, gemmi.SpaceGroup cannot be constructed anyway.
     if ss.spacegroup is None:
         raise RuntimeError("Cannot construct space group from symbols: {}".format(ss.symops))
@@ -598,7 +605,7 @@ def read_shelx_hkl(cell, sg, hklf, file_in=None, lines_in=None):
         # wavelength = l[32:40]
 
     ints = gemmi.Intensities()
-    ints.set_data(cell, sg, hkls, vals, sigs)
+    ints.set_data(cell, sg, numpy.asarray(hkls), numpy.asarray(vals), numpy.asarray(sigs))
     ints.merge_in_place(gemmi.DataType.Anomalous)
     if not (ints.isign_array < 0).any(): ints.type = gemmi.DataType.Mean
     logger.writeln(" Multiplicity: max= {} mean= {:.1f} min= {}".format(numpy.max(ints.nobs_array),