servalcat 0.4.88__cp38-cp38-macosx_11_0_arm64.whl → 0.4.99__cp38-cp38-macosx_11_0_arm64.whl

servalcat/__init__.py

@@ -6,5 +6,5 @@ This software is released under the
 Mozilla Public License, version 2.0; see LICENSE.
 """
 
-__version__ = '0.4.88'
-__date__ = '2024-09-13'
+__version__ = '0.4.99'
+__date__ = '2024-12-04'

servalcat/refine/refine.py

@@ -28,7 +28,7 @@ b_to_u = utils.model.b_to_u
 #atexit.register(profile.print_stats)
 
 class Geom:
-    def __init__(self, st, topo, monlib, adpr_w=1, shake_rms=0,
+    def __init__(self, st, topo, monlib, adpr_w=1, occr_w=1, shake_rms=0,
                  params=None, unrestrained=False, use_nucleus=False,
                  ncslist=None, atom_pos=None):
         self.st = st
@@ -53,7 +53,7 @@ class Geom:
             n_sym = len(images) + 1
             self.geom.specials.append(ext.Geometry.Special(atom, matp, mata, n_sym))
         self.adpr_w = adpr_w
-        self.occr_w = 1.
+        self.occr_w = occr_w
         self.unrestrained = unrestrained
         if shake_rms > 0:
             numpy.random.seed(0)
@@ -225,6 +225,26 @@ def write_stats_json_safe(stats, json_out):
     logger.writeln(f"Refinement statistics saved: {json_out}")
 # write_stats_json_safe()
 
+def print_h_options(h_change, h_present, refine_h, hout, geom_only):
+    if not h_present:
+        h_change = gemmi.HydrogenChange.Remove
+    logger.writeln("Hydrogen related options")
+    logger.write(" use in refinement{}: hydrogen atoms ".format("" if geom_only else "/map calculation"))
+    logger.writeln({gemmi.HydrogenChange.ReAddButWater: "have been (re)generated",
+                    gemmi.HydrogenChange.ReAdd:         "(including water) have been (re)generated",
+                    gemmi.HydrogenChange.ReAddKnown:    "(except for rotatable) have been (re) generated",
+                    gemmi.HydrogenChange.NoChange:      "from the input model have been retained",
+                    gemmi.HydrogenChange.Remove:        "have either been removed or were not present"}[h_change])
+    if h_present:
+        logger.write(" target: hydrogen atoms will be ")
+        if geom_only or not refine_h:
+            logger.writeln("just optimized according to geometric restraints")
+        else:
+            logger.writeln("refined against experimental data")
+    logger.writeln(" in output model: " + ("written" if hout and h_present else "not written"))
+    logger.writeln("")
+# print_hydrogen_options()
+
 class GroupOccupancy:
     # TODO max may not be one. should check multiplicity
     def __init__(self, st, params):
@@ -293,6 +313,7 @@ class GroupOccupancy:
         vals = []
         for _, atoms in self.groups:
             occ = numpy.mean([a.occ for a in atoms])
+            occ = min(1, max(1e-3, occ))
             vals.append(occ)
         for is_comp, idxes in self.consts:
             sum_occ = sum(vals[i] for i in idxes)
@@ -311,6 +332,7 @@ class GroupOccupancy:
         for p, (_, atoms) in zip(x, self.groups):
             for a in atoms:
                 a.occ = p
+                #a.occ = max(1, min(1e-3, p))
 
     def target(self, x, ll, ls, u):
         self.set_x(x)
@@ -425,7 +447,7 @@ class Refine:
             self.geom.set_h_parents()
         if params and params.get("write_trajectory"):
             self.st_traj = self.st.clone()
-            self.st_traj[-1].name = "0"
+            self.st_traj[-1].num = 0
         assert self.geom.group_occ.groups or self.n_params() > 0
     # __init__()
     
@@ -442,6 +464,9 @@ class Refine:
                 logger.writeln("  sigmas: {}".format(" ".join("{:.2f}".format(x) for x in g.adpr_kl_sigs)))
             else:
                 raise LookupError("unknown adpr_mode")
+        if self.refine_occ:
+            logger.writeln(" Occupancy restraints")
+            logger.writeln("  weight: {}".format(self.geom.occr_w))
 
     def scale_shifts(self, dx, scale):
         n_atoms = self.geom.n_refine_atoms
@@ -525,7 +550,7 @@ class Refine:
             elif self.adp_mode == 2:
                 a = x[offset_b + 6 * j: offset_b + 6 * (j+1)]
                 a = gemmi.SMat33d(*a)
-                M = numpy.array(a.as_mat33())
+                M = a.as_mat33().array
                 v, Q = numpy.linalg.eigh(M) # eig() may return complex due to numerical precision?
                 v = numpy.maximum(v, 0.5) # avoid NPD with minimum B = 0.5
                 M2 = Q.dot(numpy.diag(v)).dot(Q.T)
@@ -736,7 +761,7 @@ class Refine:
                     weight /= 1.1
             if self.st_traj is not None:
                 self.st_traj.add_model(self.st[0])
-                self.st_traj[-1].name = str(len(self.st_traj))
+                self.st_traj[-1].num = len(self.st_traj)
             if stats_json_out:
                 write_stats_json_safe(stats, stats_json_out)
 
@@ -781,43 +806,101 @@ class Refine:
         lstr = utils.make_loggraph_str(df, "stats vs cycle", forplot,
                                        float_format="{:.4f}".format)
         logger.writeln(lstr)
-        self.update_meta(stats[-1])
         return stats
 
-    def update_meta(self, stats):
-        # TODO write stats. probably geom.reporting.get_summary_table should return with _refine_ls_restr.type names
-        # should remove st.mod_residues?
-        self.st.helices.clear()
-        self.st.sheets.clear()
-        raw_remarks = [f'REMARK   3',
-                       f'REMARK   3 REFINEMENT.',
-                       f'REMARK   3   PROGRAM     : SERVALCAT {servalcat.__version__}',
-                       f'REMARK   3   AUTHORS     : YAMASHITA,MURSHUDOV',
-                       f'REMARK   3',
-                       ]
-        si = gemmi.SoftwareItem()
-        si.classification = gemmi.SoftwareItem.Classification.Refinement
-        si.name = "Servalcat"
-        si.version = servalcat.__version__
-        si.date = servalcat.__date__
-        self.st.meta.software = [si]
-
-        ri = gemmi.RefinementInfo()
-        if "geom" in stats:
-            restr_stats = []
-            raw_remarks.append("REMARK   3  RMS DEVIATIONS FROM IDEAL VALUES        COUNT    RMS    WEIGHT")
-            for k, n, l, pl in (("r.m.s.d.", "Bond distances, non H", "s_bond_nonh_d", "BOND LENGTHS REFINED ATOMS        (A)"),
-                                ("r.m.s.d.", "Bond angles, non H", "s_angle_nonh_d", "BOND ANGLES REFINED ATOMS   (DEGREES)")):
-                if k in stats["geom"]["summary"] and n in stats["geom"]["summary"][k]:
-                    rr = gemmi.RefinementInfo.Restr(l)
-                    rr.dev_ideal = stats["geom"]["summary"][k].get(n)
-                    rr.count = stats["geom"]["summary"]["N restraints"].get(n)
-                    rr.weight = stats["geom"]["summary"]["Mn(sigma)"].get(n)
-                    restr_stats.append(rr)
-                    raw_remarks.append(f"REMARK   3   {pl}:{rr.count:6d} ;{rr.dev_ideal:6.3f} ;{rr.weight:6.3f}")
-            ri.restr_stats = restr_stats
-            raw_remarks.append("REMARK   3")
-        self.st.meta.refinement = [ri]
-        self.st.raw_remarks = raw_remarks
-
 # class Refine
+
+def update_meta(st, stats, ll=None):
+    # TODO write stats. probably geom.reporting.get_summary_table should return with _refine_ls_restr.type names
+    # should remove st.mod_residues?
+    st.helices.clear()
+    st.sheets.clear()
+    raw_remarks = [f'REMARK   3',
+                   f'REMARK   3 REFINEMENT.',
+                   f'REMARK   3   PROGRAM     : SERVALCAT {servalcat.__version__}',
+                   f'REMARK   3   AUTHORS     : YAMASHITA,MURSHUDOV',
+                   f'REMARK   3',
+                   ]
+    si = gemmi.SoftwareItem()
+    si.classification = gemmi.SoftwareItem.Classification.Refinement
+    si.name = "Servalcat"
+    si.version = servalcat.__version__
+    si.date = servalcat.__date__
+    st.meta.software = [si]
+
+    ri = gemmi.RefinementInfo()
+    if "geom" in stats:
+        restr_stats = []
+        raw_remarks.append("REMARK   3  RMS DEVIATIONS FROM IDEAL VALUES        COUNT    RMS    WEIGHT")
+        for k, n, l, pl in (("r.m.s.d.", "Bond distances, non H", "s_bond_nonh_d",             "BOND LENGTHS REFINED ATOMS        (A)"),
+                            ("r.m.s.d.", "Bond angles, non H", "s_angle_nonh_deg",             "BOND ANGLES REFINED ATOMS   (DEGREES)"),
+                            ("r.m.s.d.", "Torsion angles, period 1", "s_dihedral_angle_1_deg", "TORSION ANGLES, PERIOD 1    (DEGREES)"),
+                            ("r.m.s.d.", "Torsion angles, period 2", "s_dihedral_angle_2_deg", "TORSION ANGLES, PERIOD 2    (DEGREES)"),
+                            ("r.m.s.d.", "Torsion angles, period 3", "s_dihedral_angle_3_deg", "TORSION ANGLES, PERIOD 3    (DEGREES)"),
+                            ("r.m.s.d.", "Torsion angles, period 6", "s_dihedral_angle_6_deg", "TORSION ANGLES, PERIOD 6    (DEGREES)"),
+                            ("r.m.s.d.", "Chiral centres", "s_chiral_restr",                   "CHIRAL-CENTER RESTRAINTS       (A**3)"),
+                            ("r.m.s.d.", "Planar groups", "s_planes",                          "GENERAL PLANES REFINED ATOMS      (A)"),
+                            ("r.m.s.d.", "VDW nonbonded", "s_nbd",                             ""),
+                            ("r.m.s.d.", "VDW torsion", "s_nbtor",                             ""),
+                            ("r.m.s.d.", "VDW hbond", "s_hbond_nbd",                           ""),
+                            ("r.m.s.d.", "VDW metal", "s_metal_ion",                           ""),
+                            ("r.m.s.d.", "VDW dummy", "s_dummy_nbd",                           ""),
+                            ("r.m.s.d.", "VDW nonbonded, symmetry", "s_symmetry_nbd",          ""),
+                            ("r.m.s.d.", "VDW torsion, symmetry", "s_symmetry_nbtor",          ""),
+                            ("r.m.s.d.", "VDW hbond, symmetry", "s_symmetry_hbond_nbd",        ""),
+                            ("r.m.s.d.", "VDW metal, symmetry", "s_symmetry_metal_ion",        ""),
+                            ("r.m.s.d.", "VDW dummy, symmetry", "s_symmetry_dummy_nbd",        "")):
+            if k in stats["geom"]["summary"] and n in stats["geom"]["summary"][k]:
+                rr = gemmi.RefinementInfo.Restr(l)
+                rr.dev_ideal = round(stats["geom"]["summary"][k].get(n), 4)
+                rr.count = stats["geom"]["summary"]["N restraints"].get(n)
+                rr.weight = round(stats["geom"]["summary"]["Mn(sigma)"].get(n), 4)
+                restr_stats.append(rr)
+                if pl:
+                    raw_remarks.append(f"REMARK   3   {pl}:{rr.count:6d} ;{rr.dev_ideal:6.3f} ;{rr.weight:6.3f}")
+        ri.restr_stats = restr_stats
+        raw_remarks.append("REMARK   3")
+    if ll is not None:
+        ri.id = ll.refine_id()
+        ri.mean_b = round(numpy.mean([cra.atom.b_iso for cra in st[0].all()]), 2)
+        if ll.b_aniso is not None:
+            ri.aniso_b = ll.b_aniso
+        for k, kd, nd in (("Rwork", "r_work", 4), ("Rfree", "r_free", 4), ("R", "r_all", 4),
+                          ("FSCaverage", "fsc_work", 4),
+                          ("FSCaverage_half1", "fsc_work", 4), ("FSCaverage_half2", "fsc_free", 4)):
+            if k in stats["data"]["summary"]:
+                setattr(ri, kd, round(stats["data"]["summary"][k], nd))
+        bins = []
+        n_all = 0
+        for b in stats["data"]["binned"]:
+            bri = gemmi.BasicRefinementInfo()
+            bri.resolution_high = round(b["d_min"], 3)
+            bri.resolution_low = round(b["d_max"], 3)
+            for k, kd, nd in (("Rwork", "r_work", 4), ("Rfree", "r_free", 4),
+                              ("R1work", "r_work", 4), ("R1free", "r_free", 4),
+                              ("R", "r_all", 4), ("R1", "r_all", 4),
+                              ("CCI", "cc_intensity_work", 4), ("CCF", "cc_fo_fc_work", 4),
+                              ("CCIwork", "cc_intensity_work", 4), ("CCIfree", "cc_intensity_free", 4),
+                              ("CCFwork", "cc_fo_fc_work", 4), ("CCFfree", "cc_fo_fc_free", 4),
+                              ("fsc_FC_full", "fsc_work", 4), ("fsc_model", "fsc_work", 4),
+                              ("fsc_model_half1", "fsc_work", 4), ("fsc_model_half2", "fsc_free", 4),
+                              ("n_work", "work_set_count", 0), ("n_free", "rfree_set_count", 0),
+                              ("n_obs", "reflection_count", 0), ("ncoeffs", "reflection_count", 0)):
+                if k in b: setattr(bri, kd, round(b[k], nd))
+            if "n_all" in b and "n_obs" in b:
+                bri.completeness = round(b["n_obs"] / b["n_all"] * 100, 2)
+                n_all += b["n_all"]
+            bins.append(bri)
+        ri.rfree_set_count = max(-1, sum(b.rfree_set_count for b in bins))
+        ri.work_set_count = max(-1, sum(b.work_set_count for b in bins))
+        ri.reflection_count = max(-1, sum(b.reflection_count for b in bins))
+        ri.resolution_high = round(min(b.resolution_high for b in bins), 3)
+        ri.resolution_low = round(max(b.resolution_low for b in bins), 3)
+        if ri.reflection_count > 0 and n_all > 0:
+            ri.completeness = round(ri.reflection_count / n_all * 100, 2)
+        ri.bins = bins
+        if ri.rfree_set_count > 0:
+            ri.cross_validation_method = "THROUGHOUT"
+    st.meta.refinement = [ri]
+    st.raw_remarks = raw_remarks
+# update_meta()

servalcat/refine/refine_geom.py

@@ -14,7 +14,7 @@ import json
 import servalcat # for version
 from servalcat.utils import logger
 from servalcat import utils
-from servalcat.refine.refine import Geom, Refine, convert_stats_to_dicts
+from servalcat.refine.refine import Geom, Refine, convert_stats_to_dicts, update_meta, print_h_options
 from servalcat.refmac import refmac_keywords
 
 def add_arguments(parser):
@@ -93,9 +93,14 @@ def refine_and_update_dictionary(cif_in, monomer_dir, output_prefix, randomize=0
         if len(st) > 0: break
     else:
         raise SystemExit("No model in the cif file")
-    monlib = utils.restraints.load_monomer_library(st, monomer_dir=monomer_dir, # monlib is needed for ener_lib
-                                                   cif_files=[cif_in],
-                                                   stop_for_unknowns=True)
+    for i in range(len(st)-1):
+        del st[1]
+    try:
+        monlib = utils.restraints.load_monomer_library(st, monomer_dir=monomer_dir, # monlib is needed for ener_lib
+                                                       cif_files=[cif_in],
+                                                       stop_for_unknowns=True)
+    except RuntimeError as e:
+        raise SystemExit("Error: {}".format(e))
     all_stats = []
     for i_macro in 0, 1:
         try:
@@ -121,16 +126,19 @@ def refine_and_update_dictionary(cif_in, monomer_dir, output_prefix, randomize=0
     for row in block.find("_chem_comp_atom.", ["atom_id", "?x", "?y", "?z",
                                                "?pdbx_model_Cartn_x_ideal",
                                                "?pdbx_model_Cartn_y_ideal",
-                                               "?pdbx_model_Cartn_z_ideal"]):
+                                               "?pdbx_model_Cartn_z_ideal",
+                                               "?model_Cartn_x", "?model_Cartn_y", "?model_Cartn_z"]):
         p = pos[row.str(0)]
         for i in range(3):
             if row.has(i+1):
                 row[i+1] = "{:.3f}".format(p[i])
             if row.has(i+4):
                 row[i+4] = "{:.3f}".format(p[i])
+            if row.has(i+7):
+                row[i+7] = "{:.3f}".format(p[i])
     # add description
     add_program_info_to_dictionary(block, st[0][0][0].name)
-    doc.write_file(output_prefix + "_updated.cif", style=gemmi.cif.Style.Aligned)
+    doc.write_file(output_prefix + "_updated.cif", options=gemmi.cif.Style.Aligned)
     logger.writeln("Updated dictionary saved: {}".format(output_prefix + "_updated.cif"))
     with open(output_prefix + "_stats.json", "w") as ofs:
         json.dump([convert_stats_to_dicts(x) for x in all_stats],
@@ -147,11 +155,13 @@ def refine_geom(model_in, monomer_dir, cif_files, h_change, ncycle, output_prefi
         logger.writeln("Take NCS constraints into account.")
         st2.expand_ncs(gemmi.HowToNameCopiedChain.Dup, merge_dist=0)
         utils.fileio.write_model(st2, file_name="input_expanded.pdb")
-
-    monlib = utils.restraints.load_monomer_library(st, monomer_dir=monomer_dir,
-                                                   cif_files=cif_files,
-                                                   stop_for_unknowns=True,
-                                                   params=params)
+    try:
+        monlib = utils.restraints.load_monomer_library(st, monomer_dir=monomer_dir,
+                                                       cif_files=cif_files,
+                                                       stop_for_unknowns=True,
+                                                       params=params)
+    except RuntimeError as e:
+        raise SystemExit("Error: {}".format(e))
     utils.restraints.find_and_fix_links(st, monlib, find_metal_links=find_links,
                                         add_found=find_links) # should remove unknown id here?
     try:
@@ -161,6 +171,8 @@ def refine_geom(model_in, monomer_dir, cif_files, h_change, ncycle, output_prefi
     except RuntimeError as e:
         raise SystemExit("Error: {}".format(e))
 
+    print_h_options(h_change, st[0].has_hydrogen(), refine_h=True, hout=True, geom_only=True)
+        
     if use_ncsr:
         ncslist = utils.restraints.prepare_ncs_restraints(st)
     else:
@@ -169,6 +181,7 @@ def refine_geom(model_in, monomer_dir, cif_files, h_change, ncycle, output_prefi
     refiner = Refine(st, geom, params=params)
     stats = refiner.run_cycles(ncycle,
                                stats_json_out=output_prefix + "_stats.json")
+    update_meta(st, stats[-1])
     refiner.st.name = output_prefix
     utils.fileio.write_model(refiner.st, output_prefix, pdb=True, cif=True)
     if params["write_trajectory"]:

servalcat/refine/refine_spa.py

@@ -14,12 +14,12 @@ from servalcat import utils
 from servalcat.spa.run_refmac import check_args, process_input, calc_fsc, calc_fofc
 from servalcat.spa import fofc
 from servalcat.refine import spa
-from servalcat.refine.refine import Geom, Refine
+from servalcat.refine.refine import Geom, Refine, update_meta, print_h_options
 from servalcat.refmac import refmac_keywords
 b_to_u = utils.model.b_to_u
 
 def add_arguments(parser):
-    parser.description = "EXPERIMENTAL program to refine cryo-EM SPA structures"
+    parser.description = "program to refine cryo-EM SPA structures"
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument("--halfmaps", nargs=2, help="Input half map files")
     group.add_argument("--map", help="Use this only if you really do not have half maps.")
@@ -68,7 +68,7 @@ def add_arguments(parser):
                         help="Jelly body only (experimental, may not be useful)")
     utils.symmetry.add_symmetry_args(parser) # add --pg etc
     parser.add_argument('--contacting_only', action="store_true", help="Filter out non-contacting strict NCS copies")
-    parser.add_argument('--ignore_symmetry',
+    parser.add_argument('--ignore_symmetry', action='store_true',
                         help='Ignore symmetry information (MTRIX/_struct_ncs_oper) in the model file')
     parser.add_argument('--find_links', action='store_true', 
                         help='Automatically add links')
@@ -94,6 +94,8 @@ def add_arguments(parser):
                         help='Bond rmsz range for weight adjustment (default: %(default)s)')
     parser.add_argument('--adpr_weight', type=float, default=1.,
                         help="ADP restraint weight (default: %(default)f)")
+    parser.add_argument('--occr_weight', type=float, default=1.,
+                        help="Occupancy restraint weight (default: %(default)f)")
     parser.add_argument('--ncsr', action='store_true', 
                         help='Use local NCS restraints')
     parser.add_argument('--bfactor', type=float,
@@ -108,8 +110,9 @@ def add_arguments(parser):
     parser.add_argument('--adp_restraint_exp_fac', type=float)
     parser.add_argument('--adp_restraint_no_long_range', action='store_true')
     parser.add_argument('--adp_restraint_mode', choices=["diff", "kldiv"], default="diff")
+    parser.add_argument('--unrestrained',  action='store_true', help="No positional restraints")
     parser.add_argument('--refine_h', action="store_true", help="Refine hydrogen against data (default: only restraints apply)")
-    parser.add_argument("--source", choices=["electron", "xray", "neutron"], default="electron")
+    parser.add_argument("-s", "--source", choices=["electron", "xray", "neutron"], default="electron")
     parser.add_argument('-o','--output_prefix', default="refined")
     parser.add_argument('--cross_validation', action='store_true',
                         help='Run cross validation. Only "throughout" mode is available (no "shake" mode)')
@@ -142,12 +145,23 @@ def main(args):
     params["write_trajectory"] = args.write_trajectory
 
     st = utils.fileio.read_structure(args.model)
-    try:
-        monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand,
-                                                       stop_for_unknowns=not args.newligand_continue,
-                                                       params=params)
-    except RuntimeError as e:
-        raise SystemExit("Error: {}".format(e))
+    if args.unrestrained:
+        monlib = gemmi.MonLib()
+        topo = None
+        if args.hydrogen == "all":
+            logger.writeln("\nWARNING: in unrestrained refinement hydrogen atoms are not generated.\n")
+            args.hydrogen = "yes"
+        elif args.hydrogen == "no":
+            st.remove_hydrogens()
+        for i, cra in enumerate(st[0].all()):
+            cra.atom.serial = i + 1
+    else:
+        try:
+            monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand,
+                                                           stop_for_unknowns=not args.newligand_continue,
+                                                           params=params)
+        except RuntimeError as e:
+            raise SystemExit("Error: {}".format(e))
     if not args.keep_entities:
         utils.model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
     if not args.keep_charges:
@@ -186,6 +200,8 @@ def main(args):
     except RuntimeError as e:
         raise SystemExit("Error: {}".format(e))
 
+    print_h_options(h_change, st[0].has_hydrogen(), args.refine_h, args.hout, geom_only=False)
+
     # initialize ADP
     utils.model.reset_adp(st[0], args.bfactor, args.adp)
 
@@ -214,8 +230,8 @@ def main(args):
         ncslist = utils.restraints.prepare_ncs_restraints(st)
     else:
         ncslist = False
-    geom = Geom(st, topo, monlib, shake_rms=args.randomize, adpr_w=args.adpr_weight,
-                params=params, unrestrained=args.jellyonly,
+    geom = Geom(st, topo, monlib, shake_rms=args.randomize, adpr_w=args.adpr_weight, occr_w=args.occr_weight,
+                params=params, unrestrained=args.unrestrained or args.jellyonly,
                 ncslist=ncslist)
     ll = spa.LL_SPA(hkldata, st, monlib,
                     lab_obs="F_map1" if args.cross_validation else "FP",
@@ -224,6 +240,7 @@ def main(args):
                      refine_xyz=not args.fix_xyz,
                      adp_mode=dict(fix=0, iso=1, aniso=2)[args.adp],
                      refine_h=args.refine_h,
+                     unrestrained=args.unrestrained,
                      params=params,
                      refine_occ=args.refine_all_occ)
 
@@ -247,12 +264,8 @@ def main(args):
         refiner.st.cell = maps[0][0].unit_cell
         refiner.st.setup_cell_images()
 
-    refiner.st.name = args.output_prefix
-    utils.fileio.write_model(refiner.st, args.output_prefix, pdb=True, cif=True, hout=args.hout)
     if params["write_trajectory"]:
         utils.fileio.write_model(refiner.st_traj, args.output_prefix + "_traj", cif=True)
-    if args.hklin:
-        return
         
     # Expand sym here
     st_expanded = refiner.st.clone()
@@ -261,20 +274,28 @@ def main(args):
         utils.fileio.write_model(st_expanded, args.output_prefix+"_expanded", pdb=True, cif=True, hout=args.hout)
 
     # Calc FSC
-    mask = utils.fileio.read_ccp4_map(args.mask)[0] if args.mask else None
-    fscavg_text = calc_fsc(st_expanded, args.output_prefix, maps,
-                           args.resolution, mask=mask, mask_radius=args.mask_radius if not args.no_mask else None,
-                           soft_edge=args.mask_soft_edge,
-                           b_before_mask=args.b_before_mask,
-                           no_sharpen_before_mask=args.no_sharpen_before_mask,
-                           make_hydrogen="yes", # no change needed in the model
-                           monlib=monlib, 
-                           blur=args.blur,
-                           d_min_fsc=args.fsc_resolution,
-                           cross_validation=args.cross_validation,
-                           cross_validation_method=args.cross_validation_method
-                           )
-    
+    if args.hklin: # cannot update a mask
+        stats_for_meta = stats[-1]
+    else:
+        mask = utils.fileio.read_ccp4_map(args.mask)[0] if args.mask else None
+        fscavg_text, stats2 = calc_fsc(st_expanded, args.output_prefix, maps,
+                                       args.resolution, mask=mask, mask_radius=args.mask_radius if not args.no_mask else None,
+                                       soft_edge=args.mask_soft_edge,
+                                       b_before_mask=args.b_before_mask,
+                                       no_sharpen_before_mask=args.no_sharpen_before_mask,
+                                       make_hydrogen="yes", # no change needed in the model
+                                       monlib=monlib,
+                                       blur=args.blur,
+                                       d_min_fsc=args.fsc_resolution,
+                                       cross_validation=args.cross_validation,
+                                       cross_validation_method=args.cross_validation_method
+                                       )
+        stats_for_meta = {"geom": stats[-1]["geom"], "data": stats2}
+    update_meta(refiner.st, stats_for_meta, ll)
+    refiner.st.name = args.output_prefix
+    utils.fileio.write_model(refiner.st, args.output_prefix, pdb=True, cif=True, hout=args.hout)
+    if args.hklin:
+        return
     # Calc Fo-Fc (and updated) maps
     diffmap_prefix = "{}_diffmap".format(args.output_prefix)
     calc_fofc(refiner.st, st_expanded, maps, monlib, ".mmcif", args, diffmap_prefix=diffmap_prefix)

servalcat/refine/refine_xtal.py

@@ -16,13 +16,13 @@ from servalcat.utils import logger
 from servalcat import utils
 from servalcat.xtal.sigmaa import decide_mtz_labels, process_input, calculate_maps, calculate_maps_int, calculate_maps_twin
 from servalcat.refine.xtal import LL_Xtal
-from servalcat.refine.refine import Geom, Refine
+from servalcat.refine.refine import Geom, Refine, update_meta, print_h_options
 from servalcat.refmac import refmac_keywords
 from servalcat import ext
 b_to_u = utils.model.b_to_u
 
 def add_arguments(parser):
-    parser.description = "EXPERIMENTAL program to refine crystallographic structures"
+    parser.description = "program to refine crystallographic structures"
     parser.add_argument("--hklin", required=True)
     parser.add_argument("-d", '--d_min', type=float)
     parser.add_argument('--d_max', type=float)
@@ -64,10 +64,14 @@ def add_arguments(parser):
                         help="refinement weight (default: auto)")
     parser.add_argument('--no_weight_adjust', action='store_true', 
                         help='Do not adjust weight during refinement')
+    parser.add_argument('--target_bond_rmsz_range', nargs=2, type=float, default=[0.5, 1.],
+                        help='Bond rmsz range for weight adjustment (default: %(default)s)')
     parser.add_argument('--ncsr', action='store_true', 
                         help='Use local NCS restraints')
     parser.add_argument('--adpr_weight', type=float, default=1.,
                         help="ADP restraint weight (default: %(default)f)")
+    parser.add_argument('--occr_weight', type=float, default=1.,
+                        help="Occupancy restraint weight (default: %(default)f)")
     parser.add_argument('--bfactor', type=float,
                         help="reset all atomic B values to specified value")
     parser.add_argument('--fix_xyz', action="store_true")
@@ -88,6 +92,8 @@ def add_arguments(parser):
                         help="Use work reflections in ML parameter estimates")
     parser.add_argument('--keep_charges',  action='store_true',
                         help="Use scattering factor for charged atoms. Use it with care.")
+    parser.add_argument("--keep_entities", action='store_true',
+                        help="Do not override entities")
     parser.add_argument('--allow_unusual_occupancies', action="store_true", help="Allow negative or more than one occupancies")
     parser.add_argument('-o','--output_prefix')
     parser.add_argument("--write_trajectory", action='store_true',
@@ -159,10 +165,12 @@ def main(args):
     if args.unrestrained:
         monlib = gemmi.MonLib()
         topo = None
+        h_change = gemmi.HydrogenChange.NoChange
         if args.hydrogen == "all":
-            logger.writeln("WARNING: in unrestrained refinement hydrogen atoms are not generated.")
+            logger.writeln("\nWARNING: in unrestrained refinement hydrogen atoms are not generated.\n")
         elif args.hydrogen == "no":
             st.remove_hydrogens()
+            h_change = gemmi.HydrogenChange.Remove
         for i, cra in enumerate(st[0].all()):
             cra.atom.serial = i + 1
     else:
@@ -172,7 +180,8 @@ def main(args):
                                                            params=params)
         except RuntimeError as e:
             raise SystemExit("Error: {}".format(e))
-        utils.model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
+        if not args.keep_entities:
+            utils.model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
         utils.restraints.find_and_fix_links(st, monlib, find_metal_links=args.find_links,
                                             add_found=args.find_links)
         h_change = {"all":gemmi.HydrogenChange.ReAddKnown,
@@ -185,6 +194,8 @@ def main(args):
         except RuntimeError as e:
             raise SystemExit("Error: {}".format(e))
 
+    print_h_options(h_change, st[0].has_hydrogen(), args.refine_h, args.hout, geom_only=False)
+    
     # initialize ADP
     utils.model.reset_adp(st[0], args.bfactor, args.adp)
         
@@ -199,7 +210,7 @@ def main(args):
         ncslist = utils.restraints.prepare_ncs_restraints(st)
     else:
         ncslist = False
-    geom = Geom(st, topo, monlib, shake_rms=args.randomize, adpr_w=args.adpr_weight, params=params,
+    geom = Geom(st, topo, monlib, shake_rms=args.randomize, adpr_w=args.adpr_weight, occr_w=args.occr_weight, params=params,
                 unrestrained=args.unrestrained or args.jellyonly, use_nucleus=(args.source=="neutron"),
                 ncslist=ncslist)
     geom.geom.angle_von_mises = args.vonmises
@@ -225,7 +236,9 @@ def main(args):
 
     stats = refiner.run_cycles(args.ncycle, weight=args.weight,
                                weight_adjust=not args.no_weight_adjust,
+                               weight_adjust_bond_rmsz_range=args.target_bond_rmsz_range,
                                stats_json_out=args.output_prefix + "_stats.json")
+    update_meta(st, stats[-1], ll)
     refiner.st.name = args.output_prefix
     utils.fileio.write_model(refiner.st, args.output_prefix, pdb=True, cif=True, hout=args.hout)
     if params["write_trajectory"]:
@@ -244,7 +257,7 @@ def main(args):
 
     # Write mtz file
     if ll.twin_data:
-        labs = ["F_est"]
+        labs = ["F_est", "F_exp", "FOM"]
     elif is_int:
         labs = ["I", "SIGI", "FOM"]
     else:
@@ -258,7 +271,7 @@ def main(args):
         labs.append("FREE")
     labs += ll.D_labs + ["S"] # for debugging, for now
     mtz_out = args.output_prefix+".mtz"
-    hkldata.write_mtz(mtz_out, labs=labs, types={"FOM": "W", "FP":"F", "SIGFP":"Q", "I":"J", "SIGI":"Q", "F_est": "F"})
+    hkldata.write_mtz(mtz_out, labs=labs, types={"FOM": "W", "FP":"F", "SIGFP":"Q", "I":"J", "SIGI":"Q", "F_est": "F", "F_exp": "F"})
 
 # main()
 

servalcat/refine/spa.py

@@ -38,8 +38,14 @@ class LL_SPA:
         self.lab_obs = lab_obs
         self.st = st
         self.monlib = monlib
-        self.d_min = hkldata.d_min_max()[0]
+        self.d_min_max = hkldata.d_min_max()
         self.ll = None
+        self.b_aniso = None
+
+    def refine_id(self):
+        if self.source == "electron":
+            return "ELECTRON MICROSCOPY"
+        return "NON-EM SPA" # does not happen, I guess
 
     def update_ml_params(self):
         # FIXME make sure D > 0
@@ -52,7 +58,7 @@ class LL_SPA:
         else:
             st = self.st
 
-        self.hkldata.df["FC"] = utils.model.calc_fc_fft(st, self.d_min - 1e-6,
+        self.hkldata.df["FC"] = utils.model.calc_fc_fft(st, self.d_min_max[0] - 1e-6,
                                                         monlib=self.monlib,
                                                         source=self.source,
                                                         mott_bethe=self.mott_bethe,
@@ -92,15 +98,17 @@ class LL_SPA:
         stats = fsc.calc_fsc_all(self.hkldata, labs_fc=["FC"], lab_f=self.lab_obs)
         fsca = fsc.fsc_average(stats.ncoeffs, stats.fsc_FC_full)
         logger.writeln("FSCaverage = {:.4f}".format(fsca))
+        ret = {"summary": {"FSCaverage": fsca, "-LL": self.calc_target()}}
         # XXX in fsc object, _full is misleading - it's not full in cross validation mode
         if "D" in self.hkldata.binned_df and "S" in self.hkldata.binned_df:
             stats[["D", "S"]] = self.hkldata.binned_df[["D", "S"]]
-        return {"bin_stats": stats, "summary": {"FSCaverage": fsca, "-LL": self.calc_target()}}
+        ret["bin_stats"] = stats
+        return ret
 
     def calc_grad(self, atom_pos, refine_xyz, adp_mode, refine_occ, refine_h, specs):
         dll_dab = numpy.empty_like(self.hkldata.df[self.lab_obs])
         d2ll_dab2 = numpy.zeros(len(self.hkldata.df.index))
-        blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min / 3) # TODO need more work
+        blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min_max[0] / 3) # TODO need more work
         logger.writeln("blur for deriv= {:.2f}".format(blur))
         for i_bin, idxes in self.hkldata.binned():
             D = self.hkldata.binned_df.D[i_bin]