servalcat 0.4.72__cp312-cp312-macosx_11_0_arm64.whl → 0.4.88__cp312-cp312-macosx_11_0_arm64.whl

servalcat/__init__.py

@@ -6,5 +6,5 @@ This software is released under the
 Mozilla Public License, version 2.0; see LICENSE.
 """
 
-__version__ = '0.4.72'
-__date__ = '2024-06-06'
+__version__ = '0.4.88'
+__date__ = '2024-09-13'

servalcat/refine/refine.py

@@ -40,7 +40,12 @@ class Geom:
             self.atom_pos = list(range(len(self.atoms)))
         self.n_refine_atoms = max(self.atom_pos) + 1
         self.lookup = {x.atom: x for x in self.st[0].all()}
-        self.geom = ext.Geometry(self.st, self.atom_pos, monlib.ener_lib)
+        try:
+            self.geom = ext.Geometry(self.st, self.atom_pos, monlib.ener_lib)
+        except TypeError as e:
+            raise SystemExit(f"An error occurred while creating the Geometry object:\n{e}\n\n"
+                             "This likely indicates an installation issue. "
+                             "Please verify that you have the correct version of gemmi installed and that both gemmi and servalcat were compiled in the same environment.")
         self.specs = utils.model.find_special_positions(self.st)
         #cs_count = len(self.st.find_spacegroup().operations())
         for atom, images, matp, mata in self.specs:
@@ -58,7 +63,6 @@ class Geom:
         self.calc_kwds = {"use_nucleus": self.use_nucleus}
         if params is None:
             params = {}
-        exte.read_external_restraints(params.get("exte", []), self.st, self.geom)
         for k in ("wbond", "wangle", "wtors", "wplane", "wchir", "wvdw", "wncs"):
             if k in params:
                 self.calc_kwds[k] = params[k]
@@ -70,26 +74,13 @@ class Geom:
         self.group_occ = GroupOccupancy(self.st, params.get("occu"))
         if not self.unrestrained:
             self.geom.load_topo(topo)
-            self.check_chemtypes(os.path.join(monlib.path(), "ener_lib.cif"), topo)
+        exte.read_external_restraints(params.get("exte", []), self.st, self.geom)
         self.geom.finalize_restraints()
         self.outlier_sigmas = dict(bond=5, angle=5, torsion=5, vdw=5, ncs=5, chir=5, plane=5, staca=5, stacd=5, per_atom=5)
         self.parents = {}
         self.ncslist = ncslist
     # __init__()
 
-    def check_chemtypes(self, enerlib_path, topo):
-        block = gemmi.cif.read(enerlib_path).sole_block()
-        all_types = set(block.find_values("_lib_atom.type"))
-        for ci in topo.chain_infos:
-            for ri in ci.res_infos:
-                cc_all = {x: ri.get_final_chemcomp(x) for x in set(a.altloc for a in ri.res)}
-                for a in ri.res:
-                    cca = cc_all[a.altloc].find_atom(a.name)
-                    if cca is None: # I believe it won't happen..
-                        logger.writeln("WARNING: restraint for {} not found.".format(self.lookup[a]))
-                    elif cca.chem_type not in all_types:
-                        raise RuntimeError("Energy type {} of {} not found in ener_lib.".format(cca.chem_type,
-                                                                                                self.lookup[a]))
     def set_h_parents(self):
         self.parents = {}
         for bond in self.geom.bonds:
@@ -176,15 +167,16 @@ class Geom:
                         logger.writeln(df.to_string(float_format="{:.3f}".format, index=False) + "\n")
 
         # Per-atom score
-        peratom = self.geom.reporting.per_atom_score(len(self.atoms), self.use_nucleus, "mean")
-        df = pandas.DataFrame(peratom)
-        df.insert(0, "atom", [str(self.lookup[x]) for x in self.atoms])
-        df = df[df["total"] >= self.outlier_sigmas["per_atom"]]
-        if show_outliers and len(df.index) > 0:
-            df.sort_values("total", ascending=False, inplace=True)
-            ret["outliers"]["per_atom"] = df
-            logger.writeln(" *** Per-atom violations (Z >= {}) ***\n".format(self.outlier_sigmas["per_atom"]))
-            logger.writeln(df.to_string(float_format="{:.2f}".format, index=False) + "\n")
+        if 0:
+            peratom = self.geom.reporting.per_atom_score(len(self.atoms), self.use_nucleus, "mean")
+            df = pandas.DataFrame(peratom)
+            df.insert(0, "atom", [str(self.lookup[x]) for x in self.atoms])
+            df = df[df["total"] >= self.outlier_sigmas["per_atom"]]
+            if show_outliers and len(df.index) > 0:
+                df.sort_values("total", ascending=False, inplace=True)
+                ret["outliers"]["per_atom"] = df
+                logger.writeln(" *** Per-atom violations (Z >= {}) ***\n".format(self.outlier_sigmas["per_atom"]))
+                logger.writeln(df.to_string(float_format="{:.2f}".format, index=False) + "\n")
 
         df = pandas.DataFrame(self.geom.reporting.get_summary_table(self.use_nucleus))
         df = df.set_index("Restraint type").rename_axis(index=None)
@@ -556,6 +548,8 @@ class Refine:
         self.geom.setup_nonbonded(self.refine_xyz) # if refine_xyz=False, no need to do it every time
         self.geom.geom.setup_target(self.refine_xyz, self.adp_mode, self.refine_occ, self.use_occr)
         logger.writeln("vdws = {}".format(len(self.geom.geom.vdws)))
+        logger.writeln(f"atoms = {self.geom.geom.target.n_atoms()}")
+        logger.writeln(f"pairs = {self.geom.geom.target.n_pairs()}")
 
     def get_x(self):
         n_atoms = self.geom.n_refine_atoms
@@ -675,6 +669,8 @@ class Refine:
         self.geom.setup_nonbonded(self.refine_xyz)
         self.geom.geom.setup_target(self.refine_xyz, self.adp_mode, self.refine_occ, self.use_occr)
         logger.writeln("vdws = {}".format(len(self.geom.geom.vdws)))
+        logger.writeln(f"atoms = {self.geom.geom.target.n_atoms()}")
+        logger.writeln(f"pairs = {self.geom.geom.target.n_pairs()}")
         stats[-1]["geom"] = self.geom.show_model_stats(refine_xyz=self.refine_xyz and not self.unrestrained,
                                                        adp_mode=self.adp_mode,
                                                        use_occr=self.refine_occ,
@@ -683,9 +679,12 @@ class Refine:
             self.ll.update_fc()
             self.ll.overall_scale()
             self.ll.update_ml_params()
+            self.ll.prepare_target()
             llstats = self.ll.calc_stats(bin_stats=True)
             stats[-1]["data"] = {"summary": llstats["summary"],
                                  "binned": llstats["bin_stats"].to_dict(orient="records")}
+            if "twin_alpha" in llstats:
+                stats[-1]["twin_alpha"] = llstats["twin_alpha"]
             show_binstats(llstats["bin_stats"], 0)
         if self.adp_mode > 0:
             utils.model.adp_analysis(self.st)
@@ -713,12 +712,15 @@ class Refine:
                 self.ll.overall_scale()
                 f0 = self.ll.calc_target()
                 self.ll.update_ml_params()
+                self.ll.prepare_target()
                 llstats = self.ll.calc_stats(bin_stats=True)#(i==ncycles-1))
                 if llstats["summary"]["-LL"] > f0:
                     logger.writeln("WARNING: -LL has increased after ML parameter optimization:"
                                    "{} to {}".format(f0, llstats["summary"]["-LL"]))
                 stats[-1]["data"] = {"summary": llstats["summary"],
                                      "binned": llstats["bin_stats"].to_dict(orient="records")}
+                if "twin_alpha" in llstats:
+                    stats[-1]["twin_alpha"] = llstats["twin_alpha"]
                 show_binstats(llstats["bin_stats"], i+1)
             if self.adp_mode > 0:
                 utils.model.adp_analysis(self.st)
@@ -734,7 +736,7 @@ class Refine:
                     weight /= 1.1
             if self.st_traj is not None:
                 self.st_traj.add_model(self.st[0])
-                self.st_traj[-1].name = str(i+1)
+                self.st_traj[-1].name = str(len(self.st_traj))
             if stats_json_out:
                 write_stats_json_safe(stats, stats_json_out)
 

servalcat/refine/refine_geom.py

@@ -44,6 +44,8 @@ def add_arguments(parser):
                         help="refmac keyword file(s)")
     parser.add_argument('-o','--output_prefix',
                         help="Output prefix")
+    parser.add_argument("--write_trajectory", action='store_true',
+                        help="Write all output from cycles")
 
 # add_arguments()
 
@@ -150,7 +152,8 @@ def refine_geom(model_in, monomer_dir, cif_files, h_change, ncycle, output_prefi
                                                    cif_files=cif_files,
                                                    stop_for_unknowns=True,
                                                    params=params)
-    utils.restraints.find_and_fix_links(st, monlib, add_found=find_links) # should remove unknown id here?
+    utils.restraints.find_and_fix_links(st, monlib, find_metal_links=find_links,
+                                        add_found=find_links) # should remove unknown id here?
     try:
         topo, _ = utils.restraints.prepare_topology(st, monlib, h_change=h_change,
                                                     check_hydrogen=(h_change==gemmi.HydrogenChange.NoChange),
@@ -163,11 +166,13 @@ def refine_geom(model_in, monomer_dir, cif_files, h_change, ncycle, output_prefi
     else:
         ncslist = False
     geom = Geom(st, topo, monlib, shake_rms=randomize, params=params, ncslist=ncslist)
-    refiner = Refine(st, geom)
+    refiner = Refine(st, geom, params=params)
     stats = refiner.run_cycles(ncycle,
                                stats_json_out=output_prefix + "_stats.json")
     refiner.st.name = output_prefix
     utils.fileio.write_model(refiner.st, output_prefix, pdb=True, cif=True)
+    if params["write_trajectory"]:
+        utils.fileio.write_model(refiner.st_traj, output_prefix + "_traj", cif=True)
 # refine_geom()
 
 def main(args):
@@ -177,6 +182,7 @@ def main(args):
     decide_prefix = lambda f: utils.fileio.splitext(os.path.basename(f))[0] + "_refined"
     if args.model:
         params = refmac_keywords.parse_keywords(keywords)
+        params["write_trajectory"] = args.write_trajectory
         if not args.output_prefix:
             args.output_prefix = decide_prefix(args.model)
         if args.ligand:

servalcat/refine/refine_spa.py

@@ -53,8 +53,10 @@ def add_arguments(parser):
                         help="Monomer library path. Default: $CLIBD_MON")
     parser.add_argument('--ligand', nargs="*", action="append",
                         help="restraint dictionary cif file(s)")
+    parser.add_argument('--newligand_continue', action='store_true',
+                        help="Make ad-hoc restraints for unknown ligands (not recommended)")
     parser.add_argument('--hydrogen', default="all", choices=["all", "yes", "no"],
-                        help="all: add riding hydrogen atoms, yes: use hydrogen atoms if present, no: remove hydrogen atoms in input. "
+                        help="all: (re)generate hydrogen atoms, yes: use hydrogen atoms if present, no: remove hydrogen atoms in input. "
                         "Default: %(default)s")
     parser.add_argument('--hout', action='store_true', help="write hydrogen atoms in the output model")
     parser.add_argument('--jellybody', action='store_true',
@@ -65,15 +67,15 @@ def add_arguments(parser):
     parser.add_argument('--jellyonly', action='store_true',
                         help="Jelly body only (experimental, may not be useful)")
     utils.symmetry.add_symmetry_args(parser) # add --pg etc
-    parser.add_argument('--contacting_only', action="store_true", help="Filter out non-contacting NCS")
+    parser.add_argument('--contacting_only', action="store_true", help="Filter out non-contacting strict NCS copies")
     parser.add_argument('--ignore_symmetry',
                         help='Ignore symmetry information (MTRIX/_struct_ncs_oper) in the model file')
     parser.add_argument('--find_links', action='store_true', 
                         help='Automatically add links')
     parser.add_argument('--no_check_ncs_overlaps', action='store_true', 
-                        help='Disable model overlap (e.g. expanded model is used with --pg) test')
+                        help='Disable model overlap test due to strict NCS')
     parser.add_argument('--no_check_ncs_map', action='store_true', 
-                        help='Disable map NCS consistency test')
+                        help='Disable map symmetry test due to strict NCS')
     parser.add_argument('--no_check_mask_with_model', action='store_true', 
                         help='Disable mask test using model')
     parser.add_argument('--keywords', nargs='+', action="append",
@@ -81,7 +83,7 @@ def add_arguments(parser):
     parser.add_argument('--keyword_file', nargs='+', action="append",
                         help="refmac keyword file(s)")
     parser.add_argument('--randomize', type=float, default=0,
-                        help='Shake coordinates with specified rmsd')
+                        help='Shake coordinates with the specified rmsd value')
     parser.add_argument('--ncycle', type=int, default=10,
                         help="number of CG cycles (default: %(default)d)")
     parser.add_argument('--weight', type=float,
@@ -91,20 +93,22 @@ def add_arguments(parser):
     parser.add_argument('--target_bond_rmsz_range', nargs=2, type=float, default=[0.5, 1.],
                         help='Bond rmsz range for weight adjustment (default: %(default)s)')
     parser.add_argument('--adpr_weight', type=float, default=1.,
-                        help="ADP restraint weight in B (default: %(default)f)")
+                        help="ADP restraint weight (default: %(default)f)")
     parser.add_argument('--ncsr', action='store_true', 
                         help='Use local NCS restraints')
     parser.add_argument('--bfactor', type=float,
-                        help="reset all atomic B values to specified value")
-    parser.add_argument('--fix_xyz', action="store_true")
-    parser.add_argument('--adp',  choices=["fix", "iso", "aniso"], default="iso")
+                        help="reset all atomic B values to the specified value")
+    parser.add_argument('--fix_xyz', action="store_true",
+                        help="Fix atomic coordinates")
+    parser.add_argument('--adp',  choices=["fix", "iso", "aniso"], default="iso",
+                        help="ADP parameterization")
     parser.add_argument('--refine_all_occ', action="store_true")
     parser.add_argument('--max_dist_for_adp_restraint', type=float, default=4.)
     parser.add_argument('--adp_restraint_power', type=float)
     parser.add_argument('--adp_restraint_exp_fac', type=float)
     parser.add_argument('--adp_restraint_no_long_range', action='store_true')
     parser.add_argument('--adp_restraint_mode', choices=["diff", "kldiv"], default="diff")
-    parser.add_argument('--refine_h', action="store_true", help="Refine hydrogen (default: restraints only)")
+    parser.add_argument('--refine_h', action="store_true", help="Refine hydrogen against data (default: only restraints apply)")
     parser.add_argument("--source", choices=["electron", "xray", "neutron"], default="electron")
     parser.add_argument('-o','--output_prefix', default="refined")
     parser.add_argument('--cross_validation', action='store_true',
@@ -140,7 +144,8 @@ def main(args):
     st = utils.fileio.read_structure(args.model)
     try:
         monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand,
-                                                       stop_for_unknowns=True, params=params)
+                                                       stop_for_unknowns=not args.newligand_continue,
+                                                       params=params)
     except RuntimeError as e:
         raise SystemExit("Error: {}".format(e))
     if not args.keep_entities:
@@ -161,7 +166,8 @@ def main(args):
         st.spacegroup_hm = hkldata.sg.xhm()
         st.setup_cell_images()
         info = {}
-        utils.restraints.find_and_fix_links(st, monlib, add_found=args.find_links)
+        utils.restraints.find_and_fix_links(st, monlib, find_metal_links=args.find_links,
+                                            add_found=args.find_links)
     else:
         if args.halfmaps:
             maps = utils.fileio.read_halfmaps(args.halfmaps, pixel_size=args.pixel_size)
@@ -316,6 +322,9 @@ Weight used: {final_weight:.3e}
 Open refined model and {diffmap_prefix}.mtz with COOT:
 coot --script {prefix}_coot.py
 
+Open refined model, map and difference map with ChimeraX/ISOLDE:
+chimerax {prefix}_chimerax.cxc
+
 {map_peaks_msg}
 =============================================================================
 """.format(rmsbond=rmsbond,

servalcat/refine/refine_xtal.py

@@ -8,15 +8,17 @@ Mozilla Public License, version 2.0; see LICENSE.
 from __future__ import absolute_import, division, print_function, generators
 import gemmi
 import numpy
+import pandas
 import os
 import shutil
 import argparse
 from servalcat.utils import logger
 from servalcat import utils
-from servalcat.xtal.sigmaa import decide_mtz_labels, process_input, calculate_maps, calculate_maps_int
+from servalcat.xtal.sigmaa import decide_mtz_labels, process_input, calculate_maps, calculate_maps_int, calculate_maps_twin
 from servalcat.refine.xtal import LL_Xtal
 from servalcat.refine.refine import Geom, Refine
 from servalcat.refmac import refmac_keywords
+from servalcat import ext
 b_to_u = utils.model.b_to_u
 
 def add_arguments(parser):
@@ -35,6 +37,8 @@ def add_arguments(parser):
                         help="Monomer library path. Default: $CLIBD_MON")
     parser.add_argument('--ligand', nargs="*", action="append",
                         help="restraint dictionary cif file(s)")
+    parser.add_argument('--newligand_continue', action='store_true',
+                        help="Make ad-hoc restraints for unknown ligands (not recommended)")
     parser.add_argument('--hydrogen', default="all", choices=["all", "yes", "no"],
                         help="all: add riding hydrogen atoms, yes: use hydrogen atoms if present, no: remove hydrogen atoms in input. "
                         "Default: %(default)s")
@@ -76,6 +80,7 @@ def add_arguments(parser):
     parser.add_argument('--adp_restraint_mode', choices=["diff", "kldiv"], default="kldiv")
     parser.add_argument('--unrestrained',  action='store_true', help="No positional restraints")
     parser.add_argument('--refine_h', action="store_true", help="Refine hydrogen (default: restraints only)")
+    parser.add_argument('--twin', action="store_true", help="Turn on twin refinement")
     parser.add_argument("-s", "--source", choices=["electron", "xray", "neutron"], required=True)
     parser.add_argument('--no_solvent',  action='store_true',
                         help="Do not consider bulk solvent contribution")
@@ -83,9 +88,12 @@ def add_arguments(parser):
                         help="Use work reflections in ML parameter estimates")
     parser.add_argument('--keep_charges',  action='store_true',
                         help="Use scattering factor for charged atoms. Use it with care.")
+    parser.add_argument('--allow_unusual_occupancies', action="store_true", help="Allow negative or more than one occupancies")
     parser.add_argument('-o','--output_prefix')
     parser.add_argument("--write_trajectory", action='store_true',
                         help="Write all output from cycles")
+    parser.add_argument("--vonmises", action='store_true',
+                        help="Experimental: von Mises type restraint for angles")
 # add_arguments()
 
 def parse_args(arg_list):
@@ -140,7 +148,8 @@ def main(args):
                                                                                  n_per_bin=n_per_bin,
                                                                                  use=use_in_est,
                                                                                  max_bins=30,
-                                                                                 keep_charges=args.keep_charges)
+                                                                                 keep_charges=args.keep_charges,
+                                                                                 allow_unusual_occupancies=args.allow_unusual_occupancies)
     except RuntimeError as e:
         raise SystemExit("Error: {}".format(e))
 
@@ -159,11 +168,13 @@ def main(args):
     else:
         try:
             monlib = utils.restraints.load_monomer_library(st, monomer_dir=args.monlib, cif_files=args.ligand,
-                                                           stop_for_unknowns=True, params=params)
+                                                           stop_for_unknowns=not args.newligand_continue,
+                                                           params=params)
         except RuntimeError as e:
             raise SystemExit("Error: {}".format(e))
         utils.model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
-        utils.restraints.find_and_fix_links(st, monlib, add_found=args.find_links)
+        utils.restraints.find_and_fix_links(st, monlib, find_metal_links=args.find_links,
+                                            add_found=args.find_links)
         h_change = {"all":gemmi.HydrogenChange.ReAddKnown,
                     "yes":gemmi.HydrogenChange.NoChange,
                     "no":gemmi.HydrogenChange.Remove}[args.hydrogen]
@@ -191,6 +202,7 @@ def main(args):
     geom = Geom(st, topo, monlib, shake_rms=args.randomize, adpr_w=args.adpr_weight, params=params,
                 unrestrained=args.unrestrained or args.jellyonly, use_nucleus=(args.source=="neutron"),
                 ncslist=ncslist)
+    geom.geom.angle_von_mises = args.vonmises
     geom.geom.adpr_max_dist = args.max_dist_for_adp_restraint
     if args.adp_restraint_power is not None: geom.geom.adpr_d_power = args.adp_restraint_power
     if args.adp_restraint_exp_fac is not None: geom.geom.adpr_exp_fac = args.adp_restraint_exp_fac
@@ -201,7 +213,8 @@ def main(args):
     if args.jellyonly: geom.geom.ridge_exclude_short_dist = False
 
     ll = LL_Xtal(hkldata, centric_and_selections, args.free, st, monlib, source=args.source,
-                 use_solvent=not args.no_solvent, use_in_est=use_in_est, use_in_target=use_in_target)
+                 use_solvent=not args.no_solvent, use_in_est=use_in_est, use_in_target=use_in_target,
+                 twin=args.twin)
     refiner = Refine(st, geom, ll=ll,
                      refine_xyz=not args.fix_xyz,
                      adp_mode=dict(fix=0, iso=1, aniso=2)[args.adp],
@@ -218,7 +231,11 @@ def main(args):
     if params["write_trajectory"]:
         utils.fileio.write_model(refiner.st_traj, args.output_prefix + "_traj", cif=True)
 
-    if is_int:
+    if ll.twin_data:
+        # replace hkldata
+        hkldata = calculate_maps_twin(ll.hkldata, ll.b_aniso, ll.fc_labs, ll.D_labs, ll.twin_data,
+                                      centric_and_selections, use=use_in_target)
+    elif is_int:
         calculate_maps_int(ll.hkldata, ll.b_aniso, ll.fc_labs, ll.D_labs, centric_and_selections,
                            use=use_in_target)
     else:
@@ -226,7 +243,9 @@ def main(args):
                        use=use_in_target)
 
     # Write mtz file
-    if is_int:
+    if ll.twin_data:
+        labs = ["F_est"]
+    elif is_int:
         labs = ["I", "SIGI", "FOM"]
     else:
         labs = ["FP", "SIGFP", "FOM"]
@@ -239,7 +258,7 @@ def main(args):
         labs.append("FREE")
     labs += ll.D_labs + ["S"] # for debugging, for now
     mtz_out = args.output_prefix+".mtz"
-    hkldata.write_mtz(mtz_out, labs=labs, types={"FOM": "W", "FP":"F", "SIGFP":"Q", "I":"J", "SIGI":"Q"})
+    hkldata.write_mtz(mtz_out, labs=labs, types={"FOM": "W", "FP":"F", "SIGFP":"Q", "I":"J", "SIGI":"Q", "F_est": "F"})
 
 # main()
 

servalcat/refine/spa.py

@@ -58,6 +58,9 @@ class LL_SPA:
                                                         mott_bethe=self.mott_bethe,
                                                         miller_array=self.hkldata.miller_array())
 
+    def prepare_target(self):
+        pass
+
     def overall_scale(self, min_b=0.5):
         k, b = self.hkldata.scale_k_and_b(lab_ref=self.lab_obs, lab_scaled="FC")
         min_b_iso = self.st[0].calculate_b_aniso_range()[0] # actually min of aniso too