servalcat 0.4.72__cp312-cp312-macosx_11_0_arm64.whl → 0.4.88__cp312-cp312-macosx_11_0_arm64.whl

servalcat/refine/xtal.py

@@ -14,18 +14,18 @@ from servalcat.utils import logger
 from servalcat.xtal import sigmaa
 from servalcat import utils
 from servalcat import ext
+from servalcat.xtal.twin import find_twin_domains_from_data, estimate_twin_fractions_from_model
 b_to_u = utils.model.b_to_u
 u_to_b = utils.model.u_to_b
 integr = sigmaa.integr
 
 class LL_Xtal:
     def __init__(self, hkldata, centric_and_selections, free, st, monlib, source="xray", mott_bethe=True,
-                 use_solvent=False, use_in_est="all", use_in_target="all"):
+                 use_solvent=False, use_in_est="all", use_in_target="all", twin=False):
         assert source in ("electron", "xray", "neutron")
         self.source = source
         self.mott_bethe = False if source != "electron" else mott_bethe
         self.hkldata = hkldata
-        self.is_int = "I" in self.hkldata.df
         self.centric_and_selections = centric_and_selections
         self.free = free
         self.st = st
@@ -44,39 +44,58 @@ class LL_Xtal:
         self.use_in_target = use_in_target
         self.ll = None
         self.scaling = sigmaa.LsqScale()
+        if twin:
+            self.twin_data = find_twin_domains_from_data(self.hkldata)
+        else:
+            self.twin_data = None
+        if self.twin_data:
+            self.twin_data.setup_f_calc(len(self.fc_labs))    
+        self.is_int = "I" in self.hkldata.df
         logger.writeln("will use {} reflections for parameter estimation".format(self.use_in_est))
         logger.writeln("will use {} reflections for refinement".format(self.use_in_target))
 
     def update_ml_params(self):
         self.b_aniso = sigmaa.determine_ml_params(self.hkldata, self.is_int, self.fc_labs, self.D_labs, self.b_aniso,
-                                                   self.centric_and_selections, use=self.use_in_est,
-                                                  )#D_trans="splus", S_trans="splus")
+                                                  self.centric_and_selections, use=self.use_in_est,
+                                                  twin_data=self.twin_data)#D_trans="splus", S_trans="splus")
         self.hkldata.df["k_aniso"] = self.hkldata.debye_waller_factors(b_cart=self.b_aniso)
         #determine_mlf_params_from_cc(self.hkldata, self.fc_labs, self.D_labs,
         #                             self.centric_and_selections)
-
-
     def update_fc(self):
-        if self.st.ncs:
-            st = self.st.clone()
-            st.expand_ncs(gemmi.HowToNameCopiedChain.Dup, merge_dist=0)
-        else:
-            st = self.st
+        sigmaa.update_fc(st_list=[self.st], fc_labs=self.fc_labs,
+                         d_min=self.d_min, monlib=self.monlib,
+                         source=self.source, mott_bethe=self.mott_bethe,
+                         hkldata=self.hkldata, twin_data=self.twin_data)
 
-        self.hkldata.df[self.fc_labs[0]] = utils.model.calc_fc_fft(st, self.d_min - 1e-6,
-                                                                   monlib=self.monlib,
-                                                                   source=self.source,
-                                                                   mott_bethe=self.mott_bethe,
-                                                                   miller_array=self.hkldata.miller_array())
-        self.hkldata.df["FC"] = self.hkldata.df[self.fc_labs].sum(axis=1)
+    def prepare_target(self):
+        if self.twin_data:
+            if self.use_in_target == "all":
+                idxes = numpy.concatenate([sel[i] for i_bin, _ in self.hkldata.binned()
+                                           for sel in self.centric_and_selections[i_bin] for i in (1,2)])
+            else:
+                i = 1 if self.use_in_target == "work" else 2
+                idxes = numpy.concatenate([sel[i] for i_bin, _ in self.hkldata.binned()
+                                           for sel in self.centric_and_selections[i_bin]])
+            mask = numpy.empty(len(self.hkldata.df.index)) * numpy.nan
+            mask[idxes] = 1 / self.hkldata.debye_waller_factors(b_cart=self.b_aniso)[idxes]**2
+            self.twin_data.est_f_true(self.hkldata.df.I * mask,
+                                      self.hkldata.df.SIGI * mask)
         
     def overall_scale(self, min_b=0.1):
-        fc_list = [self.hkldata.df[self.fc_labs[0]].to_numpy()]
+        miller_array = self.twin_data.asu if self.twin_data else self.hkldata.miller_array()
         if self.use_solvent:
-            Fmask = sigmaa.calc_Fmask(self.st, self.d_min - 1e-6, self.hkldata.miller_array())
-            fc_list.append(Fmask)
-
-        self.scaling.set_data(self.hkldata, fc_list, self.is_int, sigma_cutoff=0)
+            Fmask = sigmaa.calc_Fmask(self.st, self.d_min, miller_array)
+            if self.twin_data:
+                fc_sum = self.twin_data.f_calc[:,:-1].sum(axis=1)
+            else:
+                fc_sum = self.hkldata.df[self.fc_labs[:-1]].sum(axis=1).to_numpy()
+            fc_list = [fc_sum, Fmask]
+        else:
+            if twin_data:
+                fc_list = [self.twin_data.f_calc.sum(axis=1)]
+            else:
+                fc_list = [self.hkldata.df[self.fc_labs].sum(axis=1).to_numpy()]
+        self.scaling.set_data(self.hkldata, fc_list, self.is_int, sigma_cutoff=0, twin_data=self.twin_data)
         self.scaling.scale()
         self.b_aniso = self.scaling.b_aniso
         b = self.scaling.b_iso
@@ -90,9 +109,15 @@ class LL_Xtal:
         k_iso = self.hkldata.debye_waller_factors(b_iso=b)
         self.hkldata.df["k_aniso"] = self.hkldata.debye_waller_factors(b_cart=self.b_aniso)
         if self.use_solvent:
-            solvent_scale = self.scaling.get_solvent_scale(self.scaling.k_sol, self.scaling.b_sol,
-                                                      1. / self.hkldata.d_spacings().to_numpy()**2)
-            self.hkldata.df[self.fc_labs[-1]] = Fmask * solvent_scale
+            if self.twin_data:
+                s2 = numpy.asarray(self.twin_data.s2_array)
+            else:
+                s2 = 1. / self.hkldata.d_spacings().to_numpy()**2
+            Fbulk = Fmask * self.scaling.get_solvent_scale(self.scaling.k_sol, self.scaling.b_sol, s2)
+            if self.twin_data:
+                self.twin_data.f_calc[:,-1] = Fbulk
+            else:
+                self.hkldata.df[self.fc_labs[-1]] = Fbulk
         if self.is_int:
             o_labs = self.hkldata.df.columns.intersection(["I", "SIGI",
                                                            "I(+)","SIGI(+)", "I(-)", "SIGI(-)"])
@@ -102,103 +127,125 @@ class LL_Xtal:
                                                            "F(+)","SIGF(+)", "F(-)", "SIGF(-)"])
             self.hkldata.df[o_labs] /= self.scaling.k_overall
 
-        for lab in self.fc_labs: self.hkldata.df[lab] *= k_iso
-        self.hkldata.df["FC"] = self.hkldata.df[self.fc_labs].sum(axis=1)
+        if self.twin_data:
+            self.twin_data.f_calc[:] *= self.twin_data.debye_waller_factors(b_iso=b)[:,None]
+        else:
+            for lab in self.fc_labs: self.hkldata.df[lab] *= k_iso
+            self.hkldata.df["FC"] = self.hkldata.df[self.fc_labs].sum(axis=1)
 
         # for next cycle
         self.scaling.k_overall = 1.
         self.scaling.b_iso = 0.
+        if self.twin_data:
+            estimate_twin_fractions_from_model(self.twin_data, self.hkldata)
     # overall_scale()
 
     def calc_target(self): # -LL target for MLF or MLI
         ret = 0
-        k_aniso = self.hkldata.debye_waller_factors(b_cart=self.b_aniso)
-        f = sigmaa.mli if self.is_int else sigmaa.mlf
-        for i_bin, _ in self.hkldata.binned():
-            if self.use_in_target == "all":
-                idxes = numpy.concatenate([sel[i] for sel in self.centric_and_selections[i_bin] for i in (1,2)])
-            else:
-                i = 1 if self.use_in_target == "work" else 2
-                idxes = numpy.concatenate([sel[i] for sel in self.centric_and_selections[i_bin]])
-            ret += f(self.hkldata.df,
-                     self.fc_labs,
-                     numpy.vstack([self.hkldata.df[lab].to_numpy()[idxes] for lab in self.D_labs]).T,
-                     self.hkldata.df.S.to_numpy()[idxes],
-                     k_aniso,
-                     idxes)
+        if self.twin_data:
+            ret = self.twin_data.ll()
+        else:
+            k_aniso = self.hkldata.debye_waller_factors(b_cart=self.b_aniso)
+            f = sigmaa.mli if self.is_int else sigmaa.mlf
+            for i_bin, _ in self.hkldata.binned():
+                if self.use_in_target == "all":
+                    idxes = numpy.concatenate([sel[i] for sel in self.centric_and_selections[i_bin] for i in (1,2)])
+                else:
+                    i = 1 if self.use_in_target == "work" else 2
+                    idxes = numpy.concatenate([sel[i] for sel in self.centric_and_selections[i_bin]])
+                ret += f(self.hkldata.df,
+                         self.fc_labs,
+                         numpy.vstack([self.hkldata.df[lab].to_numpy()[idxes] for lab in self.D_labs]).T,
+                         self.hkldata.df.S.to_numpy()[idxes],
+                         k_aniso,
+                         idxes)
         return ret * 2 # friedel mates
     # calc_target()
 
     def calc_stats(self, bin_stats=False):
-        stats, overall = sigmaa.calc_r_and_cc(self.hkldata, self.centric_and_selections)
+        stats, overall = sigmaa.calc_r_and_cc(self.hkldata, self.centric_and_selections, self.twin_data)
         ret = {"summary": overall}
         ret["summary"]["-LL"] = self.calc_target()
+        if self.twin_data:
+            ret["twin_alpha"] = self.twin_data.alphas
         if bin_stats:
             ret["bin_stats"] = stats
         for lab in "R", "CC":
             logger.writeln(" ".join("{} = {:.4f}".format(x, overall[x]) for x in overall if x.startswith(lab)))
+        if self.is_int:
+            logger.writeln("R1 is calculated for reflections with I/sigma>2.")
         return ret
 
     def calc_grad(self, atom_pos, refine_xyz, adp_mode, refine_occ, refine_h, specs=None):
-        dll_dab = numpy.zeros(len(self.hkldata.df.FC), dtype=numpy.complex128)
-        d2ll_dab2 = numpy.empty(len(self.hkldata.df.index))
-        d2ll_dab2[:] = numpy.nan
         blur = utils.model.determine_blur_for_dencalc(self.st, self.d_min / 3) # TODO need more work
         logger.writeln("blur for deriv= {:.2f}".format(blur))
-        k_ani = self.hkldata.debye_waller_factors(b_cart=self.b_aniso)
-        for i_bin, _ in self.hkldata.binned():
-            for c, work, test in self.centric_and_selections[i_bin]:
-                if self.use_in_target == "all":
-                    cidxes = numpy.concatenate([work, test])
-                else:
-                    cidxes = work if self.use_in_target == "work" else test
-                epsilon = self.hkldata.df.epsilon.to_numpy()[cidxes]
-                Fcs = numpy.vstack([self.hkldata.df[lab].to_numpy()[cidxes] for lab in self.fc_labs]).T
-                Ds = numpy.vstack([self.hkldata.df[lab].to_numpy()[cidxes] for lab in self.D_labs]).T
-                S = self.hkldata.df["S"].to_numpy()[cidxes]
-                Fc = (Ds * Fcs).sum(axis=1)
-                Fc_abs = numpy.abs(Fc)
-                expip = numpy.exp(1j * numpy.angle(Fc))
-                if self.is_int:
-                    Io = self.hkldata.df.I.to_numpy()
-                    sigIo = self.hkldata.df.SIGI.to_numpy()
-                    to = Io[cidxes] / sigIo[cidxes] - sigIo[cidxes] / (c+1) / k_ani[cidxes]**2 / S / epsilon
-                    tf = k_ani[cidxes] * Fc_abs / numpy.sqrt(sigIo[cidxes])
-                    sig1 = k_ani[cidxes]**2 * epsilon * S / sigIo[cidxes]
-                    k_num = 0.5 if c == 0 else 0. # acentric:0.5, centric: 0.
-                    r = ext.integ_J_ratio(k_num, k_num - 0.5, True, to, tf, sig1, c+1,
-                                          integr.exp2_threshold, integr.h, integr.N, integr.ewmax)
-                    r *= numpy.sqrt(sigIo[cidxes]) / k_ani[cidxes]
-                    g = (2-c) * (Fc_abs - r) / epsilon / S  * Ds[:,0]
-                    dll_dab[cidxes] = g * expip
-                    #d2ll_dab2[cidxes] = (2-c)**2 / S / epsilon * Ds[0]**2 # approximation
-                    #d2ll_dab2[cidxes] = ((2-c) / S / epsilon + ((2-c) * r / k_ani[cidxes] / epsilon / S)**2) * Ds[0]**2
-                    d2ll_dab2[cidxes] =  g**2
-                else:
-                    Fo = self.hkldata.df.FP.to_numpy()[cidxes] / k_ani[cidxes]
-                    SigFo = self.hkldata.df.SIGFP.to_numpy()[cidxes] / k_ani[cidxes]
-                    if c == 0: # acentric
-                        Sigma = 2 * SigFo**2 + epsilon * S
-                        X = 2 * Fo * Fc_abs / Sigma
-                        m = gemmi.bessel_i1_over_i0(X)
-                        g = 2 * (Fc_abs - m * Fo) / Sigma * Ds[:,0]  # XXX assuming 0 is atomic structure
-                        dll_dab[cidxes] = g * expip
-                        d2ll_dab2[cidxes] = (2 / Sigma - (1 - m / X - m**2) * (2 * Fo / Sigma)**2) * Ds[:,0]**2
+        if self.twin_data:
+            dll_dab, d2ll_dab2 = self.twin_data.ll_der_fc0()
+            dll_dab *= self.twin_data.debye_waller_factors(b_iso=-blur)
+        else:
+            dll_dab = numpy.zeros(len(self.hkldata.df.FC), dtype=numpy.complex128)
+            d2ll_dab2 = numpy.empty(len(self.hkldata.df.index))
+            d2ll_dab2[:] = numpy.nan
+            k_ani = self.hkldata.debye_waller_factors(b_cart=self.b_aniso)
+            for i_bin, _ in self.hkldata.binned():
+                for c, work, test in self.centric_and_selections[i_bin]:
+                    if self.use_in_target == "all":
+                        cidxes = numpy.concatenate([work, test])
                     else:
-                        Sigma = SigFo**2 + epsilon * S
-                        X = Fo * Fc_abs / Sigma
-                        #X = X.astype(numpy.float64)
-                        m = numpy.tanh(X)
-                        g = (Fc_abs - m * Fo) / Sigma * Ds[:,0]
+                        cidxes = work if self.use_in_target == "work" else test
+                    epsilon = self.hkldata.df.epsilon.to_numpy()[cidxes]
+                    Fcs = numpy.vstack([self.hkldata.df[lab].to_numpy()[cidxes] for lab in self.fc_labs]).T
+                    Ds = numpy.vstack([self.hkldata.df[lab].to_numpy()[cidxes] for lab in self.D_labs]).T
+                    S = self.hkldata.df["S"].to_numpy()[cidxes]
+                    Fc = (Ds * Fcs).sum(axis=1)
+                    Fc_abs = numpy.abs(Fc)
+                    expip = numpy.exp(1j * numpy.angle(Fc))
+                    if self.is_int:
+                        Io = self.hkldata.df.I.to_numpy()
+                        sigIo = self.hkldata.df.SIGI.to_numpy()
+                        to = Io[cidxes] / sigIo[cidxes] - sigIo[cidxes] / (c+1) / k_ani[cidxes]**2 / S / epsilon
+                        tf = k_ani[cidxes] * Fc_abs / numpy.sqrt(sigIo[cidxes])
+                        sig1 = k_ani[cidxes]**2 * epsilon * S / sigIo[cidxes]
+                        k_num = 0.5 if c == 0 else 0. # acentric:0.5, centric: 0.
+                        r = ext.integ_J_ratio(k_num, k_num - 0.5, True, to, tf, sig1, c+1,
+                                              integr.exp2_threshold, integr.h, integr.N, integr.ewmax)
+                        r *= numpy.sqrt(sigIo[cidxes]) / k_ani[cidxes]
+                        g = (2-c) * (Fc_abs - r) / epsilon / S  * Ds[:,0]
                         dll_dab[cidxes] = g * expip
-                        d2ll_dab2[cidxes] = (1. / Sigma - (Fo / (Sigma * numpy.cosh(X)))**2) * Ds[:,0]**2
+                        #d2ll_dab2[cidxes] = (2-c)**2 / S / epsilon * Ds[0]**2 # approximation
+                        #d2ll_dab2[cidxes] = ((2-c) / S / epsilon + ((2-c) * r / k_ani[cidxes] / epsilon / S)**2) * Ds[0]**2
+                        d2ll_dab2[cidxes] =  g**2
+                    else:
+                        Fo = self.hkldata.df.FP.to_numpy()[cidxes] / k_ani[cidxes]
+                        SigFo = self.hkldata.df.SIGFP.to_numpy()[cidxes] / k_ani[cidxes]
+                        if c == 0: # acentric
+                            Sigma = 2 * SigFo**2 + epsilon * S
+                            X = 2 * Fo * Fc_abs / Sigma
+                            m = gemmi.bessel_i1_over_i0(X)
+                            g = 2 * (Fc_abs - m * Fo) / Sigma * Ds[:,0]  # XXX assuming 0 is atomic structure
+                            dll_dab[cidxes] = g * expip
+                            d2ll_dab2[cidxes] = (2 / Sigma - (1 - m / X - m**2) * (2 * Fo / Sigma)**2) * Ds[:,0]**2
+                        else:
+                            Sigma = SigFo**2 + epsilon * S
+                            X = Fo * Fc_abs / Sigma
+                            #X = X.astype(numpy.float64)
+                            m = numpy.tanh(X)
+                            g = (Fc_abs - m * Fo) / Sigma * Ds[:,0]
+                            dll_dab[cidxes] = g * expip
+                            d2ll_dab2[cidxes] = (1. / Sigma - (Fo / (Sigma * numpy.cosh(X)))**2) * Ds[:,0]**2
+            dll_dab *= self.hkldata.debye_waller_factors(b_iso=-blur)
 
         if self.mott_bethe:
-            dll_dab *= self.hkldata.d_spacings()**2 * gemmi.mott_bethe_const()
+            d2 = 1 / self.twin_data.s2_array if self.twin_data else self.hkldata.d_spacings()**2
+            dll_dab *= d2 * gemmi.mott_bethe_const()
             d2ll_dab2 *= gemmi.mott_bethe_const()**2
 
+            
         # we need V**2/n for gradient.
-        dll_dab_den = self.hkldata.fft_map(data=dll_dab * self.hkldata.debye_waller_factors(b_iso=-blur))
+        if self.twin_data:
+            dll_dab_den = utils.hkl.fft_map(self.hkldata.cell, self.hkldata.sg, self.twin_data.asu, data=dll_dab)
+        else:
+            dll_dab_den = self.hkldata.fft_map(data=dll_dab)
         dll_dab_den.array[:] *= self.hkldata.cell.volume**2 / dll_dab_den.point_count
         #asu = dll_dab_den.masked_asu()
         #dll_dab_den.array[:] *= 1 - asu.mask_array # 0 to use
@@ -211,13 +258,12 @@ class LL_Xtal:
             self.ll.calc_grad_it92(dll_dab_den, blur)
 
         # second derivative
+        s_array = numpy.sqrt(self.twin_data.s2_array) if self.twin_data else 1./self.hkldata.d_spacings().to_numpy()
         if self.source == "neutron":
-            self.ll.make_fisher_table_diag_direct_n92(1./self.hkldata.d_spacings().to_numpy(),
-                                                      d2ll_dab2)
+            self.ll.make_fisher_table_diag_direct_n92(s_array, d2ll_dab2)
             self.ll.fisher_diag_from_table_n92()
         else:
-            self.ll.make_fisher_table_diag_direct_it92(1./self.hkldata.d_spacings().to_numpy(),
-                                                       d2ll_dab2)
+            self.ll.make_fisher_table_diag_direct_it92(s_array, d2ll_dab2)
             self.ll.fisher_diag_from_table_it92()
         #json.dump(dict(b=ll.table_bs, pp1=ll.pp1, bb=ll.bb),
         #          open("ll_fisher.json", "w"), indent=True)

servalcat/refmac/exte.py

@@ -120,18 +120,20 @@ def read_external_restraints(params, st, geom):
         if r["rest_type"] == "dist":
             if not (defs["dist_min_external"] < r["restr"]["value"] < defs["dist_max_external"]):
                 continue
-            if ex.atoms[0].serial > ex.atoms[1].serial:
-                ex.atoms = ex.atoms[::-1]
             ex.alpha = r["restr"].get("alpha_in", defs["alpha_default"])
             ex.type = r["restr"].get("itype_in", defs["type_default"])
             symm1 = any([spec.get("symm") for spec in r["restr"]["specs"]]) # is it the intention?
             if r["restr"].get("symm_in", defs["symall_block"]) or symm1:
                 asu = gemmi.Asu.Different if defs["exclude_self_block"] else gemmi.Asu.Any
-                im = st.cell.find_nearest_image(ex.atoms[0].pos, ex.atoms[1].pos, asu)
-                ex.set_image(im)
+                ex.set_image(st.cell, asu)
             #print("dist=", ex.alpha, ex.type, ex.values[-1].value, ex.values[-1].sigma, ex.sym_idx, ex.pbc_shift, ex.atoms)
         elif r["rest_type"] == "angl":
-            pass
+            if any(spec.get("symm") for spec in r["restr"]["specs"]):
+                asus = [gemmi.Asu.Different if r["restr"]["specs"][i].get("symm") else gemmi.Asu.Same
+                        for i in range(3)]
+                if atoms[0].serial > atoms[2].serial:
+                    asus = asus[::-1]
+                ex.set_images(st.cell, asus[0], asus[2])
             #print("angl=", ex.values[-1].value, ex.values[-1].sigma, ex.atoms)
         elif r["rest_type"] == "tors":
             pass

servalcat/refmac/refmac_keywords.py

@@ -37,6 +37,7 @@ def parse_atom_spec(s, itk):
             itk += 2
         elif s[itk].lower().startswith("symm"):
             ret["symm"] = s[itk+1][0].lower() == "y"
+            itk += 2
         else:
             break
 
@@ -167,10 +168,10 @@ def read_exte(s):
                     except ValueError:
                         ret["restr"]["itype_in"] = dict(o=0, f=2).get(s[itk+1][0].lower(), 1)
                     if not (0 <= ret["restr"]["itype_in"] <= 2):
-                        logger.writeln("WARNING: wrong type is given. setting to 2.\n=> {}".format(l))
+                        logger.writeln("WARNING: wrong type is given. setting to 2.\n=> {}".format(" ".join(s)))
                         ret["restr"]["itype_in"] = 2
                     itk += 2
-                elif s[itk].lower().startswith("symm"): # only for distance
+                elif s[itk].lower().startswith("symm"): # only for distance and angle
                     ret["restr"]["symm_in"] = s[itk+1][0].lower() == "y"
                     itk += 2
                 else:
@@ -181,7 +182,8 @@ def read_exte(s):
                         ret["restr"][d[k]] = float(s[itk+1])
                         itk += 2
                     else:
-                        logger.writeln("unrecognised key: {}\n=> {}".format(s[itk], l))
+                        logger.writeln("unrecognised key: {}\n=> {}".format(s[itk], " ".join(s)))
+                        break
         elif s[1].lower().startswith("stac"):
             ret["rest_type"] = "stac"
             ret["restr"] = {}
@@ -194,7 +196,7 @@ def read_exte(s):
                     ip = int(s[itk+1])
                     itk += 2
                     if ip not in (1, 2):
-                        raise RuntimeError("Problem with stacking instructions. Plane number can be 1 or 2.\n=> {}".format(l))
+                        raise RuntimeError("Problem with stacking instructions. Plane number can be 1 or 2.\n=> {}".format(" ".join(s)))
                 elif s[itk].lower().startswith(("firs", "next")):
                     atoms, itk = parse_atom_spec(s, itk+1)
                     ret["restr"]["specs"][ip-1] = atoms
@@ -203,7 +205,7 @@ def read_exte(s):
                     ret["restr"][k] = float(s[itk+1]) if k != "type_r" else int(s[itk+1])
                     itk += 2
                 else:
-                    logger.writeln("WARNING: unrecognised keyword: {}\n=> {}".format(s[itk], l))
+                    logger.writeln("WARNING: unrecognised keyword: {}\n=> {}".format(s[itk], " ".join(s)))
                     itk += 1
         elif s[1].lower().startswith(("harm", "spec")):
             ret["rest_type"] = s[1][:4].lower() # in Refmac, irest_type = 1 if harm else 2
@@ -241,11 +243,11 @@ def read_exte(s):
                     ret["restr"]["sigma_u"] = float(s[itk+1]) * b_to_u
                     itk += 2
                 else:
-                    logger.writeln("WARNING: unrecognised keyword: {}\n=> {}".format(s[itk], l))
+                    logger.writeln("WARNING: unrecognised keyword: {}\n=> {}".format(s[itk], " ".join(s)))
                     itk += 1
 
         else:
-            logger.writeln("WARNING: cannot parse: {}".format(l))
+            logger.writeln("WARNING: cannot parse: {}".format(" ".join(s)))
     return ret
 # read_exte()
 
@@ -543,8 +545,8 @@ def parse_line(l, ret):
         ret.setdefault("refi", {})
         itk = 1
         while itk < ntok:
-            if s[itk].startswith("type"):
-                if itk+1 < ntok and s[itk+1].startswith("unre"):
+            if s[itk].lower().startswith("type"):
+                if itk+1 < ntok and s[itk+1].lower().startswith("unre"):
                     ret["refi"]["type"] = "unre"
                 itk += 2
             else:

servalcat/refmac/refmac_wrapper.py

@@ -11,10 +11,12 @@ import numpy
 import json
 import os
 import sys
+import io
 import tempfile
 import subprocess
 import argparse
 from collections import OrderedDict
+import servalcat # for version
 from servalcat.utils import logger
 from servalcat.refmac import refmac_keywords
 from servalcat import utils
@@ -64,7 +66,7 @@ def read_stdin(stdin):
 
 def prepare_crd(st, crdout, ligand, make, monlib_path=None, h_pos="elec",
                 no_adjust_hydrogen_distances=False, fix_long_resnames=True,
-                keep_entities=False):
+                keep_entities=False, unre=False):
     assert h_pos in ("elec", "nucl")
     h_change = dict(a=gemmi.HydrogenChange.ReAddButWater,
                     y=gemmi.HydrogenChange.NoChange,
@@ -80,67 +82,87 @@ def prepare_crd(st, crdout, ligand, make, monlib_path=None, h_pos="elec",
                 if at.occ > 1: # XXX should I check special positions?
                     at.occ = 1.
 
-    if not keep_entities:
+    refmac_fixes = utils.refmac.FixForRefmac()
+    if keep_entities:
+        refmac_fixes.store_res_labels(st)
+    else:
         utils.model.setup_entities(st, clear=True, force_subchain_names=True, overwrite_entity_type=True)
-    # TODO read dictionary from xyzin (priority: user cif -> monlib -> xyzin
-    try:
-        monlib = utils.restraints.load_monomer_library(st,
-                                                       monomer_dir=monlib_path,
-                                                       cif_files=ligand,
-                                                       stop_for_unknowns=not make.get("newligand"))
-    except RuntimeError as e:
-        raise SystemExit("Error: {}".format(e))
 
-    use_cispeps = make.get("cispept", "y") != "y"
-    make_link = make.get("link", "n")
-    make_ss = make.get("ss", "y")
-    only_from = set()
-    if make_link == "y":
-        # add all links
-        add_found = True
-    elif make_ss == "y":
-        add_found = True
-        only_from.add("disulf")
+    if unre:
+        logger.writeln("Monomer library will not be loaded due to unrestrained refinement request")
+        monlib = gemmi.MonLib()
     else:
-        add_found = False
-    
-    utils.restraints.fix_elements_in_model(monlib, st)
-    utils.restraints.find_and_fix_links(st, monlib, add_found=add_found, find_symmetry_related=False, add_only_from=only_from)
-    for con in st.connections:
-        if con.link_id not in ("?", "", "gap") and con.link_id not in monlib.links:
-            logger.writeln(" removing unknown link id ({}). Ad-hoc link will be generated.".format(con.link_id))
-            con.link_id = ""
+        # TODO read dictionary from xyzin (priority: user cif -> monlib -> xyzin
+        try:
+            monlib = utils.restraints.load_monomer_library(st,
+                                                           monomer_dir=monlib_path,
+                                                           cif_files=ligand,
+                                                           stop_for_unknowns=not make.get("newligand"))
+        except RuntimeError as e:
+            raise SystemExit("Error: {}".format(e))
+
+        use_cispeps = make.get("cispept", "y") != "y"
+        make_link = make.get("link", "n")
+        make_ss = make.get("ss", "y")
+        only_from = set()
+        if make_link == "y":
+            # add all links
+            add_found = True
+        elif make_ss == "y":
+            add_found = True
+            only_from.add("disulf")
+        else:
+            add_found = False
+
+        utils.restraints.fix_elements_in_model(monlib, st)
+        utils.restraints.find_and_fix_links(st, monlib, add_found=add_found,
+                                            find_metal_links=(make_link == "y"),
+                                            find_symmetry_related=False, add_only_from=only_from)
+        for con in st.connections:
+            if con.link_id not in ("?", "", "gap") and con.link_id not in monlib.links:
+                logger.writeln(" removing unknown link id ({}). Ad-hoc link will be generated.".format(con.link_id))
+                con.link_id = ""
 
-    refmac_fixes = utils.refmac.FixForRefmac()
     max_seq_num = max([max(res.seqid.num for res in chain) for model in st for chain in model])
     if max_seq_num > 9999:
         logger.writeln("Max residue number ({}) exceeds 9999. Needs workaround.".format(max_seq_num))
-        topo = gemmi.prepare_topology(st, monlib, ignore_unknown_links=True)
+        sio = io.StringIO()
+        topo = gemmi.prepare_topology(st, monlib, warnings=sio, ignore_unknown_links=True)
         refmac_fixes.fix_before_topology(st, topo, 
                                          fix_microheterogeneity=False,
                                          fix_resimax=True,
                                          fix_nonpolymer=False)
 
-    if make.get("hydr") == "a": logger.writeln("(re)generating hydrogen atoms")
-    try:
-        topo, metal_kws = utils.restraints.prepare_topology(st, monlib, h_change=h_change, ignore_unknown_links=False,
-                                                            check_hydrogen=(h_change==gemmi.HydrogenChange.NoChange),
-                                                            use_cispeps=use_cispeps)
-    except RuntimeError as e:
-        raise SystemExit("Error: {}".format(e))
+    if unre:
+        # Refmac5 does not seem to do anything to hydrogen when unre regardless of "make hydr"
+        sio = io.StringIO()
+        topo = gemmi.prepare_topology(st, monlib, warnings=sio, ignore_unknown_links=True)
+        metal_kws = []
+    else:
+        if make.get("hydr") == "a": logger.writeln("(re)generating hydrogen atoms")
+        try:
+            topo, metal_kws = utils.restraints.prepare_topology(st, monlib, h_change=h_change, ignore_unknown_links=False,
+                                                                check_hydrogen=(h_change==gemmi.HydrogenChange.NoChange),
+                                                                use_cispeps=use_cispeps)
+        except RuntimeError as e:
+            raise SystemExit("Error: {}".format(e))
 
-    if make.get("hydr") != "n" and st[0].has_hydrogen():
-        if h_pos == "nucl" and (make.get("hydr") == "a" or not no_adjust_hydrogen_distances):
-            resnames = st[0].get_all_residue_names()
-            utils.restraints.check_monlib_support_nucleus_distances(monlib, resnames)
-            logger.writeln("adjusting hydrogen position to nucleus")
-            topo.adjust_hydrogen_distances(gemmi.Restraints.DistanceOf.Nucleus, default_scale=1.1)
-        elif h_pos == "elec" and make.get("hydr") == "y" and not no_adjust_hydrogen_distances:
-            logger.writeln("adjusting hydrogen position to electron cloud")
-            topo.adjust_hydrogen_distances(gemmi.Restraints.DistanceOf.ElectronCloud)
+        if make.get("hydr") != "n" and st[0].has_hydrogen():
+            if h_pos == "nucl" and (make.get("hydr") == "a" or not no_adjust_hydrogen_distances):
+                resnames = st[0].get_all_residue_names()
+                utils.restraints.check_monlib_support_nucleus_distances(monlib, resnames)
+                logger.writeln("adjusting hydrogen position to nucleus")
+                topo.adjust_hydrogen_distances(gemmi.Restraints.DistanceOf.Nucleus, default_scale=1.1)
+            elif h_pos == "elec" and make.get("hydr") == "y" and not no_adjust_hydrogen_distances:
+                logger.writeln("adjusting hydrogen position to electron cloud")
+                topo.adjust_hydrogen_distances(gemmi.Restraints.DistanceOf.ElectronCloud)
 
     if fix_long_resnames: refmac_fixes.fix_long_resnames(st)
 
+    # remove "given" ncs matrices
+    # TODO write them back to the output files
+    st.ncs = gemmi.NcsOpList(x for x in st.ncs if not x.given)
+
     # for safety
     if "_entry.id" in st.info:
         st.info["_entry.id"] = st.info["_entry.id"].replace(" ", "")
@@ -221,11 +243,22 @@ def modify_output(pdbout, cifout, fixes, hout, cispeps, keep_original_output=Fal
         # should we check metals and put MetalC?
 
     # fix entity (Refmac seems to make DNA non-polymer; as seen in 1fix)
-    utils.model.setup_entities(st, clear=True, overwrite_entity_type=True, force_subchain_names=True)
-    for e in st.entities:
-        if not e.full_sequence and e.entity_type == gemmi.EntityType.Polymer and e.subchains:
-            rspan = st[0].get_subchain(e.subchains[0])
-            e.full_sequence = [r.name for r in rspan]
+    if not fixes or not fixes.res_labels:
+        utils.model.setup_entities(st, clear=True, overwrite_entity_type=True, force_subchain_names=True)
+        for e in st.entities:
+            if not e.full_sequence and e.entity_type == gemmi.EntityType.Polymer and e.subchains:
+                rspan = st[0].get_subchain(e.subchains[0])
+                e.full_sequence = [r.name for r in rspan]
+
+        # fix label_seq_id
+        for chain in st[0]:
+            for res in chain:
+                res.label_seq = None
+        st.assign_label_seq_id()
+
+    # add servalcat version
+    if len(st.meta.software) > 0 and st.meta.software[-1].name == "refmac":
+        st.meta.software[-1].version += f" (refmacat {servalcat.__version__})"
     
     suffix = ".org"
     os.rename(cifout, cifout + suffix)
@@ -292,11 +325,12 @@ def main(args):
     crdout = None
     refmac_fixes = None
     cispeps = []
-    if xyzin is not None and keywords["refi"].get("type") != "unre":
+    unre = keywords["refi"].get("type") == "unre"
+    if xyzin is not None:
         #tmpfd, crdout = tempfile.mkstemp(prefix="gemmi_", suffix=".crd") # TODO use dir=CCP4_SCR
         #os.close(tmpfd)
         st = utils.fileio.read_structure(xyzin)
-        if not st.cell.is_crystal():
+        if not st.cell.is_crystal() and not unre:
             if args.auto_box_with_padding is not None:
                 st.cell = utils.model.box_from_model(st[0], args.auto_box_with_padding)
                 st.spacegroup_hm = "P 1"
@@ -309,7 +343,8 @@ def main(args):
         refmac_fixes, metal_kws = prepare_crd(st, crdout, args.ligand, make=keywords["make"], monlib_path=args.monlib,
                                               h_pos="nucl" if keywords.get("source")=="ne" else "elec",
                                               no_adjust_hydrogen_distances=args.no_adjust_hydrogen_distances,
-                                              keep_entities=args.keep_entities)
+                                              keep_entities=args.keep_entities,
+                                              unre=unre)
         inputs = metal_kws + inputs # add metal exte first; otherwise it may be affected by user-defined inputs
         opts["xyzin"] = crdout
         cispeps = st.cispeps