servalcat 0.4.99__cp310-cp310-macosx_10_14_x86_64.whl → 0.4.100__cp310-cp310-macosx_10_14_x86_64.whl

servalcat/__init__.py

@@ -6,5 +6,5 @@ This software is released under the
 Mozilla Public License, version 2.0; see LICENSE.
 """
 
-__version__ = '0.4.99'
-__date__ = '2024-12-04'
+__version__ = '0.4.100'
+__date__ = '2025-01-22'

servalcat/refine/refine_spa.py

@@ -271,7 +271,6 @@ def main(args):
     st_expanded = refiner.st.clone()
     if not all(op.given for op in st.ncs):
         utils.model.expand_ncs(st_expanded)
-        utils.fileio.write_model(st_expanded, args.output_prefix+"_expanded", pdb=True, cif=True, hout=args.hout)
 
     # Calc FSC
     if args.hklin: # cannot update a mask
@@ -294,6 +293,10 @@ def main(args):
     update_meta(refiner.st, stats_for_meta, ll)
     refiner.st.name = args.output_prefix
     utils.fileio.write_model(refiner.st, args.output_prefix, pdb=True, cif=True, hout=args.hout)
+    if not all(op.given for op in st.ncs): # to apply updated metadata
+        st_expanded = refiner.st.clone()
+        utils.model.expand_ncs(st_expanded)
+        utils.fileio.write_model(st_expanded, args.output_prefix+"_expanded", pdb=True, cif=True, hout=args.hout)
     if args.hklin:
         return
     # Calc Fo-Fc (and updated) maps

servalcat/refine/refine_xtal.py

@@ -129,19 +129,6 @@ def main(args):
         hklin = utils.fileio.read_mmhkl(hklin)
         labin = decide_mtz_labels(hklin)
         
-    if labin and len(labin) == 3: # with test flags
-        use_in_target = "work"
-        if args.use_work_in_est:
-            use_in_est = "work"
-            n_per_bin = 100
-        else:
-            use_in_est = "test"
-            n_per_bin = 50
-    else:
-        use_in_est = "all"
-        use_in_target = "all"
-        n_per_bin = 100
-
     try:
         hkldata, sts, fc_labs, centric_and_selections, args.free = process_input(hklin=hklin,
                                                                                  labin=labin,
@@ -151,14 +138,23 @@ def main(args):
                                                                                  source=args.source,
                                                                                  d_max=args.d_max,
                                                                                  d_min=args.d_min,
-                                                                                 n_per_bin=n_per_bin,
-                                                                                 use=use_in_est,
+                                                                                 use="work" if args.use_work_in_est else "test",
                                                                                  max_bins=30,
                                                                                  keep_charges=args.keep_charges,
                                                                                  allow_unusual_occupancies=args.allow_unusual_occupancies)
     except RuntimeError as e:
         raise SystemExit("Error: {}".format(e))
 
+    if "FREE" in hkldata.df:
+        use_in_target = "work"
+        if args.use_work_in_est:
+            use_in_est = "work"
+        else:
+            use_in_est = "test"
+    else:
+        use_in_est = "all"
+        use_in_target = "all"
+
     is_int = "I" in hkldata.df
     st = sts[0]
     utils.model.fix_deuterium_residues(st)

servalcat/refine/xtal.py

@@ -45,7 +45,7 @@ class LL_Xtal:
         self.ll = None
         self.scaling = sigmaa.LsqScale()
         if twin:
-            self.twin_data = find_twin_domains_from_data(self.hkldata)
+            self.twin_data, _ = find_twin_domains_from_data(self.hkldata)
         else:
             self.twin_data = None
         if self.twin_data:
@@ -94,7 +94,7 @@ class LL_Xtal:
                 fc_sum = self.hkldata.df[self.fc_labs[:-1]].sum(axis=1).to_numpy()
             fc_list = [fc_sum, Fmask]
         else:
-            if twin_data:
+            if self.twin_data:
                 fc_list = [self.twin_data.f_calc.sum(axis=1)]
             else:
                 fc_list = [self.hkldata.df[self.fc_labs].sum(axis=1).to_numpy()]
@@ -235,11 +235,11 @@ class LL_Xtal:
                             m = numpy.tanh(X)
                             g = (Fc_abs - m * Fo) / Sigma * Ds[:,0]
                             dll_dab[cidxes] = g * expip
-                            d2ll_dab2[cidxes] = (1. / Sigma - (Fo / (Sigma * numpy.cosh(X)))**2) * Ds[:,0]**2
+                            d2ll_dab2[cidxes] = (1. / Sigma - (Fo / Sigma)**2 * (1. - m**2)) * Ds[:,0]**2
             dll_dab *= self.hkldata.debye_waller_factors(b_iso=-blur)
 
         if self.mott_bethe:
-            d2 = 1 / self.twin_data.s2_array if self.twin_data else self.hkldata.d_spacings()**2
+            d2 = numpy.reciprocal(self.twin_data.s2_array) if self.twin_data else self.hkldata.d_spacings()**2
             dll_dab *= d2 * gemmi.mott_bethe_const()
             d2ll_dab2 *= gemmi.mott_bethe_const()**2
 

servalcat/utils/hkl.py

@@ -493,6 +493,31 @@ class HklData:
             self.df = self.df[~sel]
     # remove_nonpositive()
 
+    def mask_invalid_obs_values(self, labels):
+        assert 1 < len(labels) < 6
+        assert labels[1].startswith("SIG")
+        def do_mask(label, target_labels):
+            sel = self.df[label] <= 0
+            n_bad = sel.sum()
+            if n_bad > 0:
+                logger.writeln("Removing {} reflections with {}<=0".format(n_bad, label))
+                self.df.loc[sel, target_labels] = numpy.nan
+            # If any element within target_labels is non-finite, mask all elements
+            self.df.loc[(~numpy.isfinite(self.df[target_labels])).any(axis=1), target_labels] = numpy.nan
+
+        if len(labels) < 4: # F/SIGF or I/SIGI
+            if labels[0].startswith("F"):
+                do_mask(labels[0], labels[:2]) # bad F
+            do_mask(labels[1], labels[:2]) # bad sigma
+        else: # I(+)/SIGI(+)/I(-)/SIGI(-) or F...
+            assert labels[3].startswith("SIG")
+            if labels[0].startswith("F"):
+                do_mask(labels[0], labels[:2]) # bad F+
+                do_mask(labels[2], labels[2:4]) # bad F-
+            do_mask(labels[1], labels[:2]) # bad sigma+
+            do_mask(labels[3], labels[2:4]) # bad sigma-
+    # mask_invalid_obs_values()
+
     def remove_systematic_absences(self):
         is_absent = self.sg.operations().systematic_absences(self.miller_array())
         n_absent = numpy.sum(is_absent)
@@ -501,12 +526,22 @@ class HklData:
             self.df = self.df[~is_absent]
     # remove_systematic_absences()
 
-    def merge_anomalous(self, labs, newlabs):
+    def merge_anomalous(self, labs, newlabs, method="weighted"):
+        assert method in ("weighted", "simple")
         assert len(labs) == 4 # i+,sigi+,i-,sigi- for example
         assert len(newlabs) == 2
-        # skipna=True is default, so missing value is handled nicely.
-        self.df[newlabs[0]] = self.df[[labs[0], labs[2]]].mean(axis=1)
-        self.df[newlabs[1]] = self.df[[labs[1], labs[3]]].pow(2).mean(axis=1).pow(0.5)
+        if method == "simple":
+            # skipna=True is default, so missing value is handled nicely.
+            self.df[newlabs[0]] = self.df[[labs[0], labs[2]]].mean(axis=1)
+            self.df[newlabs[1]] = self.df[[labs[1], labs[3]]].pow(2).mean(axis=1).pow(0.5)
+        else:
+            obs = self.df[[labs[0], labs[2]]].to_numpy()
+            weights = 1. / self.df[[labs[1], labs[3]]].to_numpy()**2
+            sum_w = numpy.nansum(weights, axis=1)
+            sum_w[sum_w == 0] = numpy.nan # mask when both are nan
+            self.df[newlabs[0]] = numpy.nansum(obs * weights, axis=1) / sum_w
+            self.df[newlabs[1]] = numpy.sqrt(1. / sum_w)
+    # merge_anomalous()
 
     def as_asu_data(self, label=None, data=None, label_sigma=None):
         if label is None: assert data is not None

servalcat/utils/logger.py

@@ -37,7 +37,7 @@ class Logger(object):
     def write(self, l, end="", flush=True, fs=None, print_fs=sys.stdout):
         if self.stopped: return
         if self.prefix:
-            l = "\n".join(self.prefix + x for x in l.splitlines(keepends=True))
+            l = "".join(self.prefix + x for x in l.splitlines(keepends=True))
         print(l, end=end, file=print_fs, flush=flush)
         for f in (self.ofs, fs):
             if f is not None:

servalcat/utils/model.py

@@ -73,11 +73,12 @@ def remove_charge(sts):
 def check_atomsf(sts, source, mott_bethe=True):
     assert source in ("xray", "electron", "neutron")
     if source != "electron": mott_bethe = False
-    logger.writeln("Atomic scattering factors for {}".format("electron (Mott-Bethe)" if mott_bethe else source))
+    logger.writeln("Atomic scattering factors for {}".format("xray (use Mott-Bethe to convert to electrons)" if mott_bethe else source))
     if source != "xray" and not mott_bethe:
         logger.writeln("  Note that charges will be ignored")
     el_charges = {(cra.atom.element, cra.atom.charge) for st in sts for cra in st[0].all()}
     elems = {x[0] for x in el_charges}
+    tmp = {}
     if source == "xray" or mott_bethe:
         shown = set()
         for el, charge in sorted(el_charges, key=lambda x: (x[0].atomic_number, x[1])):
@@ -88,12 +89,16 @@ def check_atomsf(sts, source, mott_bethe=True):
                 charge = 0
             if (el, charge) in shown: continue
             label = el.name if charge == 0 else "{}{:+}".format(el.name, charge)
-            logger.writeln("  {} {}".format(label, tuple(sf.get_coefs())))
             shown.add((el, charge))
+            tmp[label] = {**{f"{k}{i+1}": x for k in ("a", "b") for i, x in enumerate(getattr(sf, k))}, "c": sf.c}
     else:
         for el in sorted(elems, key=lambda x: x.atomic_number):
-            sf = el.c4322 if source == "electron" else el.neutron92
-            logger.writeln("  {} {}".format(el.name, tuple(sf.get_coefs())))
+            if source == "electron":
+                tmp[el.name] = {f"{k}{i+1}": x for k in ("a", "b") for i, x in enumerate(getattr(el.c4322, k))}
+            else:
+                tmp[el.name] = {"a": el.neutron92.get_coefs()[0]}
+    with logger.with_prefix("  "):
+        logger.writeln(pandas.DataFrame(tmp).T.to_string())
     logger.writeln("")
 # check_atomsf()
 
@@ -553,7 +558,9 @@ def reset_adp(model, bfactor=None, adp_mode="iso"):
         if adp_mode == "iso" or (adp_mode == "fix" and bfactor is not None):
             cra.atom.aniso = gemmi.SMat33f(0,0,0,0,0,0)
         elif adp_mode == "aniso":
-            if not cra.atom.aniso.nonzero() or bfactor is not None:
+            if cra.atom.aniso.nonzero() and bfactor is None: # just in case
+                cra.atom.b_iso = numpy.mean(cra.atom.aniso.calculate_eigenvalues()) * u_to_b
+            else:
                 u = cra.atom.b_iso * b_to_u
                 cra.atom.aniso = gemmi.SMat33f(u, u, u, 0, 0, 0)
 # reset_adp()

servalcat/xtal/sigmaa.py

@@ -1056,10 +1056,7 @@ def calculate_maps_int(hkldata, b_aniso, fc_labs, D_labs, centric_and_selections
             S = hkldata.df["S"].to_numpy()[cidxes]
             f, m_proxy = expected_F_from_int(Io[cidxes], sigIo[cidxes], k_ani[cidxes], DFc[cidxes], eps[cidxes], c, S)
             exp_ip = numpy.exp(numpy.angle(DFc[cidxes])*1j)
-            if c == 0:
-                hkldata.df.loc[cidxes, "FWT"] = 2 * f * exp_ip - DFc[cidxes]
-            else:
-                hkldata.df.loc[cidxes, "FWT"] = f * exp_ip
+            hkldata.df.loc[cidxes, "FWT"] = 2 * f * exp_ip - DFc[cidxes]
             hkldata.df.loc[cidxes, "DELFWT"] = f * exp_ip - DFc[cidxes]
             hkldata.df.loc[cidxes, "FOM"] = m_proxy
             if has_ano:
@@ -1167,7 +1164,6 @@ def process_input(hklin, labin, n_bins, free, xyzins, source, d_max=None, d_min=
                   allow_unusual_occupancies=False, space_group=None):
     if labin: assert 1 < len(labin) < 6
     assert use in ("all", "work", "test")
-    assert n_bins or n_per_bin #if n_bins not set, n_per_bin should be given
 
     if len(xyzins) > 0 and type(xyzins[0]) is gemmi.Structure:
         sts = xyzins
@@ -1205,6 +1201,7 @@ def process_input(hklin, labin, n_bins, free, xyzins, source, d_max=None, d_min=
     logger.writeln("Observation type: {}".format(name))
     if len(newlabels) < len(labin): newlabels.append("FREE")
     hkldata = utils.hkl.hkldata_from_mtz(mtz, labin, newlabels=newlabels, require_types=require_types)
+    hkldata.mask_invalid_obs_values(newlabels)
     if newlabels[0] == "F(+)":
         hkldata.merge_anomalous(newlabels[:4], ["FP", "SIGFP"])
         newlabels = ["FP", "SIGFP"] + newlabels[4:]
@@ -1256,9 +1253,6 @@ def process_input(hklin, labin, n_bins, free, xyzins, source, d_max=None, d_min=
 
     if sg_use is not None:
         hkldata.sg = sg_use
-    if newlabels[0] == "FP":
-        hkldata.remove_nonpositive(newlabels[0])
-    hkldata.remove_nonpositive(newlabels[1])
     hkldata.switch_to_asu()
     hkldata.remove_systematic_absences()
     #hkldata.df = hkldata.df.astype({name: 'float64' for name in ["I","SIGI","FP","SIGFP"] if name in hkldata.df})
@@ -1280,6 +1274,17 @@ def process_input(hklin, labin, n_bins, free, xyzins, source, d_max=None, d_min=
         free = hkldata.guess_free_number(newlabels[0])
 
     if n_bins is None:
+        if n_per_bin is None:
+            if use == "all" or "FREE" not in hkldata.df:
+                n_per_bin = 100
+                use = "all"
+            elif use == "work":
+                n_per_bin = 100
+            elif use == "test":
+                n_per_bin = 50
+            else:
+                raise RuntimeError(f"should not happen: {use=}")
+        
         sel = hkldata.df[newlabels[0]].notna()
         if use == "work":
             sel &= hkldata.df.FREE != free
@@ -1473,13 +1478,11 @@ def calculate_maps(hkldata, b_aniso, centric_and_selections, fc_labs, D_labs, lo
                 Sigma = 2 * SigFo**2 + epsilon * S
                 X = 2 * Fo * DFc_abs / Sigma
                 m = gemmi.bessel_i1_over_i0(X)
-                hkldata.df.loc[cidxes, "FWT"] = (2 * m * Fo - DFc_abs) * expip
             else:
                 Sigma = SigFo**2 + epsilon * S
                 X = Fo * DFc_abs / Sigma
                 m = numpy.tanh(X)
-                hkldata.df.loc[cidxes, "FWT"] = (m * Fo) * expip
-
+            hkldata.df.loc[cidxes, "FWT"] = (2 * m * Fo - DFc_abs) * expip
             hkldata.df.loc[cidxes, "DELFWT"] = (m * Fo - DFc_abs) * expip
             hkldata.df.loc[cidxes, "FOM"] = m
             hkldata.df.loc[cidxes, "X"] = X
@@ -1500,7 +1503,8 @@ def calculate_maps(hkldata, b_aniso, centric_and_selections, fc_labs, D_labs, lo
         Fc = hkldata.df.FC.to_numpy()[idxes] * k_ani[idxes]
         Fo = hkldata.df.FP.to_numpy()[idxes]
         mean_DFc2 = numpy.nanmean(numpy.abs((Ds[idxes,:] * Fcs[idxes,:]).sum(axis=1) * k_ani[idxes])**2)
-        mean_log_DFcs = numpy.log(numpy.nanmean(numpy.abs(Ds[idxes,:] * Fcs[idxes,:] * k_ani[idxes,None]), axis=0)).tolist()
+        with numpy.errstate(divide="ignore"):
+            mean_log_DFcs = numpy.log(numpy.nanmean(numpy.abs(Ds[idxes,:] * Fcs[idxes,:] * k_ani[idxes,None]), axis=0)).tolist()
         mean_Ds = numpy.nanmean(Ds[idxes,:], axis=0).tolist()
         if sum(nrefs) > 0:
             r = numpy.nansum(numpy.abs(numpy.abs(Fc)-Fo)) / numpy.nansum(Fo)
@@ -1535,7 +1539,6 @@ def calculate_maps(hkldata, b_aniso, centric_and_selections, fc_labs, D_labs, lo
 # calculate_maps()
 
 def main(args):
-    n_per_bin = {"all": 500, "work": 500, "test": 50}[args.use]
     try:
         hkldata, sts, fc_labs, centric_and_selections,free = process_input(hklin=args.hklin,
                                                                            labin=args.labin.split(",") if args.labin else None,
@@ -1545,7 +1548,6 @@ def main(args):
                                                                            source=args.source,
                                                                            d_max=args.d_max,
                                                                            d_min=args.d_min,
-                                                                           n_per_bin=n_per_bin,
                                                                            use=args.use,
                                                                            max_bins=30,
                                                                            keep_charges=args.keep_charges,
@@ -1554,7 +1556,7 @@ def main(args):
         raise SystemExit("Error: {}".format(e))
 
     if args.twin:
-        twin_data = find_twin_domains_from_data(hkldata)
+        twin_data, _ = find_twin_domains_from_data(hkldata)
     else:
         twin_data = None
     if twin_data:

servalcat/xtal/twin.py

@@ -21,39 +21,43 @@ def find_twin_domains_from_data(hkldata, max_oblique=5, min_alpha=0.05):
     #for op in ops:
     #    logger.writeln(f"  {op.triplet()}")
     if not ops:
-        return
+        logger.writeln("")
+        return None, None
     twin_data = ext.TwinData()
     twin_data.setup(hkldata.miller_array(), hkldata.df.bin, hkldata.sg, hkldata.cell, ops)
     if "I" in hkldata.df:
         Io = hkldata.df.I.to_numpy()
     else:
         Io = hkldata.df.FP.to_numpy()**2
-    alphas = []
     ccs, nums = [], []
+    tmp = []
     for i_bin, bin_idxes in hkldata.binned():
         ratios = [1.]
         ccs.append([])
         nums.append([])
+        rs = []
         for i_op, op in enumerate(ops):
             ii = numpy.array(twin_data.pairs(i_op, i_bin))
             val = numpy.all(numpy.isfinite(Io[ii]), axis=1)
             if numpy.sum(val) == 0:
-                cc = numpy.nan
+                cc = r = numpy.nan
             else:
                 cc = numpy.corrcoef(Io[ii][val].T)[0,1]
-            rr = (1 - numpy.sqrt(1 - cc**2)) / cc
-            ratios.append(rr)
+                r = numpy.sum(numpy.abs(Io[ii][val, 0] - Io[ii][val, 1])) / numpy.sum(Io[ii][val])
+            ratio = (1 - numpy.sqrt(1 - cc**2)) / cc
+            ratios.append(ratio)
             ccs[-1].append(cc)
+            rs.append(r)
             nums[-1].append(len(val))
-        alphas.append(numpy.array(ratios) / numpy.nansum(ratios))
-    alphas = numpy.maximum(0, numpy.mean(alphas, axis=0))
-    alphas /= numpy.nansum(alphas)
+        tmp.append(rs + ccs[-1] + nums[-1] + (numpy.array(ratios) / numpy.nansum(ratios)).tolist()[1:])
+    df = pandas.DataFrame(tmp, columns=[f"{n}_op{i+1}" for n in ("R", "CC", "num", "raw_est") for i in range(len(ops))])
+    with logger.with_prefix(" "):
+        logger.writeln(df.to_string(float_format="%.4f"))
     ccs = numpy.array(ccs)
     nums = numpy.array(nums)
     tmp = [{"Operator": gemmi.Op().triplet(),
             "R_twin_obs": 0,
-            "CC_mean": 1,
-            "Alpha_from_CC": alphas[0]}]
+            "CC_mean": 1}]
     for i_op, op in enumerate(ops):
         ii = numpy.array(twin_data.pairs(i_op))
         val = numpy.all(numpy.isfinite(Io[ii]), axis=1)
@@ -61,61 +65,80 @@ def find_twin_domains_from_data(hkldata, max_oblique=5, min_alpha=0.05):
             r_obs = numpy.nan
         else:
             r_obs = numpy.sum(numpy.abs(Io[ii][val, 0] - Io[ii][val, 1])) / numpy.sum(Io[ii][val])
+        cc = numpy.sum(nums[:,i_op] * ccs[:,i_op]) / numpy.sum(nums[:,i_op])
         tmp.append({"Operator": op.triplet(),
-                    "CC_mean": numpy.sum(nums[:,i_op] * ccs[:,i_op]) / numpy.sum(nums[:,i_op]),
+                    "CC_mean": cc, 
                     "R_twin_obs": r_obs,
-                    "Alpha_from_CC": alphas[i_op+1],
                     })
     df = pandas.DataFrame(tmp)
-    logger.writeln(df.to_string(float_format="%.2f"))
+    df["Alpha_from_CC"] = (1 - numpy.sqrt(1 - df["CC_mean"]**2)) / df["CC_mean"]
+    df["Alpha_from_CC"] /= numpy.nansum(df["Alpha_from_CC"])
+    logger.writeln("\n Initial twin fraction estimates:")
+    with logger.with_prefix(" "):
+        logger.writeln(df.to_string(float_format="%.2f"))
 
-    sel_idxes = [i for i, a in enumerate(alphas) if i > 0 and a > min_alpha]
-    if not sel_idxes:
-        logger.writeln(" No twinning detected")
-        return
+    sel = df["Alpha_from_CC"].to_numpy() > min_alpha
+    if sel[1:].sum() == 0:
+        logger.writeln(" No twinning detected\n")
+        return None, None
     
-    if len(sel_idxes) + 1 != len(alphas):
-        ops = [ops[i-1] for i in sel_idxes]
-        logger.writeln(" Twin operators after filtering small fractions")
-        alphas = numpy.array([alphas[0]] + [alphas[i] for i in sel_idxes])
-        alphas /= numpy.sum(alphas)
-        df = pandas.DataFrame({"Operator": [x.triplet() for x in [gemmi.Op()]+ops],
-                               "Alpha": alphas})
-        logger.writeln(df.to_string(float_format="%.2f"))
+    if not sel.all():
+        ops = [ops[i] for i in range(len(ops)) if sel[i+1]]
+        logger.writeln(f"\n Twin operators after filtering small fractions (<= {min_alpha})")
+        df = df[sel]
+        df["Alpha_from_CC"] /= numpy.nansum(df["Alpha_from_CC"])
+        with logger.with_prefix(" "):
+            logger.writeln(df.to_string(float_format="%.2f"))
         twin_data = ext.TwinData()
         twin_data.setup(hkldata.miller_array(), hkldata.df.bin, hkldata.sg, hkldata.cell, ops)
-    twin_data.alphas = alphas
+    twin_data.alphas = df["Alpha_from_CC"].tolist()
     if "I" not in hkldata.df:
         logger.writeln('Generating "observed" intensities for twin refinement: Io = Fo**2, SigIo = 2*F*SigFo')
         hkldata.df["I"] = hkldata.df.FP**2
         hkldata.df["SIGI"] = 2 * hkldata.df.FP * hkldata.df.SIGFP
-    return twin_data
+    logger.writeln("")
+    return twin_data, df
 
 # find_twin_domains_from_data()
 
 def estimate_twin_fractions_from_model(twin_data, hkldata):
     logger.writeln("Estimating twin fractions")
     Ic = numpy.abs(twin_data.f_calc.sum(axis=1))**2
-    Ic_all = Ic[twin_data.twin_related(hkldata.sg)]
+    idx_all = twin_data.twin_related(hkldata.sg)
+    Ic_all = Ic[idx_all]
+    Ic_all[(idx_all < 0).any(axis=1)] = numpy.nan
     rr = twin_data.obs_related_asu()
     tmp = []
+    P_list, cc_oc_list, weight_list = [], [], []
+    n_ops = len(twin_data.ops) + 1
+    tidxes = numpy.triu_indices(n_ops, 1)
     for i_bin, bin_idxes in hkldata.binned():
-        cc_o_c = []
         i_tmp = Ic_all[numpy.asarray(twin_data.bin)==i_bin,:]
+        i_tmp = i_tmp[numpy.isfinite(i_tmp).all(axis=1)]
         P = numpy.corrcoef(i_tmp.T)
         iobs = hkldata.df.I.to_numpy()[bin_idxes]
         ic_bin = Ic[rr[bin_idxes,:]]
-        val = numpy.isfinite(iobs) & numpy.isfinite(ic_bin).all(axis=1)
+        val = numpy.isfinite(iobs) & numpy.isfinite(ic_bin).all(axis=1) & numpy.all(rr[bin_idxes,:]>=0, axis=1)
         iobs, ic_bin = iobs[val], ic_bin[val,:]
-        cc_o_c = [numpy.corrcoef(iobs, ic_bin[:,i])[0,1] for i in range(len(twin_data.ops)+1)]
-        frac_est = numpy.dot(numpy.linalg.pinv(P), cc_o_c)
-        tmp.append(frac_est.tolist())
-
-    df = pandas.DataFrame(tmp)
-    df.iloc[:,:] /= df.sum(axis=1).to_numpy()[:,None]
-    mean_alphas = numpy.maximum(0, df.mean())
-    mean_alphas /= numpy.sum(mean_alphas)
-    logger.write(" Estimated fractions from data-model correlations: ")
-    logger.writeln(" ".join("%.2f"%x for x in mean_alphas))
-    twin_data.alphas = mean_alphas
+        cc_oc = [numpy.corrcoef(iobs, ic_bin[:,i])[0,1] for i in range(n_ops)]
+        P_list.append(P)
+        cc_oc_list.append(cc_oc)
+        weight_list.append(numpy.sum(val))
+        frac_est = numpy.dot(numpy.linalg.pinv(P), cc_oc)
+        frac_est /= frac_est.sum()
+        tmp.append(P[tidxes].tolist() + cc_oc + [weight_list[-1]] + frac_est.tolist())
 
+    P = numpy.average(P_list, axis=0, weights=weight_list)
+    cc_oc = numpy.average(cc_oc_list, axis=0, weights=weight_list)
+    frac_est = numpy.dot(numpy.linalg.pinv(P), cc_oc)
+    frac_est = numpy.maximum(0, frac_est)
+    frac_est /= frac_est.sum()
+    df = pandas.DataFrame(tmp, columns=[f"cc_{i+1}_{j+1}" for i, j in zip(*tidxes)] +
+                          [f"cc_o_{i+1}" for i in range(n_ops)] +
+                          ["nref"] + [f"raw_est_{i+1}" for i in range(n_ops)])
+    with logger.with_prefix(" "):
+        logger.writeln(df.to_string(float_format="%.4f"))
+    logger.write(" Final twin fraction estimate: ")
+    logger.writeln(" ".join("%.2f"%x for x in frac_est))
+    twin_data.alphas = frac_est
+    return df

{servalcat-0.4.99.dist-info → servalcat-0.4.100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: servalcat
-Version: 0.4.99
+Version: 0.4.100
 Summary: Structure refinement and validation for crystallography and single particle analysis
 Author: Keitaro Yamashita, Garib N. Murshudov
 License: MPL-2.0

{servalcat-0.4.99.dist-info → servalcat-0.4.100.dist-info}/RECORD

@@ -1,30 +1,30 @@
-servalcat-0.4.99.dist-info/RECORD,,
-servalcat-0.4.99.dist-info/WHEEL,sha256=HzwzW7yP_LKe5H32o-KYfVu2qyVc2oQicrxMg8jcdCM,116
-servalcat-0.4.99.dist-info/entry_points.txt,sha256=G1mDxhOCdF3umYz4k0kfwJbSdYSKqhvQdGCmrP8FRAY,111
-servalcat-0.4.99.dist-info/METADATA,sha256=OxT6LeODHswv4-Yg1Xtv6pjkvHjriRcZNFfq5asJObs,2741
-servalcat-0.4.99.dist-info/licenses/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
-servalcat/ext.cpython-310-darwin.so,sha256=0cUcBIRNBhhkxsQABirzPX55UKkrR565wAJfFW3mvGg,1327104
-servalcat/__init__.py,sha256=Wzar8K8OYk-xh70qv6hGajtSKjXH3qL93fHLMDTEh4w,231
+servalcat-0.4.100.dist-info/RECORD,,
+servalcat-0.4.100.dist-info/WHEEL,sha256=HzwzW7yP_LKe5H32o-KYfVu2qyVc2oQicrxMg8jcdCM,116
+servalcat-0.4.100.dist-info/entry_points.txt,sha256=G1mDxhOCdF3umYz4k0kfwJbSdYSKqhvQdGCmrP8FRAY,111
+servalcat-0.4.100.dist-info/METADATA,sha256=fY8VFfVql5PFlkJdbmAFaPOH9WVm8MJ7oKedUyyL9ys,2742
+servalcat-0.4.100.dist-info/licenses/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725
+servalcat/ext.cpython-310-darwin.so,sha256=sn9Xey4KltZNwjTmYVOy3khIC69YdZHAoHZqFXEElZQ,1327104
+servalcat/__init__.py,sha256=QKT0vTBwtQtat3LpKrUkOQ5o2aA_FjjhNjTdQPhyes8,232
 servalcat/__main__.py,sha256=bNEZKrG5765uOp32UiBAJmeJi3O0sk4I5QjrXUbngIE,4070
 servalcat/refmac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 servalcat/refmac/refmac_keywords.py,sha256=yzYSJELMjdTnf4pT0_8EdIOjCDncqkjpPJ3VNcsFDrU,26649
 servalcat/refmac/exte.py,sha256=HtmlTHRzCeCN-vSJZdKCD1GIhQO6zTe1OKLUObC3gZ8,9019
 servalcat/refmac/refmac_wrapper.py,sha256=euuhBv2F_ArmUvlfAA9fld4oCDkWMZENP73ioTtv118,17838
-servalcat/xtal/twin.py,sha256=lmBz0no9lfjq6MNkK9Q-9R47a0xGkDfaBYrOWunqyBI,4900
+servalcat/xtal/twin.py,sha256=FfmVIdwve7MCCFXu019MYKSeWX8_Vy0SrHjqh1_sxBs,6181
 servalcat/xtal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-servalcat/xtal/sigmaa.py,sha256=mTGKfjnrZGAee-TNZUovOd5mTCDZ10Xri8t2mY4l8Tg,75326
+servalcat/xtal/sigmaa.py,sha256=ZkswqjUw-ISOjsRiwJC4FcsR_RlRvv6eYIzBbeKVTIc,75258
 servalcat/xtal/french_wilson.py,sha256=GE3WEPhuizDVSPq1Afyb1O-OPTlzKDXYhkDCIELWI_0,11787
 servalcat/xtal/run_refmac_small.py,sha256=Yg-bEPuOdQxCwdwY8StCvXabcWHrRl6A22gJgLsypeU,10394
 servalcat/utils/symmetry.py,sha256=ahMiaEDMyfV2YW0BkikF_ldMtIN4-usAGsNRCXGVbug,11869
 servalcat/utils/generate_operators.py,sha256=fCrcTcciRn-6fpFtHYvf1oPXAObjuqR3WQejz5d0RrA,10980
 servalcat/utils/maps.py,sha256=0hen0Pt9IxbGQOLZyZaGiNWBAttc6aMqb6NzrCKkZMQ,13225
 servalcat/utils/__init__.py,sha256=j-fMAqHvzyMMQXb2Sf7Urnk4oQ31pT-g_NXf3K-NM4c,1137
-servalcat/utils/logger.py,sha256=c84HQf7Hb1yzXZ3Syog0f5vNRnozl0OCpqi0qIMoBlY,4589
-servalcat/utils/model.py,sha256=wmwYUacysc9zAXIS807sNlZGpc2cjJIa5QvUQb7eEYU,30177
+servalcat/utils/logger.py,sha256=eglkG7RxCJl4Q5P0aQGp4SVauoWh36nQgFvSJJLpyrM,4587
+servalcat/utils/model.py,sha256=78flu7sl-MmlN_ItY4YSWxyvTxuSsRMgBPotzh9MHqc,30556
 servalcat/utils/restraints.py,sha256=V2QBfgSH636upbLtGany4QyBk3r05CLxqvJYiM1SLSQ,36960
 servalcat/utils/refmac.py,sha256=LKwgCtLg7R5D8mjI7yCGd2lrjcjTeEJVAo1pey2u3IU,30922
 servalcat/utils/commands.py,sha256=dxsbPMZl-1BVKKr0rXzGF08k-Aco5qwzEHRWxZlcxek,70956
-servalcat/utils/hkl.py,sha256=aH2SMAccU-cdBqSgUJ7O6D7PpzO7m_1HlRVzg9Q_4Ds,28062
+servalcat/utils/hkl.py,sha256=dPwHxCRx33j_Egd9lKRWlUbk5MecziSiqW6vJz_eeUk,29787
 servalcat/utils/fileio.py,sha256=YytwlL0SeaxHCgIzFEjfS_tRpLDmvhVDjwEnkDXw1_E,29502
 servalcat/spa/shiftback.py,sha256=A9OfaZ8r2BC6A2uGXxNNhmyBNORUB_MeqJC29ZRebnw,4664
 servalcat/spa/localcc.py,sha256=0otKfGKH33cFsD6Qvh8iL1mINlWqb3gqAbVKqy8fQPY,7985
@@ -35,11 +35,11 @@ servalcat/spa/fsc.py,sha256=ISsBOl5JJZA6yT2yuDHNQ4I780TrmuYW6CPvimCwDqo,17454
 servalcat/spa/realspcc_from_var.py,sha256=VpdvNEY4zRza08Vz727JI6Seq-ix-oujx4Gvzy_VkYI,5275
 servalcat/spa/shift_maps.py,sha256=iP1w4LNXETvZblElJ64AFMcE-lPrhx1s4SfacGktg4o,13268
 servalcat/spa/fofc.py,sha256=Q97XkJX_4KWmxCIzq16R5BTMUY28V_4CPdbzF6p5Oxo,23113
-servalcat/refine/xtal.py,sha256=2Sr3sqO0nfIPNS4Ry_jmDimez3uRIWUsEExRC6ps1So,14617
+servalcat/refine/xtal.py,sha256=F_inGxiUJ1FabeJe2IMZiqJbJRuDrvJedYAoCwWnDSY,14635
 servalcat/refine/refine.py,sha256=W3oduNRNgc05mqMO1wYUdgN3pRrUuR9kEJ3yhAJd9jQ,44035
-servalcat/refine/refine_spa.py,sha256=Iq9lDvWC8IcOzHwx5FRylBUF3mb7YgtyzICsQTSm7Tw,19162
+servalcat/refine/refine_spa.py,sha256=nTNw2c1HU6sCbdfjR7gPyWiKDpvhep8wNnL4d5Mfjnc,19318
 servalcat/refine/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 servalcat/refine/spa.py,sha256=lmh4P6nQNFIdaZUKFIc_ES6B6X-FG-JkxyfpqHgtk9g,6472
 servalcat/refine/cgsolve.py,sha256=tHuWXr76x4sRAMUQ4dGVJzyHF_n1LiwKHTjAwh8iFMg,3072
 servalcat/refine/refine_geom.py,sha256=uATaTCEVkPJw3DjpVX66j7mhYs9Mp21rDwBfumrpwfw,11352
-servalcat/refine/refine_xtal.py,sha256=DFy7FFRBP3fhcHl8JzI0FqBfdfAHE2vJNyH7kp9lXtQ,14736
+servalcat/refine/refine_xtal.py,sha256=dxJsS73frNbjgQOwiMopv4ijed23-AXxmhZv5v0-kGk,14564