servalcat 0.4.60__cp312-cp312-win_amd64.whl

servalcat/utils/refmac.py

@@ -0,0 +1,747 @@
+"""
+Author: "Keitaro Yamashita, Garib N. Murshudov"
+MRC Laboratory of Molecular Biology
+    
+This software is released under the
+Mozilla Public License, version 2.0; see LICENSE.
+"""
+from __future__ import absolute_import, division, print_function, generators
+import gemmi
+import numpy
+import subprocess
+import shlex
+import json
+import copy
+import re
+import os
+import string
+import itertools
+import tempfile
+from servalcat.utils import logger
+from servalcat.utils import fileio
+
+re_version = re.compile("#.* Refmac *version ([^ ]+) ")
+re_error = re.compile('(warn|error *[:]|error *==|^error)', re.IGNORECASE)
+re_outlier_start = re.compile("\*\*\*\*.*outliers")
+
+def check_version(exe="refmac5"):
+    ver = ()
+    try:
+        output = ""
+        p = subprocess.Popen([exe], shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                             universal_newlines=True)
+        p.stdin.write("end\n")
+        p.stdin.close()
+        for l in iter(p.stdout.readline, ""):
+            output += l
+            r_ver = re_version.search(l)
+            if r_ver:
+                logger.writeln("Refmac version: {}".format(r_ver.group(1)))
+                ver = tuple(map(int, r_ver.group(1).split(".")))
+        p.wait()
+        if not ver and output:
+            logger.writeln("\nError: failed to check the Refmac version. The raw output:\n")
+            logger.writeln(output)
+    except OSError as e:
+        logger.writeln("Error: Cannot execute {}".format(exe))
+    return ver
+# check_version()
+
+def ensure_ccp4scr():
+    tmpdir = os.environ.get("CCP4_SCR")
+    if tmpdir:
+        if os.path.isdir(tmpdir): # TODO check writability
+            try:
+                t = tempfile.TemporaryFile(dir=tmpdir)
+                t.close()
+                return
+            except OSError:
+                logger.writeln("Warning: cannot write files in CCP4_SCR= {}".format(tmpdir))
+        else:
+            try:
+                os.makedirs(tmpdir)
+                return
+            except:
+                logger.writeln("Warning: cannot create CCP4_SCR= {}".format(tmpdir))
+
+    os.environ["CCP4_SCR"] = tempfile.mkdtemp(prefix="ccp4tmp")
+    logger.writeln("Updated CCP4_SCR= {}".format(os.environ["CCP4_SCR"]))
+# ensure_ccp4scr()
+
+def external_restraints_json_to_keywords(json_in):
+    ret = []
+    with open(json_in) as f: exte_list = json.load(f)
+    for e in exte_list:
+        if "use" in e:
+            ret.append("EXTERNAL USE {}".format(e["use"]))
+        if "dmax" in e:
+            ret.append("EXTERNAL DMAX {0}".format(e["dmax"]))
+        if "weight_scale" in e:
+            ret.append("EXTERNAL WEIGHT SCALE {0}".format(e["weight_scale"]))
+        if "weight_gmwt" in e:
+            ret.append("EXTERNAL WEIGHT GMWT {0}".format(e["weight_gmwt"]))
+        if "file" in e:
+            ret.append("@"+e["file"])
+
+    return "\n".join(ret) + "\n"
+# external_restraints_json_to_keywords()
+
+def read_tls_file(tlsin):
+    # TODO sort out L/S units - currently use Refmac tlsin/out as is
+    # TODO change to gemmi::TlsGroup?
+    
+    groups = []
+    with open(tlsin) as ifs:
+        for l in ifs:
+            l = l.strip()
+            if l.startswith("TLS"):
+                title = l[4:]
+                groups.append(dict(title=title, ranges=[], origin=None, T=None, L=None, S=None))
+            elif l.startswith("RANG"):
+                r = l[l.index(" "):].strip()
+                groups[-1]["ranges"].append(r)
+            elif l.startswith("ORIG"):
+                try:
+                    groups[-1]["origin"] = gemmi.Position(*(float(x) for x in l.split()[1:]))
+                except:
+                    raise ValueError("Problem with TLS file: {}".format(l))
+            elif l.startswith("T   "):
+                try:
+                    groups[-1]["T"] = [float(x) for x in l.split()[1:7]]
+                except:
+                    raise ValueError("Problem with TLS file: {}".format(l))
+            elif l.startswith("L   "):
+                try:
+                    groups[-1]["L"] = [float(x) for x in l.split()[1:7]]
+                except:
+                    raise ValueError("Problem with TLS file: {}".format(l))
+            elif l.startswith("S   "):
+                try:
+                    groups[-1]["S"] = [float(x) for x in l.split()[1:10]]
+                except:
+                    raise ValueError("Problem with TLS file: {}".format(l))
+
+    return groups
+# read_tls_file()
+
+def write_tls_file(groups, tlsout):
+    with open(tlsout, "w") as f:
+        for g in groups:
+            f.write("TLS {}\n".format(g["title"]))
+            for r in g["ranges"]:
+                f.write("RANGE {}\n".format(r))
+            if g["origin"] is not None:
+                f.write("ORIGIN ")
+                f.write(" ".join("{:8.4f}".format(x) for x in g["origin"].tolist()))
+                f.write("\n")
+            for k in "TLS":
+                if g[k] is not None:
+                    f.write("{:4s}".format(k))
+                    f.write(" ".join("{:8.4f}".format(x) for x in g[k]))
+                    f.write("\n")
+# write_tls_file()
+
+class FixForRefmac:
+    """
+    Workaround for Refmac limitations
+    - microheterogeneity
+    - residue number > 9999
+
+    XXX fix external restraints accordingly
+    TODO fix _struct_conf, _struct_sheet_range, _pdbx_struct_sheet_hbond
+    """
+    def __init__(self):
+        self.MAXNUM = 9999
+        self.fixes = []
+        self.resn_old_new = []
+        
+    def fix_before_topology(self, st, topo, fix_microheterogeneity=True, fix_resimax=True, fix_nonpolymer=True, add_gaps=False):
+        self.chainids = set(chain.name for chain in st[0])
+        if fix_microheterogeneity:
+            self.fix_microheterogeneity(st, topo)
+        if add_gaps:
+            self.add_gaps(st, topo)
+        if fix_resimax: # This modifies chains, so topo will be broken
+            self.fix_too_large_seqnum(st, topo)
+        if fix_nonpolymer: # This modifies chains, so topo will be broken
+            self.fix_nonpolymer(st)
+
+    def new_chain_id(self, original_chain_id):
+        # decide new chain ID
+        for i in itertools.count(start=1):
+            new_id = "{}{}".format(original_chain_id, i)
+            if new_id not in self.chainids:
+                self.chainids.add(new_id)
+                return new_id
+
+    def fix_metadata(self, st, changedict):
+        # fix connections
+        # changedict = dict(changes)
+        aa2tuple = lambda aa: (aa.chain_name, aa.res_id.seqid.num, chr(ord(aa.res_id.seqid.icode)|0x20))
+        for con in st.connections:
+            for aa in (con.partner1, con.partner2):
+                changeto = changedict.get(aa2tuple(aa))
+                if changeto is not None:
+                    aa.chain_name = changeto[0]
+                    aa.res_id.seqid.num = changeto[1]
+                    aa.res_id.seqid.icode = changeto[2]
+
+    def add_gaps(self, st, topo):
+        # Refmac (as of 5.8.0352) has a bug that makes two links for IAS (IAS-pept and usual TRANS/CIS)
+        # However this implementation is even more harmful.. if gap is inserted to real gaps then necessary p link is also gone!
+        for chain in st[0]:
+            rs = chain.get_polymer()
+            for i in range(1, len(rs)):
+                res0 = rs[i-1]
+                res = rs[i]
+                links = topo.links_to_previous(res)
+                if len(links) == 0 or links[0].link_id in ("gap", "?"):
+                    con = gemmi.Connection()
+                    con.asu = gemmi.Asu.Same
+                    con.type = gemmi.ConnectionType.Unknown
+                    con.link_id = "gap"
+                    con.partner1 = gemmi.AtomAddress(chain.name, res0.seqid, res0.name, "", "\0")
+                    con.partner2 = gemmi.AtomAddress(chain.name, res.seqid, res.name, "", "\0")
+                    logger.writeln("Refmac workaround (gap link): {}".format(con))
+                    st.connections.append(con)
+
+    def fix_microheterogeneity(self, st, topo):
+        mh_res = []
+        chains = []
+        icodes = {} # to avoid overlaps
+        modifications = [] # return value
+
+        # Check if microheterogeneity exists
+        for chain in st[0]:
+            for rg in chain.get_polymer().residue_groups():
+                if len(rg) > 1:
+                    ress = [r for r in rg]
+                    chains.append(chain.name)
+                    mh_res.append(ress)
+                    ress_str = "/".join([str(r) for r in ress])
+                    logger.writeln("Microheterogeneity detected in chain {}: {}".format(chain.name, ress_str))
+
+        if not mh_res: return
+
+        for chain in st[0]:
+            for res in chain:
+                if res.seqid.icode != " ":
+                    icodes.setdefault(chain.name, {}).setdefault(res.seqid.num, []).append(res.seqid.icode)
+
+        def append_links(bond, prr, toappend):
+            atoms = bond.atoms
+            assert len(atoms) == 2
+            found = None
+            for i in range(2):
+                if any(filter(lambda ra: atoms[i]==ra, prr)): found = i
+            if found is not None:
+                toappend.append([atoms[i], atoms[1-i]]) # prev atom, current atom
+        # append_links()
+
+        mh_res_all = sum(mh_res, [])
+        mh_link = {}
+
+        # Check links
+        for chain in st[0]:
+            for res in chain:
+                # If this residue is microheterogeneous
+                if res in mh_res_all:
+                    for link in topo.links_to_previous(res):
+                        mh_link.setdefault(id(res), []).append([link.res1, "prev", link.link_id, []])
+                        append_links(topo.first_bond_in_link(link), link.res1, mh_link[id(res)][-1][-1])
+
+                # Check if previous residue(s) is microheterogeneous
+                for link in topo.links_to_previous(res):
+                    prr = link.res1
+                    if prr in mh_res_all:
+                        mh_link.setdefault(id(prr), []).append([res, "next", link.link_id, []])
+                        append_links(topo.first_bond_in_link(link), prr, mh_link[id(prr)][-1][-1])
+
+        # Change IDs
+        for chain_name, rr in zip(chains, mh_res):
+            chars = string.ascii_uppercase
+            # avoid already used inscodes
+            if chain_name in icodes and rr[0].seqid.num in icodes[chain_name]:
+                used_codes = set(icodes[chain_name][rr[0].seqid.num])
+                chars = list(filter(lambda x: x not in used_codes, chars))
+            for ir, r in enumerate(rr[1:]):
+                modifications.append([(chain_name, r.seqid.num, r.seqid.icode),
+                                      (chain_name, r.seqid.num, chars[ir])])
+                r.seqid.icode = chars[ir]
+
+        logger.writeln("DEBUG: mh_link= {}".format(mh_link))
+        # Update connections (LINKR)
+        for chain_name, rr in zip(chains, mh_res):
+            for r in rr:
+                for p in mh_link.get(id(r), []):
+                    for atoms in p[-1]:
+                        con = gemmi.Connection()
+                        con.asu = gemmi.Asu.Same
+                        con.type = gemmi.ConnectionType.Covale
+                        con.link_id = p[2]
+                        if p[1] == "prev":
+                            p1 = gemmi.AtomAddress(chain_name, p[0].seqid, p[0].name, atoms[1].name, atoms[1].altloc)
+                            p2 = gemmi.AtomAddress(chain_name, r.seqid, r.name, atoms[0].name, atoms[0].altloc)
+                        else:
+                            p1 = gemmi.AtomAddress(chain_name, r.seqid, r.name, atoms[1].name, atoms[1].altloc)
+                            p2 = gemmi.AtomAddress(chain_name, p[0].seqid, p[0].name, atoms[0].name, atoms[0].altloc)
+
+                        con.partner1 = p1
+                        con.partner2 = p2
+                        logger.writeln(" Adding link: {}".format(con))
+                        st.connections.append(con)
+            for r1, r2 in itertools.combinations(rr, 2):
+                for a1 in set([a.altloc for a in r1]):
+                    for a2 in set([a.altloc for a in r2]):
+                        con = gemmi.Connection()
+                        con.asu = gemmi.Asu.Same
+                        con.link_id = "gap"
+                        # XXX altloc will be ignored when atom does not match.. grrr
+                        con.partner1 = gemmi.AtomAddress(chain_name, r1.seqid, r1.name, "", a1)
+                        con.partner2 = gemmi.AtomAddress(chain_name, r2.seqid, r2.name, "", a2)
+                        st.connections.append(con)
+
+        self.fixes.append(modifications)
+    # fix_microheterogeneity()
+
+    def fix_nonpolymer(self, st):
+        # Refmac (as of 5.8.0352) has a bug that links non-neighbouring nucleotides
+        # It only happens with mmCIF file
+        newchains = []
+        changes = []
+        for chain in st[0]:
+            polymer = chain.get_polymer()
+            if len(polymer) == len(chain): continue
+            if len(polymer) == 0: continue
+            del_idxes = []
+            newchains.append(gemmi.Chain(self.new_chain_id(chain.name)))
+            logger.writeln("Refmac workaround (nonpolymer-fix) {} => {} ({} residues)".format(chain.name, newchains[-1].name,
+                                                                                            len(chain) - len(polymer)))
+            for i, res in enumerate(chain):
+                if res in polymer: continue
+                newchains[-1].add_residue(res)
+                del_idxes.append(i)
+                changes.append([(chain.name, res.seqid.num, res.seqid.icode),
+                                (newchains[-1].name, newchains[-1][-1].seqid.num, newchains[-1][-1].seqid.icode)])
+            for i in reversed(del_idxes):
+                del chain[i]
+                
+        for c in newchains:
+            st[0].add_chain(c)
+        if changes:
+            st.remove_empty_chains()
+            self.fix_metadata(st, dict(changes))
+        self.fixes.append(changes)
+
+    def fix_too_large_seqnum(self, st, topo):
+        # Refmac cannot handle residue id > 9999
+        # What to do:
+        # - move to new chains
+        # - modify link records (and others?)
+        # - add link record if needed
+        newchains = []
+        changes = []
+        
+        for chain in st[0]:
+            maxseqnum = max([r.seqid.num for r in chain])
+            if maxseqnum > self.MAXNUM:
+                offset = 0
+                #target = [res for res in chain if res.seqid.num > 9999]
+                del_idxes = []
+                for ires, res in enumerate(chain):
+                    if res.seqid.num <= self.MAXNUM: continue
+                    if res.seqid.num - offset > self.MAXNUM:
+                        newchains.append(gemmi.Chain(self.new_chain_id(chain.name)))
+                        offset = res.seqid.num - 1
+                        # need to keep link to previous residue if exists
+                        for link in topo.links_to_previous(res):
+                            logger.writeln("Link: {} {} {} alt= {} {}".format(link.link_id, link.res1, link.res2, 
+                                                                            link.alt1, link.alt2))
+
+                            con = gemmi.Connection()
+                            con.type = gemmi.ConnectionType.Covale
+                            con.link_id = link.link_id
+                            #return link
+                            bond = topo.first_bond_in_link(link)
+                            if bond is not None:
+                                con.partner1 = gemmi.AtomAddress(chain.name, link.res1.seqid, link.res1.name, bond.atoms[0].name, bond.atoms[0].altloc)
+                                con.partner2 = gemmi.AtomAddress(chain.name, link.res2.seqid, link.res2.name, bond.atoms[1].name, bond.atoms[1].altloc)
+                                st.connections.append(con)
+
+                    newchains[-1].add_residue(res)
+                    newchains[-1][-1].seqid.num -= offset
+                    del_idxes.append(ires)
+                    prev = chain[ires-1].seqid if ires > 0 else None
+                    changes.append([(chain.name, res.seqid.num, res.seqid.icode),
+                                    (newchains[-1].name, newchains[-1][-1].seqid.num, newchains[-1][-1].seqid.icode)])
+                    logger.writeln("Refmac workaround (too large seq) {} => {} {}".format(changes[-1][0], changes[-1][1], res.name))
+
+                for i in reversed(del_idxes):
+                    del chain[i]
+
+        for c in newchains:
+            st[0].add_chain(c)
+        if changes:
+            st.remove_empty_chains()
+            self.fix_metadata(st, dict(changes))
+        self.fixes.append(changes)
+
+    def fix_long_resnames(self, st):
+        # this function should be called separately (after preparing topology)
+        st.shorten_ccd_codes()
+        self.resn_old_new = [x for x in st.shortened_ccd_codes]
+
+    def fix_model(self, st, changedict):
+        chain_newid = set()
+        for chain in st[0]:
+            for res in chain:
+                changeto = changedict.get((chain.name, res.seqid.num, res.seqid.icode))
+                if changeto is not None:
+                    logger.writeln("back: {} {} to {}".format(chain.name, res.seqid, changeto))
+                    #chain.name = changeto[0] # this is ok when modify back
+                    chain_newid.add((chain, changeto[0]))
+                    res.seqid.num = changeto[1]
+                    res.seqid.icode = changeto[2]
+
+        for chain, newid in chain_newid:
+            chain.name = newid
+        st.merge_chain_parts()
+        self.fix_metadata(st, changedict)
+        
+    def modify_back(self, st):
+        for fix in reversed(self.fixes):
+            reschanges = dict([x[::-1] for x in fix])
+            self.fix_model(st, reschanges)
+
+        if self.resn_old_new:
+            st.shortened_ccd_codes = self.resn_old_new
+            st.restore_full_ccd_codes()
+
+
+class Refmac:
+    def __init__(self, **kwargs):
+        self.prefix = "refmac"
+        self.hklin = self.xyzin = ""
+        self.source = "electron"
+        self.lab_f = None
+        self.lab_sigf = None
+        self.lab_phi = None
+        self.libin = None
+        self.tlsin = None
+        self.hydrogen = "all"
+        self.hout = False
+        self.ncycle = 10
+        self.tlscycle = 0
+        self.resolution = None
+        self.weight_matrix = None
+        self.weight_auto_scale = None
+        self.bfactor = None
+        self.jellybody = None
+        self.jellybody_sigma, self.jellybody_dmax = 0.01, 4.2
+        self.ncsr = None
+        self.shake = None
+        self.keyword_files = []
+        self.keywords = []
+        self.external_restraints_json = None
+        self.exe = "refmac5"
+        self.monlib_path = None
+        self.keep_chain_ids = False
+        self.show_log = False # summary only if false
+        self.global_mode = kwargs.get("global_mode")
+        
+        for k in kwargs:
+            if k == "args":
+                self.init_from_args(kwargs["args"])
+            else:
+                setattr(self, k, kwargs[k])
+
+        ensure_ccp4scr()
+    # __init__()
+
+    def init_from_args(self, args):
+        self.hklin = args.mtz
+        self.xyzin = args.model
+        self.libin = args.ligand
+        self.tlsin = args.tlsin
+        self.ncycle = args.ncycle
+        self.tlscycle = args.tlscycle
+        self.lab_f = args.lab_f
+        self.lab_phi = args.lab_phi
+        self.lab_sigf = args.lab_sigf
+        self.hydrogen = args.hydrogen
+        self.hout = args.hout
+        self.ncsr = args.ncsr
+        self.bfactor = args.bfactor
+        self.jellybody = args.jellybody
+        self.jellybody_sigma, self.jellybody_dmax = args.jellybody_params
+        self.resolution = args.resolution
+        self.weight_auto_scale = args.weight_auto_scale
+        self.keyword_files = args.keyword_file
+        self.keywords = args.keywords
+        self.external_restraints_json = args.external_restraints_json
+        self.exe = args.exe
+        self.show_log = args.show_refmac_log
+        self.monlib_path = args.monlib
+    # init_from_args()
+
+    def copy(self, **kwargs):
+        ret = copy.deepcopy(self)
+        for k in kwargs:
+            setattr(ret, k, kwargs[k])
+            
+        return ret
+    # copy()
+
+    def set_libin(self, ligands):
+        if not ligands: return
+        if len(ligands) > 1:
+            mcif = "merged_ligands.cif" # XXX directory!
+            logger.writeln("Merging ligand cif files: {}".format(ligands))
+            fileio.merge_ligand_cif(ligands, mcif)
+            self.libin = mcif
+        else:
+            self.libin = ligands[0]
+    # set_libin()
+            
+    
+    def make_keywords(self):
+        ret = ""
+        labin = []
+        if self.lab_f: labin.append("FP={}".format(self.lab_f))
+        if self.lab_sigf: labin.append("SIGFP={}".format(self.lab_sigf))
+        if self.lab_phi: labin.append("PHIB={}".format(self.lab_phi))
+        if labin:
+            ret += "labin {}\n".format(" ".join(labin))
+
+
+        ret += "make hydr {}\n".format(self.hydrogen)
+        ret += "make hout {}\n".format("yes" if self.hout else "no")
+        
+        if self.global_mode == "spa":
+            ret += "solvent no\n"
+            ret += "scale lssc isot\n"
+            ret += "source em mb\n"
+        elif self.source == "electron":
+            ret += "source ec mb\n"
+        elif self.source == "neutron":
+            ret += "source n\n"
+
+        ret += "ncycle {}\n".format(self.ncycle)
+        if self.resolution is not None:
+            ret += "reso {}\n".format(self.resolution)
+        if self.weight_matrix is not None:
+            ret += "weight matrix {}\n".format(self.weight_matrix)
+        elif self.weight_auto_scale is not None:
+            ret += "weight auto {:.2e}\n".format(self.weight_auto_scale)
+        else:
+            ret += "weight auto\n"
+
+        if self.bfactor is not None:
+            ret += "bfactor set {}\n".format(self.bfactor)
+        if self.jellybody:
+            ret += "ridge dist sigma {:.3e}\n".format(self.jellybody_sigma)
+            ret += "ridge dist dmax {:.2e}\n".format(self.jellybody_dmax)
+        if self.ncsr:
+            ret += "ncsr {}\n".format(self.ncsr)
+        if self.shake:
+            ret += "rand {}\n".format(self.shake)
+        if self.tlscycle > 0:
+            ret += "refi tlsc {}\n".format(self.tlscycle)
+            ret += "tlsout addu\n"
+        if self.keep_chain_ids:
+            ret += "pdbo keep auth\n"
+
+        if self.external_restraints_json:
+            ret += external_restraints_json_to_keywords(self.external_restraints_json)
+            
+        if self.keyword_files:
+            for f in self.keyword_files:
+                ret += "@{}\n".format(f)
+
+        if self.keywords:
+            ret += "\n".join(self.keywords).strip() + "\n"
+
+        return ret
+    # make_keywords()
+
+    def xyzout(self): return self.prefix + ".pdb"
+    def hklout(self): return self.prefix + ".mtz"
+    def tlsout(self): return self.prefix + ".tls"
+
+    def make_cmd(self):
+        cmd = [self.exe]
+        cmd.extend(["hklin", self.hklin])
+        cmd.extend(["hklout", self.hklout()])
+        cmd.extend(["xyzin", self.xyzin])
+        cmd.extend(["xyzout", self.xyzout()])
+        if self.libin:
+            cmd.extend(["libin", self.libin])
+        if self.tlsin:
+            cmd.extend(["tlsin", self.tlsin])
+        if self.tlscycle > 0:
+            cmd.extend(["tlsout", self.tlsout()])
+        if self.source == "neutron":
+            cmd.extend(["atomsf", os.path.join(os.environ["CLIBD"], "atomsf_neutron.lib")])
+            
+        return cmd
+    # make_cmd()
+
+    def run_refmac(self, write_summary_json=True):
+        cmd = self.make_cmd()
+        stdin = self.make_keywords()
+        with open(self.prefix+".inp", "w") as ofs: ofs.write(stdin)
+
+        logger.writeln("Running REFMAC5..")
+        logger.writeln("{} <<__eof__ > {}".format(" ".join(shlex.quote(x) for x in cmd), self.prefix+".log"))
+        logger.write(stdin)
+        logger.writeln("__eof__")
+
+        env = os.environ
+        if self.monlib_path: env["CLIBD_MON"] = os.path.join(self.monlib_path, "") # should end with /
+        
+        p = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                             universal_newlines=True, env=env)
+        p.stdin.write(stdin)
+        p.stdin.close()
+
+        log = open(self.prefix+".log", "w")
+        cycle = 0
+        re_cycle_table = re.compile("Cycle *([0-9]+). Rfactor analysis")
+        re_actual_weight = re.compile("Actual weight *([^ ]+) *is applied to the X-ray term")
+        rmsbond = ""
+        rmsangle = ""
+        log_delay = []
+        summary_write = (lambda x: log_delay.append(x)) if self.show_log else logger.writeln
+        outlier_flag = False
+        last_table_flag = False
+        last_table_keys = []
+        occ_flag = False
+        occ_cycles = 0
+        ret = {"version":None,
+               "cycles": [{"cycle":i} for i in range(self.ncycle+self.tlscycle+1)],
+               } # metadata
+        
+        for l in iter(p.stdout.readline, ""):
+            log.write(l)
+
+            if self.show_log:
+                print(l, end="")
+            
+            r_ver = re_version.search(l)
+            if r_ver:
+                ret["version"] = r_ver.group(1)
+                summary_write("Starting Refmac {} (PID: {})".format(r_ver.group(1), p.pid))
+
+            # print error/warning
+            r_err = re_error.search(l)
+            if r_err:
+                if self.global_mode == "spa":
+                    if "Figure of merit of phases has not been assigned" in l:
+                        continue
+                    elif "They will be assumed to be equal to 1.0" in l:
+                        continue
+                summary_write(l.rstrip())
+
+            # print outliers
+            r_outl = re_outlier_start.search(l)
+            if r_outl:
+                outlier_flag = True
+                summary_write(l.rstrip())
+            elif outlier_flag:
+                if l.strip() == "" or "monitored" in l or "dev=" in l or "sigma=" in l.lower() or "sigma.=" in l:
+                    summary_write(l.rstrip())
+                else:
+                    outlier_flag = False
+
+            if "TLS refinement cycle" in l:
+                cycle = int(l.split()[-1])
+            elif "----Group occupancy refinement----" in l:
+                occ_flag = True
+                occ_cycles += 1
+                cycle += 1
+            elif "CGMAT cycle number =" in l:
+                cycle = int(l[l.index("=")+1:]) + self.tlscycle + occ_cycles
+                occ_flag = False
+            
+            r_cycle = re_cycle_table.search(l)
+            if r_cycle: cycle = int(r_cycle.group(1))                
+
+            for i in range(len(ret["cycles"]), cycle):
+                ret["cycles"].append({"cycle":i})
+
+            if "Overall R factor                     =" in l and cycle > 0:
+                rfac = l[l.index("=")+1:].strip()
+                if self.global_mode != "spa":
+                    ret["cycles"][cycle-1]["r_factor"] = rfac
+                    summary_write(" cycle= {:3d} R= {}".format(cycle-1, rfac))
+            elif "Average Fourier shell correlation    =" in l and cycle > 0:
+                fsc = l[l.index("=")+1:].strip()
+                if occ_flag:
+                    note = "(occupancy)"
+                elif cycle == 1:
+                    note = "(initial)"
+                elif cycle <= self.tlscycle+1:
+                    note = "(TLS)"
+                elif cycle > self.ncycle + occ_cycles + self.tlscycle:
+                    note = "(final)"
+                else:
+                    note = ""
+
+                if self.global_mode == "spa":
+                    ret["cycles"][cycle-1]["fsc_average"] = fsc
+                    summary_write(" cycle= {:3d} FSCaverage= {} {}".format(cycle-1, fsc, note))
+            elif "Rms BondLength" in l:
+                rmsbond = l
+            elif "Rms BondAngle" in l:
+                rmsangle = l
+
+            r_actual_weight = re_actual_weight.search(l)
+            if r_actual_weight:
+                ret["cycles"][cycle-1]["actual_weight"] = r_actual_weight.group(1)
+
+            # Final table
+            if "    Ncyc    Rfact    Rfree     FOM" in l:
+                last_table_flag = True
+                last_table_keys = l.split()
+                if last_table_keys[-1] == "$$": del last_table_keys[-1]
+            elif last_table_flag:
+                if "$$ Final results $$" in l:
+                    last_table_flag = False
+                    continue
+                sp = l.split()
+                if len(sp) == len(last_table_keys) and sp[0] != "$$":
+                    cyc = int(sp[last_table_keys.index("Ncyc")])
+                    key_name = dict(rmsBOND="rms_bond", zBOND="rmsz_bond",
+                                    rmsANGL="rms_angle", zANGL="rmsz_angle",
+                                    rmsCHIRAL="rms_chiral")
+                    for k in key_name:
+                        if k in last_table_keys:
+                            ret["cycles"][cyc][key_name[k]] = sp[last_table_keys.index(k)]
+                        else:
+                            logger.error("table does not have key {}?".format(k))
+                            
+        retcode = p.wait()
+        log.close()
+        if log_delay:
+            logger.writeln("== Summary of Refmac ==")
+            logger.writeln("\n".join(log_delay))
+                         
+        if rmsbond:
+            logger.writeln("                      Initial    Final")
+            logger.writeln(rmsbond.rstrip())
+            logger.writeln(rmsangle.rstrip())
+            
+        logger.writeln("REFMAC5 finished with exit code= {}".format(retcode))
+
+        if write_summary_json:
+            json.dump(ret,
+                      open("{}_summary.json".format(self.prefix), "w"),
+                      indent=True)
+
+        # TODO check timestamp
+        if not os.path.isfile(self.xyzout()) or not os.path.isfile(self.hklout()):
+            raise RuntimeError("REFMAC5 did not produce output files. Check {}".format(self.prefix+".log"))
+
+        return ret
+    # run_refmac()
+# class Refmac