modelcraft 5.0.3__py3-none-any.whl → 6.0.0__py3-none-any.whl

modelcraft/modelcraftxray.py

@@ -1,19 +1,28 @@
 import os
+import shutil
 import time
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
 import gemmi
+
 from . import __version__
+from .cell import max_distortion, remove_scale, update_cell
+from .combine import combine_results
 from .jobs.buccaneer import Buccaneer
-from .jobs.coot import FixSideChains, Prune
 from .jobs.ctruncate import CTruncate
 from .jobs.findwaters import FindWaters
 from .jobs.nautilus import Nautilus
+from .jobs.nucleofind import NucleoFindBuild, NucleoFindPredict
 from .jobs.parrot import Parrot
 from .jobs.refmac import Refmac
 from .jobs.sheetbend import Sheetbend
-from .cell import max_distortion, remove_scale, update_cell
-from .combine import combine_results
+from .monlib import MonLib
 from .pipeline import Pipeline
+from .prune import prune
 from .reflections import DataItem, write_mtz
+from .scripts.sidechains import any_missing_side_chains
+from .scripts.sidechains import main as fix_side_chains
 from .structure import ModelStats, remove_residues, write_mmcif
 
 
@@ -38,6 +47,10 @@ class ModelCraftXray(Pipeline):
         self.last_refmac = None
         self.output_refmac = None
         self.cycles_without_improvement = 0
+        resnames = self.args.contents.monomer_codes()
+        if self.args.model:
+            resnames |= set(self.args.model[0].get_all_residue_names())
+        self.monlib = MonLib(resnames, self.args.restraints, include_standard=True)
 
     @property
     def resolution(self):
@@ -52,23 +65,23 @@ class ModelCraftXray(Pipeline):
         if self.args.model is not None:
             self._refine_input_model()
         for self.cycle in range(1, self.args.cycles + 1):
-            print("\n## Cycle %d\n" % self.cycle, flush=True)
+            print(f"\n## Cycle {self.cycle}\n", flush=True)
             self.run_cycle()
             self.process_cycle_output(self.last_refmac)
             if self.cycles_without_improvement == self.args.auto_stop_cycles > 0:
                 break
         if (
             not self.args.basic
-            and self.output_refmac.rwork < 0.3
-            and self.resolution < 2.5
+            and not self.args.disable_side_chain_fixing
+            and any_missing_side_chains(self.output_refmac.structure)
         ):
-            print("\n## Finalisations\n", flush=True)
+            print("\n## Adding missing side chains\n", flush=True)
             self.cycle += 1
             self.update_current_from_refmac_result(self.output_refmac)
             self.fixsidechains()
             self.process_cycle_output(self.last_refmac)
         print("\n## Best Model:", flush=True)
-        _print_refmac_result(self.output_refmac)
+        self._print_refmac_result(self.output_refmac)
         self._remove_current_files()
         self.terminate(reason="Normal")
 
@@ -101,13 +114,13 @@ class ModelCraftXray(Pipeline):
         self.args.model = self.current_structure
         if self.args.phases is not None:
             self.current_phases = self.args.phases
-        _print_refmac_result(self.last_refmac)
+        self._print_refmac_result(self.last_refmac)
 
     def run_cycle(self):
         if self.args.basic:
             if self.cycle == 1:
                 self.parrot()
-            self.run_buccaneer_and_nautilus()
+            self.run_model_building()
         else:
             if self.cycle > 1 and self.resolution < 2.3:
                 self.prune()
@@ -116,19 +129,25 @@ class ModelCraftXray(Pipeline):
                 if self.cycle > 1 or self.args.phases is None:
                     self.findwaters(dummy=True)
                 remove_residues(structure=self.current_structure, names={"HOH", "DUM"})
-            self.run_buccaneer_and_nautilus()
+            self.run_model_building()
             self.prune(chains_only=True)
             self.findwaters()
 
-    def run_buccaneer_and_nautilus(self):
+    def run_model_building(self):
         buccaneer = self.buccaneer()
+        if shutil.which("nucleofind"):
+            try:
+                return self.nucleofind(buccaneer)
+            except FileNotFoundError:
+                pass
         nautilus = self.nautilus()
         if buccaneer is None and nautilus is None:
             self.terminate(reason="No residues built")
         if buccaneer is None or nautilus is None:
             self.update_current_from_refmac_result(buccaneer or nautilus)
         else:
-            combined = self.run_refmac(combine_results(buccaneer, nautilus), cycles=5)
+            combined_structure = combine_results(buccaneer, nautilus, self.monlib)
+            combined = self.run_refmac(combined_structure, cycles=5)
             best = min((buccaneer, nautilus, combined), key=lambda result: result.rfree)
             self.update_current_from_refmac_result(best)
 
@@ -149,11 +168,32 @@ class ModelCraftXray(Pipeline):
             cycles=3 if self.cycle == 1 else 2,
             threads=self.args.threads,
         ).run(self)
-        if result.structure is None or ModelStats(result.structure).residues == 0:
+        if (
+            result.structure is None
+            or ModelStats(result.structure, self.monlib).residues == 0
+        ):
             return None
         write_mmcif(self.path("current.cif"), result.structure)
         return self.run_refmac(result.structure, cycles=10)
 
+    def nucleofind(self, refmac):
+        if not (self.args.contents.rnas or self.args.contents.dnas):
+            return None
+        prediction = NucleoFindPredict(fphi=refmac.fphi_best).run(self)
+        result = NucleoFindBuild(
+            contents=self.args.contents,
+            fphi=refmac.fphi_best,
+            structure=refmac.structure,
+            prediction=prediction,
+        ).run(self)
+        if (
+            result.structure is None
+            or ModelStats(result.structure, self.monlib).residues == 0
+        ):
+            return None
+        write_mmcif(self.path("current.cif"), result.structure)
+        self.refmac(result.structure, cycles=10, auto_accept=True)
+
     def nautilus(self):
         if not (self.args.contents.rnas or self.args.contents.dnas):
             return None
@@ -165,7 +205,10 @@ class ModelCraftXray(Pipeline):
             freer=self.args.freer,
             structure=self.current_structure,
         ).run(self)
-        if result.structure is None or ModelStats(result.structure).residues == 0:
+        if (
+            result.structure is None
+            or ModelStats(result.structure, self.monlib).residues == 0
+        ):
             return None
         write_mmcif(self.path("current.cif"), result.structure)
         return self.run_refmac(result.structure, cycles=10)
@@ -181,7 +224,7 @@ class ModelCraftXray(Pipeline):
             write_mmcif(self.path("current.cif"), self.current_structure)
 
     def run_refmac(self, structure: gemmi.Structure, cycles: int):
-        if ModelStats(structure).residues == 0:
+        if ModelStats(structure, self.monlib).residues == 0:
             self.terminate(reason="No residues to refine")
         use_phases = self.args.unbiased and (
             self.output_refmac is None or self.output_refmac.rwork > 0.35
@@ -216,6 +259,7 @@ class ModelCraftXray(Pipeline):
             phases=self.current_phases,
             fphi=self.current_fphi_best,
             structure=self.current_structure,
+            monlib=self.monlib,
         ).run(self)
         self.current_phases = result.abcd
         self.current_fphi_best = result.fphi
@@ -224,25 +268,30 @@ class ModelCraftXray(Pipeline):
     def prune(self, chains_only=False):
         if self.args.disable_pruning or not self.args.contents.proteins:
             return
-        result = Prune(
+        pruned = prune(
             structure=self.current_structure,
             fphi_best=self.current_fphi_best,
             fphi_diff=self.current_fphi_diff,
-            chains_only=chains_only,
-        ).run(self)
-        write_mmcif(self.path("current.cif"), result.structure)
-        self.refmac(result.structure, cycles=5, auto_accept=True)
+            fphi_calc=self.current_fphi_calc,
+            residues=not chains_only,
+            monlib=self.monlib,
+        )
+        if pruned:
+            write_mmcif(self.path("current.cif"), pruned)
+            self.refmac(pruned, cycles=5, auto_accept=True)
 
     def fixsidechains(self):
-        if self.args.disable_side_chain_fixing or not self.args.contents.proteins:
-            return
-        result = FixSideChains(
-            structure=self.current_structure,
-            fphi_best=self.current_fphi_best,
-            fphi_diff=self.current_fphi_diff,
-        ).run(self)
-        write_mmcif(self.path("current.cif"), result.structure)
-        self.refmac(result.structure, cycles=5, auto_accept=False)
+        with TemporaryDirectory() as tempdir:
+            xyzin = str(Path(tempdir, "input.cif"))
+            hklin = str(Path(tempdir, "input.mtz"))
+            xyzout = str(Path(tempdir, "output.cif"))
+            write_mmcif(xyzin, self.current_structure)
+            write_mtz(hklin, [self.current_fphi_best], ["FWT,PHWT"])
+            fix_side_chains([xyzin, hklin, xyzout])
+            if os.path.exists(xyzout):
+                structure = gemmi.read_structure(xyzout)
+                write_mmcif(self.path("current.cif"), structure)
+                self.refmac(structure, cycles=5, auto_accept=False)
 
     def findwaters(self, dummy=False):
         if dummy and self.args.disable_dummy_atoms:
@@ -258,8 +307,8 @@ class ModelCraftXray(Pipeline):
         self.refmac(result.structure, cycles=10, auto_accept=False)
 
     def process_cycle_output(self, result):
-        _print_refmac_result(result)
-        model_stats = ModelStats(result.structure)
+        self._print_refmac_result(result)
+        model_stats = ModelStats(result.structure, self.monlib)
         stats = {
             "cycle": self.cycle,
             "residues": model_stats.residues,
@@ -308,13 +357,12 @@ class ModelCraftXray(Pipeline):
             except FileNotFoundError:
                 pass
 
-
-def _print_refmac_result(result):
-    model_stats = ModelStats(result.structure)
-    print("")
-    print(f"Residues: {model_stats.residues:6d}")
-    print(f"Protein:  {model_stats.protein:6d}")
-    print(f"Nucleic:  {model_stats.nucleic:6d}")
-    print(f"Waters:   {model_stats.waters:6d}")
-    print(f"R-work:   {result.rwork:6.4f}")
-    print(f"R-free:   {result.rfree:6.4f}", flush=True)
+    def _print_refmac_result(self, result):
+        model_stats = ModelStats(result.structure, self.monlib)
+        print("")
+        print(f"Residues: {model_stats.residues:6d}")
+        print(f"Protein:  {model_stats.protein:6d}")
+        print(f"Nucleic:  {model_stats.nucleic:6d}")
+        print(f"Waters:   {model_stats.waters:6d}")
+        print(f"R-work:   {result.rwork:6.4f}")
+        print(f"R-free:   {result.rfree:6.4f}", flush=True)

modelcraft/monlib.py

@@ -1,55 +1,58 @@
-import functools
 import os
-import gemmi
-
-
-@functools.lru_cache(maxsize=None)
-def _path(code: str) -> str:
-    directory = os.path.join(os.environ["CLIBD_MON"], code[0].lower())
-    single = os.path.join(directory, f"{code.upper()}.cif")
-    double = os.path.join(directory, f"{code.upper()}_{code.upper()}.cif")
-    return double if os.path.exists(double) else single
-
-
-@functools.lru_cache(maxsize=None)
-def atom_ids(code: str) -> set:
-    return {atom.id for atom in chemcomp(code).atoms}
-
-
-@functools.lru_cache(maxsize=None)
-def chemcomp(code: str) -> gemmi.ChemComp:
-    doc = gemmi.cif.read(_path(code))
-    return gemmi.make_chemcomp_from_block(doc[-1])
-
-
-@functools.lru_cache(maxsize=None)
-def in_library(code: str) -> bool:
-    return os.path.exists(_path(code))
-
-
-@functools.lru_cache(maxsize=None)
-def group(code: str) -> gemmi.ChemComp.Group:
-    if in_library(code):
-        doc = gemmi.cif.read(_path(code))
-        monlib = gemmi.MonLib()
-        monlib.read_monomer_doc(doc)
-        return monlib.monomers[code].group
-    return None
-
-
-@functools.lru_cache(maxsize=None)
-def is_protein(code: str) -> bool:
-    return group(code) in {
-        gemmi.ChemComp.Group.Peptide,
-        gemmi.ChemComp.Group.PPeptide,
-        gemmi.ChemComp.Group.MPeptide,
-    }
+import sys
 
+import gemmi
 
-@functools.lru_cache(maxsize=None)
-def is_nucleic(code: str) -> bool:
-    return group(code) in {
-        gemmi.ChemComp.Group.Dna,
-        gemmi.ChemComp.Group.Rna,
-        gemmi.ChemComp.Group.DnaRna,
-    }
+from .sequence import DNA_CODES, PROTEIN_CODES, RNA_CODES
+
+
+class MonLib(gemmi.MonLib):
+    def __init__(self, resnames, libin: str = "", include_standard: bool = False):
+        super().__init__()
+        if libin:
+            self.read_monomer_cif(libin)
+        if include_standard:
+            resnames = set(resnames)
+            resnames |= set(PROTEIN_CODES.values())
+            resnames |= set(RNA_CODES.values())
+            resnames |= set(DNA_CODES.values())
+            resnames |= {"MSE", "HOH"}
+        ok = self.read_monomer_lib(
+            os.environ["CLIBD_MON"], list(resnames), logging=sys.stderr
+        )
+        if not ok:
+            raise ValueError("Please create definitions for missing monomers.")
+
+    def __contains__(self, code: str):
+        return code in self.monomers
+
+    def __getitem__(self, code: str):
+        if code not in self:
+            raise KeyError(f"Monomer {code} not in this monomer library instance")
+        return self.monomers[code]
+
+    def atom_ids(self, code: str):
+        return {atom.id for atom in self[code].atoms}
+
+    def group(self, code: str):
+        return self[code].group if code in self else gemmi.ChemComp.Group.Null
+
+    def is_nucleic(self, code: str) -> bool:
+        return self.group(code) in {
+            gemmi.ChemComp.Group.Dna,
+            gemmi.ChemComp.Group.Rna,
+            gemmi.ChemComp.Group.DnaRna,
+        }
+
+    def is_protein(self, code: str) -> bool:
+        return self.group(code) in {
+            gemmi.ChemComp.Group.Peptide,
+            gemmi.ChemComp.Group.PPeptide,
+            gemmi.ChemComp.Group.MPeptide,
+        }
+
+    def volume(self, code: str):
+        return sum(18 for atom in self[code].atoms if not atom.is_hydrogen())
+
+    def weight(self, code: str):
+        return sum(atom.el.weight for atom in self[code].atoms)

modelcraft/pdbe.py

@@ -0,0 +1,54 @@
+import multiprocessing
+import re
+
+import requests
+
+_MULTIPROCESSING_LOCK = multiprocessing.Lock()
+_SERVER = "https://www.ebi.ac.uk/pdbe"
+
+
+def _response_json(url, data=None):
+    with _MULTIPROCESSING_LOCK:
+        print("Requesting:", url)
+        if data is None:
+            response = requests.get(url, timeout=30)
+        else:
+            response = requests.post(url, data=data, timeout=30)
+        response.raise_for_status()
+    return response.json()
+
+
+def molecule_dicts(entry_id: str) -> list:
+    entry_id = _superceeding_entry(entry_id)
+    url = _SERVER + "/api/pdb/entry/molecules/" + entry_id
+    response = _response_json(url)
+    mols = response[entry_id]
+    if any(mol["molecule_type"] == "carbohydrate polymer" for mol in mols):
+        codes = _carb_codes(entry_id)
+        for mol in mols:
+            mol["carb_codes"] = codes.get(mol["entity_id"])
+    return mols
+
+
+def _superceeding_entry(entry_id: str) -> str:
+    entry_id = entry_id.lower()
+    url = _SERVER + "/api/pdb/entry/status/" + entry_id
+    response = _response_json(url)
+    superceded_by = response[entry_id][0].get("superceded_by", [])
+    return entry_id if len(superceded_by) == 0 else superceded_by[-1]
+
+
+def _carb_codes(entry: str) -> dict:
+    url = _SERVER + "/search/pdb/select?"
+    query = "pdb_id:" + entry
+    filter_list = "carb_compound_id_entity"
+    request_data = {"q": query, "fl": filter_list, "wt": "json"}
+    response = _response_json(url, data=request_data)
+    docs = response["response"]["docs"]
+    codes = {}
+    for doc in docs:
+        for line in doc["carb_compound_id_entity"]:
+            match = re.match(r"(.+)\((\d+)\)_(\d+)", line)
+            code, copies, entity = match.groups()
+            codes.setdefault(int(entity), {})[code] = int(copies)
+    return codes

modelcraft/pipeline.py

@@ -55,7 +55,7 @@ class Pipeline:
     def write_report(self):
         if self.json_name:
             self.seconds["total"] = time.time() - self.start_time
-            with open(self.path(self.json_name), "w") as report_file:
+            with open(self.path(self.json_name), "w", encoding="utf-8") as report_file:
                 json.dump(self.report, report_file, indent=4)
 
     def terminate(self, reason: str):

modelcraft/prune.py

@@ -0,0 +1,69 @@
+import gemmi
+
+from .monlib import MonLib
+from .reflections import DataItem
+from .structure import remove_isolated_fragments
+from .validation import validate
+
+
+def prune(
+    structure: gemmi.Structure,
+    fphi_best: DataItem,
+    fphi_diff: DataItem,
+    fphi_calc: DataItem,
+    residues: bool = True,
+    chain_threshold: float = -2,
+    residue_threshold: float = -5,
+    monlib: MonLib = None,
+) -> gemmi.Structure:
+    print("Performing validation for pruning", flush=True)
+    structure = structure.clone()
+    monlib = monlib or MonLib(structure[0].get_all_residue_names())
+    metrics = validate(structure, fphi_best, fphi_diff, fphi_calc, monlib)
+
+    max_deleted = int(len(metrics) * 0.2)
+    num_deleted = 0
+    grouped = metrics.groupby("Chain")
+    means = grouped.mean(numeric_only=True)
+    for chain_name in means.sort_values("Score").index:
+        score = means.loc[chain_name, "Score"]
+        count = grouped.size().loc[chain_name]
+        print(
+            f"Chain {chain_name} has a score of {score} over {count} residues",
+            flush=True,
+        )
+        if (
+            means.loc[chain_name, "Score"] < chain_threshold
+            and count <= 20
+            and num_deleted + count <= max_deleted
+        ):
+            print("Deleting chain", chain_name, flush=True)
+            del structure[0][chain_name]
+            num_deleted += num_deleted
+            metrics = metrics[metrics["Chain"] != chain_name]
+
+    if not residues:
+        return structure if num_deleted > 0 else None
+
+    max_deleted = int(len(metrics) * 0.2)
+    metrics = metrics[metrics["Score"] < residue_threshold]
+    metrics.sort_values("Score", inplace=True)
+    metrics = metrics.head(max_deleted)
+    if len(metrics) == 0:
+        return structure if num_deleted > 0 else None
+
+    print(
+        f"Deleting {len(metrics)} residues with scores < {residue_threshold}",
+        flush=True,
+    )
+    to_delete = {(row["Chain"], row["SeqId"]) for _, row in metrics.iterrows()}
+    for chain in structure[0]:
+        for i, residue in reversed(list(enumerate(chain))):
+            if (chain.name, str(residue.seqid)) in to_delete:
+                del chain[i]
+
+    print("Removing isolated residues (if any)", flush=True)
+    for chain in structure[0]:
+        remove_isolated_fragments(chain, monlib, max_length=1)
+
+    return structure

modelcraft/reflections.py

@@ -1,6 +1,8 @@
-from typing import Iterator, Iterable, List, Optional, Union
 import itertools
 import re
+from functools import partial
+from typing import Iterable, Iterator, List, Optional, Union
+
 import gemmi
 import numpy
 import pandas
@@ -104,6 +106,14 @@ class DataItem(gemmi.Mtz):
         data = numpy.array(self, copy=copy)
         return pandas.DataFrame(data=data, columns=self.column_labels())
 
+    def map(self, spacing: float = 1.0, size=None) -> gemmi.FloatGrid:
+        if self.types != "FP":
+            raise ValueError("DataItem must contain F and PHI columns")
+        func = partial(self.transform_f_phi_to_map, self.label(0), self.label(1))
+        if size is None:
+            return func(sample_rate=self.resolution_high() / spacing)
+        return func(exact_size=size)
+
     @classmethod
     def search(cls, mtz: gemmi.Mtz, types: str, sequential: bool = True):
         types = list(types)