DescripyTor 0.3.2__tar.gz → 0.4.3__tar.gz

{descripytor-0.3.2 → descripytor-0.4.3}/M1_pre_calculations/pipeline/build.py

@@ -76,13 +76,17 @@ def _embed(smiles: str, n_conf: int, seed: int, pair=None, pair_range=(2.5, 3.0)
     mol = Chem.AddHs(mol)
     params = AllChem.ETKDGv3()
     params.randomSeed = seed
+    pair_held = bool(pair)
     if pair:
         from rdkit import DistanceGeometry
         from rdkit.Chem import rdDistGeom
         i, j = sorted(pair)
         bm = rdDistGeom.GetMoleculeBoundsMatrix(mol)
         bm[i][j], bm[j][i] = pair_range[1], pair_range[0]          # upper triangle = max, lower = min
-        DistanceGeometry.DoTriangleSmoothing(bm)
+        if not DistanceGeometry.DoTriangleSmoothing(bm):
+            # a rigid ligand (fused biimidazoline) fixes the donor distance outside this range: embed it as it is,
+            # and let the bite-angle test decide. Another donor pair must not win just because it is flexible.
+            bm = rdDistGeom.GetMoleculeBoundsMatrix(mol); DistanceGeometry.DoTriangleSmoothing(bm); pair_held = False
         params.SetBoundsMat(bm)
         params.useExpTorsionAnglePrefs = False     # the torsion library holds bipyridine anti over the bound
     cids = list(AllChem.EmbedMultipleConfs(mol, n_conf, params))
@@ -93,9 +97,9 @@ def _embed(smiles: str, n_conf: int, seed: int, pair=None, pair_range=(2.5, 3.0)
     energy = {}
     for cid in cids:
         ff = AllChem.MMFFGetMoleculeForceField(mol, props, confId=cid) if mmff else AllChem.UFFGetMoleculeForceField(mol, confId=cid)
-        if pair and mmff:
+        if pair_held and mmff:
             ff.MMFFAddDistanceConstraint(pair[0], pair[1], False, pair_range[0], pair_range[1] - 0.1, 100.0)
-        elif pair:
+        elif pair_held:
             ff.UFFAddDistanceConstraint(pair[0], pair[1], False, pair_range[0], pair_range[1] - 0.1, 100.0)
         ff.Minimize(maxIts=2000)
         energy[cid] = ff.CalcEnergy()
@@ -131,6 +135,7 @@ def _relax_around_metal(mol, cid, m, donors, dists):
         rw.AddBond(int(d), k, Chem.BondType.SINGLE)
     cx = rw.GetMol()
     cx.UpdatePropertyCache(strict=False)
+    Chem.FastFindRings(cx)                 # the stand-in closes a new ring (the chelate); UFF typing needs ring info
     atom = cx.GetAtomWithIdx(k)
     atom.SetHybridization(Chem.HybridizationType.SP3)
     atom.SetNoImplicit(True)
@@ -182,14 +187,16 @@ def _ancillaries(m, d1, d2, ligand, dist, n, target):
     out = []
     for u in ideal:
         best, bestd = u, -1.0
+        clear = np.vstack([ligand] + out)    # the ancillaries placed so far count too: two tilting Cl ended up 1.1-2.3 A apart
         perp = np.cross(nrm, u)
         perp /= np.linalg.norm(perp)
         for tilt in np.radians([0, 10, 20, 30]):
-            for phi in np.linspace(0, 2 * np.pi, 12, endpoint=False):
+            # square planar: tilt out of the plane only; an in-plane turn walks one Cl into the other's site
+            for phi in (np.pi / 2, 3 * np.pi / 2) if n == 2 else np.linspace(0, 2 * np.pi, 12, endpoint=False):
                 axis = np.cos(phi) * nrm + np.sin(phi) * perp
                 v = u * np.cos(tilt) + np.cross(axis, u) * np.sin(tilt)
                 v /= np.linalg.norm(v)
-                score = min(np.linalg.norm(ligand - (m + v * dist), axis=1).min(), target) - 0.15 * tilt
+                score = min(np.linalg.norm(clear - (m + v * dist), axis=1).min(), target) - 0.15 * tilt
                 if score > bestd:
                     best, bestd = v, score
             if bestd > 0.9 * target:
@@ -297,7 +304,11 @@ def _build_chelate(smiles, b, n_conf, seed):
         raise BuildError("no bidentate donor pair found")
     tried = []
     for i, j in pairs:
-        mol, confs = _embed(smiles, n_conf, seed, pair=(i, j))
+        try:
+            mol, confs = _embed(smiles, n_conf, seed, pair=(i, j))
+        except BuildError as exc:                 # this pair cannot chelate: try the next candidate
+            tried.append(f"{i + 1}-{j + 1}: {exc}")
+            continue
         sym = [a.GetSymbol() for a in mol.GetAtoms()]
         n = len(sym)
         key = {}

{descripytor-0.3.2 → descripytor-0.4.3}/M1_pre_calculations/pipeline/cluster.py

@@ -39,13 +39,13 @@ class Remote:
     def __init__(self, host: str, key: str, timeout: int = 180):
         self.host, self.key, self.timeout = host, os.path.expanduser(key), timeout
 
-    def _ssh(self, command: str) -> str:
+    def _ssh(self, command: str, timeout: int | None = None) -> str:
         cmd = ["ssh", "-n", "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=30", "-o", "IdentitiesOnly=yes",
                "-i", self.key, self.host, command]
-        r = subprocess.run(cmd, capture_output=True, timeout=self.timeout)
+        r = subprocess.run(cmd, capture_output=True, timeout=timeout or self.timeout)
         return (r.stdout or b"").decode("utf-8", "replace")
 
-    def run(self, script: str, tries: int = 3) -> str:
+    def run(self, script: str, tries: int = 3, timeout: int | None = None) -> str:
         """Run a bash script remotely; returns its stdout. Retries when the reply comes back empty
         or cut short (the end marker is missing). Scripts must be safe to run twice."""
         payload = base64.b64encode(f"echo {BEGIN}\n{script}\necho {END}\n".encode()).decode()
@@ -53,7 +53,7 @@ class Remote:
             raise ClusterError("remote script too long for the tcsh login shell; upload it as a file instead")
         last = ""
         for _ in range(tries):
-            last = self._ssh(f"echo {payload} | base64 -d | bash")
+            last = self._ssh(f"echo {payload} | base64 -d | bash", timeout)
             if BEGIN in last and END in last:
                 return last.split(BEGIN, 1)[1].rsplit(END, 1)[0].strip("\n")
             time.sleep(2)
@@ -125,7 +125,10 @@ def _run_chain(r: Remote, p: Protocol, plan: str, plan_name: str) -> dict:
     """Upload a plan and submit it. Returns {task: {stage: job id}}. Raises on any qsub failure,
     reporting what was submitted, and never resubmits blindly."""
     _put_file(r, p.remote_root, plan_name, plan)
-    out = r.run(f"bash {p.remote_root}/submit_chain.sh {p.remote_root}/{plan_name}", tries=1)
+    # one qsub takes about 2 s on hpcgate, so the reply time grows with the plan: with the default timeout a plan of
+    # more than about 30 three-stage molecules was cut off part-way, its job ids lost with the reply
+    n_qsub = sum(len(line.split()) - 1 for line in plan.splitlines())
+    out = r.run(f"bash {p.remote_root}/submit_chain.sh {p.remote_root}/{plan_name}", tries=1, timeout=max(r.timeout, 60 + 10 * n_qsub))
     jobs, failed = {}, []
     for line in out.splitlines():
         f = line.split()

{descripytor-0.3.2 → descripytor-0.4.3}/M1_pre_calculations/pipeline/stages.py

@@ -257,7 +257,9 @@ print("converged" if converged else "NOT converged", f, flush=True)
 '''
 
 GAUSSIAN_BODY = r"""export g16root=@@G16ROOT@@ GAUSS_SCRDIR="$W"
+set +u                                  # g16.profile reads variables that may be unset (PERLLIB); under set -u the job dies with its marker left at "running"
 source "$g16root/g16/bsd/g16.profile"
+set -u
 {
   printf '%%nprocshared=%s\n%%mem=%sGB\n%%chk=%s.chk\n' @@CORES@@ @@MEM@@ "$NAME"
   cat <<'ROUTE'

descripytor-0.4.3/M2_data_extractor/alignment.py

@@ -0,0 +1,277 @@
+"""Atom renumbering with Conformaty (github.com/Milo-group/Conformaty): every molecule of a folder numbered like a
+reference, so a descriptor picked on the reference reads the same atom everywhere.
+
+The result per molecule is the 1-based {old: new} dict ``Molecule.renumber_atoms`` takes. Conformaty matches the
+largest common substructure, fits the molecule onto the reference and assigns the rest by position, so equivalent
+atoms (the two H of a CH2) are told apart by where they sit. The reference must be the smaller molecule (it is the
+common part); a molecule larger than it keeps its extra atoms after the reference's.
+
+    from M2_data_extractor.alignment import map_folder, write_renumbered
+    results = map_folder(mols, "basic")            # mols: a Molecules
+    write_renumbered(mols, results, "renumbered")
+"""
+from __future__ import annotations
+
+import os
+import time
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+INSTALL = "pip install git+https://github.com/Milo-group/Conformaty"   # shortcut: a git URL until Conformaty is on PyPI, then the [align] extra
+
+
+def _conformaty():
+    try:
+        import conformaty
+    except ImportError as exc:
+        raise ImportError(f"Atom renumbering needs Conformaty: {INSTALL}") from exc
+    return conformaty
+
+
+def _worker_threads(n):
+    import torch
+
+    torch.set_num_threads(n)          # one: the fits are a few dozen atoms, where more threads only add overhead
+    _conformaty()                     # import torch and Conformaty once per process, not on every molecule
+
+
+_POOL = {"pool": None, "n": 0}        # the worker processes, kept between runs: starting one and importing costs ~5 s
+
+
+def _pool(workers):
+    pool = _POOL["pool"]
+    if pool is None or _POOL["n"] != workers or getattr(pool, "_broken", False):
+        if pool is not None:
+            pool.shutdown(wait=False, cancel_futures=True)
+        pool = ProcessPoolExecutor(workers, initializer=_worker_threads, initargs=(1,))
+        _POOL.update(pool=pool, n=workers)
+    return pool
+
+
+# Conformaty's losses and Kabsch. rmsd is the default here. Since Conformaty starts its fits at the Kabsch pose (d74f074)
+# none stalls on the example set; mixed and sig settle up to 0.5 A off Kabsch on o_Br/o_Cl, where their own losses prefer
+# to be. Before, from its principal-axis seed, mixed and mad stalled about 6 A away on 5 of 9.
+FITS = ("rmsd", "msd", "mixed", "mad", "sig", "kabsch")
+
+
+def kabsch(moving, fixed, rows):
+    """moving, rotated and moved onto fixed by the closed-form least-squares fit of `rows` (the smallest RMSD)."""
+    import numpy as np
+
+    a, b = moving[rows], fixed[rows]
+    ca, cb = a.mean(0), b.mean(0)
+    u, _s, vt = np.linalg.svd((a - ca).T @ (b - cb))
+    return (moving - ca) @ (u @ np.diag([1.0, 1.0, np.sign(np.linalg.det(u @ vt))]) @ vt) + cb
+
+
+def fit_onto(ref, symbols, coords, fit="rmsd", sigma=30, rows=None):
+    """The molecule (symbols, coords in the reference's numbering) fitted onto the reference (symbols, coords) over
+    `rows` (0-based; default the heavy atoms holding the reference's element, as Conformaty's own align fits).
+    fit: one of FITS, Conformaty's rigid-body optimiser with that loss, or kabsch. Returns the fitted (n, 3) array, or
+    the coordinates unmoved if fewer than three rows match."""
+    import numpy as np
+
+    rs, rx = ref
+    rx, coords = np.asarray(rx, float), np.asarray(coords, float)
+    if rows is None:
+        rows = [i for i in range(len(rs)) if symbols[i] == rs[i]]
+        heavy = [i for i in rows if rs[i] != "H"]
+        rows = heavy if len(heavy) >= 3 else rows
+    if len(rows) < 3:
+        return coords
+    if fit == "kabsch":
+        return kabsch(coords, rx, rows)
+    cf = _conformaty()
+    moved = cf.fit_frames(cf.frame_from_atoms(rs, rx), cf.frame_from_atoms(symbols, coords), {i: i for i in rows},
+                          loss_type=fit, sigma=sigma)
+    return moved.iloc[2:][["X", "Y", "Z"]].to_numpy(float)
+
+
+def compare_fits(ref, symbols, coords, sigma=30):
+    """Every fit (FITS) of one molecule, already in the reference's numbering, onto the reference: the RMSD over the
+    positions with the reference's element and over its heavy atoms, how many of them end within 0.25 A, the seconds,
+    and the fitted xyz. Kabsch has the smallest RMSD by construction; mixed favours atoms in near-exact overlap."""
+    import numpy as np
+
+    rs, rx = ref[0], np.asarray(ref[1], float)
+    same = [i for i in range(len(rs)) if symbols[i] == rs[i]]
+    heavy = [i for i in same if rs[i] != "H"]
+    out = []
+    for fit in FITS:
+        t0 = time.perf_counter()
+        xyz = fit_onto(ref, symbols, coords, fit, sigma)
+        d = np.linalg.norm(xyz[same] - rx[same], axis=1)
+        out.append({"fit": fit, "rmsd": float(np.sqrt((d ** 2).mean())) if same else None,
+                    "rmsd_heavy": float(np.sqrt((np.linalg.norm(xyz[heavy] - rx[heavy], axis=1) ** 2).mean())) if heavy else None,
+                    "within": int((d < 0.25).sum()), "n": len(same), "seconds": time.perf_counter() - t0,
+                    "xyz": f"{len(symbols)}\nfitted ({fit})\n" + "\n".join(
+                        f"{s} {x:.6f} {y:.6f} {z:.6f}" for s, (x, y, z) in zip(symbols, xyz))})
+    return out
+
+
+def map_atoms(ref, query, anchors=None, stages=None, match_charge=False, fit="rmsd", sigma=30, fit_atoms=None):
+    """Number one molecule like the reference. ref, query: (symbols, coords) of each.
+
+    anchors: 1-based (reference atom, query atom) pairs that must correspond. stages: Conformaty's middle stages in
+    order (complete_rings, extend_mcs, coords_proximity_any; default all three). fit, sigma: the rigid-body fit of
+    the overlay (FITS); the numbering keeps Conformaty's own internal fit. fit_atoms: 1-based reference atoms the
+    overlay is fitted on (default its heavy atoms with the reference's element, as Conformaty's align).
+    Returns plain values (they cross a process boundary): old_to_new {old: new} (1-based), stage (the Conformaty stage
+    that matched each atom, in the new order), xyz (the molecule fitted onto the reference, new order), differs (reference
+    positions holding another element), rmsd over the positions with the same element (rmsd_kabsch: the same for
+    the Kabsch fit, the least-squares optimum, to tell a stalled fit), seconds.
+    """
+    import numpy as np
+
+    cf = _conformaty()
+    (rs, rx), (qs, qx) = ref, query
+    if len(qs) < len(rs):
+        raise ValueError(f"it has fewer atoms ({len(qs)}) than the reference ({len(rs)}): choose the smaller molecule "
+                         "as the reference")
+    t0 = time.perf_counter()
+    state = cf.map_frames(cf.frame_from_atoms(rs, rx), cf.frame_from_atoms(qs, qx),
+                          anchors=[(r - 1, q - 1) for r, q in anchors] if anchors else None,
+                          stage_order=stages, match_formal_charge=match_charge)
+    order = cf.query_order(state)
+    found = {q: entry.name for entry in state.log for q in entry.added}
+    syms = [str(qs[q]) for q in order]
+    same = [i for i in range(len(rs)) if syms[i] == rs[i]]
+    rows = [a - 1 for a in fit_atoms if a - 1 in same] if fit_atoms else None
+    rx = np.asarray(rx, float)
+    moving = np.asarray(qx, float)[order]
+
+    def dev(xyz):
+        return float(np.sqrt(((xyz[same] - rx[same]) ** 2).sum(1).mean())) if same else None
+
+    fitted = fit_onto((rs, rx), syms, moving, fit, sigma, rows)
+    rmsd = dev(fitted)
+    best = rmsd if fit == "kabsch" else dev(fit_onto((rs, rx), syms, moving, "kabsch", rows=rows))
+    return {
+        "old_to_new": {int(q) + 1: k + 1 for k, q in enumerate(order)},
+        "stage": [found.get(q, "") for q in order],
+        "xyz": f"{len(syms)}\nfitted onto the reference\n" + "\n".join(
+            f"{s} {x:.6f} {y:.6f} {z:.6f}" for s, (x, y, z) in zip(syms, fitted)),
+        "differs": [i + 1 for i in range(len(rs)) if syms[i] != rs[i]],
+        "rmsd": rmsd,
+        "rmsd_kabsch": best,
+        "fit": fit,
+        "seconds": time.perf_counter() - t0,
+    }
+
+
+def _atoms(mol):
+    x = mol.xyz_df
+    return x["atom"].astype(str).tolist(), x[["x", "y", "z"]].to_numpy(float).tolist()
+
+
+def map_folder(mols, reference, anchors=None, stages=None, match_charge=False, workers=None, progress=None,
+               cancelled=lambda: False, fit="rmsd", sigma=30, fit_atoms=None, isolate=False, only=None):
+    """Number every molecule of a Molecules like `reference` (one of its names). anchors: {name: [(ref atom, atom)]}.
+
+    workers > 1 runs that many molecules at once in separate processes, the cores shared between them; 1 (the default)
+    runs them here, one after the other, unless isolate (a server: RDKit's substructure search holds the interpreter
+    lock, which would stall its other threads). progress(event) gets one dict per molecule as it finishes; cancelled()
+    is asked between molecules. Returns {name: result of map_atoms, or {"error": text}};
+    the reference itself maps onto itself unchanged.
+    """
+    _conformaty()
+    by = {m.molecule_name: m for m in mols.molecules}
+    if reference not in by:
+        raise ValueError(f"no molecule named {reference!r} in this folder")
+    ref = _atoms(by[reference])
+    n = len(ref[0])
+    results = {reference: {"old_to_new": {i: i for i in range(1, n + 1)}, "stage": ["reference"] * n, "differs": [],
+                           "rmsd": 0.0, "seconds": 0.0, "xyz": None, "reference": True}}
+    todo = [name for name in by if name != reference and (only is None or name in only)]   # only: these names, again
+    workers = max(1, min(len(todo), workers or 1))
+    if progress:
+        progress({"event": "start", "n": len(todo), "names": todo, "reference": reference, "workers": workers})
+    if fit not in FITS:
+        raise ValueError(f"fit must be one of {', '.join(FITS)}")
+    args = {name: (ref, _atoms(by[name]), (anchors or {}).get(name), stages, match_charge, fit, sigma, fit_atoms)
+            for name in todo}
+
+    def done(i, name, run):
+        try:
+            results[name] = run()
+        except Exception as exc:              # one molecule that cannot be mapped does not stop the folder
+            results[name] = {"error": f"{type(exc).__name__}: {exc}"}
+        if progress:
+            progress({"event": "molecule", "i": i, "n": len(todo), "name": name, **_summary(results[name]),
+                      "result": results[name]})
+
+    if workers == 1 and not isolate:
+        for i, name in enumerate(todo, 1):
+            if cancelled():
+                break
+            done(i, name, lambda: map_atoms(*args[name]))
+        return results
+    import threading
+
+    pool = _pool(workers)
+    # the pool outlives the run, so the next one starts warm; Stop ends its processes and the next run starts new ones
+    jobs = {pool.submit(map_atoms, *args[name]): name for name in todo}
+
+    def watch():                       # one molecule can take minutes: Stop ends its process instead of waiting
+        while not all(j.done() for j in jobs):
+            if cancelled():
+                for other in jobs:
+                    other.cancel()
+                for proc in list(getattr(pool, "_processes", {}).values()):
+                    proc.terminate()
+                _POOL["pool"] = None
+                return
+            time.sleep(0.3)
+
+    threading.Thread(target=watch, daemon=True).start()
+    for i, job in enumerate(as_completed(jobs), 1):
+        if cancelled():
+            break
+        done(i, jobs[job], job.result)
+    return results
+
+
+def _summary(result):
+    if "error" in result:
+        return {"error": result["error"]}
+    return {"moved": sum(o != n for o, n in result["old_to_new"].items()), "differs": result["differs"],
+            "rmsd": result["rmsd"], "rmsd_kabsch": result.get("rmsd_kabsch"), "seconds": result["seconds"]}
+
+
+def write_renumbered(mols, results, out_dir, reference=""):
+    """Write each mapped molecule, renumbered, to out_dir/<name>.feather (the names unchanged, so outcomes.csv still
+    joins; it is copied along), and renumbering.json: the {old: new} dict of every molecule. Molecules without one
+    (failed) are not written. Returns (written names, skipped names)."""
+    import copy
+    import json
+    import shutil
+
+    from M2_data_extractor.data_extractor import _molecules_folder_files
+
+    os.makedirs(out_dir, exist_ok=True)
+    folder, files = _molecules_folder_files(mols.molecules_path)
+    xyz_in = {os.path.splitext(f)[0] for f, _s, _z in files if f.lower().endswith(".xyz")}
+    written, skipped = [], []
+    for mol in mols.molecules:
+        result = results.get(mol.molecule_name) or {}
+        if "old_to_new" not in result:
+            skipped.append(mol.molecule_name)
+            continue
+        m = copy.deepcopy(mol)
+        if mol.molecule_name in xyz_in:          # coordinates stay coordinates; .log, .out and .json become .feather
+            m.renumber_atoms(result["old_to_new"], verbose=False, save_feather=False)
+            x = m.xyz_df
+            with open(os.path.join(out_dir, f"{mol.molecule_name}.xyz"), "w", encoding="utf-8") as fh:
+                fh.write(f"{len(x)}\nrenumbered like {reference}\n" + "".join(
+                    f"{a} {float(px):.6f} {float(py):.6f} {float(pz):.6f}\n" for a, px, py, pz in x[["atom", "x", "y", "z"]].itertuples(index=False)))
+        else:
+            m.renumber_atoms(result["old_to_new"], verbose=False, save_feather=True,
+                             out_file=os.path.join(out_dir, f"{mol.molecule_name}.feather"))
+        written.append(mol.molecule_name)
+    outcomes = os.path.join(mols.molecules_path, "outcomes.csv")
+    if os.path.isfile(outcomes):
+        shutil.copy(outcomes, out_dir)
+    keep = ("old_to_new", "stage", "differs", "rmsd", "rmsd_kabsch", "fit", "seconds", "error")
+    with open(os.path.join(out_dir, "renumbering.json"), "w", encoding="utf-8") as fh:
+        json.dump({"reference": reference, "folder": mols.molecules_path,
+                   "molecules": {k: {f: v[f] for f in keep if f in v} for k, v in results.items()}}, fh, indent=1)
+    return written, skipped