supporth 0.4.0__py3-none-any.whl

supporth/__init__.py

@@ -0,0 +1,5 @@
+"""SuppOrth: support-based integration of complementary orthology predictors."""
+
+__version__ = "0.4.0"
+
+__all__ = ["__version__"]

supporth/adapters/__init__.py

@@ -0,0 +1,57 @@
+"""Predictor adapters: one module per orthology tool."""
+
+from __future__ import annotations
+
+from .base import Adapter, MethodProfile, StageContext
+from .broccoli import Broccoli
+from .oma import OMA
+from .orthofinder import OrthoFinder
+from .sonicparanoid import SonicParanoid
+
+REGISTRY: dict[str, type[Adapter]] = {
+    OrthoFinder.name: OrthoFinder,
+    SonicParanoid.name: SonicParanoid,
+    Broccoli.name: Broccoli,
+    OMA.name: OMA,
+}
+
+# Predictors SuppOrth installs and launches. OMA is not among them: the user
+# supplies an existing standalone run with --oma.
+DEFAULT_ORDER = (OrthoFinder.name, SonicParanoid.name, Broccoli.name)
+
+
+def get(name: str) -> Adapter:
+    key = name.strip().lower()
+    if key not in REGISTRY:
+        known = ", ".join(sorted(REGISTRY))
+        raise KeyError(f"unknown predictor {name!r}; available: {known}")
+    return REGISTRY[key]()
+
+
+def resolve(names: str | None) -> list[Adapter]:
+    """Adapters for a comma-separated selection, kept in run order."""
+    if not names or names.strip().lower() == "all":
+        return [get(n) for n in DEFAULT_ORDER]
+    requested = {n.strip().lower() for n in names.split(",") if n.strip()}
+    if "oma" in requested:
+        raise KeyError(
+            "OMA is not run by suppOrth; pass an existing standalone run with --oma PATH"
+        )
+    unknown = requested - set(REGISTRY)
+    if unknown:
+        raise KeyError(
+            f"unknown predictor(s): {', '.join(sorted(unknown))}; "
+            f"available: {', '.join(sorted(REGISTRY))}"
+        )
+    return [get(n) for n in DEFAULT_ORDER if n in requested]
+
+
+__all__ = [
+    "Adapter",
+    "MethodProfile",
+    "StageContext",
+    "REGISTRY",
+    "DEFAULT_ORDER",
+    "get",
+    "resolve",
+]

supporth/adapters/base.py

@@ -0,0 +1,332 @@
+"""Common interface for installing, checking, running and parsing predictors."""
+
+from __future__ import annotations
+
+import shutil
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Sequence
+
+from .. import disk, interpreters, manifest, paths, shell
+from ..canonical import PredictorResult
+from ..proteomes import Proteome
+
+
+class AdapterError(RuntimeError):
+    pass
+
+
+class NotInstalled(AdapterError):
+    pass
+
+
+@dataclass
+class MethodProfile:
+    """Methods recorded for one predictor stage."""
+
+    search: str = "unspecified"
+    clustering: str = "unspecified"
+    alignment: str = "none"
+    tree: str = "none"
+    orthology: str = "unspecified"
+    extra: dict[str, str] = field(default_factory=dict)
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "search": self.search,
+            "clustering": self.clustering,
+            "alignment": self.alignment,
+            "tree": self.tree,
+            "orthology": self.orthology,
+            **self.extra,
+        }
+
+
+@dataclass
+class StageContext:
+    """Everything a single predictor stage needs to run."""
+
+    input_dir: Path
+    work_dir: Path
+    proteomes: tuple[Proteome, ...] = ()
+    sp1: Proteome | None = None
+    sp2: Proteome | None = None
+    threads: int = 1
+    options: list[str] = field(default_factory=list)
+    alignment_dir: Path | None = None
+
+    def __post_init__(self) -> None:
+        if self.proteomes:
+            if self.sp1 is None:
+                self.sp1 = self.proteomes[0]
+            if self.sp2 is None and len(self.proteomes) > 1:
+                self.sp2 = self.proteomes[1]
+        elif self.sp1 is not None and self.sp2 is not None:
+            self.proteomes = (self.sp1, self.sp2)
+        else:
+            raise ValueError("StageContext needs at least two proteomes")
+
+    @property
+    def log(self) -> Path:
+        return self.work_dir / "stage.log"
+
+
+class Adapter(ABC):
+    name: str = ""
+    label: str = ""
+    repo: str = ""
+    default_ref: str = ""
+    entrypoint_candidates: tuple[str, ...] = ()
+    binaries: tuple[str, ...] = ()
+    # True when the predictor puts its own bundled bin directory ahead of PATH,
+    # which defeats the shared binary layer unless the bundled copies are linked.
+    shadows_path: bool = False
+    # Interpreter range the predictor supports, when it is narrower than
+    # SuppOrth's own. Empty means "whatever SuppOrth runs on will do".
+    python_requires: str = ""
+    # Set when the predictor changes behaviour depending on whether its
+    # interpreter belongs to a conda-style distribution.
+    prefers_conda_interpreter: bool = False
+
+    # ---- installation -------------------------------------------------
+
+    def install(
+        self,
+        *,
+        ref: str | None = None,
+        force: bool = False,
+        keep_bundled: bool = False,
+        python: Path | str | None = None,
+        log=print,
+        **post_install_options,
+    ) -> Path:
+        ref = ref or self.default_ref
+        root = paths.tool_dir(self.name, ref)
+
+        if root.exists() and force:
+            shutil.rmtree(root)
+
+        if not root.exists():
+            root.parent.mkdir(parents=True, exist_ok=True)
+            log(f"  cloning {self.repo} @ {ref}")
+            shell.run(
+                [
+                    "git",
+                    "clone",
+                    "--depth",
+                    "1",
+                    "--branch",
+                    ref,
+                    self.repo,
+                    str(root),
+                ],
+                log=paths.cache_root() / f"{self.name}-clone.log",
+            )
+        else:
+            log(f"  source already present at {root}")
+
+        commit = self._resolve_commit(root)
+        interpreter = self.select_interpreter(python, log=log)
+        # The build step must precede entrypoint discovery: tools installed into
+        # a private virtualenv only gain their executable once it has run.
+        self.post_install(root, interpreter=interpreter, log=log, **post_install_options)
+        entrypoint = self.locate_entrypoint(root)
+
+        if self.shadows_path and not keep_bundled:
+            self._collapse_bundled(root, log=log)
+
+        freed = disk.prune_build_artifacts(root)
+        if freed:
+            log(f"  removed the build tree ({disk.human(freed)})")
+
+        manifest.record_predictor(
+            self.name,
+            ref=ref,
+            commit=commit,
+            source=self.repo,
+            root=root,
+            entrypoint=str(entrypoint),
+            notes=self.install_notes,
+            interpreter=interpreters.describe(self.venv_python(root) or interpreter),
+        )
+        log(f"  {self.name} ready at commit {commit[:10]}")
+        return root
+
+    install_notes: str = ""
+
+    def _collapse_bundled(self, root: Path, *, log=print) -> None:
+        """Link helpers that would otherwise shadow the shared binary directory."""
+        from .. import deps
+
+        bundled = [b for b in deps.find_bundled({self.name: root}) if not b.linked]
+        if not bundled:
+            return
+        unrunnable = [b for b in bundled if b.runnable is False]
+        if unrunnable:
+            names = ", ".join(sorted({b.name for b in unrunnable}))
+            log(f"  bundled {names} cannot run on this machine; using the shared copies")
+        else:
+            log(f"  linking {len(bundled)} bundled binaries to the shared bin")
+        deps.link_bundled(bundled, log=lambda message: log(f"  {message.strip()}"))
+
+    def post_install(self, root: Path, *, interpreter: Path | None = None, log=print) -> None:
+        """Hook for tools needing a build step or Python dependencies."""
+
+    def leftovers(self, root: Path) -> list[tuple[Path, str]]:
+        """Return unused files or directories with their cleanup descriptions."""
+        return []
+
+    def select_interpreter(self, explicit: Path | str | None = None, *, log=print) -> Path:
+        """The interpreter this predictor's private environment is built from."""
+        chosen = interpreters.find(
+            self.python_requires,
+            explicit=explicit,
+            prefer_conda=self.prefers_conda_interpreter,
+        )
+        if self.python_requires or self.prefers_conda_interpreter:
+            log(f"  interpreter: {interpreters.describe(chosen)}")
+        return chosen
+
+    def install_to_prefix(
+        self,
+        root: Path,
+        interpreter: Path,
+        *,
+        log=print,
+    ) -> Path:
+        """Install source into a private pip target while retaining the selected interpreter."""
+        target = root / "site-packages"
+        target.mkdir(parents=True, exist_ok=True)
+        log(f"  installing into {target.name}/ with pip (this compiles extensions)")
+        shell.run(
+            [str(interpreter), "-m", "pip", "install", "--upgrade", "--target", str(target), "."],
+            cwd=root,
+            log=root / "install.log",
+        )
+        (root / "interpreter.txt").write_text(f"{interpreter}\n")
+        return target
+
+    def prefix_interpreter(self, root: Path) -> Path | None:
+        record = root / "interpreter.txt"
+        if not record.exists():
+            return None
+        path = Path(record.read_text().strip())
+        return path if path.exists() else None
+
+    def venv_python(self, root: Path) -> Path | None:
+        candidate = root / ".venv" / "bin" / "python"
+        return candidate if candidate.exists() else None
+
+    def build_venv(
+        self,
+        root: Path,
+        interpreter: Path | None,
+        requirements: Sequence[str] = (),
+        *,
+        editable_source: bool = False,
+        log=print,
+    ) -> Path:
+        """Create a private virtualenv and install dependencies with pip."""
+        venv = root / ".venv"
+        python = interpreter or Path(self.python())
+        if not (venv / "bin" / "python").exists():
+            log(f"  creating private virtualenv with {python}")
+            shell.run([str(python), "-m", "venv", str(venv)])
+
+        pip = venv / "bin" / "pip"
+        shell.run(
+            [str(pip), "install", "--upgrade", "pip", "wheel", "setuptools"],
+            log=root / "install-pip.log",
+        )
+        if requirements:
+            log(f"  installing {', '.join(requirements)}")
+            shell.run([str(pip), "install", *requirements], cwd=root, log=root / "install.log")
+        if editable_source:
+            log("  building the predictor from source (this compiles extensions)")
+            shell.run([str(pip), "install", "."], cwd=root, log=root / "install.log")
+        return venv / "bin" / "python"
+
+    def _resolve_commit(self, root: Path) -> str:
+        try:
+            result = shell.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=False)
+            return (result.stdout or "").strip() or "unknown"
+        except OSError:
+            return "unknown"
+
+    def locate_entrypoint(self, root: Path) -> Path:
+        for candidate in self.entrypoint_candidates:
+            matches = sorted(root.glob(candidate))
+            for match in matches:
+                if match.is_file():
+                    return match
+        listing = ", ".join(sorted(p.name for p in root.iterdir())[:12])
+        raise AdapterError(
+            f"{self.name}: no entrypoint found under {root}. "
+            f"Tried {list(self.entrypoint_candidates)}. Top level contains: {listing}. "
+            f"The upstream layout may have changed; pin a different --ref."
+        )
+
+    # ---- readiness ----------------------------------------------------
+
+    def installed_root(self) -> Path:
+        record = manifest.predictor(self.name)
+        if not record:
+            raise NotInstalled(
+                f"{self.name} is not installed; run: suppOrth install --tools {self.name}"
+            )
+        root = Path(record["root"])
+        if not root.exists():
+            raise NotInstalled(
+                f"{self.name} is recorded at {root} but that path is gone; "
+                f"re-run suppOrth install --force"
+            )
+        return root
+
+    def extra_binaries(self, options: list[str] | None = None) -> tuple[str, ...]:
+        """Binaries required only because of the options this run was given.
+
+        Alternative aligners and tree builders are not needed by every
+        configuration, so they are resolved from the run's options rather than
+        demanded up front.
+        """
+        return ()
+
+    def check(self, options: list[str] | None = None) -> tuple[bool, str]:
+        """Cheap confirmation that the tool starts, without running an analysis."""
+        try:
+            root = self.installed_root()
+        except NotInstalled as error:
+            return False, str(error)
+        try:
+            entrypoint = self.locate_entrypoint(root)
+        except AdapterError as error:
+            return False, str(error)
+
+        required = (*self.binaries, *self.extra_binaries(options))
+        missing = [b for b in dict.fromkeys(required) if shell.which(b) is None]
+        if missing:
+            return False, (
+                f"missing shared binaries: {', '.join(missing)} "
+                f"(suppOrth install --binaries {','.join(missing)})"
+            )
+        return True, f"ok ({entrypoint.name})"
+
+    # ---- execution ----------------------------------------------------
+
+    @abstractmethod
+    def run(self, context: StageContext) -> Path:
+        """Execute the predictor and return the root of its native output."""
+
+    @abstractmethod
+    def parse(self, native_root: Path, proteomes: Sequence[Proteome] | Proteome, *rest: Proteome) -> PredictorResult:
+        """Convert native output into canonical oriented pairs."""
+
+    @abstractmethod
+    def method_profile(self, context: StageContext) -> MethodProfile:
+        """Search, tree and clustering choices used for this stage."""
+
+    def python(self) -> str:
+        import sys
+
+        return sys.executable

supporth/adapters/broccoli.py

@@ -0,0 +1,79 @@
+"""Run Broccoli in its own working directory and read the step-4 pair tables."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+
+from ..canonical import PredictorResult, build_result
+from ..proteomes import Proteome
+from ..method_stack import BROCCOLI
+from .. import shell
+from .base import Adapter, AdapterError, MethodProfile, StageContext
+from .tabular import id_sets, pairs_from_table
+
+
+class Broccoli(Adapter):
+    name = "broccoli"
+    label = "B"
+    repo = "https://github.com/rderelle/Broccoli.git"
+    default_ref = "v1.4.0"
+    entrypoint_candidates = ("broccoli.py", "*/broccoli.py")
+    binaries = ("diamond", "FastTree")
+    install_notes = (
+        "run as a script from a per-stage working directory; "
+        "needs diamond and FastTree from the shared bin, plus ete3"
+    )
+
+    def post_install(self, root: Path, *, interpreter: Path | None = None, log=print) -> None:
+        self.build_venv(root, interpreter, ("ete3", "six", "numpy"), log=log)
+
+    def _interpreter(self, root: Path) -> str:
+        return str(self.venv_python(root) or self.python())
+
+    def method_profile(self, context: StageContext) -> MethodProfile:
+        return MethodProfile(
+            search="DIAMOND (via Broccoli)",
+            clustering=BROCCOLI.clustering,
+            alignment=BROCCOLI.alignment,
+            tree="FastTree, per-orthogroup",
+            orthology=BROCCOLI.orthology,
+        )
+
+    def run(self, context: StageContext) -> Path:
+        root = self.installed_root()
+        entrypoint = self.locate_entrypoint(root)
+
+        output = context.work_dir / "results"
+        output.mkdir(parents=True, exist_ok=True)
+
+        argv = [
+            self._interpreter(root),
+            str(entrypoint),
+            "-dir",
+            str(context.input_dir),
+            "-threads",
+            str(context.threads),
+            *context.options,
+        ]
+        shell.run(argv, cwd=output, log=context.log, timeout=None, threads=context.threads)
+        return output
+
+    def parse(self, native_root: Path, proteomes: Sequence[Proteome] | Proteome, *rest: Proteome) -> PredictorResult:
+        loaded = (proteomes, *rest) if isinstance(proteomes, Proteome) else tuple(proteomes)
+        tables = self._result_tables(native_root)
+        if not tables:
+            raise AdapterError(
+                f"Broccoli produced no pair table under {native_root}. "
+                f"Expected dir_step4/orthologous_pairs.txt; check the stage log."
+            )
+        raw: list[tuple[str, str]] = []
+        for table in tables:
+            raw.extend(pairs_from_table(table, *id_sets(loaded)))
+        return build_result(self.name, self.label, raw, loaded, source_files=tables)
+
+    def _result_tables(self, native_root: Path) -> list[Path]:
+        pairs = [p for p in native_root.rglob("*orthologous_pairs*") if p.is_file()]
+        if pairs:
+            return pairs
+        return [p for p in native_root.rglob("*orthologous_groups*") if p.is_file()]

supporth/adapters/of_config.py

@@ -0,0 +1,151 @@
+"""Register OrthoFinder search commands with a thread allocation based on the number of proteomes."""
+
+from __future__ import annotations
+
+import fcntl
+import json
+import shutil
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+
+PREFIX = "supporth_"
+
+# Mirrors of the stock entries, differing only in the thread count. Keeping the
+# rest of each command identical means results stay comparable to a stock run.
+SEARCH_TEMPLATES: dict[str, dict[str, str]] = {
+    "diamond": {
+        "db_cmd": "diamond makedb --ignore-warnings --in INPUT -d OUTPUT",
+        "search_cmd": (
+            "diamond blastp --ignore-warnings -d DATABASE -q INPUT -o OUTPUT "
+            "--more-sensitive -p {threads} --quiet -e 0.001 --compress 1"
+        ),
+    },
+    "diamond_ultra_sens": {
+        "db_cmd": "diamond makedb --ignore-warnings --in INPUT -d OUTPUT",
+        "search_cmd": (
+            "diamond blastp --ignore-warnings -d DATABASE -q INPUT -o OUTPUT "
+            "--ultra-sensitive -p {threads} --quiet -e 0.001 --compress 1"
+        ),
+    },
+    "blast": {
+        "db_cmd": "makeblastdb -dbtype prot -in INPUT -out OUTPUT",
+        "search_cmd": (
+            "blastp -outfmt 6 -evalue 0.001 -num_threads {threads} "
+            "-query INPUT -db DATABASE -out OUTPUT"
+        ),
+    },
+    "mmseqs": {
+        "db_cmd": "mmseqs createdb INPUT OUTPUT.fa ; mmseqs createindex OUTPUT.fa /tmp",
+        "search_cmd": (
+            "mmseqs search PATH/mmseqsDBBASENAME DATABASE.fa OUTPUT.db /tmp/tmpBASEOUTNAME "
+            "--threads {threads} ; mmseqs convertalis --threads {threads} "
+            "PATH/mmseqsDBBASENAME DATABASE.fa OUTPUT.db OUTPUT"
+        ),
+    },
+}
+
+SEARCH_BINARIES: dict[str, tuple[str, ...]] = {
+    "diamond": ("diamond",),
+    "diamond_ultra_sens": ("diamond",),
+    "blast": ("blastp", "makeblastdb"),
+    "mmseqs": ("mmseqs",),
+}
+
+
+def search_job_count(n_species: int = 2, *, double_blast: bool = True) -> int:
+    """Number of all-versus-all search jobs OrthoFinder will create.
+
+    With ``double_blast`` (the default) both directions of every species pair
+    are searched, including self-comparisons, giving ``n**2`` jobs. The ``-1``
+    option searches one direction only, giving ``n * (n + 1) / 2``.
+    """
+    if double_blast:
+        return n_species * n_species
+    return n_species * (n_species + 1) // 2
+
+
+def threads_per_job(total_threads: int, jobs: int) -> int:
+    """Allocate threads among search jobs, with at least one thread per job."""
+    return max(1, total_threads // max(1, jobs))
+
+
+def find_config(root: Path) -> Path | None:
+    """Locate config.json inside an OrthoFinder source tree."""
+    preferred = root / "scripts_of" / "config.json"
+    if preferred.is_file():
+        return preferred
+    for candidate in sorted(root.rglob("config.json")):
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def variant_name(program: str) -> str:
+    return f"{PREFIX}{program}"
+
+
+def binaries_for_search(program: str) -> tuple[str, ...]:
+    base = program[len(PREFIX):] if program.startswith(PREFIX) else program
+    return SEARCH_BINARIES.get(base, ())
+
+
+@contextmanager
+def config_lock(root: Path) -> Iterator[None]:
+    """Serialise access to one OrthoFinder install.
+
+    The thread count is baked into config.json, so two SuppOrth runs sharing an
+    install must not configure it at the same time.
+    """
+    lock_path = root / ".supporth.lock"
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with lock_path.open("w") as handle:
+        fcntl.flock(handle, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(handle, fcntl.LOCK_UN)
+
+
+def register_threaded_search(config_path: Path, program: str, threads: int) -> str:
+    """Register a threaded variant, or preserve unrecognised search methods."""
+    base = program[len(PREFIX):] if program.startswith(PREFIX) else program
+    template = SEARCH_TEMPLATES.get(base)
+    if template is None:
+        return program
+
+    name = variant_name(base)
+    backup = config_path.with_suffix(".json.supporth-orig")
+    if not backup.exists():
+        shutil.copy2(config_path, backup)
+
+    with config_path.open() as handle:
+        config = json.load(handle)
+
+    config[name] = {
+        "program_type": "search",
+        "db_cmd": template["db_cmd"],
+        "search_cmd": template["search_cmd"].format(threads=threads),
+    }
+
+    temporary = config_path.with_suffix(".json.supporth-tmp")
+    with temporary.open("w") as handle:
+        json.dump(config, handle, indent=4)
+        handle.write("\n")
+    temporary.replace(config_path)
+    return name
+
+
+def strip_option(options: list[str], flag: str) -> list[str]:
+    """Remove a flag and its value from pass-through options."""
+    out: list[str] = []
+    skip = False
+    for token in options:
+        if skip:
+            skip = False
+            continue
+        if token == flag:
+            skip = True
+            continue
+        out.append(token)
+    return out