split3c 0.0.1__py3-none-any.whl

split3c/resite/pretreatment.py

@@ -0,0 +1,299 @@
+"""
+This script is a the split3c project, designed to process paired-end FASTQ files by fragmenting DNA sequences at specified restriction enzyme sites.
+
+Copyright © 2024 Samir Bertache
+
+SPDX-License-Identifier: AGPL-3.0-or-later
+
+===============================================================================
+
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the
+Free Software Foundation, either version 3 of the License, or (at your option)
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See the GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import logging
+import re
+import sys
+from typing import List, Tuple
+
+from Bio.Restriction import RestrictionBatch
+from Bio.Seq import Seq
+
+logging.basicConfig(level=logging.INFO)
+
+
+###############################  Common part #################################
+
+
+def case_adaptation(List_Enzyme):
+    """
+    Case sensitive enzymes adaptation
+
+    Examples
+    --------
+    >>> case_adaptation(["hindiii"])
+    ['HindIII']
+    >>> case_adaptation(["dpnii", "bglii", "mboi"])
+    ['DpnII', 'BglII', 'MboI']
+    >>> case_adaptation(["arima"])
+    ['DpnII', 'HinfI']
+    >>> case_adaptation([" Foo ", "arima", "DpnII"])
+    ['Foo', 'DpnII', 'HinfI', 'DpnII']
+    """
+    adapted = []
+
+    for enzyme in List_Enzyme:
+        enz = str(enzyme).strip()
+        key = enz.lower()
+
+        if key == "hindiii":
+            adapted.append("HindIII")
+        elif key == "dpnii":
+            adapted.append("DpnII")
+        elif key == "bglii":
+            adapted.append("BglII")
+        elif key == "mboi":
+            adapted.append("MboI")
+        elif key == "arima":
+            # Double enzyme
+            adapted.extend(["DpnII", "HinfI"])
+        else:
+            adapted.append(enz)
+
+    return adapted
+
+
+def find_liga_sites(
+    List_Enzyme: List[str], borderless: bool = False
+) -> List[Tuple[re.Pattern, int]]:
+    """
+    This function finds the ligation sites for a given list of enzymes and
+    their length.
+
+    Parameters:
+    List_Enzyme (List[str]): A list of enzymes for which to find the ligation
+    sites.
+
+    borderless (bool, optional): If True, the total length of the give and
+                                        accept sites is used.
+                                 If False, only the length of the give site
+                                        is used. Default is False.
+
+    Returns:
+    List[Tuple[re.Pattern, int]]: A list of tuples, where each tuple contains
+                                    a compiled regular expression
+                                    pattern for the ligation site and the length
+                                    of the site.
+
+    Examples
+    --------
+    >>> out = find_liga_sites(["DpnII"])
+    >>> isinstance(out, list) and len(out) >= 1
+    True
+    >>> any(p.pattern == "GATCGATC" and off == 4 for p, off in out)
+    True
+
+    >>> out_b = find_liga_sites(["DpnII"], borderless=True)
+    >>> any(p.pattern == "GATCGATC" and off == 8 for p, off in out_b)
+    True
+
+    # This function is inspired by and adapted from the Cutsite function in Hicstuff
+    # (https://github.com/koszullab/hicstuff), originally under BSD license.
+    # See https://github.com/koszullab/hicstuff/blob/main/LICENSE for the full license.
+    """
+    restriction_batch = RestrictionBatch(List_Enzyme)
+    give_list = []
+    accept_list = []
+    ligation_site_list = []
+
+    for enz in restriction_batch:
+        site = enz.elucidate()
+        fw_cut = site.find("^")
+        rev_cut = site.find("_")
+
+        # Purify give site
+        give_site = site[:rev_cut].replace("^", "")
+        while give_site[0] == "N":
+            give_site = give_site[1:]
+        give_list.append(give_site)
+
+        # Purify accept site
+        accept_site = site[fw_cut + 1 :].replace("_", "")
+        while accept_site[-1] == "N":
+            accept_site = accept_site[:-1]
+        accept_list.append(accept_site)
+
+    # Find ligation site
+    for give_site in give_list:
+        for accept_site in accept_list:
+            ligation_site = (give_site + accept_site).replace("N", ".")
+            compiled_regex = re.compile(ligation_site)
+
+            # Use total lenght for borderless
+            if borderless:
+                length = len(give_site) + len(accept_site)
+            else:
+                length = len(give_site)
+            ligation_site_list.append((compiled_regex, length))
+
+            # If ligation site is not palindromic
+            reverse_complement_site = str(Seq(ligation_site).reverse_complement())
+
+            if ligation_site != reverse_complement_site:
+                compiled_reverse_regex = re.compile(reverse_complement_site)
+                # Use lenght of accept site for reverse complement site
+                if borderless:
+                    length = len(give_site) + len(accept_site)
+                else:
+                    length = len(accept_site)
+                ligation_site_list.append((compiled_reverse_regex, length))
+
+    return ligation_site_list
+
+
+def search_in_database(enzymes, borderless=False):
+    """
+    _summary_ : Search enzyme in database and retrieve ligation site
+    Examples
+    --------
+    >>> import io, contextlib
+    >>> buf = io.StringIO()
+    >>> with contextlib.redirect_stdout(buf):
+    ...     out = search_in_database("DpnII")
+    >>> isinstance(out, list) and len(out) >= 1
+    True
+    >>> any(p.pattern == "GATCGATC" and off == 4 for p, off in out)
+    True
+
+    >>> buf = io.StringIO()
+    >>> with contextlib.redirect_stdout(buf):
+    ...     out = search_in_database("DpnII", borderless=True)
+    >>> any(p.pattern == "GATCGATC" and off == 8 for p, off in out)
+    True
+    >>> "Mode Borderless" in buf.getvalue()
+    True
+
+    >>> buf = io.StringIO()
+    >>> try:
+    ...     with contextlib.redirect_stdout(buf):
+    ...         search_in_database("No restriction enzyme found")
+    ... except SystemExit as e:
+    ...     code = e.code
+    >>> code
+    0
+    >>> "No restriction enzyme found" in buf.getvalue()
+    True
+    """
+    if enzymes == "No restriction enzyme found":
+        print(enzymes)
+        sys.exit(0)
+    else:
+        if borderless:
+            print("Mode Borderless")
+        list_enz = enzymes.split(",")
+        try:
+            ligation_site_list = find_liga_sites(case_adaptation(list_enz), borderless)
+            if len(ligation_site_list) > 1:
+                for el in ligation_site_list:
+                    print(f"Ligation sites: {el[0]}", flush=True)
+            else:
+                print(f"Ligation sites: {ligation_site_list[0]}", flush=True)
+            return ligation_site_list
+
+        except Exception as e:
+            raise RuntimeError(
+                f"Error in enzyme identification for input={enzymes!r}"
+            ) from e
+
+
+def _split_two(total: int) -> tuple[int, int]:
+    """
+    Partage total en deux parts entières, différence ≤ 1, chacune ≥ 1.
+    Examples
+    --------
+    >>> _split_two(2)
+    (1, 1)
+    >>> _split_two(5)
+    (2, 3)
+    >>> _split_two(8)
+    (4, 4)
+    >>> _split_two(1)
+    Traceback (most recent call last):
+    ...
+    ValueError: total doit être ≥ 2, ici 1
+    """
+    if total < 2:
+        raise ValueError(f"total doit être ≥ 2, ici {total}")
+    a = total // 2
+    b = total - a
+    return max(1, a), max(1, b)
+
+
+def partition_threads(
+    num_threads: int, oversubscribe_factor: float = 1.35
+) -> tuple[int, int, int]:
+    """
+    Retourne (TRead_total, TFrag, TWrite_total), avec TRead/TWrite toujours PAIRS.
+    Palier minimal 5: 1R/flux, 1Frag, 1W/flux.
+    Extra par paires: d'abord W, puis R, en alternance. Reste impair → Frag.
+
+    Overallocation added (program doesn't use 1 CPU for 1 threads)
+    """
+    import math
+
+    if num_threads < 5:
+        raise ValueError(f"num_threads doit être ≥ 5, ici {num_threads}")
+
+    # base: 1 par flux pour lecture/écriture, 1 pour frag
+    rpf = 1  # read per file
+    wpf = 1  # write per file
+    frag = 1
+
+    extra = num_threads - 5
+    turn = "write"
+    while extra >= 2:
+        if turn == "write":
+            wpf += 1
+            turn = "read"
+        else:
+            rpf += 1
+            turn = "write"
+        extra -= 2
+
+    if extra == 1:
+        frag += 1
+
+    tread = 2 * rpf
+    twrite = 2 * wpf
+    tfrag = frag
+
+    nominal_total = tread + twrite + tfrag
+    target_total = math.floor(num_threads * oversubscribe_factor)
+    surplus = target_total - nominal_total
+
+    # Surallocation préférentielle vers écriture puis lecture, par paires
+    turn = "write"
+    while surplus >= 2:
+        if turn == "write":
+            twrite += 2
+            turn = "read"
+        else:
+            tread += 2
+            turn = "write"
+        surplus -= 2
+
+    # Reste impair vers fragmentation
+    if surplus == 1:
+        tfrag += 1
+
+    return tread, tfrag, twrite

split3c/resite/read.py

@@ -0,0 +1,91 @@
+"""
+This script is a the split3c project, designed to process paired-end FASTQ files by fragmenting DNA sequences at specified restriction enzyme sites.
+
+Copyright © 2024 Samir Bertache
+
+SPDX-License-Identifier: AGPL-3.0-or-later
+
+===============================================================================
+
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the
+Free Software Foundation, either version 3 of the License, or (at your option)
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See the GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import logging
+import subprocess
+
+logging.basicConfig(level=logging.INFO)
+
+
+def stop_signal(Queue, NumThreadFragmentation):
+    """
+    _summary_ : Add a stop signal to the queue for each thread
+    """
+    for _ in range(NumThreadFragmentation):
+        Queue.put(None)
+
+
+def read_fastq_gzip_simultaneously(
+    fileA: str, fileB: str, Queue, num_threads, NumThreadFragmentation
+):
+    """
+    _summary_ : Read two fastq files simultaneously, decompress them with pigz,
+    take a couple a read and put them into a queue by block
+    """
+    from .pretreatment import _split_two
+
+    tA, tB = _split_two(num_threads)
+    # Use pigz to decompress the input files
+    procA = subprocess.Popen(
+        ["pigz", "-dc", "-p", str(tA), fileA],
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    procB = subprocess.Popen(
+        ["pigz", "-dc", "-p", str(tB), fileB],
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+
+    Stacker = []
+    try:
+        while True:
+            NomA = (procA.stdout.readline()).rstrip()
+            seqA = (procA.stdout.readline()).rstrip()
+            procA.stdout.readline()  # Skip +
+            qualA = (procA.stdout.readline()).rstrip()
+
+            NomB = (procB.stdout.readline()).rstrip()
+            seqB = (procB.stdout.readline()).rstrip()
+            procB.stdout.readline()  # Skip +
+            qualB = (procB.stdout.readline()).rstrip()
+
+            if not seqA or not seqB:
+                break
+
+            Stacker.append([[NomA, NomB], [seqA, seqB], [qualA, qualB]])
+
+            if len(Stacker) > 256:
+                Queue.put(Stacker)
+                Stacker = []
+
+        if len(Stacker) > 0:
+            Queue.put(Stacker)
+
+    except Exception as e:
+        logging.error(f"Error in TakeOneItem: {e}")
+
+    finally:
+        stop_signal(Queue, NumThreadFragmentation)
+        procA.wait()
+        procB.wait()

split3c/resite/write_control.py

@@ -0,0 +1,111 @@
+"""
+This script is a the split3c project, designed to process paired-end FASTQ files by fragmenting DNA sequences at specified restriction enzyme sites.
+
+Copyright © 2024 Samir Bertache
+
+SPDX-License-Identifier: AGPL-3.0-or-later
+
+===============================================================================
+
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the
+Free Software Foundation, either version 3 of the License, or (at your option)
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+See the GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import logging
+import signal
+import subprocess
+import sys
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+
+
+def signal_handler(sig, frame, outF, outR):
+    print(f"\nReceived signal {sig}. Terminating gracefully...")
+    outF.terminate()  # Terminate the pigz processes
+    outR.terminate()
+    logging.info("\nProcess termination requested by signal")
+    sys.exit(0)
+
+
+def open_output(TWrite, output_forward, output_reverse):
+    from .pretreatment import _split_two
+
+    tF, tR = _split_two(TWrite)
+
+    # Open output files for writing
+    outF = subprocess.Popen(
+        ["pigz", "-c", "-p", str(tF)],
+        stdin=subprocess.PIPE,
+        stdout=open(output_forward, "wb"),
+    )
+
+    outR = subprocess.Popen(
+        ["pigz", "-c", "-p", str(tR)],
+        stdin=subprocess.PIPE,
+        stdout=open(output_reverse, "wb"),
+    )
+
+    # Register signal handlers
+    signal.signal(
+        signal.SIGINT,
+        lambda sig, frame: signal_handler(sig, frame, outF, outR),
+    )  # Ctrl+C
+    signal.signal(
+        signal.SIGTSTP,
+        lambda sig, frame: signal_handler(sig, frame, outF, outR),
+    )  # Ctrl+Z
+
+    return outF, outR
+
+
+def manage_pigz_problems(outF, outR, output_forward, output_reverse):
+    outF.stdin.close()
+    outR.stdin.close()
+
+    outF.wait()
+    outR.wait()
+
+    stdoutF, stderrF = outF.communicate()
+    if stderrF:
+        print(
+            f"Error in pigz command for file {output_forward}: {stderrF}",
+            flush=True,
+        )
+
+    stdoutR, stderrR = outR.communicate()
+    if stderrR:
+        print(
+            f"Error in pigz command for file {output_reverse}: {stderrR}",
+            flush=True,
+        )
+
+
+def write_pairs(
+    Output_buffer,
+    outF: subprocess.Popen,
+    outR: subprocess.Popen,
+    TFrag,
+) -> None:
+    finished_processes = 0
+    while finished_processes < TFrag:
+        try:
+            data = Output_buffer.get()
+            if data is None:
+                finished_processes += 1
+            else:
+                outF.stdin.write("".join(data[0]).encode("utf-8"))
+                outR.stdin.write("".join(data[1]).encode("utf-8"))
+
+        except Exception as e:
+            logging.error(f"Error in write_pairs: {e}")

split3c/resolve/bam.py

@@ -0,0 +1,129 @@
+from typing import Iterator
+
+import pysam
+
+
+def get_bam_headers(bam_for_path: str, bam_rev_path: str) -> tuple[dict, dict]:
+    """
+    Read both BAM headers and return them as dictionaries.
+
+    Examples
+    --------
+    No doctest here because it requires real BAM files.
+    """
+    with pysam.AlignmentFile(bam_for_path, "rb") as bf:
+        header_for = bf.header.to_dict()
+    with pysam.AlignmentFile(bam_rev_path, "rb") as br:
+        header_rev = br.header.to_dict()
+    return header_for, header_rev
+
+
+def get_bam_header_single(bam_path: str) -> dict:
+    """
+    Read one BAM header and return it as a dictionary.
+
+    Examples
+    --------
+    No doctest here because it requires a real BAM file.
+    """
+    with pysam.AlignmentFile(bam_path, "rb") as bam:
+        return bam.header.to_dict()
+
+
+def iter_bam_pairs(
+    bam_for_path: str,
+    bam_rev_path: str,
+    bam_threads: int = 1,
+) -> Iterator[tuple[pysam.AlignedSegment, pysam.AlignedSegment]]:
+    """
+    Iterate over synchronized forward/reverse BAM records.
+
+    The two BAMs must contain the same qname-sorted records in the same order.
+
+    Examples
+    --------
+    No doctest here because it requires real BAM files.
+    """
+    with (
+        pysam.AlignmentFile(bam_for_path, "rb", threads=bam_threads) as bam_for,
+        pysam.AlignmentFile(bam_rev_path, "rb", threads=bam_threads) as bam_rev,
+    ):
+        for idx, (read_for, read_rev) in enumerate(zip(bam_for, bam_rev), start=1):
+            if read_for is None or read_rev is None:
+                raise ValueError(
+                    "Forward and reverse BAMs do not have the same number of records "
+                    f"(first mismatch at record {idx})."
+                )
+            if read_for.query_name != read_rev.query_name:
+                raise ValueError(
+                    "Forward and reverse BAMs are not synchronized by qname "
+                    f"at record {idx}: {read_for.query_name!r} != {read_rev.query_name!r}."
+                )
+            yield read_for, read_rev
+
+
+def iter_bam_pairs_single(
+    bam_path: str,
+    bam_threads: int = 1,
+) -> Iterator[tuple[pysam.AlignedSegment, pysam.AlignedSegment]]:
+    """
+    Iterate over pairs from one interleaved BAM.
+
+    Assumptions
+    -----------
+    - records are written as consecutive pairs
+    - the two mates of one logical pair have the same query_name
+    - the BAM contains an even number of records
+
+    Examples
+    --------
+    No doctest here because it requires real BAM files.
+    """
+    with pysam.AlignmentFile(bam_path, "rb", threads=int(bam_threads * 2)) as bam:
+        it = iter(bam)
+        pair_idx = 0
+
+        while True:
+            try:
+                read1 = next(it)
+            except StopIteration:
+                break
+
+            try:
+                read2 = next(it)
+            except StopIteration as exc:
+                raise ValueError(
+                    "Single BAM contains an odd number of records; "
+                    f"dangling read at pair index {pair_idx + 1}: "
+                    f"{read1.query_name!r}."
+                ) from exc
+
+            pair_idx += 1
+
+            if read1.query_name != read2.query_name:
+                raise ValueError(
+                    "Single BAM is not properly interleaved by qname "
+                    f"at pair index {pair_idx}: "
+                    f"{read1.query_name!r} != {read2.query_name!r}."
+                )
+
+            yield read1, read2
+
+
+def chromsizes_from_header(header_dict: dict) -> list[tuple[str, int]]:
+    """
+    Extract chromosome names and lengths from a BAM header dictionary.
+
+    Examples
+    --------
+    >>> chromsizes_from_header({"SQ": [{"SN": "chr1", "LN": 100}, {"SN": "chr2", "LN": 50}]})
+    [('chr1', 100), ('chr2', 50)]
+    """
+    chromsizes: list[tuple[str, int]] = []
+    for sq in header_dict.get("SQ", []):
+        sn = sq.get("SN")
+        ln = sq.get("LN")
+        if sn is None or ln is None:
+            continue
+        chromsizes.append((str(sn), int(ln)))
+    return chromsizes

split3c/resolve/io_utils.py

@@ -0,0 +1,77 @@
+import io
+import os
+import shlex
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TextIO
+
+
+@dataclass
+class TextWriter:
+    handle: TextIO
+    process: subprocess.Popen | None = None
+    outfile: None = None
+
+    def write(self, text: str) -> int:
+        return self.handle.write(text)
+
+    def flush(self) -> None:
+        self.handle.flush()
+        if self.outfile is not None:
+            self.outfile.flush()
+
+    def close(self) -> None:
+        try:
+            self.handle.close()
+        finally:
+            if self.process is not None:
+                ret = self.process.wait()
+                if ret != 0:
+                    raise RuntimeError(
+                        f"Compression command failed with exit code {ret}."
+                    )
+            if self.outfile is not None and not self.outfile.closed:
+                self.outfile.close()
+
+
+def _pick_gzip_command(nproc: int) -> list[str] | None:
+    candidates = []
+    if shutil.which("pbgzip"):
+        candidates.append(["pbgzip", "-c", "-n", str(max(1, nproc))])
+    if shutil.which("bgzip"):
+        candidates.append(["bgzip", "-c", "-@", str(max(1, nproc))])
+    if shutil.which("pigz"):
+        candidates.append(["pigz", "-c", "-p", str(max(1, nproc))])
+    if shutil.which("gzip"):
+        candidates.append(["gzip", "-c"])
+    return candidates[0] if candidates else None
+
+
+def open_text_output(path: str | Path, nproc: int = 1) -> TextWriter:
+    path = str(path)
+    if path == "-":
+        return TextWriter(
+            handle=io.TextIOWrapper(os.fdopen(os.dup(1), "wb"), encoding="utf-8")
+        )
+
+    if path.endswith(".gz"):
+        cmd = _pick_gzip_command(nproc)
+        if cmd is None:
+            raise RuntimeError(
+                "No gzip-compatible compressor found (tried pbgzip, bgzip, pigz, gzip)."
+            )
+        outfile = open(path, "wb")
+        proc = subprocess.Popen(
+            cmd, stdin=subprocess.PIPE, stdout=outfile, stderr=subprocess.PIPE
+        )
+        if proc.stdin is None:
+            outfile.close()
+            raise RuntimeError(
+                f"Failed to open compressor stdin for: {shlex.join(cmd)}"
+            )
+        handle = io.TextIOWrapper(proc.stdin, encoding="utf-8")
+        return TextWriter(handle=handle, process=proc, outfile=outfile)
+
+    return TextWriter(handle=open(path, "w", encoding="utf-8"))