clipkit 2.7.0__py3-none-any.whl

clipkit/__init__.py

@@ -0,0 +1 @@
+from .api import clipkit

clipkit/__main__.py

@@ -0,0 +1,6 @@
+"""clipkit.__main__: executed when clipkit is called as script"""
+import sys
+
+from .clipkit import main
+
+main(sys.argv[1:])

clipkit/api.py

@@ -0,0 +1,71 @@
+from typing import TextIO, Union
+from tempfile import NamedTemporaryFile
+
+from .clipkit import run
+from .files import FileFormat
+from .helpers import SeqType, write_msa
+from .logger import logger
+from .modes import TrimmingMode
+
+
+def clipkit(
+    *,
+    raw_alignment: Union[str, None] = None,
+    input_file_path: Union[str, None] = None,
+    output_file_path: Union[str, None] = None,
+    mode: TrimmingMode = TrimmingMode.smart_gap,
+    gaps: Union[float, None] = None,
+    gap_characters=None,
+    input_file_format=FileFormat.fasta,
+    output_file_format=FileFormat.fasta,
+    sequence_type=SeqType.aa,
+    codon: bool = False,
+    ends_only=False,
+    threads: int = 1,
+) -> TextIO:
+    """
+    If input_file_path is given with no output_file_path -> Bio MSA (multiple sequence alignment object)
+    If input_file_path is given and output_file_path is given -> write to output file
+    If raw_alignment is given we write it to NamedTemporaryFile and then pass to execute
+        * handles when output_file_path is given and also when not given
+    """
+    logger.disabled = True
+    output_temp_file = None
+    input_temp_file = None
+    if raw_alignment:
+        input_temp_file = NamedTemporaryFile()
+        input_temp_file.write(bytes(raw_alignment, "utf-8"))
+        input_temp_file.flush()
+
+    if not output_file_path:
+        output_temp_file = NamedTemporaryFile()
+
+    # override some options not currently available through programmatic interface
+    complement = False
+    use_log = False
+    quiet = True
+    auxiliary_file = None  # TODO: implement?
+
+    trim_run, stats = run(
+        input_temp_file.name if input_temp_file else input_file_path,
+        input_file_format,
+        output_temp_file.name if output_temp_file else output_file_path,
+        output_file_format,
+        auxiliary_file,
+        sequence_type,
+        gaps,
+        gap_characters,
+        complement,
+        codon,
+        TrimmingMode(mode),
+        use_log,
+        quiet,
+        ends_only,
+        threads,
+    )
+
+    if not output_file_path:
+        return trim_run, stats
+    else:
+        write_msa(trim_run.msa, output_file_path, trim_run.output_file_format)
+        return output_file_path, stats

clipkit/args_processing.py

@@ -0,0 +1,65 @@
+import logging
+import os.path
+import sys
+
+from .helpers import SeqType
+from .modes import TrimmingMode
+from .settings import DEFAULT_AA_GAP_CHARS
+
+logger = logging.getLogger(__name__)
+
+
+def process_args(args) -> dict:
+    """
+    Process args from argparser and set defaults
+    """
+    input_file = args.input
+    output_file = args.output or f"{input_file}.clipkit"
+
+    if not os.path.isfile(input_file):
+        logger.warning("Input file does not exist")
+        sys.exit()
+
+    if input_file == output_file:
+        logger.warning("Input and output files can't have the same name.")
+        sys.exit()
+
+    # assign optional arguments
+    complement = args.complementary or False
+    codon = args.codon or False
+    mode = TrimmingMode(args.mode) if args.mode else TrimmingMode.smart_gap
+    gaps = float(args.gaps) if args.gaps is not None else 0.9
+    gap_characters = (
+        [c for c in args.gap_characters] if args.gap_characters is not None else None
+    )
+    auxiliary_file = args.auxiliary_file
+    use_log = args.log or False
+    quiet = args.quiet or False
+    sequence_type = SeqType(args.sequence_type.lower()) if args.sequence_type else None
+
+    if codon and mode == TrimmingMode.c3:
+        logger.warning(
+            "C3 and codon-based trimming are incompatible.\nCodon-based trimming removes whole codons while C3 removes every third codon position."
+        )
+        sys.exit()
+
+    ends_only = args.ends_only or False
+    threads = args.threads if hasattr(args, 'threads') else 1
+
+    return dict(
+        input_file=input_file,
+        output_file=output_file,
+        input_file_format=args.input_file_format,
+        output_file_format=args.output_file_format,
+        auxiliary_file=auxiliary_file,
+        codon=codon,
+        sequence_type=sequence_type,
+        complement=complement,
+        gaps=gaps,
+        gap_characters=gap_characters,
+        mode=mode,
+        use_log=use_log,
+        quiet=quiet,
+        ends_only=ends_only,
+        threads=threads,
+    )

clipkit/clipkit.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python
+
+import logging
+import sys
+import time
+from typing import Union
+
+from Bio.Align import MultipleSeqAlignment
+from .args_processing import process_args
+from .exceptions import InvalidInputFileFormat
+from .files import (
+    get_alignment_and_format,
+    FileFormat,
+    write_debug_log_file,
+    get_custom_sites_to_trim,
+)
+from .helpers import (
+    create_msa,
+    get_seq_type,
+    get_gap_chars,
+    write_msa,
+    write_complement,
+    SeqType,
+)
+from .logger import logger, log_file_logger
+from .modes import TrimmingMode
+from .msa import MSA
+from .parser import create_parser
+from .settings import DEFAULT_AA_GAP_CHARS, DEFAULT_NT_GAP_CHARS
+from .smart_gap_helper import smart_gap_threshold_determination
+from .version import __version__ as current_version
+from .warnings import (
+    warn_if_all_sites_were_trimmed,
+    warn_if_entry_contains_only_gaps,
+)
+from .write import (
+    write_user_args,
+    write_output_stats,
+    write_output_files_message,
+)
+
+from dataclasses import dataclass
+
+
+@dataclass
+class TrimRun:
+    alignment: MultipleSeqAlignment
+    msa: MSA
+    gap_characters: list
+    sequence_type: SeqType
+    input_file_format: FileFormat
+    output_file_format: FileFormat
+    gaps: float
+    codon: bool
+    version: str = current_version
+
+    @property
+    def complement(self):
+        return self.msa.complement_to_bio_msa()
+
+    @property
+    def trimmed(self):
+        return self.msa.to_bio_msa()
+
+
+def run(
+    input_file: str,
+    input_file_format: FileFormat,
+    output_file: str,
+    output_file_format: FileFormat,
+    auxiliary_file: str,
+    sequence_type: Union[SeqType, None],
+    gaps: float,
+    gap_characters: Union[list, None],
+    complement: bool,
+    codon: bool,
+    mode: TrimmingMode,
+    use_log: bool,
+    quiet: bool,
+    ends_only: bool,
+    threads: int = 1,
+):
+    try:
+        alignment, input_file_format = get_alignment_and_format(
+            input_file, input_file_format
+        )
+    except InvalidInputFileFormat:
+        return logger.error(
+            f"""Format type could not be read.\nPlease check acceptable input file formats: {", ".join([file_format.value for file_format in FileFormat])}"""
+        )
+
+    sequence_type = sequence_type or get_seq_type(alignment)
+
+    if not gap_characters:
+        gap_characters = get_gap_chars(sequence_type)
+
+    if not output_file_format:
+        output_file_format = input_file_format
+    else:
+        output_file_format = FileFormat(output_file_format)
+
+    # determine smart_gap threshold
+    if mode in {
+        TrimmingMode.smart_gap,
+        TrimmingMode.kpi_smart_gap,
+        TrimmingMode.kpic_smart_gap,
+    }:
+        gaps = smart_gap_threshold_determination(alignment, gap_characters)
+
+    site_positions_to_trim = None
+    if mode == TrimmingMode.cst:
+        aln_length = alignment.get_alignment_length()
+        site_positions_to_trim = (
+            get_custom_sites_to_trim(auxiliary_file, aln_length) or []
+        )
+
+    msa = create_msa(alignment, gap_characters, threads)
+    msa.trim(
+        mode,
+        gap_threshold=gaps,
+        site_positions_to_trim=site_positions_to_trim,
+        codon=codon,
+        ends_only=ends_only,
+    )
+
+    trim_run = TrimRun(
+        alignment,
+        msa,
+        gap_characters,
+        sequence_type,
+        input_file_format,
+        output_file_format,
+        gaps,
+        codon,
+    )
+
+    return trim_run, msa.stats
+
+
+def execute(
+    input_file: str,
+    input_file_format: FileFormat,
+    output_file: str,
+    output_file_format: FileFormat,
+    sequence_type: Union[SeqType, None],
+    gaps: float,
+    gap_characters: Union[list, None],
+    complement: bool,
+    codon: bool,
+    ends_only: bool,
+    mode: TrimmingMode,
+    use_log: bool,
+    quiet: bool,
+    auxiliary_file: str = None,
+    threads: int = 1,
+    **kwargs,
+) -> None:
+    if use_log:
+        log_file_logger.setLevel(logging.DEBUG)
+        log_file_logger.propagate = False
+        fh = logging.FileHandler(f"{output_file}.log", mode="w")
+        fh.setLevel(logging.DEBUG)
+        log_file_logger.addHandler(fh)
+
+    if quiet:
+        logger.disabled = True
+
+    # for reporting runtime duration to user
+    start_time = time.time()
+
+    trim_run, stats = run(
+        input_file,
+        input_file_format,
+        output_file,
+        output_file_format,
+        auxiliary_file,
+        sequence_type,
+        gaps,
+        gap_characters,
+        complement,
+        codon,
+        mode,
+        use_log,
+        quiet,
+        ends_only,
+        threads,
+    )
+
+    # display to user what args are being used in stdout
+    write_user_args(
+        input_file,
+        trim_run.input_file_format,
+        output_file,
+        trim_run.output_file_format,
+        trim_run.sequence_type,
+        trim_run.gaps,
+        trim_run.gap_characters,
+        mode,
+        complement,
+        codon,
+        use_log,
+        ends_only,
+    )
+
+    write_output_files_message(output_file, complement, use_log)
+
+    if use_log:
+        warn_if_all_sites_were_trimmed(trim_run.msa)
+        warn_if_entry_contains_only_gaps(trim_run.msa)
+        write_debug_log_file(trim_run.msa)
+
+    write_msa(trim_run.msa, output_file, trim_run.output_file_format)
+
+    # if the -c/--complementary argument was used, create an alignment of the trimmed sequences
+    if complement:
+        write_complement(trim_run.msa, output_file, trim_run.output_file_format)
+
+    write_output_stats(stats, start_time)
+
+
+def main(argv=None):
+    """
+    Function that parses and collects arguments
+    """
+    parser = create_parser()
+    args = parser.parse_args()
+
+    execute(**process_args(args))
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

clipkit/exceptions.py

@@ -0,0 +1,6 @@
+class ClipKITException(Exception):
+    pass
+
+
+class InvalidInputFileFormat(ClipKITException):
+    pass

clipkit/files.py

@@ -0,0 +1,75 @@
+from enum import Enum
+from .logger import log_file_logger
+
+from Bio import AlignIO
+from Bio.Align import MultipleSeqAlignment
+import numpy as np
+
+from .exceptions import InvalidInputFileFormat
+
+
+class FileFormat(Enum):
+    fasta = "fasta"
+    clustal = "clustal"
+    maf = "maf"
+    mauve = "mauve"
+    phylip = "phylip"
+    phylip_sequential = "phylip_sequential"
+    phylip_relaxed = "phylip_relaxed"
+    stockholm = "stockholm"
+
+
+def get_alignment_and_format(
+    input_file_name: str, file_format: FileFormat
+) -> tuple[MultipleSeqAlignment, FileFormat]:
+    """
+    Automatically determines what type of input file was used
+    and reads in the alignment file
+    """
+
+    if file_format:
+        file_format = FileFormat(file_format)
+        alignment = AlignIO.read(open(input_file_name), file_format.value)
+        return alignment, file_format
+    else:
+        # attempt to auto-detect file format
+        for fileFormat in FileFormat:
+            try:
+                alignment = AlignIO.read(open(input_file_name), fileFormat.value)
+                return alignment, fileFormat
+            # the following exceptions refer to skipping over errors
+            # associated with reading the wrong input file
+            except ValueError:
+                continue
+            except AssertionError:
+                continue
+
+        raise InvalidInputFileFormat("File could not be read")
+
+
+def get_custom_sites_to_trim(file_path: str, aln_length: int) -> list:
+    with open(file_path) as f:
+        lines = f.read().splitlines()
+
+    sites_to_trim = []
+    sites_to_keep = []
+    for line in lines:
+        site = line.split("\t")
+        pos = int(site[0]) - 1
+        if site[1] == "trim":
+            sites_to_trim.append(pos)
+        else:
+            sites_to_keep.append(pos)
+
+    if len(sites_to_trim) == 0:
+        # we only had keeps so treat every other site as a trim
+        sites_to_trim = list(
+            np.setdiff1d(np.arange(aln_length), np.array(sites_to_keep))
+        )
+
+    return sites_to_trim
+
+
+def write_debug_log_file(msa):
+    for info in msa.generate_debug_log_info():
+        log_file_logger.debug(f"{str(info[0] + 1)} {info[1]} {info[2].value} {info[3]}")

clipkit/helpers.py

@@ -0,0 +1,78 @@
+import re
+
+from Bio import SeqIO
+from Bio.Align import MultipleSeqAlignment
+import numpy as np
+
+from .msa import MSA
+from .modes import TrimmingMode
+from .settings import DEFAULT_AA_GAP_CHARS, DEFAULT_NT_GAP_CHARS
+from .files import FileFormat
+from .stats import TrimmingStats
+
+from enum import Enum
+
+
+class SeqType(Enum):
+    aa = "aa"
+    nt = "nt"
+
+
+def remove_gaps(seq: str, gap_chars: list[str] = DEFAULT_AA_GAP_CHARS) -> str:
+    pattern = "|".join([re.escape(char) for char in gap_chars])
+    return re.sub(pattern, "", seq)
+
+
+def get_seq_type(alignment: MultipleSeqAlignment) -> SeqType:
+    seq = str(alignment[0].seq)
+    seq = remove_gaps(seq)
+    if len(seq) < 200:
+        seq = "".join([str(record.seq) for record in alignment])
+        seq = remove_gaps(seq)
+
+    if len(set(seq.upper())) > 5:
+        sequence_type = SeqType.aa
+    else:
+        sequence_type = SeqType.nt
+
+    return sequence_type
+
+
+def get_gap_chars(seq_type: SeqType) -> list[str]:
+    if seq_type == SeqType.nt:
+        return DEFAULT_NT_GAP_CHARS
+    else:
+        return DEFAULT_AA_GAP_CHARS
+
+
+def create_msa(alignment: MultipleSeqAlignment, gap_chars: list[str] = None, threads: int = 1) -> MSA:
+    """
+    Create MSA class
+    """
+    return MSA.from_bio_msa(alignment, gap_chars, threads)
+
+
+def write_msa(msa: MSA, out_file_name: str, out_file_format: FileFormat) -> None:
+    """
+    msa is populated with sites that are kept after trimming is finished
+    """
+    output_msa = msa.to_bio_msa()
+    if out_file_format.value == "phylip_relaxed":
+        SeqIO.write(output_msa, out_file_name, "phylip-relaxed")
+    elif out_file_format.value == "phylip_sequential":
+        SeqIO.write(output_msa, out_file_name, "phylip-sequential")
+    else:
+        SeqIO.write(output_msa, out_file_name, out_file_format.value)
+
+
+def write_complement(msa: MSA, out_file: str, out_file_format: FileFormat) -> None:
+    """
+    msa is populated with sites that are trimmed after trimming is finished
+    """
+    output_msa = msa.complement_to_bio_msa()
+    completmentOut = str(out_file) + ".complement"
+    if out_file_format.value == "phylip_relaxed":
+        SeqIO.write(output_msa, out_file, "phylip-relaxed")
+    elif out_file_format.value == "phylip_sequential":
+        SeqIO.write(output_msa, out_file, "phylip-sequential")
+    SeqIO.write(output_msa, completmentOut, out_file_format.value)

clipkit/logger.py

@@ -0,0 +1,6 @@
+import logging
+import sys
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+log_file_logger = logging.getLogger("log_file")

clipkit/modes.py

@@ -0,0 +1,20 @@
+from enum import Enum
+from typing import TYPE_CHECKING
+from .logger import log_file_logger
+
+if TYPE_CHECKING:
+    from Bio.Align import MultipleSeqAlignment
+    from .msa import MSA
+
+
+class TrimmingMode(Enum):
+    gappy = "gappy"
+    smart_gap = "smart-gap"
+    kpi = "kpi"  # keep parsimony informative sites
+    kpi_gappy = "kpi-gappy"
+    kpi_smart_gap = "kpi-smart-gap"
+    kpic = "kpic"  # keep parsimony informative and constant sites
+    kpic_gappy = "kpic-gappy"
+    kpic_smart_gap = "kpic-smart-gap"
+    cst = "cst"  # custom site trimming
+    c3 = "c3"