eefinder 1.1.2__py3-none-any.whl

eefinder/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("eefinder")
+except PackageNotFoundError:  # pragma: no cover - package not installed
+    __version__ = "unknown"

eefinder/bed.py

@@ -0,0 +1,206 @@
+import pandas as pd
+import numpy as np
+import shlex
+import subprocess
+import re
+
+
+class GetFasta:
+    """
+    This function execute the bedtools getfasta.
+
+    Keyword arguments:
+    input_file: input_file, parsed with -in argument.
+    bed_file: bed file, genereated along the pipeline
+    out_file: output file
+    """
+
+    def __init__(self, input_file: str, bed_file: str, out_file: str) -> object:
+        self.input_file = input_file
+        self.bed_file = bed_file
+        self.out_file = out_file
+
+        self.get_fasta()
+
+    def get_fasta(self) -> None:
+        get_fasta = f"bedtools getfasta -fi {self.input_file} -bed {self.bed_file} -fo {self.out_file}"
+        get_fasta = shlex.split(get_fasta)
+        cmd_get_fasta = subprocess.Popen(
+            get_fasta, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+        )
+        cmd_get_fasta.wait()
+
+
+class GetAnnotBed:
+    """
+    Create a bed file that will be used to merge truncated EVEs of the same
+    family in the same sense based on a limite length treshold.
+
+    Keyword arguments:
+    blast_tax_info: csv file generated in the get_taxonomy_info function on get_taxonomy.py
+    merge_level: genus or family, choose which level going to merge nearby elements
+    """
+
+    def __init__(self, blast_tax_info: str, merge_level: str) -> object:
+        self.blast_tax_info = blast_tax_info
+        self.merge_level = merge_level
+
+        self.get_annotated_bed()
+
+    def get_annotated_bed(self) -> None:
+        df_blast_tax_info = pd.read_csv(self.blast_tax_info, sep=",")
+        df_blast_tax_info["qseqid"] = df_blast_tax_info["qseqid"].str.replace(
+            r"\:.*", "", regex=True
+        )
+        df_blast_tax_info["sseqid"] = (
+            df_blast_tax_info["sseqid"]
+            + "|"
+            + df_blast_tax_info["sense"]
+            + "|"
+            + df_blast_tax_info["pident"].astype(str)
+        )
+        df_blast_tax_info["Family"] = df_blast_tax_info["Family"].fillna("Unknown")
+        df_blast_tax_info["Genus"] = df_blast_tax_info["Genus"].fillna("Unknown")
+
+        if self.merge_level == "genus":
+            df_blast_tax_info["formated_name"] = np.where(
+                df_blast_tax_info["Genus"] != "Unknown",
+                df_blast_tax_info["qseqid"]
+                + "|"
+                + df_blast_tax_info["Family"]
+                + "|"
+                + df_blast_tax_info["Genus"]
+                + "|"
+                + df_blast_tax_info["sense"],
+                df_blast_tax_info["qseqid"]
+                + "|"
+                + df_blast_tax_info["sseqid"]
+                + "|"
+                + df_blast_tax_info["Genus"],
+            )
+        else:
+            df_blast_tax_info["formated_name"] = np.where(
+                df_blast_tax_info["Family"] != "Unknown",
+                df_blast_tax_info["qseqid"]
+                + "|"
+                + df_blast_tax_info["Family"]
+                + "|"
+                + df_blast_tax_info["sense"],
+                df_blast_tax_info["qseqid"]
+                + "|"
+                + df_blast_tax_info["sseqid"]
+                + "|"
+                + df_blast_tax_info["Family"],
+            )
+        bed_blast_info = df_blast_tax_info[
+            ["formated_name", "qstart", "qend", "sseqid"]
+        ].copy()
+        bed_blast_info = bed_blast_info.sort_values(
+            ["formated_name", "qstart"], ascending=(True, True)
+        )
+        bed_blast_info.to_csv(
+            f"{self.blast_tax_info}.bed", index=False, header=False, sep="\t"
+        )
+
+
+class RemoveAnnotation:
+    """
+    Remove the annotated information generate into the get_annotated_bed function.
+
+    Keyword arguments:
+    bed_annotated_merged_file: tsv file generated in the merge_bedfile function on bed_merge.py
+    """
+
+    def __init__(self, bed_annotated_merged_file: str) -> object:
+        self.bed_annotated_merged_file = bed_annotated_merged_file
+
+        self.reformat_bed()
+
+    def reformat_bed(self) -> None:
+        df_merge_file = pd.read_csv(
+            self.bed_annotated_merged_file, sep="\t", header=None
+        )
+        df_merge_file.iloc[:, 0] = df_merge_file.iloc[:, 0].str.replace(
+            "\|.*", "", regex=True
+        )
+        df_merge_file.to_csv(
+            f"{self.bed_annotated_merged_file}.fmt", index=False, header=False, sep="\t"
+        )
+
+
+class MergeBed:
+    """
+    Execute the bedtools merge.
+
+    Keyword arguments:
+    bed_annotated_file: annotated bed file created at get_annotated_bed function
+    limit_merge: Limit of bases to merge regions, parsed with -lm argument
+    """
+
+    def __init__(self, bed_annotated_file: str, limit_merge: int) -> object:
+        self.bed_annotated_file = bed_annotated_file
+        self.limit_merge = limit_merge
+
+        self.merge_bed()
+
+    def merge_bed(self) -> None:
+        bed_merge_output = open(f"{self.bed_annotated_file}.merge", "w")
+        bed_merge_cmd = f'bedtools merge -d {int(self.limit_merge)} -i {self.bed_annotated_file} -c 4 -o collapse -delim " AND "'
+        bed_merge_cmd = shlex.split(bed_merge_cmd)
+        bed_merge_process = subprocess.Popen(bed_merge_cmd, stdout=bed_merge_output)
+        bed_merge_process.wait()
+
+
+class BedFlank:
+    """
+    Extract flanking regions of EEs using bedtools slop.
+
+    Keyword arguments:
+    input_file: bed file generated by get_bed function
+    lenght_file: lenght file produced by get_length function
+    flank_region: desired lenght regions for extraction, parsed from
+    """
+
+    def __init__(self, input_file: str, length_file: str, flank_region: int) -> object:
+        self.input_file = input_file
+        self.length_file = length_file
+        self.flank_region = flank_region
+
+        self.bedtools_flank()
+
+    def bedtools_flank(self) -> None:
+        with open(f"{self.input_file}.flank", "w") as flank_out:
+            bed_flank_cmd = f"bedtools slop -i {self.input_file} -g {self.length_file} -b {str(self.flank_region)}"
+            bed_flank_cmd = shlex.split(bed_flank_cmd)
+            bed_flank_process = subprocess.Popen(bed_flank_cmd, stdout=flank_out)
+            bed_flank_process.wait()
+
+
+class GetBed:
+    """
+    Create a bed file from fasta file using replace logic.
+
+    Keyword arguments:
+    input_file: fasta file for desired bed file
+    """
+
+    def __init__(self, input_file: str) -> object:
+        self.input_file = input_file
+
+        self.get_bed()
+
+    def get_bed(self) -> None:
+        with open(f"{self.input_file}", "r") as repeat_eves, open(
+            f"{self.input_file}.bed", "w"
+        ) as repeat_eves_bed_out:
+            repeat_eves_lines = repeat_eves.readlines()
+            for line in repeat_eves_lines:
+                if ">" in line:
+                    line_name = line.replace(">", "")
+                    line_name = re.sub(":.*", "", line_name).rstrip("\n")
+                    line_start = re.sub(".*:", "", line)
+                    line_start = re.sub("-.*", "", line_start).rstrip("\n")
+                    line_end = re.sub(".*-", "", line).rstrip("\n")
+                    repeat_eves_bed_out.write(
+                        f"{line_name}\t{line_start}\t{line_end}\n"
+                    )

eefinder/clean_data.py

@@ -0,0 +1,64 @@
+from Bio import SeqIO
+
+
+class RemoveShortSequences:
+    """
+    Remove sequences bellow the cutoff threshold.
+
+    Keyword arguments:
+    input_file: input fasta file
+    cutoff: cutoff length, parsed by -ln
+    """
+
+    def __init__(self, input_file: str, cutoff: int) -> object:
+        self.input_file = input_file
+        self.cutoff = cutoff
+
+        self.cut_seq()
+
+    def cut_seq(self) -> None:
+        new_sequences = []
+        input_handle = open(self.input_file, "r")
+        output_handle = open(self.input_file + ".fmt", "w")
+        for record in SeqIO.parse(input_handle, "fasta"):
+            if len(record.seq) >= int(self.cutoff):
+                new_sequences.append(record)
+        SeqIO.write(new_sequences, output_handle, "fasta")
+
+
+class MaskClean:
+    """
+    Remove sequences of EE on regions with a certain % of soft masked bases.
+
+    Keyword arguments:
+    input_file: fasta file, with putative EEs
+    m_per: treshold masked percentage value, parsed with -mp argument
+    """
+
+    def __init__(self, input_file: str, m_per: int) -> object:
+        self.input_file = input_file
+        self.m_per = m_per
+
+        self.mask_clean()
+
+    def mask_clean(self) -> None:
+        sequences = {}
+        for seq_record in SeqIO.parse(self.input_file, "fasta"):
+            sequence = str(seq_record.seq)
+            sequence_id = str(seq_record.id)
+            if (
+                float(
+                    sequence.count("a")
+                    + sequence.count("t")
+                    + sequence.count("c")
+                    + sequence.count("g")
+                    + sequence.count("n")
+                    + sequence.count("N")
+                )
+                / float(len(sequence))
+            ) * 100 <= float(self.m_per):
+                if sequence_id not in sequences:
+                    sequences[sequence_id] = sequence
+        with open(self.input_file + ".cl", "w+") as output_file:
+            for sequence_id, sequence in sequences.items():
+                output_file.write(f">{sequence_id}\n{sequence}\n")

eefinder/compare_results.py

@@ -0,0 +1,30 @@
+import pandas as pd
+
+
+class CompareResults:
+    """
+    This function compares 2 blast results, for queries with same ID, only the one
+    with the major bitscore is keept. In a final step only queries with tag EE are maintained
+
+    Keyword arguments:
+    vir_result: filtred blast against ee database
+    host_result: filtred blast against filter database
+    """
+
+    def __init__(self, vir_result: str, host_result: str) -> object:
+        self.vir_result = vir_result
+        self.host_result = host_result
+
+        self.compare_results()
+
+    def compare_results(self) -> None:
+        df_vir = pd.read_csv(self.vir_result, sep="\t")
+        df_vir["qseqid"] = df_vir["bed_name"]
+        df_host = pd.read_csv(self.host_result, sep="\t")
+        df_hybrid = pd.concat([df_vir, df_host], ignore_index=True)
+        df_hybrid = df_hybrid.sort_values(by=["qseqid", "bitscore"], ascending=False)
+        df_hybrid.to_csv(self.host_result + ".concat", sep="\t", index=False)
+        df_nr = df_hybrid.drop_duplicates(subset=["qseqid"])
+        df_nr.to_csv(self.host_result + ".concat.nr", sep="\t", index=False)
+        df_nr_vir = df_nr[df_nr.tag == "EE"]
+        df_nr_vir.to_csv(self.host_result + ".concat.nr", sep="\t", index=False)

eefinder/filter_table.py

@@ -0,0 +1,135 @@
+import pandas as pd
+import csv
+import os
+import re
+import glob
+import shutil
+
+
+class FilterTable:
+    """
+    Receives a blastx result and filter based on query ID and ranges of qstart and qend.
+
+    Keyword arguments:
+    blast_result: input blastx result
+    rangejunction: range for filter redundant hits
+    tag: HOST or EE, tells which blastx is
+    out_dir: output directory, parsed by -od
+    """
+
+    def __init__(
+        self, blast_result: str, rangejunction: int, tag: str, out_dir: str
+    ) -> object:
+        self.blast_result = blast_result
+        self.rangejunction = rangejunction
+        self.tag = tag
+        self.out_dir = out_dir
+
+        self.filter_blast()
+
+    def filter_blast(self) -> None:
+        header_outfmt6 = [
+            "qseqid",
+            "sseqid",
+            "pident",
+            "length",
+            "mismatch",
+            "gapopen",
+            "qstart",
+            "qend",
+            "sstart",
+            "send",
+            "evalue",
+            "bitscore",
+        ]  # creates a blast header output in format = 6
+        df = pd.read_csv(
+            self.blast_result, sep="\t", header=None, names=header_outfmt6
+        ).sort_values(by="bitscore", ascending=False)
+        df["sense"] = ""
+        df["bed_name"] = ""
+        df["tag"] = ""
+        df["new_qstart"] = df["qstart"]
+        df["new_qend"] = df["qend"]
+        df.to_csv(self.blast_result + ".csv", sep="\t")
+        chunks = df = pd.read_csv(
+            f"{self.blast_result}.csv", sep="\t", chunksize=200000
+        )
+        count = 0
+        tmp_path = f"{self.out_dir}/tmp/"
+        if os.path.exists(tmp_path) == False:
+            os.mkdir(tmp_path)
+        for df in chunks:
+            df["sense"] = df["sense"].astype(object)
+            df.loc[
+                df["qstart"].astype(int) > df["qend"].values.astype(int), "sense"
+            ] = "neg"
+            df.loc[
+                df["qend"].values.astype(int) > df["qstart"].astype(int), "sense"
+            ] = "pos"
+            df.loc[df["sense"] == "neg", "new_qstart"] = df["qend"]
+            df.loc[df["sense"] == "neg", "new_qend"] = df["qstart"]
+            df.loc[df["sense"] == "neg", "qstart"] = df["new_qstart"]
+            df.loc[df["sense"] == "neg", "qend"] = df["new_qend"]
+            df.drop(columns=["new_qstart", "new_qend"], inplace=True)
+            if self.tag == "EE":
+                df["tag"] = "EE"
+                df["bed_name"] = df.apply(
+                    lambda x: "%s:%s-%s" % (x["qseqid"], x["qstart"], x["qend"]), axis=1
+                )
+            else:
+                df["tag"] = "HOST"
+                df["bed_name"] = df["qseqid"]
+            pd.options.display.float_format = "{:,.2f}".format
+            df["evalue"] = pd.to_numeric(df["evalue"], downcast="float")
+            df = df[df.length >= 33]
+            header = [
+                "qseqid",
+                "sseqid",
+                "pident",
+                "length",
+                "mismatch",
+                "gapopen",
+                "qstart",
+                "qend",
+                "sstart",
+                "send",
+                "evalue",
+                "bitscore",
+                "sense",
+                "bed_name",
+                "tag",
+            ]
+            df = df[header]
+            with open(f"{tmp_path}chunk.{count}.tsv", "w") as chunk_writer:
+                df.to_csv(chunk_writer, sep="\t", index=False)
+            count += 1
+        all_chunks = glob.glob(f"{tmp_path}/*.tsv")
+        final_filtred_file = pd.DataFrame()
+        chunks_list = []
+        for chunk in all_chunks:
+            df = pd.read_csv(chunk, sep="\t")
+            chunks_list.append(df)
+        final_filtred_file = pd.concat(chunks_list, ignore_index=True)
+        final_filtred_file["qstart_rng"] = final_filtred_file.qstart.floordiv(
+            self.rangejunction
+        )
+        final_filtred_file["qend_rng"] = final_filtred_file.qend.floordiv(
+            self.rangejunction
+        )
+        final_filtred_file = (
+            final_filtred_file.drop_duplicates(subset=["qseqid", "qstart_rng", "sense"])
+            .drop_duplicates(subset=["qseqid", "qstart_rng", "sense"])
+            .sort_values(by=["qseqid"])
+        )
+        final_filtred_file.to_csv(
+            f"{self.blast_result}.filtred", sep="\t", index=False, columns=header
+        )
+        if self.tag == "EE":
+            final_filtred_file.to_csv(
+                f"{self.blast_result}.filtred.bed",
+                header=False,
+                sep="\t",
+                index=False,
+                columns=["qseqid", "qstart", "qend"],
+            )
+        shutil.rmtree(tmp_path, ignore_errors=True)

eefinder/get_length.py

@@ -0,0 +1,21 @@
+from Bio import SeqIO
+
+
+class GetLength:
+    """
+    Creates a length file with the module SeqIO.
+
+    Keywords arguments:
+    input_file: formated genome, generated by cut_seq function
+    """
+
+    def __init__(self, input_file: str) -> object:
+        self.input_file = input_file
+
+        self.get_length()
+
+    def get_length(self) -> None:
+        with open(f"{self.input_file}.rn.fmt.lenght", "w") as output_length:
+            length_list = []
+            for seq_record in SeqIO.parse(self.input_file, "fasta"):
+                output_length.write(f"{seq_record.id}\t{str(len(seq_record))}\n")

eefinder/get_taxonomy.py

@@ -0,0 +1,179 @@
+import pandas as pd
+import re, csv
+from Bio import SeqIO
+
+
+class GetTaxonomy:
+    """
+    Merge the filtred blast results with taxonomy information, creating a taxonomy signature for being used in get_final_taxonomy.
+
+    Keyword arguments:
+    blast_file: tsv filtred blast results
+    tax_file: table with taxonomy and other metadata, parsed with -mt parameter
+    """
+
+    def __init__(self, blast_file: str, tax_file: str) -> object:
+        self.blast_file = blast_file
+        self.tax_file = tax_file
+
+        self.get_taxonomy()
+
+    def get_taxonomy(self) -> None:
+        df_blast_file = pd.read_csv(self.blast_file, sep="\t")
+        df_tax_file = pd.read_csv(self.tax_file)
+        df_tax_file.rename(columns={"Accession": "sseqid"}, inplace=True)
+        df_merged = pd.merge(df_blast_file, df_tax_file, on="sseqid", how="left")
+        df_merged.to_csv(f"{self.blast_file}.tax", index=False, header=True)
+
+
+class GetFinalTaxonomy:
+    """
+    Mount taxonomy for each putative EEs.
+
+    Keyword arguments:
+    bed_formated: bed file, generated by cut_seq function
+    taxonomy_info: taxonomy signature generated by get_taxonomy function
+    """
+
+    def __init__(self, bed_formated: str, taxonomy_info: str) -> object:
+        self.bed_formated = bed_formated
+        self.taxonomy_info = taxonomy_info
+
+        self.get_final_taxonomy()
+
+    def get_final_taxonomy(self) -> None:
+        with open(self.bed_formated, "r") as bed_merge_file, open(
+            f"{self.bed_formated}.fa.tax", "w"
+        ) as bed_merge_tax_out:
+            bed_merge_tax_list = []
+            bed_merge_file_reader = csv.reader(bed_merge_file, delimiter="\t")
+            bed_merge_tax_out_writer = csv.writer(bed_merge_tax_out, delimiter="\t")
+            bed_merge_tax_out_writer.writerow(
+                [
+                    "Element-ID",
+                    "Sense",
+                    "Protein-IDs",
+                    "Protein-Products",
+                    "Molecule_type",
+                    "Family",
+                    "Genus",
+                    "Species",
+                    "Host",
+                ]
+            )
+            for line in bed_merge_file_reader:
+                element_merged_id = (
+                    line[0].rstrip("\n")
+                    + ":"
+                    + line[1].strip("\n")
+                    + "-"
+                    + line[2].strip("\n")
+                )
+                if "pos" in line[3]:
+                    sense = "pos"
+                    line[3] = re.sub("\|pos", "", line[3]).rstrip("\n")
+                elif "neg" in line[3]:
+                    sense = "neg"
+                    line[3] = re.sub("\|neg", "", line[3]).rstrip("\n")
+                protein_ids = line[3].rstrip("\n")
+                with open(self.taxonomy_info, "r") as prot_info:
+                    prot_info_reader = csv.reader(prot_info, delimiter=",")
+                    protein_terms = ""
+                    genus = ""
+                    species = ""
+                    host = ""
+                    if "AND" in protein_ids:
+                        for line_prot in prot_info_reader:
+                            protein_ids = re.sub("AND", "|", line[3]).rstrip("\n")
+                            if line_prot[1].rstrip("\n") in protein_ids:
+                                if line_prot[19].rstrip("\n") not in protein_terms:
+                                    protein_terms += line_prot[19] + " AND "
+                                    mol_type = line_prot[18]
+                                    family = line_prot[17]
+                                if line_prot[16].rstrip("\n") not in genus:
+                                    genus += line_prot[16] + " AND "
+                                if line_prot[15].rstrip("\n") not in species:
+                                    species += line_prot[15] + " AND "
+                                if line_prot[20].rstrip("\n") not in host:
+                                    host += line_prot[20] + " AND "
+                    else:
+                        for line_prot in prot_info_reader:
+                            if line_prot[1].rstrip("\n") in protein_ids:
+                                protein_terms = line_prot[19]
+                                mol_type = line_prot[18]
+                                family = line_prot[17]
+                                genus = line_prot[16]
+                                species = line_prot[15]
+                                host = line_prot[20]
+
+                protein_terms = re.sub(r" AND $", "", protein_terms)
+                genus = re.sub(r" AND $", "", genus)
+                species = re.sub(r" AND $", "", species)
+                host = re.sub(r" AND $", "", host)
+                if mol_type == "":
+                    vir_order = "Undefined"
+                if family == "":
+                    family = "Unclassified"
+                if genus == "":
+                    genus = "Unclassified"
+                if species == "":
+                    species = "Unclassified"
+                if host == "":
+                    host = "Undefined"
+
+                bed_merge_tax_list.append(
+                    [
+                        element_merged_id,
+                        sense,
+                        protein_ids,
+                        protein_terms,
+                        mol_type,
+                        family,
+                        genus,
+                        species,
+                        host,
+                    ]
+                )
+            bed_merge_tax_out_writer.writerows(bed_merge_tax_list)
+
+
+class GetCleanedTaxonomy:
+    """
+    Mount cleaned taxonomy for each putative EEs, parsed by -cm parameter.
+
+    Keyword arguments:
+    bed_formated: bed file, generated by cut_seq function
+    taxonomy_info: taxonomy signature generated by get_taxonomy function
+    """
+
+    def __init__(self, cleaned_file: str, taxonomy_file: str) -> object:
+        self.cleaned_file = cleaned_file
+        self.taxonomy_file = taxonomy_file
+
+        self.get_cleaned_taxonomy()
+
+    def get_cleaned_taxonomy(self) -> None:
+        output_list = [
+            [
+                "Element-ID",
+                "Sense",
+                "Protein-IDs",
+                "Protein-Products",
+                "Molecule_type",
+                "Family",
+                "Genus",
+                "Species",
+                "Host",
+            ]
+        ]
+
+        for seq_record in SeqIO.parse(self.cleaned_file, "fasta"):
+            with open(self.taxonomy_file, "r") as tax_file:
+                taxonomy_file_reader = csv.reader(tax_file, delimiter="\t")
+                for line in taxonomy_file_reader:
+                    if line[0] == seq_record.id:
+                        output_list.append(line)
+
+        with open(f"{self.cleaned_file}.tax", "w") as output_file:
+            output_file_writer = csv.writer(output_file, delimiter="\t")
+            output_file_writer.writerows(output_list)

eefinder/log.py

@@ -0,0 +1,4 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("eefinder")