mgnify-pipelines-toolkit 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl

mgnify_pipelines_toolkit/analysis/assembly/add_rhea_chebi_annotation.py

@@ -78,7 +78,11 @@ def main():
         "--output",
         required=True,
         type=Path,
-        help="Output TSV file with columns: contig_id, protein_id, UniRef90 cluster, rhea_ids, CHEBI reaction participants",
+        help=(
+            "Output TSV file with columns: contig_id, protein_id, protein hash, "
+            "Rhea IDs, CHEBI reaction, reaction definition, 'top hit' if it is "
+            "the first hit for the protein"
+        ),
     )
     parser.add_argument(
         "-p",

mgnify_pipelines_toolkit/analysis/assembly/gff_annotation_utils.py

@@ -17,8 +17,19 @@
 
 import re
 import sys
+import fileinput
 
-from mgnify_pipelines_toolkit.constants.thresholds import EVALUE_CUTOFF_IPS, EVALUE_CUTOFF_EGGNOG
+from mgnify_pipelines_toolkit.constants.thresholds import (
+    EVALUE_CUTOFF_IPS,
+    EVALUE_CUTOFF_EGGNOG,
+)
+
+DBCAN_CLASSES_DICT = {
+    "TC": "dbcan_transporter_classification",
+    "TF": "dbcan_transcription_factor",
+    "STP": "dbcan_signal_transduction_prot",
+    "CAZyme": "dbcan_prot_family",
+}
 
 
 def get_iprs(ipr_annot):
@@ -26,7 +37,8 @@ def get_iprs(ipr_annot):
     antifams = list()
     if not ipr_annot:
         return iprs, antifams
-    with open(ipr_annot) as f:
+    with fileinput.hook_compressed(ipr_annot, "r", encoding="utf-8") as f:
+
         for line in f:
             cols = line.strip().split("\t")
             protein = cols[0]
@@ -55,7 +67,8 @@ def get_eggnog(eggnog_annot):
     eggnogs = {}
     if not eggnog_annot:
         return eggnogs
-    with open(eggnog_annot, "r") as f:
+    with fileinput.hook_compressed(eggnog_annot, "r", encoding="utf-8") as f:
+
         for line in f:
             line = line.rstrip()
             cols = line.split("\t")
@@ -104,7 +117,8 @@ def get_bgcs(bgc_file, prokka_gff, tool):
         return bgc_annotations
     # save positions of each BGC cluster to dictionary cluster_positions
     # and save the annotations to dictionary bgc_result
-    with open(bgc_file, "r") as bgc_in:
+    with fileinput.hook_compressed(bgc_file, "r", encoding="utf-8") as bgc_in:
+
         for line in bgc_in:
             if not line.startswith("#"):
                 (
@@ -138,7 +152,7 @@ def get_bgcs(bgc_file, prokka_gff, tool):
                     type_value = ""
                     as_product = ""
                     for a in annotations.split(
-                            ";"
+                        ";"
                     ):  # go through all parts of the annotation field
                         if a.startswith("as_type="):
                             type_value = a.split("=")[1]
@@ -170,9 +184,12 @@ def get_bgcs(bgc_file, prokka_gff, tool):
                         {"bgc_function": type_value},
                     )
                     if as_product:
-                        tool_result[contig]["_".join([start_pos, end_pos])]["bgc_product"] = as_product
+                        tool_result[contig]["_".join([start_pos, end_pos])][
+                            "bgc_product"
+                        ] = as_product
     # identify CDSs that fall into each of the clusters annotated by the BGC tool
-    with open(prokka_gff, "r") as gff_in:
+    with fileinput.hook_compressed(prokka_gff, "r", encoding="utf-8") as gff_in:
+
         for line in gff_in:
             if not line.startswith("#"):
                 matching_interval = ""
@@ -228,8 +245,9 @@ def get_bgcs(bgc_file, prokka_gff, tool):
                             },
                         )
                         if "bgc_product" in tool_result[contig][matching_interval]:
-                            bgc_annotations[cds_id]["antismash_product"] = tool_result[contig][matching_interval][
-                                "bgc_product"]
+                            bgc_annotations[cds_id]["antismash_product"] = tool_result[
+                                contig
+                            ][matching_interval]["bgc_product"]
             elif line.startswith("##FASTA"):
                 break
     return bgc_annotations
@@ -239,7 +257,7 @@ def get_amr(amr_file):
     amr_annotations = {}
     if not amr_file:
         return amr_annotations
-    with open(amr_file, "r") as f:
+    with fileinput.hook_compressed(amr_file, "r", encoding="utf-8") as f:
         for line in f:
             if line.startswith("Protein identifier"):
                 continue
@@ -286,7 +304,7 @@ def get_dbcan(dbcan_file):
     substrates = dict()
     if not dbcan_file:
         return dbcan_annotations
-    with open(dbcan_file, "r") as f:
+    with fileinput.hook_compressed(dbcan_file, "r", encoding="utf-8") as f:
         for line in f:
             if "predicted PUL" in line:
                 annot_fields = line.strip().split("\t")[8].split(";")
@@ -314,13 +332,45 @@ def get_dbcan(dbcan_file):
                         elif a.startswith("Parent"):
                             parent = a.split("=")[1]
                     dbcan_annotations[acc] = (
-                        "dbcan_prot_type={};dbcan_prot_family={};substrate_dbcan-pul={};substrate_dbcan-sub={}".format(
+                        "dbcan_prot_type={};{}={};substrate_dbcan-pul={};substrate_dbcan-sub={}".format(
                             prot_type,
+                            DBCAN_CLASSES_DICT[prot_type],
                             prot_fam,
                             substrates[parent]["substrate_pul"],
                             substrates[parent]["substrate_ecami"],
                         )
                     )
+
+    return dbcan_annotations
+
+
+def get_dbcan_individual_cazys(dbcan_cazys_file):
+    dbcan_annotations = dict()
+    if not dbcan_cazys_file:
+        return dbcan_annotations
+    with fileinput.hook_compressed(dbcan_cazys_file, "r", encoding="utf-8") as f:
+        for line in f:
+            if line.startswith("#"):
+                continue
+            attributes = line.strip().split("\t")[8]
+            attributes_dict = dict(
+                re.split(r"(?<!\\)=", item)
+                for item in re.split(r"(?<!\\);", attributes.rstrip(";"))
+            )
+            if "num_tools" in attributes_dict and int(attributes_dict["num_tools"]) < 2:
+                continue  # don't keep annotations supported by only one tool within dbcan
+            cds_pattern = r"\.CDS\d+$"
+            protein = re.sub(
+                cds_pattern, "", attributes_dict["ID"]
+            )  # remove the CDS number
+            annotation_text = "dbcan_prot_type=CAZyme;"
+            for field in ["protein_family", "substrate_dbcan-sub", "eC_number"]:
+                if field in attributes_dict:
+                    annotation_text += (
+                        f"{'dbcan_prot_family' if field == 'protein_family' else field}"
+                        f"={attributes_dict[field]};"
+                    )
+            dbcan_annotations[protein] = annotation_text.strip(";")
     return dbcan_annotations
 
 
@@ -329,7 +379,8 @@ def get_defense_finder(df_file):
     type_info = dict()
     if not df_file:
         return defense_finder_annotations
-    with open(df_file, "r") as f:
+    with fileinput.hook_compressed(df_file, "r", encoding="utf-8") as f:
+
         for line in f:
             if "Anti-phage system" in line:
                 annot_fields = line.strip().split("\t")[8].split(";")
@@ -366,6 +417,7 @@ def load_annotations(
     antismash_file,
     gecco_file,
     dbcan_file,
+    dbcan_cazys_file,
     defense_finder_file,
     pseudofinder_file,
 ):
@@ -376,6 +428,7 @@ def load_annotations(
     antismash_bgcs = get_bgcs(antismash_file, in_gff, tool="antismash")
     amr_annotations = get_amr(amr_file)
     dbcan_annotations = get_dbcan(dbcan_file)
+    dbcan_cazys_annotations = get_dbcan_individual_cazys(dbcan_cazys_file)
     defense_finder_annotations = get_defense_finder(defense_finder_file)
     pseudogenes = get_pseudogenes(pseudofinder_file)
     pseudogene_report_dict = dict()
@@ -384,7 +437,7 @@ def load_annotations(
     header = []
     fasta = []
     fasta_flag = False
-    with open(in_gff) as f:
+    with fileinput.hook_compressed(in_gff, "r", encoding="utf-8") as f:
         for line in f:
             line = line.strip()
             if line[0] != "#" and not fasta_flag:
@@ -496,6 +549,11 @@ def load_annotations(
                         added_annot[protein]["dbCAN"] = dbcan_annotations[protein]
                     except KeyError:
                         pass
+                    try:
+                        dbcan_cazys_annotations[protein]
+                        added_annot[protein]["dbCAN"] = dbcan_cazys_annotations[protein]
+                    except KeyError:
+                        pass
                     try:
                         defense_finder_annotations[protein]
                         added_annot[protein]["defense_finder"] = (
@@ -530,7 +588,7 @@ def load_annotations(
 def get_ncrnas(ncrnas_file):
     ncrnas = {}
     counts = 0
-    with open(ncrnas_file, "r") as f:
+    with fileinput.hook_compressed(ncrnas_file, "r", encoding="utf-8") as f:
         for line in f:
             if not line.startswith("#"):
                 cols = line.strip().split()
@@ -543,7 +601,9 @@ def get_ncrnas(ncrnas_file):
                     # Skip tRNAs, we add them from tRNAscan-SE
                     continue
                 strand = cols[11]
-                start, end = (int(cols[9]), int(cols[10])) if strand == "+" else (int(cols[10]), int(cols[9]))
+                start, end = int(cols[10]), int(cols[9])
+                if strand == "+":
+                    start, end = end, start
                 rna_feature_name, ncrna_class = prepare_rna_gff_fields(cols)
                 annot = [
                     "ID=" + locus,
@@ -718,7 +778,10 @@ def prepare_rna_gff_fields(cols):
     }
 
     if rna_feature_name == "ncRNA":
-        ncrna_class = next((rna_type for rna_type, rfams in rna_types.items() if cols[2] in rfams), None)
+        ncrna_class = next(
+            (rna_type for rna_type, rfams in rna_types.items() if cols[2] in rfams),
+            None,
+        )
         if not ncrna_class:
             if "microRNA" in cols[-1]:
                 ncrna_class = "pre_miRNA"
@@ -729,7 +792,7 @@ def prepare_rna_gff_fields(cols):
 
 def get_trnas(trnas_file):
     trnas = {}
-    with open(trnas_file, "r") as f:
+    with fileinput.hook_compressed(trnas_file, "r", encoding="utf-8") as f:
         for line in f:
             if not line.startswith("#"):
                 cols = line.split("\t")
@@ -738,13 +801,13 @@ def get_trnas(trnas_file):
                     line = line.replace("tRNAscan-SE", "tRNAscan-SE:2.0.9")
                     trnas.setdefault(contig, dict()).setdefault(
                         int(start), list()
-                    ).append(line.strip())
+                    ).append(line.strip().strip(";"))
     return trnas
 
 
 def load_crispr(crispr_file):
     crispr_annotations = dict()
-    with open(crispr_file, "r") as f:
+    with fileinput.hook_compressed(crispr_file, "r", encoding="utf-8") as f:
         record = list()
         left_coord = ""
         loc_contig = ""
@@ -791,7 +854,7 @@ def get_pseudogenes(pseudofinder_file):
     pseudogenes = dict()
     if not pseudofinder_file:
         return pseudogenes
-    with open(pseudofinder_file) as file_in:
+    with fileinput.hook_compressed(pseudofinder_file, "r", encoding="utf-8") as file_in:
         for line in file_in:
             if not line.startswith("#"):
                 col9 = line.strip().split("\t")[8]

mgnify_pipelines_toolkit/analysis/assembly/gff_file_utils.py

@@ -28,6 +28,17 @@ def write_results_to_file(
         contig_list = check_for_additional_keys(
             ncrnas, trnas, crispr_annotations, contig_list
         )
+        # sort contigs by digit at the end of contig/genome accession
+        if contig_list[0].startswith(
+            "MGYG"
+        ):  # e.g. 'MGYG000500002_1', 'MGYG000500002_2', 'MGYG000500002_3'
+            contig_list = sorted(list(contig_list), key=lambda x: int(x.split("_")[-1]))
+        elif contig_list[0].startswith(
+            "ERZ"
+        ):  # e.g. 'ERZ1049444', 'ERZ1049445', 'ERZ1049446'
+            contig_list = sorted(
+                list(contig_list), key=lambda x: int(x.split("ERZ")[-1])
+            )
         for contig in contig_list:
             sorted_pos_list = sort_positions(
                 contig, main_gff_extended, ncrnas, trnas, crispr_annotations

mgnify_pipelines_toolkit/analysis/assembly/gff_toolkit.py

@@ -17,8 +17,16 @@
 
 import argparse
 
-from gff_annotation_utils import get_ncrnas, get_trnas, load_annotations, load_crispr
-from gff_file_utils import write_results_to_file, print_pseudogene_report
+from mgnify_pipelines_toolkit.analysis.assembly.gff_annotation_utils import (
+    get_ncrnas,
+    get_trnas,
+    load_annotations,
+    load_crispr,
+)
+from mgnify_pipelines_toolkit.analysis.assembly.gff_file_utils import (
+    write_results_to_file,
+    print_pseudogene_report,
+)
 
 
 def main(
@@ -31,6 +39,7 @@ def main(
     antismash_file,
     gecco_file,
     dbcan_file,
+    dbcan_cazys_file,
     defense_finder_file,
     pseudofinder_file,
     rfam_file,
@@ -53,6 +62,7 @@ def main(
         antismash_file,
         gecco_file,
         dbcan_file,
+        dbcan_cazys_file,
         defense_finder_file,
         pseudofinder_file,
     )
@@ -66,7 +76,9 @@ def main(
     if crispr_file:
         crispr_annotations = load_crispr(crispr_file)
 
-    write_results_to_file(outfile, header, main_gff_extended, fasta, ncrnas, trnas, crispr_annotations)
+    write_results_to_file(
+        outfile, header, main_gff_extended, fasta, ncrnas, trnas, crispr_annotations
+    )
     if pseudogene_report_file:
         print_pseudogene_report(pseudogene_report_dict, pseudogene_report_file)
 
@@ -74,7 +86,7 @@ def main(
 def parse_args():
     parser = argparse.ArgumentParser(
         description="The script extends a user-provided base GFF annotation file by incorporating "
-                    "information extracted from the user-provided outputs of supplementary annotation tools.",
+        "information extracted from the user-provided outputs of supplementary annotation tools.",
     )
     parser.add_argument(
         "-g",
@@ -124,7 +136,12 @@ def parse_args():
     )
     parser.add_argument(
         "--dbcan",
-        help="The GFF file produced by dbCAN post-processing script",
+        help="The GFF file produced by dbCAN post-processing script that uses cluster annotations",
+        required=False,
+    )
+    parser.add_argument(
+        "--dbcan-cazys",
+        help="The GFF file produced by dbCAN-CAZYs post-processing script",
         required=False,
     )
     parser.add_argument(
@@ -149,7 +166,7 @@ def parse_args():
     return parser.parse_args()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     args = parse_args()
     main(
         args.gff_input,
@@ -161,10 +178,11 @@ if __name__ == '__main__':
         args.antismash,
         args.gecco,
         args.dbcan,
+        args.dbcan_cazys,
         args.defense_finder,
         args.pseudofinder,
         args.rfam,
         args.trnascan,
         args.outfile,
         args.pseudogene_report,
-         )
+    )

mgnify_pipelines_toolkit/analysis/assembly/krona_txt_from_cat_classification.py

@@ -40,10 +40,12 @@ def import_nodes(nodes_dmp):
     taxid2rank = {}
 
     with open(nodes_dmp) as f1:
-        reader = csv.reader(f1, delimiter="\t")
-        for line in reader:
-            taxid = line[0]
-            rank = line[4]
+        for line in f1:
+            fields = [part.strip() for part in line.split("|")]
+            if len(fields) != 14:
+                raise ValueError(f"Unexpected number of columns in line: {line}")
+            taxid = fields[0]
+            rank = fields[2]
             taxid2rank[taxid] = rank
 
     return taxid2rank
@@ -54,11 +56,13 @@ def import_names(names_dmp):
     taxid2name = {}
 
     with open(names_dmp, newline="") as f1:
-        reader = csv.reader(f1, delimiter="\t")
-        for line in reader:
-            if line[6] == "scientific name":
-                taxid = line[0]
-                name = line[2]
+        for line in f1:
+            fields = [part.strip() for part in line.split("|")]
+            if len(fields) != 5:
+                raise ValueError(f"Unexpected number of columns in line: {line}")
+            if fields[3] == "scientific name":
+                taxid = fields[0]
+                name = fields[1]
                 taxid2name[taxid] = name
 
     return taxid2name

mgnify_pipelines_toolkit/analysis/assembly/process_dbcan_result_cazys.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2023-2025 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import fileinput
+import logging
+from pathlib import Path
+import re
+
+logging.basicConfig(level=logging.INFO)
+
+
+def main(hmm_file, overview_file, genome_gff, outfile, dbcan_version):
+
+    hmm_path = Path(hmm_file)
+    overview_path = Path(overview_file)
+
+    if not hmm_path.is_file():
+        raise FileNotFoundError(f"Input hmm path does not exist: {hmm_file}")
+
+    if not overview_path.is_file():
+        raise FileNotFoundError(f"Input overview path does not exist: {overview_file}")
+
+    substrates = load_substrates(hmm_path)
+    genome_gff_lines = load_gff(genome_gff)
+
+    print_gff(overview_file, outfile, dbcan_version, substrates, genome_gff_lines)
+
+
+def load_gff(gff):
+    genome_gff_lines = dict()
+    with fileinput.hook_compressed(gff, "rt") as gff:
+        for line in gff:
+            if line.startswith("##FASTA"):
+                return genome_gff_lines
+
+            fields = line.strip().split("\t")
+            if len(fields) != 9 or fields[2] != "CDS":
+                continue
+
+            if "Parent=" in line:
+                # Get transcript name from the 9th column for mettannotator
+                match = re.search(r"Parent=([^;]+)", fields[8])
+            elif "ID=" in line:
+                # Get transcript name from the 9th column for ASA
+                match = re.search(r"ID=([^;]+)", fields[8])
+            else:
+                logging.error(
+                    "Not sure what gff annotation delimiter is in use. Exiting"
+                )
+                exit(1)
+
+            transcript_name = match.group(1)
+            genome_gff_lines.setdefault(transcript_name, []).append(line)
+    return genome_gff_lines
+
+
+def print_gff(overview_file, outfile, dbcan_version, substrates, genome_gff_lines):
+    with open(outfile, "w") as file_out:
+        file_out.write("##gff-version 3\n")
+        with fileinput.hook_compressed(overview_file, "rt") as file_in:
+            for line in file_in:
+                if line.startswith("MGYG") or line.startswith("ERZ"):
+                    (
+                        transcript,
+                        ec_number_raw,
+                        dbcan_hmmer,
+                        dbcan_sub_ecami,
+                        diamond,
+                        num_of_tools,
+                    ) = line.strip().split("\t")
+                    # EC is reported as 2.4.99.-:5 with :5 meaning 5 proteins in the subfamily have EC 2.4.99.-
+
+                    ec_number = ""
+                    ec_list = ec_number_raw.split("|")
+                    for ec in ec_list:
+                        if ec != "-":
+                            ec_number += ec.split(":")[0] + "|"
+
+                    ec_number = ec_number.strip("|")
+
+                    # Dbcan recommends to use subfamily preference as dbcan_hmmer > dbcan_sub_ecami > diamond
+                    # diamond is messier, so we don't report it here
+                    if dbcan_hmmer != "-":
+                        # the field dbcan_hmmer reports match positions in parentheses, clear them out first:
+                        subfamily = dbcan_hmmer.split("(")[0]
+                    elif dbcan_sub_ecami != "-":
+                        subfamily = dbcan_sub_ecami
+                    else:
+                        continue
+                    cleaned_substrates = ",".join(
+                        sorted(
+                            {
+                                subsrate.strip()
+                                for subsrate in substrates.get(transcript, "N/A").split(
+                                    ","
+                                )
+                            }
+                        )
+                    )
+                    # Assemble information to add to the 9th column
+                    col9_parts = [
+                        f"protein_family={subfamily}",
+                        f"substrate_dbcan-sub={cleaned_substrates}",
+                    ]
+
+                    if ec_number:
+                        col9_parts.append(f"eC_number={ec_number}")
+
+                    col9_parts.append(f"num_tools={num_of_tools}")
+                    col9_text = ";".join(col9_parts)
+
+                    for gff_line in genome_gff_lines[transcript]:
+                        fields = gff_line.strip().split("\t")
+                        # Replace the tool
+                        fields[1] = f"dbCAN:{dbcan_version}"
+                        # Replace the feature
+                        fields[2] = "CAZyme"
+                        # Replace the confidence value
+                        fields[5] = "."
+                        # Keep only the ID in the 9th column
+                        attributes = fields[8].split(";")[0]
+                        # Add dbcan information to the 9th column
+                        attributes = f"{attributes};{col9_text};"
+                        fields[8] = attributes
+                        file_out.write("\t".join(fields) + "\n")
+
+
+def load_substrates(hmm_path):
+    substrates = dict()
+    with fileinput.hook_compressed(hmm_path, "rt") as file_in:
+        header = next(file_in)
+        header_fields = header.strip().split("\t")
+        substrate_idx = header_fields.index("Substrate")
+        gene_idx = header_fields.index("Gene ID")
+        evalue_idx = header_fields.index("E Value")
+        for line in file_in:
+            fields = line.strip().split("\t")
+            if float(fields[evalue_idx]) < 1e-15:  # evalue is the default from dbcan
+                substrate = fields[substrate_idx]
+                if not substrate == "-":
+                    gene_id = fields[gene_idx]
+                    substrates.setdefault(gene_id, []).append(substrate)
+    # resolve cases with multiple substrates
+    for gene_id, substrate_list in substrates.items():
+        substrate_list = list(set(substrate_list))
+        if len(substrate_list) == 1:
+            substrates[gene_id] = substrate_list[0]
+        else:
+            substrates[gene_id] = ",".join(substrate_list)
+    return substrates
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=(
+            "The script takes dbCAN output for a eukaryotic genome and parses it to create a standalone GFF."
+        )
+    )
+    parser.add_argument(
+        "-hmm",
+        dest="hmm_file",
+        required=True,
+        help="Path to the hmm file.",
+    )
+    parser.add_argument(
+        "-ov",
+        dest="overview_file",
+        required=True,
+        help="Path to the overview file.",
+    )
+    parser.add_argument(
+        "-g",
+        dest="genome_gff",
+        required=True,
+        help="Path to the genome GFF.",
+    )
+    parser.add_argument(
+        "-o",
+        dest="outfile",
+        required=True,
+        help="Path to the output file.",
+    )
+    parser.add_argument(
+        "-v",
+        dest="dbcan_ver",
+        required=True,
+        help="dbCAN version used.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(
+        args.hmm_file, args.overview_file, args.genome_gff, args.outfile, args.dbcan_ver
+    )