mgnify-pipelines-toolkit 0.2.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

mgnify_pipelines_toolkit/analysis/assembly/gff_file_utils.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2025 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import csv
+
+
+def write_results_to_file(
+    outfile, header, main_gff_extended, fasta, ncrnas, trnas, crispr_annotations
+):
+    with open(outfile, "w") as file_out:
+        file_out.write("\n".join(header) + "\n")
+        contig_list = list(main_gff_extended.keys())
+        # check if there are any contigs that don't have CDS; if so add them in
+        contig_list = check_for_additional_keys(
+            ncrnas, trnas, crispr_annotations, contig_list
+        )
+        for contig in contig_list:
+            sorted_pos_list = sort_positions(
+                contig, main_gff_extended, ncrnas, trnas, crispr_annotations
+            )
+            for pos in sorted_pos_list:
+                for my_dict in (ncrnas, trnas, crispr_annotations, main_gff_extended):
+                    if contig in my_dict and pos in my_dict[contig]:
+                        for line in my_dict[contig][pos]:
+                            if type(line) is str:
+                                file_out.write(f"{line}\n")
+                            else:
+                                for element in line:
+                                    file_out.write(element)
+        for line in fasta:
+            file_out.write(f"{line}\n")
+
+
+def sort_positions(contig, main_gff_extended, ncrnas, trnas, crispr_annotations):
+    sorted_pos_list = list()
+    for my_dict in (main_gff_extended, ncrnas, trnas, crispr_annotations):
+        if contig in my_dict:
+            sorted_pos_list += list(my_dict[contig].keys())
+    return sorted(list(set(sorted_pos_list)))
+
+
+def check_for_additional_keys(ncrnas, trnas, crispr_annotations, contig_list):
+    for my_dict in (ncrnas, trnas, crispr_annotations):
+        dict_keys = set(my_dict.keys())
+        absent_keys = dict_keys - set(contig_list)
+        if absent_keys:
+            contig_list = contig_list + list(absent_keys)
+    return contig_list
+
+
+def print_pseudogene_report(pseudogene_report_dict, pseudogene_report_file):
+    with open(pseudogene_report_file, "w") as file_out:
+        writer = csv.writer(file_out, delimiter="\t", lineterminator="\n")
+        # Print header
+        writer.writerow(
+            [
+                "ID",
+                "Pseudogene according to Bakta/Prokka",
+                "Pseudogene according to Pseudofinder",
+                "AntiFam hit",
+            ]
+        )
+
+        all_keys = ["gene_caller", "pseudofinder", "antifams"]
+        for protein, attributes in pseudogene_report_dict.items():
+            # Fill in missing attributes with False
+            line = [protein] + [str(attributes.get(key, False)) for key in all_keys]
+            writer.writerow(line)

mgnify_pipelines_toolkit/analysis/assembly/gff_toolkit.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2025 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+
+from gff_annotation_utils import get_ncrnas, get_trnas, load_annotations, load_crispr
+from gff_file_utils import write_results_to_file, print_pseudogene_report
+
+
+def main(
+    gff,
+    ipr_file,
+    eggnog_file,
+    sanntis_file,
+    crispr_file,
+    amr_file,
+    antismash_file,
+    gecco_file,
+    dbcan_file,
+    defense_finder_file,
+    pseudofinder_file,
+    rfam_file,
+    trnascan_file,
+    outfile,
+    pseudogene_report_file,
+):
+    # load annotations and add them to existing CDS
+    # here header contains leading GFF lines starting with "#",
+    # main_gff_extended is a dictionary that contains GFF lines with added in additional annotations
+    # fasta is the fasta portion of the original GFF file
+    # pseudogene_report_dict is the information on detected pseudogene which can be optionally printed
+    # to a separate output file
+    header, main_gff_extended, fasta, pseudogene_report_dict = load_annotations(
+        gff,
+        eggnog_file,
+        ipr_file,
+        sanntis_file,
+        amr_file,
+        antismash_file,
+        gecco_file,
+        dbcan_file,
+        defense_finder_file,
+        pseudofinder_file,
+    )
+    ncrnas = {}
+    if rfam_file:
+        ncrnas = get_ncrnas(rfam_file)
+    trnas = {}
+    if trnascan_file:
+        trnas = get_trnas(trnascan_file)
+    crispr_annotations = {}
+    if crispr_file:
+        crispr_annotations = load_crispr(crispr_file)
+
+    write_results_to_file(outfile, header, main_gff_extended, fasta, ncrnas, trnas, crispr_annotations)
+    if pseudogene_report_file:
+        print_pseudogene_report(pseudogene_report_dict, pseudogene_report_file)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="The script extends a user-provided base GFF annotation file by incorporating "
+                    "information extracted from the user-provided outputs of supplementary annotation tools.",
+    )
+    parser.add_argument(
+        "-g",
+        dest="gff_input",
+        required=True,
+        help="GFF input file containing the base annotation",
+    )
+    parser.add_argument(
+        "-i",
+        dest="ips",
+        help="InterProScan annotation results (TSV)",
+        required=False,
+    )
+    parser.add_argument(
+        "-e",
+        dest="eggnog",
+        help="EggNOG mapper annotation results (TSV)",
+        required=False,
+    )
+    parser.add_argument(
+        "-s",
+        dest="sanntis",
+        help="SanntiS results",
+        required=False,
+    )
+    parser.add_argument(
+        "-c",
+        dest="crispr",
+        help="CRISPRCasFinder results for the cluster rep (pre-filtered high quality GFF)",
+        required=False,
+    )
+    parser.add_argument(
+        "-a",
+        dest="amr",
+        help="The TSV file produced by AMRFinderPlus",
+        required=False,
+    )
+    parser.add_argument(
+        "--antismash",
+        help="The GFF file produced by AntiSMASH post-processing script",
+        required=False,
+    )
+    parser.add_argument(
+        "--gecco",
+        help="The GFF file produced by GECCO",
+        required=False,
+    )
+    parser.add_argument(
+        "--dbcan",
+        help="The GFF file produced by dbCAN post-processing script",
+        required=False,
+    )
+    parser.add_argument(
+        "--defense-finder",
+        help="The GFF file produced by Defense Finder post-processing script",
+        required=False,
+    )
+    parser.add_argument(
+        "--pseudofinder",
+        help="The GFF file produced by the Pseudofinder post-processing script",
+        required=False,
+    )
+    parser.add_argument("-r", dest="rfam", help="Rfam results", required=False)
+    parser.add_argument(
+        "-t", dest="trnascan", help="tRNAScan-SE results", required=False
+    )
+    parser.add_argument("-o", dest="outfile", help="Outfile name", required=True)
+    parser.add_argument(
+        "--pseudogene-report", help="Pseudogene report filename", required=False
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    main(
+        args.gff_input,
+        args.ips,
+        args.eggnog,
+        args.sanntis,
+        args.crispr,
+        args.amr,
+        args.antismash,
+        args.gecco,
+        args.dbcan,
+        args.defense_finder,
+        args.pseudofinder,
+        args.rfam,
+        args.trnascan,
+        args.outfile,
+        args.pseudogene_report,
+         )

mgnify_pipelines_toolkit/analysis/assembly/go_utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2024 EMBL - European Bioinformatics Institute
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.

mgnify_pipelines_toolkit/analysis/assembly/summarise_goslims.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2024 EMBL - European Bioinformatics Institute
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.

mgnify_pipelines_toolkit/analysis/shared/dwc_summary_generator.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from collections import defaultdict
+import pathlib
+import logging
+import requests
+
+import pandas as pd
+import pyfastx
+
+logging.basicConfig(level=logging.DEBUG)
+
+URL = "https://www.ebi.ac.uk/ena/portal/api/search?result"
+RUNS_URL = f"{URL}=read_run&fields=secondary_study_accession,sample_accession&limit=10&format=json&download=false"
+SAMPLES_URL = f"{URL}=sample&fields=lat,lon,collection_date,depth&limit=10&format=json&download=false"
+HEADERS = {"Accept": "application/json"}
+
+
+def parse_args():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-i",
+        "--input_path",
+        required=True,
+        type=str,
+        help="Directory where ASV files are.",
+    )
+    parser.add_argument(
+        "-r",
+        "--runs",
+        required=True,
+        type=str,
+        help="Path to CSV file containing successful analyses generated by the pipeline (columns: `run, status`)",
+    )
+    parser.add_argument(
+        "-o", "--output", required=True, type=str, help="Path to output directory."
+    )
+
+    args = parser.parse_args()
+
+    input_path = args.input_path
+    runs = args.runs
+    output = args.output
+
+    return input_path, runs, output
+
+
+def get_metadata_from_run_acc(run_acc):
+
+    query = f"{RUNS_URL}&includeAccessions={run_acc}"
+    res_run = requests.get(query, headers=HEADERS)
+
+    if res_run.status_code != 200:
+        logging.error(f"Data not found for run {run_acc}")
+        return False
+
+    sample_acc = res_run.json()[0]["sample_accession"]
+
+    query = f"{SAMPLES_URL}&includeAccessions={sample_acc}"
+    res_sample = requests.get(query, headers=HEADERS)
+
+    full_res_dict = res_run.json()[0] | res_sample.json()[0]
+
+    fields_to_clean = ["lat", "lon", "depth"]
+
+    for field in fields_to_clean:
+        val = full_res_dict[field]
+        if val == "":
+            full_res_dict[field] = "NA"
+
+    if full_res_dict["collection_date"] == "":
+        full_res_dict["collectionDate"] = "NA"
+    else:
+        full_res_dict["collectionDate"] = full_res_dict["collection_date"]
+
+    del full_res_dict["collection_date"]
+
+    res_df = pd.DataFrame(full_res_dict, index=[0])
+    res_df.columns = [
+        "RunID",
+        "SampleID",
+        "StudyID",
+        "decimalLongitude",
+        "depth",
+        "decimalLatitude",
+        "collectionDate",
+    ]
+
+    return res_df
+
+
+def get_all_metadata_from_runs(runs):
+
+    run_metadata_dict = defaultdict(dict)
+
+    for run in runs:
+        res_df = get_metadata_from_run_acc(run)
+        if res_df is not False:
+            run_metadata_dict[run] = res_df
+
+    return run_metadata_dict
+
+
+def cleanup_taxa(df):
+
+    df.pop("Kingdom")
+    cleaned_df = df.rename(columns={"Superkingdom": "Kingdom", "asv": "ASVID"})
+
+    ranks = ["Kingdom", "Phylum", "Class", "Order", "Family", "Genus", "Species"]
+
+    for rank in ranks:
+        cleaned_df[rank] = cleaned_df[rank].apply(
+            lambda x: x.split("__")[1] if pd.notnull(x) else "NA"
+        )
+
+    for rank in ranks:
+        cleaned_df[rank] = cleaned_df[rank].apply(lambda x: x if x != "" else "NA")
+
+    cleaned_df = cleaned_df[
+        [
+            "ASVID",
+            "StudyID",
+            "SampleID",
+            "RunID",
+            "decimalLongitude",
+            "decimalLatitude",
+            "depth",
+            "collectionDate",
+            "Kingdom",
+            "Phylum",
+            "Class",
+            "Order",
+            "Family",
+            "Genus",
+            "Species",
+            "ASVSeq",
+        ]
+    ]
+
+    return cleaned_df
+
+
+def get_asv_dict(runs_df, root_path):
+
+    asv_dict = {}
+    for i in range(0, len(runs_df)):
+        run_acc = runs_df.loc[i, "run"]
+        status = runs_df.loc[i, "status"]
+
+        if status != "all_results":
+            continue
+
+        tax_file = sorted(
+            list(
+                (pathlib.Path(root_path) / run_acc / "asv").glob(
+                    "*_DADA2-SILVA_asv_tax.tsv"
+                )
+            )
+        )[0]
+        count_files = sorted(
+            list(pathlib.Path(f"{root_path}/{run_acc}/asv").glob("*S-V*/*.tsv"))
+        )
+
+        asv_fasta_file = sorted(
+            list(pathlib.Path(f"{root_path}/{run_acc}/asv").glob("*_asv_seqs.fasta"))
+        )[0]
+        fasta = pyfastx.Fasta(str(asv_fasta_file), build_index=False)
+        asv_fasta_dict = {name: seq for name, seq in fasta}
+        asv_fasta_df = pd.DataFrame(asv_fasta_dict, index=["ASVSeq"]).transpose()
+        asv_fasta_df["asv"] = asv_fasta_df.index
+        run_tax_df = pd.read_csv(tax_file, sep="\t")
+
+        count_dfs = []
+
+        for count_file in count_files:
+            count_df = pd.read_csv(count_file, sep="\t")
+            count_dfs.append(count_df)
+
+        all_ampregions_count_df = pd.concat(count_dfs)
+        merged_df = all_ampregions_count_df.merge(
+            run_tax_df, left_on="asv", right_on="ASV"
+        )
+        merged_df.pop("ASV")
+        run_col = [run_acc] * len(merged_df)
+        merged_df["RunID"] = run_col
+        merged_df = merged_df.merge(asv_fasta_df, on="asv")
+        asv_dict[run_acc] = merged_df
+
+    return asv_dict
+
+
+def main():
+
+    input_path, runs, output = parse_args()
+
+    root_path = pathlib.Path(input_path)
+
+    if not root_path.exists():
+        logging.error(f"Results path does not exist: {root_path}")
+        exit(1)
+
+    runs_df = pd.read_csv(runs, names=["run", "status"])
+
+    all_runs = runs_df.run.to_list()
+    run_metadata_dict = get_all_metadata_from_runs(all_runs)
+    asv_dict = get_asv_dict(runs_df, root_path)
+
+    all_merged_df = []
+
+    for run in all_runs:
+        if run in asv_dict.keys() and run in run_metadata_dict.keys():
+            run_asv_data = asv_dict[run]
+            run_metadata = run_metadata_dict[run]
+            run_merged_result = run_metadata.merge(run_asv_data, on="RunID")
+            all_merged_df.append(run_merged_result)
+
+    final_df = pd.concat(all_merged_df, ignore_index=True)
+    final_df = cleanup_taxa(final_df)
+
+    final_df.to_csv(f"{output}_dwcready.csv", index=False, na_rep="NA")
+
+
+if __name__ == "__main__":
+    main()

mgnify_pipelines_toolkit/analysis/shared/fastq_suffix_header_check.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2024 EMBL - European Bioinformatics Institute
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mgnify_pipelines_toolkit/analysis/shared/get_subunits.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2024 EMBL - European Bioinformatics Institute
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mgnify_pipelines_toolkit/analysis/shared/get_subunits_coords.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2024 EMBL - European Bioinformatics Institute
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mgnify_pipelines_toolkit/analysis/shared/library_strategy_check.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2024 EMBL - European Bioinformatics Institute
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

mgnify_pipelines_toolkit/analysis/shared/mapseq2biom.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2024 EMBL - European Bioinformatics Institute
+# Copyright 2024-2025 EMBL - European Bioinformatics Institute
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.