mgnify-pipelines-toolkit 0.1.9__py3-none-any.whl → 0.2.0__py3-none-any.whl

mgnify_pipelines_toolkit/analysis/assembly/cgc_merge.py

@@ -0,0 +1,424 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import logging
+import os
+import re
+
+from Bio import SeqIO
+
+__version__ = "1.0.4"
+
+
+class Region:
+    def __init__(self, start, end):
+        # if end < start: # assuming that for +/- start always lower
+        #    start, end = end, start
+        self.start = int(start)
+        self.end = int(end)
+
+    def __str__(self):
+        return "[" + str(self.start) + "," + str(self.end) + "]"
+
+    def __ge__(self, other):
+        return self.start >= other.end
+
+    def __gt__(self, other):
+        return self.start > other.end
+
+    def __le__(self, other):
+        return self.end <= other.start
+
+    def __lt__(self, other):
+        return self.end < other.start
+
+    def length(self):
+        return self.end - self.start + 1
+
+    # If 'other' overlaps and has a greater end position
+    def extends_right(self, other):
+        if self.overlaps(other) and self.end > other.end:
+            return True
+        return False
+
+    # For overlapping fragments extend start and end to match other
+    def extend(self, other):
+        if self.overlaps(other):
+            if other.end > self.end:
+                self.end = other.end
+            if other.start < self.start:
+                self.start = other.start
+
+    def within(self, other):
+        if self.start >= other.start and self.end <= other.end:
+            return True
+        return False
+
+    # Return length of overlap between regions
+    def overlaps(self, other):
+        if self > other or other > self:
+            return False
+        # overlap = sum of the individual lengths ...
+        ltot = self.length() + other.length()
+        # ... minus length of the combined region (i.e. min start to max end)
+        lmax = max(self.end, other.end) - min(self.start, other.start) + 1
+        return ltot - lmax
+
+
+# FGS has seq_id/start/end in the fasta files - use those to extract the sequences we want to keep;
+# for prodigal it uses a seq_id/index_number, so need to add an extra field
+class NumberedRegion(Region):
+    def __init__(self, start, end, nid):
+        super().__init__(start, end)
+        self.nid = nid
+
+
+def flatten_regions(regions):
+    """Take a list of regions (possibly overlapping) and return the non-overlapping set"""
+    if len(regions) < 2:
+        return regions
+
+    flattened = []
+    regions = sorted(regions, key=lambda x: x.start)  # sort by start
+    flattened = [regions[0]]
+    regions = regions[1:]  # store the first
+    for region in regions:
+        if not region.overlaps(flattened[-1]):  # doesn't overlap: store new region
+            flattened.append(region)
+        elif region.extends_right(flattened[-1]):  # overlaps to the right: extend previous region
+            flattened[-1].extend(region)
+            # else end < prev end => new region within old: do nothing
+    return flattened
+
+
+def check_against_gaps(regions, candidates):
+    """Given a set of non-overlapping gaps and a list of candidate regions, return the candidates that do not overlap"""
+    regions = sorted(regions, key=lambda line: line.start)
+    candidates = sorted(candidates, key=lambda line: line.start)
+    selected = []
+    r = 0
+    if not len(regions):
+        return candidates  # no existing predictions - all candidates accepted
+
+    for c in candidates:
+        if c < regions[0] or c > regions[-1]:  # outside any of the regions: just append
+            selected.append(c)
+        else:
+            while r < len(regions) - 1 and c >= regions[r]:
+                r += 1
+            if c < regions[r]:  # found a gap
+                selected.append(c)
+
+    return selected
+
+
+def output_prodigal(predictions, files, outputs):
+    """From the combined predictions output the prodigal data"""
+
+    sequence_set = set()
+    for seq in predictions:
+        for strand in ["-", "+"]:
+            for region in predictions[seq][strand]:
+                sequence_set.add("_".join([seq, str(region.nid)]))
+
+    # files contains the .faa and .ffn fasta files
+    for index in [1, 2]:
+        sequences = []
+        for record in SeqIO.parse(files[index], "fasta"):
+            # remove anything after the first space
+            seq_name = record.id.split(" ")[0]
+            # Replace ending * #
+            record.seq = record.seq.rstrip("*")
+            if seq_name in sequence_set:
+                sequences.append(record)
+
+        with open(outputs[index], "a") as output_handle:
+            SeqIO.write(sequences, output_handle, "fasta")
+
+
+def output_fgs(predictions, files, outputs):
+    """From the combined predictions output the FGS data"""
+    sequence_set = set()
+    for seq in predictions:
+        for strand in ["-", "+"]:
+            for region in predictions[seq][strand]:
+                sequence_set.add("_".join([seq, str(region.start), str(region.end), strand]))
+
+    # files contains the .faa and .ffn fasta files
+    for index in [1, 2]:
+        sequences = []
+        for record in SeqIO.parse(files[index], "fasta"):
+            # remove anything after the first space
+            seq_name = record.id.split(" ")[0]
+            # Replace "*" with "X"
+            record.seq = record.seq.replace("*", "X")
+            if seq_name in sequence_set:
+                sequences.append(record)
+
+        with open(outputs[index], "a") as output_handle:
+            SeqIO.write(sequences, output_handle, "fasta")
+
+
+def output_files(predictions, summary, files):
+    """Output all files"""
+    # To avoid that sequences get appended to the merged output files after restart,
+    # make sure the files get deleted if they exist
+    logging.info("Removing output files if they exist.")
+    for file_ in files["merged"]:
+        if os.path.exists(file_):
+            logging.info(f"Removing {file_}")
+            os.remove(file_)
+
+    for caller in predictions:
+        if caller == "fgs":
+            output_fgs(predictions["fgs"], files["fgs"], files["merged"])
+        if caller == "prodigal":
+            output_prodigal(predictions["prodigal"], files["prodigal"], files["merged"])
+
+    with open(files["merged"][0], "w") as sf:
+        sf.write(json.dumps(summary, sort_keys=True, indent=4) + "\n")
+
+
+def get_regions_fgs(fn):
+    """Parse FGS output.
+    Example:
+    # >Bifidobacterium-longum-subsp-infantis-MC2-contig1
+    # 256	2133	-	1	1.263995	I:	D:
+    """
+    regions = {}
+    with open(fn) as f:
+        for line in f:
+            if line[0] == ">":
+                id_ = line.split()[0][1:]
+                regions[id_] = {}
+                regions[id_]["+"] = []
+                regions[id_]["-"] = []
+            else:
+                r = line.split()  # start end strand
+                s = int(r[0])
+                e = int(r[1])
+                regions[id_][r[2]].append(Region(s, e))
+    return regions
+
+
+"""
+# noqa: E501
+This is from cmsearch
+ERR855786.1000054-HWI-M02024:111:000000000-A8H14:1:1115:23473:14586-1 -         LSU_rRNA_bacteria    RF02541   hmm     1224     1446        5      227      +     -    6 0.61   0.8  135.2   2.8e-38 !   -
+"""
+
+
+def get_regions_mask(mask_file):
+    """Parse masked region file (i.e. ncRNA)"""
+    regions = {}
+    with open(mask_file) as f:
+        for line in f:
+            if line[:1] == "#":
+                continue
+            r = line.rstrip().split()
+            id_ = r[0]
+            start = int(r[7])
+            end = int(r[8])
+            if id_ not in regions:
+                regions[id_] = []
+            if start > end:
+                start, end = end, start
+            regions[id_].append(Region(start, end))
+    return regions
+
+
+# # Sequence Data: seqnum=1;seqlen=25479;seqhdr="Bifidobacterium-longum-subsp-infantis-MC2-contig1"
+# # Model Data: version=Prodigal.v2.6.3;run_type=Single;model="Ab initio";gc_cont=59.94;transl_table=11;uses_sd=1
+# >1_1_279_+
+def get_regions_prodigal(fn):
+    """Parse prodigal output"""
+    regions = {}
+    with open(fn) as f:
+        for line in f:
+            if line[:12] == "# Model Data":
+                continue
+            if line[:15] == "# Sequence Data":
+                m = re.search(r'seqhdr="(\S+)"', line)
+                if m:
+                    id_ = m.group(1)
+                regions[id_] = {}
+                regions[id_]["+"] = []
+                regions[id_]["-"] = []
+            else:
+                r = line[1:].rstrip().split("_")
+                n = int(
+                    r[0]
+                )  # also store the index of the fragment - prodigal uses these (rather than coords) to identify sequences in the fasta output
+                s = int(r[1])
+                e = int(r[2])
+                regions[id_][r[3]].append(NumberedRegion(s, e, n))
+    return regions
+
+
+def mask_regions(regions, mask):
+    """Look for overlaps of more than 5 base pairs of the supplied regions against a set of masks
+    This is probably O(N^2) but, in theory, there shouldn't be many mask regions
+    """
+    new_regions = {}
+    for seq in regions:
+        new_regions[seq] = {}
+        for strand in ["-", "+"]:
+            new_regions[seq][strand] = []
+            for r in regions[seq][strand]:
+                if seq in mask:
+                    overlap = 0
+                    for r2 in mask[seq]:
+                        if r.overlaps(r2) > 5:
+                            overlap = 1
+                    if not overlap:
+                        new_regions[seq][strand].append(r)
+                else:
+                    new_regions[seq][strand].append(r)
+
+    return new_regions
+
+
+# FIXME - This won't work if we have only a single set of predictions, but then
+# there's no point in trying to merge
+def merge_predictions(predictions, callers):
+    """Check that we have priorities set of for all callers we have data for"""
+    p = set(callers)
+    new_predictions = {}
+    for type_ in predictions:
+        if type_ not in p:
+            return None
+            # throw here? - if we've used a caller that we don't have a priority for
+
+    # first set of predictions takes priority - just transfer them
+    new_predictions[callers[0]] = predictions[callers[0]]
+
+    # for now assume only two callers, but can be extended
+    new_predictions[callers[1]] = {}  # empty set for second priority caller
+    for seq in predictions[callers[1]]:
+        new_predictions[callers[1]][seq] = {}
+        for strand in ["-", "+"]:
+            new_predictions[callers[1]][seq][strand] = []
+            if seq in predictions[callers[0]]:  # if this sequence already has predictions
+                prev_predictions = flatten_regions(
+                    predictions[callers[0]][seq][strand]
+                )  # non-overlapping set of existing predictions/regions
+                new_predictions[callers[1]][seq][strand] = check_against_gaps(
+                    prev_predictions, predictions[callers[1]][seq][strand]
+                )  # plug new predictions/regions into gaps
+            else:  # no existing predictions: just add them
+                new_predictions[callers[1]][seq][strand] = predictions[callers[1]][seq][strand]
+
+    return new_predictions
+
+
+def get_counts(predictions):
+    total = {}
+    for caller in predictions:
+        total[caller] = 0
+        for sample in predictions[caller]:
+            for strand in ["-", "+"]:
+                total[caller] += len(predictions[caller][sample][strand])
+    return total
+
+
+def combine_main():
+    parser = argparse.ArgumentParser(
+        "MGnify gene caller combiner. This script will merge the gene called by prodigal and fraggenescan (in any order)"
+    )
+    parser.add_argument("-n", "--name", action="store", dest="name", required=True, help="basename")
+    parser.add_argument("-k", "--mask", action="store", dest="mask", required=False, help="Sequence mask file")
+
+    parser.add_argument("-a", "--prodigal-out", action="store", dest="prodigal_out", required=False, help="Stats out prodigal")
+    parser.add_argument("-b", "--prodigal-ffn", action="store", dest="prodigal_ffn", required=False, help="Stats ffn prodigal")
+    parser.add_argument("-c", "--prodigal-faa", action="store", dest="prodigal_faa", required=False, help="Stats faa prodigal")
+
+    parser.add_argument("-d", "--fgs-out", action="store", dest="fgs_out", required=False, help="Stats out FGS")
+    parser.add_argument("-e", "--fgs-ffn", action="store", dest="fgs_ffn", required=False, help="Stats ffn FGS")
+    parser.add_argument("-f", "--fgs-faa", action="store", dest="fgs_faa", required=False, help="Stats faa FGS")
+
+    parser.add_argument(
+        "-p",
+        "--caller-priority",
+        action="store",
+        dest="caller_priority",
+        required=False,
+        choices=["prodigal_fgs", "fgs_prodigal"],
+        default="prodigal_fgs",
+        help="Caller priority.",
+    )
+
+    parser.add_argument("-v", "--verbose", help="verbose output", dest="verbose", action="count", required=False)
+
+    parser.add_argument("--version", action="version", version=f"{__version__}")
+
+    args = parser.parse_args()
+
+    # Set up logging system
+    verbose_mode = args.verbose or 0
+
+    log_level = logging.WARNING
+    if verbose_mode:
+        log_level = logging.DEBUG if verbose_mode > 1 else logging.INFO
+
+    logging.basicConfig(level=log_level, format="%(levelname)s %(asctime)s - %(message)s", datefmt="%Y/%m/%d %I:%M:%S %p")
+
+    summary = {}
+    all_predictions = {}
+    files = {}
+    caller_priority = []
+    if args.caller_priority:
+        caller_priority = args.caller_priority.split("_")
+    else:
+        caller_priority = ["prodigal", "fgs"]
+
+    logging.info(f"Caller priority: 1. {caller_priority[0]}, 2. {caller_priority[1]}")
+
+    if args.prodigal_out:
+        logging.info("Prodigal presented")
+        logging.info("Getting Prodigal regions...")
+        all_predictions["prodigal"] = get_regions_prodigal(args.prodigal_out)
+
+        files["prodigal"] = [args.prodigal_out, args.prodigal_ffn, args.prodigal_faa]
+
+    if args.fgs_out:
+        logging.info("FGS presented")
+        logging.info("Getting FragGeneScan regions ...")
+        all_predictions["fgs"] = get_regions_fgs(args.fgs_out)
+
+        files["fgs"] = [args.fgs_out, args.fgs_ffn, args.fgs_faa]
+
+    summary["all"] = get_counts(all_predictions)
+
+    # Apply mask of ncRNA search
+    logging.info("Masking non coding RNA regions...")
+    if args.mask:
+        logging.info("Reading regions for masking...")
+        mask = get_regions_mask(args.mask)
+        if "prodigal" in all_predictions:
+            logging.info("Masking Prodigal outputs...")
+            all_predictions["prodigal"] = mask_regions(all_predictions["prodigal"], mask)
+        if "fgs" in all_predictions:
+            logging.info("Masking FragGeneScan outputs...")
+            all_predictions["fgs"] = mask_regions(all_predictions["fgs"], mask)
+        summary["masked"] = get_counts(all_predictions)
+
+    # Run the merging step
+    if len(all_predictions) > 1:
+        logging.info("Merging combined gene caller results...")
+        merged_predictions = merge_predictions(all_predictions, caller_priority)
+    else:
+        logging.info("Skipping merging step...")
+        merged_predictions = all_predictions
+    summary["merged"] = get_counts(merged_predictions)
+
+    # Output fasta files and summary (json)
+    logging.info("Writing output files...")
+
+    files["merged"] = [args.name + ext for ext in [".out", ".ffn", ".faa"]]
+
+    output_files(merged_predictions, summary, files)
+
+
+if __name__ == "__main__":
+    combine_main()

mgnify_pipelines_toolkit/analysis/assembly/generate_gaf.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import logging
+
+from mgnify_pipelines_toolkit.analysis.assembly.go_utils import parse_interproscan_tsv
+
+
+def parse_args():
+
+    description = "Go slim pipeline for processing InterProScan results"
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument(
+        "-i", "--ips_input", help="InterProScan result file", required=True
+    )
+    parser.add_argument("-o", "--output", help="GO summary output file", required=True)
+    args = parser.parse_args()
+
+    ips_input = args.ips_input
+    output = args.output
+
+    return ips_input, output
+
+
+# Constants
+PROJECT_NAME = "EBI Metagenomics"
+PROJECT_URL = "http://www.ebi.ac.uk/metagenomics"
+PROJECT_CONTACT = "metagenomics-help@ebi.ac.uk"
+FIXED_TIMESTAMP = "20160528"  # What is this timestamp?
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s: %(message)s"
+)
+
+
+def write_gaf_file(gaf_input_file_path: str, go_id_set: set[str]) -> None:
+    """
+    Create a GO Annotation File (GAF) from a set of GO IDs.
+
+    :param gaf_input_file_path: Path to output GAF file
+    :param go_id_set: Set of GO IDs to include in the file
+    """
+    with open(gaf_input_file_path, "w") as fw:
+        # Write GAF header
+        fw.write("!gaf-version: 2.1\n")
+        fw.write(f"!Project_name: {PROJECT_NAME}\n")
+        fw.write(f"!URL: {PROJECT_URL}\n")
+        fw.write(f"!Contact Email: {PROJECT_CONTACT}\n")
+
+        # Write GO entries
+        for go_id in go_id_set:
+            gaf_entry = "\t".join(
+                [
+                    "EMG",
+                    go_id,
+                    "GO",
+                    "",
+                    go_id,
+                    "PMID:12069591",
+                    "IEA",
+                    "",
+                    "P",
+                    "",
+                    "",
+                    "protein",
+                    "taxon:1310605",
+                    FIXED_TIMESTAMP,
+                    "InterPro",
+                    "",
+                ]
+            )
+            fw.write(gaf_entry + "\n")
+
+    logging.info(f"GAF file created successfully: {gaf_input_file_path}")
+
+
+def main():
+    """
+    Process the InterProScan TSV output and generate a GO annotation file (GAF)).
+    """
+
+    ips_input, output = parse_args()
+
+    # Validate input file
+    if not os.path.exists(ips_input):
+        raise FileNotFoundError(f"Input file not found: {ips_input}")
+
+    if os.path.getsize(ips_input) == 0:
+        logging.warning("Input file is empty. Skipping processing.")
+        return
+
+    # Parse InterProScan result file
+    logging.info(f"Parsing InterProScan input: {ips_input}")
+    go2protein_count_dict = parse_interproscan_tsv(ips_input)
+    logging.info("Finished parsing InterProScan file")
+
+    logging.info("Writing the GAF file")
+    write_gaf_file(f"{output}_ips_annotations.gaf", go2protein_count_dict.keys())
+
+
+if __name__ == "__main__":
+    main()

mgnify_pipelines_toolkit/analysis/assembly/go_utils.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+import logging
+import os
+from pathlib import Path
+import re
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s: %(message)s"
+)
+
+
+def count_and_assign_go_annotations(
+    go2protein_count: defaultdict[int],
+    go_annotations: set[str],
+    num_of_proteins: int,
+    mapped_go_terms: defaultdict[set] = None,
+) -> defaultdict[int]:
+    """Increments counts dictionary for GO terms found on a protein.
+        If used for GO-slim terms, then a mapped_go_terms dictionary is required
+        (with default value of None).
+    :param go2protein_count: Current state of the count dictionary
+    :type go2protein_count: defaultdict[int]
+    :param go_annotations: GO-terms to be incremented
+    :type go_annotations: set[str]
+    :param num_of_proteins: Number of proteins to be incremented (not sure if we need this, see TODO below)
+    :type num_of_proteins: int
+    :param mapped_go_terms: Dictionary containin the GO-slim conversion
+    :type mapped_go_terms: defaultdict(set)
+    :return: _description_
+    :rtype: _type_
+    """
+
+    if not mapped_go_terms:
+        for go_id in go_annotations:
+            go2protein_count[go_id] += num_of_proteins
+    else:
+        slim_go_ids_set = set()
+        for go_annotation in go_annotations:
+            mapped_go_ids = mapped_go_terms.get(go_annotation)
+            if mapped_go_ids:
+                slim_go_ids_set.update(mapped_go_ids)
+        for slim_go_id in slim_go_ids_set:
+            go2protein_count[slim_go_id] += num_of_proteins
+
+    return go2protein_count
+
+
+def parse_interproscan_tsv(ips_file: Path, mapped_go_terms: dict = None) -> dict:
+    """Parses an InterProScan output line by line and return a dictionary of counts for the different GO terms.
+        The structure of the IPS file is one annotation per line, some of which will be GO terms. If a protein
+        has multiple annotations, then those annotations will follow one by one in order. This function therefore
+        parses the file by keeping some flags to track which proteins it's currently on, and which GO terms were found
+        for said protein. It then finally increments the count of said protein's GO terms when it's done being parsed.
+    :param ips_file: InterProScan .tsv file
+    :type ips_file: Path
+    :return: Dictionary containing GO term counts in the input InterProScan file
+    :rtype: dict
+    """
+
+    go2protein_count = defaultdict(int)
+    if not os.path.exists(ips_file):
+        logging.error(f"The InterProScan file {ips_file} could not be found. Exiting.")
+        exit(1)
+
+    num_of_proteins_with_go = 0
+    total_num_of_proteins = 0
+    line_counter = 0
+    previous_protein_acc = None
+    go_annotations_single_protein = set()
+
+    fr = open(ips_file, "r")
+    go_pattern = re.compile("GO:\\d+")
+
+    for line in fr:
+        # IPS files are parsed line by line - the same protein accession will appear multiple lines in a row with different annotation
+        line_counter += 1
+        line = line.strip()
+        chunks = line.split("\t")
+        # Get protein accession
+        current_protein_acc = chunks[0]
+
+        # TODO: not sure if this line is needed - do we ever have more than one protein in a single line of IPS?
+        # Will keep just in case
+        num_of_proteins = len(current_protein_acc.split("|"))
+
+        # If we're at a new protein accession in the IPS file then we finally increment
+        # the go2protein_count dictionary for each term that was found in that protein
+        if current_protein_acc != previous_protein_acc:
+            total_num_of_proteins += 1
+            if len(go_annotations_single_protein) > 0:
+                num_of_proteins_with_go += 1
+                go2protein_count = count_and_assign_go_annotations(
+                    go2protein_count,
+                    go_annotations_single_protein,
+                    num_of_proteins,
+                    mapped_go_terms,
+                )
+            # reset GO id set because we hit a new protein accession
+            go_annotations_single_protein = set()
+            previous_protein_acc = current_protein_acc
+
+        # Parse out GO annotations
+        # GO annotations are associated to InterPro entries (InterPro entries start with 'IPR')
+        # Than use the regex to extract the GO Ids (e.g. GO:0009842)
+        if len(chunks) >= 13 and chunks[11].startswith("IPR"):
+            for go_annotation in go_pattern.findall(line):
+                go_annotations_single_protein.add(go_annotation)
+
+    # Do final counting for the last protein
+    go2protein_count = count_and_assign_go_annotations(
+        go2protein_count,
+        go_annotations_single_protein,
+        num_of_proteins,
+        mapped_go_terms,
+    )
+
+    fr.close()
+
+    return go2protein_count

mgnify_pipelines_toolkit/analysis/assembly/summarise_goslims.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2024 EMBL - European Bioinformatics Institute
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from collections import defaultdict
+import logging
+import os
+from pathlib import Path
+
+from mgnify_pipelines_toolkit.analysis.assembly.go_utils import parse_interproscan_tsv
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s: %(message)s"
+)
+
+
+def parse_args():
+
+    description = "Go slim pipeline."
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument(
+        "-go", "--go_obo", help="Gene Ontology basic file.", required=True
+    )
+    parser.add_argument(
+        "-gb", "--go_banding", help="Subset GO banding file.", required=True
+    )
+    parser.add_argument(
+        "-gaf",
+        "--gaf_input",
+        help="GAF file, generated by generate_gaf.py",
+        required=True,
+    )
+    parser.add_argument(
+        "-i", "--ips_input", help="InterProScan result file.", required=True
+    )
+    parser.add_argument("-o", "--output", help="GO summary output file.", required=True)
+    args = parser.parse_args()
+
+    go_obo = args.go_obo
+    go_banding = args.go_banding
+    gaf_input = args.gaf_input
+    ips_input = args.ips_input
+    output = args.output
+
+    return go_obo, go_banding, gaf_input, ips_input, output
+
+
+def parse_mapped_gaf_file(gaf_file: Path) -> defaultdict[set]:
+
+    mapped_go_dict = defaultdict(set)
+    if os.path.exists(gaf_file):
+        handle = open(gaf_file, "r")
+        for line in handle:
+            if not line.startswith("!"):
+                line = line.strip()
+                splitted_line = line.split("\t")
+                go_id = splitted_line[1]
+                mapped_go_id = splitted_line[4]
+                mapped_go_dict[go_id].add(mapped_go_id)
+
+    return mapped_go_dict
+
+
+def get_go_slim_summary(go_slim_banding_file, goslims2_protein_count):
+    summary = []
+
+    fr = open(go_slim_banding_file, "r")
+
+    for line in fr:
+        if line.startswith("GO"):
+            line = line.strip()
+            line_chunks = line.split("\t")
+            go_id = line_chunks[0]
+            term = line_chunks[1]
+            category = line_chunks[2]
+            # Default value for the count
+            count = 0
+            if go_id in goslims2_protein_count:
+                count = goslims2_protein_count[go_id]
+            summary.append((go_id, term, category, count))
+    return summary
+
+
+def write_go_summary_to_file(go_summary, output_file):
+    fw = open(output_file, "w")
+    for go, term, category, count in go_summary:
+        fw.write('","'.join(['"' + go, term, category, str(count) + '"']) + "\n")
+    fw.close()
+
+
+def parse_gene_ontology(obo_file):
+    """
+    Parses OBO formatted file.
+    :param obo_file:
+    :return:
+    """
+    go_term_tuples = []
+    fr = open(obo_file, "r")
+    id, term, category = "", "", ""
+    for line in fr:
+        line = line.strip()
+        split_line = line.split(": ")
+        if line.startswith("id:"):
+            id = split_line[1]
+        elif line.startswith("name:"):
+            term = split_line[1]
+        elif line.startswith("namespace"):
+            category = split_line[1]
+        else:
+            if id.startswith("GO:") and id and term and category:
+                item = (id, term, category)
+                go_term_tuples.append(item)
+                id, term, category = "", "", ""
+    fr.close()
+    return go_term_tuples
+
+
+def get_full_go_summary(core_gene_ontology, go2protein_count_dict, top_level_go_ids):
+    summary = []
+
+    for go_id, term, category in core_gene_ontology:
+
+        if (go_id in go2protein_count_dict) and (
+            go_id not in top_level_go_ids
+        ):  # make sure that top level terms are not included (they tell you nothing!)
+            count = go2protein_count_dict[go_id]
+            summary.append((go_id, term, category, count))
+    summary.sort(key=lambda x: (x[2], -x[3]))
+    return summary
+
+
+def main():
+
+    go_obo, go_banding, gaf_input, ips_input, output = parse_args()
+
+    logging.info("Parsing the InterProScan input: " + ips_input)
+    go2protein_count_dict = parse_interproscan_tsv(ips_input)
+    logging.info("Finished parsing.")
+
+    # Generate GO summary
+    logging.info("Loading full Gene ontology: " + go_obo)
+    go_term_tuples = parse_gene_ontology(go_obo)
+    logging.info("Finished loading.")
+
+    logging.info("Generating full GO summary...")
+    top_level_go_ids = ["GO:0008150", "GO:0003674", "GO:0005575"]
+    full_go_summary = get_full_go_summary(
+        go_term_tuples, go2protein_count_dict, top_level_go_ids
+    )
+    logging.info("Finished generation.")
+
+    logging.info("Writing full GO summary: " + output)
+    write_go_summary_to_file(full_go_summary, output)
+    logging.info("Finished writing.")
+
+    mapped_go_terms = parse_mapped_gaf_file(gaf_input)
+    logging.info("Getting GO slim counts")
+    goslims2_protein_count = parse_interproscan_tsv(ips_input, mapped_go_terms)
+
+    go_slim_summary = get_go_slim_summary(go_banding, goslims2_protein_count)
+    go_slim_output_file = output + "_slim"
+    logging.info("Writing GO slim summary: " + go_slim_output_file)
+    write_go_summary_to_file(go_slim_summary, go_slim_output_file)
+    logging.info("Finished writing.")
+
+
+if __name__ == "__main__":
+    main()

{mgnify_pipelines_toolkit-0.1.9.dist-info → mgnify_pipelines_toolkit-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mgnify_pipelines_toolkit
-Version: 0.1.9
+Version: 0.2.0
 Summary: Collection of scripts and tools for MGnify pipelines
 Author-email: MGnify team <metagenomics-help@ebi.ac.uk>
 License: Apache Software License 2.0

{mgnify_pipelines_toolkit-0.1.9.dist-info → mgnify_pipelines_toolkit-0.2.0.dist-info}/RECORD

@@ -14,6 +14,10 @@ mgnify_pipelines_toolkit/analysis/amplicon/rev_comp_se_primers.py,sha256=19NgCYE
 mgnify_pipelines_toolkit/analysis/amplicon/standard_primer_matching.py,sha256=RDPsaWKf0wIDwvCHXyRCh2zSJf3y9E7uOhHjaAeX8bY,11099
 mgnify_pipelines_toolkit/analysis/assembly/add_rhea_chebi_annotation.py,sha256=69iK8vtG5xFgYQ-KJiSQlaxuhSoxzcO59eNLyDS3nm0,4323
 mgnify_pipelines_toolkit/analysis/assembly/antismash_gff_builder.py,sha256=OODl3XhLvksvG5RZn1iHZlg9L3DXiWIkyxJ6o-y6oeg,6949
+mgnify_pipelines_toolkit/analysis/assembly/cgc_merge.py,sha256=u6r_1GRGgBAJQvU_t5Rtl3ZYjTtGJGd5yHCobtL9ob0,15405
+mgnify_pipelines_toolkit/analysis/assembly/generate_gaf.py,sha256=U1Ls3O0CQmukmoyUwEAEN11jHUKuCdS-qVkr5ai243I,3582
+mgnify_pipelines_toolkit/analysis/assembly/go_utils.py,sha256=vsYaFJ_cmbo6DXlWs_X8wpZJfMQOq1CrLX4-3owmYjI,5447
+mgnify_pipelines_toolkit/analysis/assembly/summarise_goslims.py,sha256=RthgLO3YTO_JGMC7Nx2JDrowXRimnOtVUDkM1l31rt4,5834
 mgnify_pipelines_toolkit/analysis/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mgnify_pipelines_toolkit/analysis/shared/fastq_suffix_header_check.py,sha256=H5ccd1e_e5dk8vhVOvHLK1lknYbRPbnqPjULCYnU0FQ,4021
 mgnify_pipelines_toolkit/analysis/shared/get_subunits.py,sha256=xl5HduWtGPWiI9yqsjQ3itIzwHSxF2ig5KgjLXmj9EE,4772
@@ -31,9 +35,9 @@ mgnify_pipelines_toolkit/schemas/schemas.py,sha256=fd2xCoA1Ty-XaMG9U_gxNcBokHiYE
 mgnify_pipelines_toolkit/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mgnify_pipelines_toolkit/utils/fasta_to_delimited.py,sha256=GbNT7clHso21w_1PbPpWKVRd5bNs_MDbGXt8XVIGl2o,3991
 mgnify_pipelines_toolkit/utils/get_mpt_version.py,sha256=zsQ4TuR4vpqYa67MgIdopdscsS0DVJdy4enRe1nCjSs,793
-mgnify_pipelines_toolkit-0.1.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-mgnify_pipelines_toolkit-0.1.9.dist-info/METADATA,sha256=JttpU3vw2IbWoETCweOs6T5guAH_ip3aiEQ2HhJmGVo,6068
-mgnify_pipelines_toolkit-0.1.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-mgnify_pipelines_toolkit-0.1.9.dist-info/entry_points.txt,sha256=SHRTFOo4hHWv3ORwhjnKQY554et4dXS1vVwUVmfKFHM,1844
-mgnify_pipelines_toolkit-0.1.9.dist-info/top_level.txt,sha256=xA_wC7C01V3VwuDnqwRM2QYeJJ45WtvF6LVav4tYxuE,25
-mgnify_pipelines_toolkit-0.1.9.dist-info/RECORD,,
+mgnify_pipelines_toolkit-0.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+mgnify_pipelines_toolkit-0.2.0.dist-info/METADATA,sha256=TR0FyKtC0Xyj0zvDCPiYsI6bGbZI9GkQ8fiC1WWomEk,6068
+mgnify_pipelines_toolkit-0.2.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+mgnify_pipelines_toolkit-0.2.0.dist-info/entry_points.txt,sha256=60Nov738JAon-uZXUqqjOGy4TXxgS4xtxqYhAi12HY0,2084
+mgnify_pipelines_toolkit-0.2.0.dist-info/top_level.txt,sha256=xA_wC7C01V3VwuDnqwRM2QYeJJ45WtvF6LVav4tYxuE,25
+mgnify_pipelines_toolkit-0.2.0.dist-info/RECORD,,

{mgnify_pipelines_toolkit-0.1.9.dist-info → mgnify_pipelines_toolkit-0.2.0.dist-info}/entry_points.txt

@@ -3,10 +3,12 @@ add_rhea_chebi_annotation = mgnify_pipelines_toolkit.analysis.assembly.add_rhea_
 are_there_primers = mgnify_pipelines_toolkit.analysis.amplicon.are_there_primers:main
 assess_inflection_point_mcp = mgnify_pipelines_toolkit.analysis.amplicon.assess_inflection_point_mcp:main
 assess_mcp_proportions = mgnify_pipelines_toolkit.analysis.amplicon.assess_mcp_proportions:main
+cgc_merge = mgnify_pipelines_toolkit.analysis.assembly.cgc_merge:combine_main
 classify_var_regions = mgnify_pipelines_toolkit.analysis.amplicon.classify_var_regions:main
 fasta_to_delimited = mgnify_pipelines_toolkit.utils.fasta_to_delimited:main
 fastq_suffix_header_check = mgnify_pipelines_toolkit.analysis.shared.fastq_suffix_header_check:main
 find_mcp_inflection_points = mgnify_pipelines_toolkit.analysis.amplicon.find_mcp_inflection_points:main
+generate_gaf = mgnify_pipelines_toolkit.analysis.assembly.generate_gaf:main
 get_mpt_version = mgnify_pipelines_toolkit.utils.get_mpt_version:main
 get_subunits = mgnify_pipelines_toolkit.analysis.shared.get_subunits:main
 get_subunits_coords = mgnify_pipelines_toolkit.analysis.shared.get_subunits_coords:main
@@ -19,3 +21,4 @@ remove_ambiguous_reads = mgnify_pipelines_toolkit.analysis.amplicon.remove_ambig
 rev_comp_se_primers = mgnify_pipelines_toolkit.analysis.amplicon.rev_comp_se_primers:main
 standard_primer_matching = mgnify_pipelines_toolkit.analysis.amplicon.standard_primer_matching:main
 study_summary_generator = mgnify_pipelines_toolkit.analysis.shared.study_summary_generator:main
+summarise_goslims = mgnify_pipelines_toolkit.analysis.assembly.summarise_goslims:main