GraDiv 0.1.0__py3-none-any.whl

gradiv/__init__.py

@@ -0,0 +1,7 @@
+def get_version():
+    from importlib.metadata import version
+    return version("gradiv")
+
+__version__ = get_version()
+
+#add import classes

gradiv/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)

gradiv/algo.py

@@ -0,0 +1,163 @@
+import gradiv.pggraph as pggraph
+import gradiv.polymorphism as polymorphism
+from pybedtools import BedTool
+import typing
+import logging
+
+def call_polymorphisms (graph_index:'pggraph.GraphIndex', id_table:bool=True, seq_table:bool=True) -> int :
+    """
+    First version of the "child" algorithm. Detects polymorphisms in a pangenome graph and outputs them in CSV files.
+    :param graph_index: GraphIndex instance
+    :param id_table : Flag to output genotype table with node IDs
+    :param seq_table : Flag to output genotype table with sequences
+    :return: None
+    """
+    logger = logging.getLogger("gradiv")
+    logger.debug("Calling polymorphism functions")
+
+    # output polymorphisms to a genotype table file
+    header = "previous_node"
+    for sample in graph_index.samples:
+        header += f',{sample}'
+    header += '\n'
+    f_id = None
+    f_seq = None
+    if id_table:
+        f_id = open(str(graph_index.id_table_path), 'w')
+        f_id.write(header)
+    if seq_table:
+        f_seq = open(str(graph_index.seq_table_path), 'w')
+        f_seq.write(header)
+
+    # scan the graph for atomic polymorphisms
+    nb_poly = 0
+    for segment in graph_index.bam_file_for_parsing :
+        # initialize variables
+        reversed_parent = None
+        is_reversed = False
+
+        parent = pggraph.Node(bam_segment=segment, sense=1)
+        parent.current_nodes = {parent.get_full_id(): parent}
+        # get all nodes implicated
+        links, forward_nodes, reverse_nodes, stop = parent.get_links(graph_index=graph_index)
+        if stop:
+            continue
+        nodes_both_senses = forward_nodes.intersection(reverse_nodes)
+        if parent.identifier in nodes_both_senses:
+            is_reversed = True
+            reversed_parent = pggraph.Node(bam_segment=segment, sense=-1)
+            reversed_parent.current_nodes = parent.current_nodes
+            parent.current_nodes[reversed_parent.get_full_id()] = reversed_parent
+
+        # create nodes
+        parent.create_nodes(nodes_id_set=forward_nodes, sense=1, graph_index=graph_index)
+        parent.create_nodes(nodes_id_set=reverse_nodes, sense=-1, graph_index=graph_index)
+
+        # resolve segments' samples when in both senses
+        for node_id in nodes_both_senses:
+            parent.current_nodes[f'+{node_id}'].samples, parent.current_nodes[f'-{node_id}'].samples = get_samples_per_sense(parent.current_nodes[f'+{node_id}'], graph_index=graph_index)
+
+        parent.add_children(links=links)
+        parent.add_parents(graph_index=graph_index)
+
+        # call polymorphisms
+        poly = call_polymorphism_from_parent(parent)
+        if poly.node_per_sample:
+            write_out_polymorphism(poly, parent, graph_index, f_id, f_seq)
+            nb_poly += 1
+        if is_reversed:
+            poly = call_polymorphism_from_parent(reversed_parent)
+            if poly.node_per_sample:
+                write_out_polymorphism(poly, reversed_parent, graph_index, f_id, f_seq)
+                nb_poly += 1
+    if f_id:
+         f_id.close()
+    if f_seq:
+         f_seq.close()
+    return nb_poly
+
+
+def get_samples_per_sense (node:pggraph.Node, graph_index:pggraph.GraphIndex) -> tuple[set, set] :
+    """
+    Associate samples to the forward or reverse segment. Used only when both segments are present.
+    :param node: Node instance
+    :param graph_index: GraphIndex instance
+    :return: tuple of sets of samples (forward, reverse)
+    """
+    segment_info = (set(), set())
+    for sample in node.samples:
+        bed_file = BedTool(str(graph_index.bed_files[sample]))
+        sample_node_presence = search_bed(node.identifier, bed_file)
+        if sample_node_presence[0] :
+            segment_info[0].add(sample)
+        elif sample_node_presence[1] :
+            segment_info[1].add(sample)
+        else:
+            raise ValueError("Segment not found")
+    return segment_info
+
+def search_bed (node_id:str, bed_file:BedTool) -> tuple[bool, bool] :
+    """
+    Browse BED files and return presence or absence of forward and reverse given segment per sample
+    :param node_id: Full node ID (sign and ID)
+    :param bed_file: pybedtools BED file
+    :return: tuple of booleans (forward, reverse)
+    """
+    presence_forward = False
+    presence_reverse = False
+    for interval in bed_file:
+        if len(interval.fields) >= 4:
+            segment_id = interval.name
+            if segment_id[1:] == node_id:
+                if segment_id[0] == '+':
+                    presence_forward = True
+                else:
+                    presence_reverse = True
+        else:
+            raise ValueError("Incorrect BED format")
+        if presence_reverse and presence_forward:
+            return presence_forward, presence_reverse
+    return presence_forward, presence_reverse
+
+def call_polymorphism_from_parent(parent:pggraph.Node) -> polymorphism.Polymorphism :
+    """
+    Call potential polymorphism right after parent node.
+    :param parent: Node instance
+    :return: Polymorphism instance
+    """
+    poly = polymorphism.Polymorphism()
+    grand_children = parent.get_grand_children()
+    for gc_id in grand_children:
+        substitution_nodes = parent.current_nodes[gc_id].parents.intersection(parent.children)
+        if len(substitution_nodes) > 1:
+            poly.add_substitution(parent, substitution_nodes, parent.current_nodes[gc_id])
+        if gc_id in parent.children:
+            poly.add_deletion(parent, parent.current_nodes[gc_id])
+            if len(substitution_nodes) == 1:
+                poly.add_substitution(parent, substitution_nodes, parent.current_nodes[gc_id])
+    return poly
+
+def write_out_polymorphism(poly:polymorphism.Polymorphism, parent:pggraph.Node, graph_index:pggraph.GraphIndex, f_id:typing.TextIO=None, f_seq:typing.TextIO=None) -> None:
+    """
+    Write a single polymorphism to genotype tables.
+    :param poly: Polymorphism instance
+    :param parent: Node instance associated with the polymorphism
+    :param graph_index: GraphIndex instance
+    :param f_id: Flag to output genotype table with node IDs
+    :param f_seq: Flag to output genotype table with sequences
+    :return:  None
+    """
+    poly.add_undefined_values(graph_index.samples)
+    if f_id:
+        f_id.write(parent.get_full_id())
+        for sample in graph_index.samples:
+            f_id.write(f',{poly.node_per_sample[sample]}')
+        f_id.write('\n')
+    if f_seq:
+        f_seq.write(parent.get_full_id())
+        for sample in graph_index.samples:
+            if poly.node_per_sample[sample] in ["?", "*"]:
+                f_seq.write(f',{poly.node_per_sample[sample]}')
+            else:
+                f_seq.write(f',{parent.current_nodes[poly.node_per_sample[sample]].sequence}')
+        f_seq.write('\n')

gradiv/functions.py

@@ -0,0 +1,91 @@
+from sys import argv
+from cloup import HelpFormatter, HelpTheme, Style, Context
+import logging
+from rich.logging import RichHandler
+from rich.traceback import install
+from gradiv.__init__ import __version__
+
+
+# context settings for gradiv command
+CONTEXT_SETTINGS = Context.settings(
+    help_option_names=('-h', '--help'),
+    ignore_unknown_options=False,
+    align_option_groups=True,
+    align_sections=True,
+    show_constraints=True,
+    formatter_settings=HelpFormatter.settings(
+        max_width=120,
+        col1_max_width=40,
+        col2_min_width=60,
+        indent_increment=2,
+        col_spacing=1,
+        row_sep=None,
+        theme=HelpTheme.dark().with_(
+            invoked_command=Style(fg='bright_yellow'),
+            command_help=Style(fg='bright_cyan'),
+            heading=Style(fg='green', bold=True),
+            constraint=Style(fg='magenta'),
+            section_help=Style(fg='red'),
+            col1=Style(fg='bright_cyan'),
+            col2=Style(fg='white'),
+            epilog=Style(fg="bright_white", italic=True)
+        )
+    )
+)
+
+def init_log(verbosity:str="INFO") -> logging.Logger:
+    """
+    Initialize the logger and configure console and file output.
+    :param str verbosity: Logging verbosity level.
+    :return: Logger object.
+    """
+    verbosity = verbosity.upper()
+    debug = False
+    if verbosity == 'DEBUG':
+        debug = True
+        install(show_locals=True)
+
+    # create console handler
+    rich_handler = RichHandler(
+        rich_tracebacks=True if debug else False,
+        markup=True,
+        show_time=False,
+        show_level=False,
+        show_path=True if debug else False,
+        # console=shared_console
+    )
+
+    # create file handler (a: append / w: write)
+    file_handler = logging.FileHandler('gradiv.log', mode='w', encoding='utf-8')
+    file_handler.setLevel(logging.DEBUG)  # level debug and above
+
+    # set format
+    formatter = logging.Formatter(
+        fmt=f'%(asctime)s | {"%(name)-8s | %(funcName)-22s |" if debug else ""} %(levelname)-8s | %(message)s',
+        datefmt='%m-%d %H:%M'
+    )
+    file_handler.setFormatter(formatter)
+    rich_handler.setFormatter(formatter)
+
+    # initialize logger and set verbosity level
+    logger = logging.getLogger("gradiv")
+    logger.setLevel(getattr(logging, verbosity))
+
+    if not logger.handlers:
+        logger.addHandler(file_handler)
+        logger.addHandler(rich_handler)
+
+    logger.info("Starting GraDiv")
+    logger.info(f"GraDiv version: {__version__}")
+    logger.info(f"Verbosity level: {verbosity}")
+    logger.info(f"Command line run: {' '.join(argv)}")
+    return logger
+
+
+
+def format_stat_output(x) -> str:
+    """Format statistics in output file for gradiv compute."""
+    if isinstance(x, float): return format(x, '.2f')
+    if x is None: return 'NA'
+    if isinstance(x, (list, tuple)): return f'({" ".join(map(format_stat_output, x))})'
+    return str(x)

gradiv/main.py

@@ -0,0 +1,241 @@
+import click
+from cloup import group, option, command, option_group
+from cloup.constraints import ErrorFmt, RequireExactly
+import egglib
+from time import time
+from gradiv.__init__ import __version__
+import gradiv.algo as algo
+import gradiv.pggraph as pggraph
+from gradiv.functions import *
+
+start_time = time()
+
+
+
+# OPTION GROUPS
+
+common_options = [
+    option("-v", "--verbosity", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False), default="INFO"),
+    option("-g", "--gratools-index", type=click.Path(exists=True, file_okay=False), required=True, help="Path to GraTools index"),
+]
+
+stat_options = RequireExactly(1).rephrased(
+    help="""The compute command needs a list of statistics to compute. One of those options has to be provided.""",
+    error=f'Exactly one statistic parameter must be provided:\n{ErrorFmt.param_list}'
+)
+
+
+
+# GRADIV COMMANDS
+
+@group("gradiv",
+       help="A tool for diversity analysis of pangenome graphs",
+       no_args_is_help=True,
+       invoke_without_command=True,
+       context_settings=CONTEXT_SETTINGS)
+@click.version_option(__version__, "-v", "--version", message="%(prog)s, version %(version)s")
+def main_command(verbosity:str="INFO"):
+    pass
+
+
+@command("call",
+         short_help="Call polymorphisms from a GraTools pangenome graph index",
+         help="Call polymorphisms from a GraTools index and output them as genotype tables in a GraDiv directory",
+         no_args_is_help=True)
+@option_group("call options", *common_options)
+def call(gratools_index, verbosity:str="INFO"):
+    """Generate genotype tables from a GraTools index and outputs them in a GraDiv directory"""
+    logger = init_log(verbosity)
+
+    gfa_index = pggraph.GraphIndex(gratools_index)
+    gfa_index.make_gradiv_directory()
+    gfa_index.make_bam_index_on_reads()
+    gfa_index.open_bam_for_parsing()
+
+    logger.info(f'Calling polymorphisms from "{gfa_index.gratools_directory}"')
+    logger.info(f'Output directory : {gfa_index.gradiv_directory}')
+
+    # call polymorphisms
+    nb_poly = algo.call_polymorphisms(graph_index=gfa_index, id_table=True, seq_table=True)
+
+    execution_time = time() - start_time
+    logger.info(f'Found {nb_poly} polymorphisms')
+    logger.info(f'Executed in {execution_time:.4f} seconds')
+
+    gfa_index.close()
+
+
+@command("compute",
+         short_help="Compute statistics from a genotype table",
+         help="""Compute given statistics from a GraTools index and a genotype table generated with gradiv call.
+              To know which statistics ans options to provide, check out EggLib's documentation : https://www.egglib.org/.
+              Analysis can be run on a specific region (query sample will be required).
+              For more information, see GraDiv's documentation : https://forge.ird.fr/phim/gradiv.
+              """,
+         no_args_is_help=True,
+         show_constraints=True)
+@option_group("compute options",
+              *common_options,
+              stat_options(
+                  option("--stat-list", type=str, required=False, help="Comma separated list of statistics to compute"),
+                  option("--stat-type", type=str, required=False, help="Type of statistic to compute (can be a comma-separated list)"),
+                  option("--stat-file", type=click.Path(exists=True), required=False, help="Path to file providing statistics to compute")
+              ),
+              option("-q", "--query", type=str, required=False, help="Sample name of query sequence to use as reference coordinates"),
+              option("-c", "--chrom", type=str, required=False, help="Chromosome to use as reference coordinates"),
+              option("-s", "--start-position", type=click.IntRange(min=0), required=False, help="Start position of specific region to compute statistics on"),
+              option("-e", "--end-position", type=click.IntRange(min=0), required=False, help="End position of specific region to compute statistics on"),
+              option("--structure", type=click.Path(exists=True), required=False, help="Path to file providing sample structure"),
+              option("--max-missing", type=int, required=False, help="Ignore polymorphisms with more missing data than this threshold"),
+              option("--maf", type=float, default=0, required=False, help="Ignore polymorphisms where the relative minority allele frequency is below this threshold"),
+              option("--only-diallelic", is_flag=True, required=False, help="Ignore polymorphisms with more than two alleles"))
+def compute(gratools_index, query: str = None, chrom: str = None, start_position: int = None, end_position: int = None,
+            structure=None, stat_list: str = None, stat_type: str = None,
+            stat_file=None, max_missing: int = None, maf: float = 0, only_diallelic: bool = False,
+            verbosity: str = "INFO"):
+    """Compute statistics from a GraDiv directory."""
+    logger = init_log(verbosity)
+
+    gfa_index = pggraph.GraphIndex(gratools_index)
+    gfa_index.make_bam_index_on_reads()
+
+    logger.info(f'Computing statistics from {gfa_index.id_table_path}')
+    logger.info(f'Output file : {gfa_index.stats_path}')
+
+    # open genotype table
+    f = open(str(gfa_index.id_table_path), "r")
+    f.readline()
+
+    # configure ComputeStats
+    cs = egglib.stats.ComputeStats(multi=True, multi_hits=False)
+    cs.configure(maf=maf)
+
+    # get structure
+    if structure:
+        structure_file = open(structure, "r")
+        ingroup_dict = {}
+        outgroup_dict = {}
+        samples = []
+        fields = structure_file.readline().strip().split(',')
+        field_to_index = {}
+        sample_index = 0
+        for i in range(len(fields)):
+            field_to_index[fields[i]] = i
+
+        legal_fields = {'cluster', 'population', 'outgroup'}
+        if fields[0] != 'sample':
+            logger.error('Structure file : First field must be "sample"')
+            raise ValueError('Structure file : First field must be "sample"')
+        for field in fields[1:]:
+            if field not in legal_fields:
+                logger.error(f'Structure file : "{field}" is not a valid field')
+                raise ValueError(f'Structure file : "{field}" is not a valid field')
+
+        for line in structure_file:
+            line = line.strip().split(',')
+            samples.append(line[0])
+            if 'outgroup' in fields and line[field_to_index['outgroup']] == "yes":
+                outgroup_dict[line[0]] = [sample_index] # list of a single element as only haploidy is supported
+            else:
+                if 'cluster' not in fields:
+                    cluster = None
+                else:
+                    cluster = line[field_to_index['cluster']]
+                if cluster not in ingroup_dict.keys():
+                    ingroup_dict[cluster] = {}
+                if line[field_to_index['population']] not in ingroup_dict[cluster].keys():
+                    ingroup_dict[cluster][line[field_to_index['population']]] = {}
+                ingroup_dict[cluster][line[field_to_index['population']]][line[0]] = [sample_index] # list of a single element as only haploidy is supported
+            sample_index += 1
+
+        # check if all samples are included
+        samples.sort()
+        if samples != gfa_index.samples:
+            logging.error('Structure file : incorrect sample names or missing samples')
+            raise ValueError('Structure file : incorrect sample names or missing samples')
+
+        struct = egglib.Structure()
+        struct.from_dict(ingroup_dict, outgroup=outgroup_dict)
+        cs.configure(struct=struct)
+
+    # passing statistics to compute
+    if stat_list :
+        cs.add_stats(*stat_list.split(","))
+    elif stat_type :
+        cs.add_stats(*[f"+{item}" for item in stat_type.split(",")])
+    elif stat_file :
+        open_stat_file = open(str(stat_file), "r")
+        stat_list_from_file = open_stat_file.readline().strip().split(",")
+        cs.add_stats(*stat_list_from_file)
+        open_stat_file.close()
+    #TODO: add debug : computed stats
+
+    by_position = False
+    if query and chrom:
+        by_position = True
+        logger.debug('Computing statistics by position')
+    else:
+        if query or chrom or start_position or end_position:
+            logger.error('Both query and chrom options are needed to compute statistics by region.')
+            raise ValueError('Both query and chrom options are needed to compute statistics by region.')
+
+    # compute stats
+    for line in f:
+        anchor_node = line.rstrip().split(',')[0]
+        allele_position = None
+        if by_position:
+            allele_position = gfa_index.get_site_position(node_id=anchor_node, query=query, chrom=chrom)
+            if allele_position == -1:
+                continue
+            if start_position and end_position and (allele_position < start_position or allele_position > end_position):
+                continue
+        split_line = line.rstrip().split(',')[1:]
+        sequences = []
+        for part in split_line:
+            if part not in ["?", "*"] :
+                segment = gfa_index.query_segment(part[1:])
+                if part[0] == '+' :
+                    sequences.append(segment.query_sequence)
+                else:
+                    sequences.append(pggraph.Node.reverse_complement(segment.query_sequence))
+            else :
+                sequences.append(part)
+        alleles = set(sequences) #TODO: get polymorphism type from it (filter on type/length)
+        alleles.discard("?")
+        alph = egglib.Alphabet("string", list(alleles), ["?"])
+        site = egglib.site_from_list(sequences, alph)
+        if by_position:
+            site.position = allele_position
+        # filter out polymorphisms
+        ignore_site = False
+        if max_missing is not None and site.num_missing > max_missing:
+            ignore_site=True
+        if only_diallelic and alph.num_exploitable > 2 :
+            ignore_site=True
+        if not ignore_site:
+            cs.process_site(site)
+    logger.debug('Computing all statistics')
+    stats = cs.results()
+    logger.debug('Statistics computed')
+
+    # outputs stats in CSV file
+    stat_names = []
+    for stat_name in stats.keys():
+        stat_names.append(stat_name)
+
+    canon_stat_order = [key for grp in ('site', 'unphased', 'phased', 'allelesize') for key in cs.stats_group(grp)]
+    stat_names.sort(key=canon_stat_order.index)
+
+    f_stats = open(str(gfa_index.stats_path), 'w')
+    f_stats.write("stat_name,value\n")
+    for stat_name in stat_names:
+        f_stats.write(f'{stat_name},{format_stat_output(stats[stat_name])}\n')
+    f_stats.close()
+
+    execution_time = time() - start_time
+    logger.info(f'Executed in {execution_time:.4f} seconds')
+
+
+# add commands to main
+main_command.add_command(call)
+main_command.add_command(compute)