gaftools 1.0.0__tar.gz

gaftools-1.0.0/.attic/.vcf_based_sorting_scripts/add-bubble-info.py

@@ -0,0 +1,235 @@
+import sys
+import argparse
+import logging
+from collections import defaultdict
+import gzip
+import re
+from pysam import libcbgzf
+import resource
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logging(debug):
+    handler = logging.StreamHandler()
+    root = logging.getLogger()
+    root.addHandler(handler)
+    root.setLevel(logging.DEBUG if debug else logging.INFO)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--gfa",
+        required=True,
+        help="GFA file which was used to create the VCF in the GFA-to-VCF pipeline",
+    )
+    parser.add_argument(
+        "--vcf", required=True, help="Input VCF file (output of the GFA-to-VCF conversion pipeline)"
+    )
+    parser.add_argument("--output", default=None, help="Output GFA File path (Default: sys.stdout)")
+    parser.add_argument(
+        "--bgzip",
+        action="store_true",
+        help="Flag to bgzip the output. Can only be given with --output.",
+    )
+    parser.add_argument("--debug", action="store_true", default=False, help="Print debug messages")
+    options = parser.parse_args()
+    setup_logging(options.debug)
+
+    # Checks for argument parsing
+    if options.bgzip and not options.output:
+        raise RuntimeError(
+            "--bgzip flag has been specified but not output path has been defined. Please define the output path."
+        )
+
+    add_bubble_tags(options)
+
+    memory_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    logger.info("\n##### Memory Information #####")
+    logger.info("Maximum memory usage: %.3f GB", memory_kb / 1e6)
+
+
+def add_bubble_tags(options):
+    """
+    Adding the bubble tags to the nodes in the GFA file.
+    Sort Keys will be added to the GFA S-lines. For this bubble tag, only the top level bubbles (LV=0) are considered.
+
+    Two tags will be added:
+    1. BO:i:<integer> - This tag gives the bubble order. Odd tag means that the node is a scaffold node (at the start or end of the bubble). Even tag means that the node is inside a bubble.
+    2. NO:i:<integer> - This tag gives the node order. For nodes with the same BO tag, they are distinguished based on this NO tag to order each node uniquely.
+
+    Properties of BO tag:
+    1. BO tag of the form 2n indicates that node is present in the nth bubble defined in the VCF. Consequently, BO tags 2n-1 and 2n+1 are the scaffold nodes for the nth bubble of the VCF.
+    2. This tag has a proper ordering based on the genome coordinate system (Node with BO tag n+1 can be confidently put after Node with BO tag n). There might be some cases where the bubble definition is violated and this order cannot be established. Nodes can be part of two bubbles. Such inconsistent nodes are given a 0 tag.
+
+    Properties of NO tag:
+    1. This does not give ordering based on the genome coordinate system.
+    2. The ordering is based on the appearance of the node in the Allele Traversal INFO in the VCF file. Sequentially, the non scaffold nodes will be provided a tag starting from 1.
+    3. Though this is not ordered based on genome coordinate, this will be a deterministic ordering of some sort.
+    4. NO tag will be 0 for scaffold nodes and inconsistent nodes (nodes present in two top level variants)
+
+    General Notes:
+    1. So inconsistent nodes will have 0 for both BO and NO tag.
+    """
+
+    if options.output is None:
+        writer = sys.stdout
+    else:
+        if options.bgzip:
+            writer = libcbgzf.BGZFile(options.output, "wb")
+        else:
+            writer = open(options.output, "w")
+    node = defaultdict(lambda: [-1, -1])
+    read_vcf(options.vcf, node)
+    modify_gfa(options.gfa, node, writer)
+    writer.close()
+
+
+def write_to_file(line, writer):
+    try:
+        writer.write(line)
+    except TypeError:
+        writer.write(str.encode(line))
+
+
+def modify_gfa(gfa, node, writer):
+    logger.info("\n##### Parsing GFA file and adding sort keys #####")
+    if is_file_gzipped(gfa):
+        reader = gzip.open(gfa, "rt")
+    else:
+        reader = open(gfa, "r")
+
+    S_lines_not_in_vcf = 0
+    line_count = defaultdict(lambda: 0)
+    while True:
+        line = reader.readline()
+        if not line:
+            break
+        line_count[line[0]] += 1
+        if line[0] != "S":
+            write_to_file(line, writer)
+            continue
+        fields = line.split("\t")
+        n = fields[1]
+        if node[n] == [-1, -1]:
+            S_lines_not_in_vcf += 1
+            write_to_file(line, writer)
+            continue
+        bo_tag = "BO:i:"
+        no_tag = "NO:i:"
+        assert node[n][0] != -1 and node[n][1] != -1, "Node %s has -1,-1 tag" % (n)
+        bo_tag += str(node[n][0])
+        no_tag += str(node[n][1])
+        new_line = line.rstrip() + "\t" + bo_tag + "\t" + no_tag + "\n"
+        write_to_file(new_line, writer)
+
+    for line_type, value in line_count.items():
+        logger.info("Number of %s lines: %d" % (line_type, value))
+    logger.info("Number of S lines not present in VCF: %d" % (S_lines_not_in_vcf))
+
+
+def read_vcf(vcf, node):
+    logger.info("\n##### Parsing VCF file and reading bubble information #####")
+    if is_file_gzipped(vcf):
+        reader = gzip.open(vcf, "rt")
+    else:
+        reader = open(vcf, "r")
+
+    total_variants = 0
+    inconsistent_bubble_starts = 0
+    inconsistent_nodes = defaultdict(lambda: 0)
+    bo = 1
+    while True:
+        line = reader.readline()
+        if not line:
+            break
+        if line[0] == "#":
+            continue
+        fields = line.split("\t")
+        info = fields[7].split(";")
+        lv = None
+        lv = int([x[3:] for x in info if x[0:2] == "LV"][0])
+        assert lv is not None, (
+            "Variant at chromosome %s position %s does not have a LV field. Need LV info to process"
+            % (fields[0], fields[1])
+        )
+
+        # Not considering any bubble at LV>0
+        if lv != 0:
+            continue
+        total_variants += 1
+
+        at = None
+        at = [x[3:].split(",") for x in info if x[0:2] == "AT"][0]
+        assert at is not None, (
+            "Variant at chromosome %s position %s does not have a AT field. Need AT info to process"
+            % (fields[0], fields[1])
+        )
+
+        flanks = list(filter(None, re.split("(>)|(<)", fields[2])))
+        assert flanks[0] == flanks[2], "Bad orientation for the bubble."
+        if flanks[0] == "<":
+            flanks = [">", flanks[-1], ">", flanks[1]]
+        assert len(flanks) == 4
+
+        if bo == 1:
+            node[flanks[1]][0] = (
+                bo  # Since there is no previous bubble, this value has to be assigned
+            )
+            node[flanks[1]][1] = 0
+        else:
+            if (
+                node[flanks[1]][0] != bo
+            ):  # By definition, the end node of bubble 1 should be start node of bubble 2. Checking that and counting such inconsistent starts.
+                inconsistent_bubble_starts += 1
+                node[flanks[1]][0] = bo
+                node[flanks[1]][1] = 0
+
+        no = 1
+        for traversal in at:
+            tn = list(filter(None, re.split("(>)|(<)", traversal)))
+            assert (tn[0] == flanks[0] and tn[1] == flanks[1]) and (
+                tn[-2] == flanks[2] and tn[-1] == flanks[3]
+            )
+            tn = tn[2:-2]
+            for n in tn:
+                if n == ">" or n == "<":
+                    continue
+                # This condition checks for the case where the scaffold node is also present inside the bubble.
+                if node[n][0] == bo:
+                    node[n][1] = (
+                        1  # Assigning an NO tag of 1. Usually scaffold nodes are tagged with 0. Here it is tagged with 1 to show that this is also inside a bubble. Hence its orientation cannot be trusted.
+                    )
+                    continue
+                if not (node[n][0] == -1 or node[n][0] == bo + 1):
+                    logger.debug(
+                        "[0] Node %s present in Bubble %s is also present in Bubble %s at level 0. Anomaly found in variant at chromosome %s:%s."
+                        % (n, str((bo + 1) / 2), str(node[n][0] / 2), fields[0], fields[1])
+                    )  # Asserting that the node n has not been called before. This will violate the definition of top level bubble.
+                    inconsistent_nodes[0] += 1
+                    node[n][0] = 0
+                    node[n][1] = 0
+                    continue
+                node[n][0] = bo + 1
+                if node[n][1] != -1:
+                    continue
+                node[n][1] = no
+                no += 1
+        node[flanks[3]][0] = bo + 2  # Assigning the sort key to the end node of the buble
+        node[flanks[3]][1] = 0
+        bo += 2
+    logger.info("Total Variant Bubbles Processed: %d" % (total_variants))
+    logger.info("Inconsistent Bubble Starts: %d" % (inconsistent_bubble_starts))
+    for key, value in inconsistent_nodes.items():
+        logger.info("Number of nodes found in 2 separate bubbles (at level %d): %d" % (key, value))
+    reader.close()
+
+
+def is_file_gzipped(src):
+    with open(src, "rb") as inp:
+        return inp.read(2) == b"\x1f\x8b"
+
+
+if __name__ == "__main__":
+    main()

gaftools-1.0.0/.attic/.vcf_based_sorting_scripts/bubble-sort.py

@@ -0,0 +1,249 @@
+"""
+This uses the BO and NO tags defined by Samarendra Pani through processing the bubbles in the VCF produced by Glenn Hickey for the MC graph. The script to add these BO and NO tags are in add-bubble-info.py in this directory.
+
+Here since the bubbles are already defined, there is no need to process the entire alignment to find orientation of alignment since inversions are already taken into consideration.
+"""
+
+import sys
+import argparse
+import logging
+from collections import defaultdict, namedtuple
+import functools
+import gzip
+import re
+from pysam import libcbgzf
+import resource
+from whatshap.timer import StageTimer
+
+logger = logging.getLogger(__name__)
+timers = StageTimer()
+
+
+def setup_logging(debug):
+    handler = logging.StreamHandler()
+    root = logging.getLogger()
+    root.addHandler(handler)
+    root.setLevel(logging.DEBUG if debug else logging.INFO)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--debug", action="store_true", default=False, help="Print debug messages")
+    parser.add_argument(
+        "--gfa", required=True, help="GFA file with the sort keys (BO and NO tagged)"
+    )
+    parser.add_argument("--gaf", required=True, help="Input GAF File")
+    parser.add_argument("--output", default=None, help="Output GAF File path (Default: sys.stdout)")
+    parser.add_argument(
+        "--bgzip",
+        action="store_true",
+        help="Flag to bgzip the output. Can only be given with --output.",
+    )
+
+    options = parser.parse_args()
+    setup_logging(options.debug)
+
+    validate_arguments(options)
+
+    bubble_sort(options)
+
+    memory_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    logger.info("\nMemory Information")
+    logger.info("  Maximum memory usage: %.3f GB", memory_kb / 1e6)
+    logger.info("\nTime Summary:")
+    logger.info("  Time to parse GFA file: %.3f" % (timers.elapsed("read_gfa")))
+    logger.info("  Total time to sort GAF file: %.3f" % (timers.elapsed("total_sort")))
+    logger.info("    Time to parse GAF file: %.3f" % (timers.elapsed("read_gaf")))
+    logger.info("    Time to sort GAF file: %.3f" % (timers.elapsed("sort_gaf")))
+    logger.info("    Time to write GAF file: %.3f" % (timers.elapsed("write_gaf")))
+
+
+def bubble_sort(options):
+    if options.output is None:
+        writer = sys.stdout
+    else:
+        if options.bgzip:
+            writer = libcbgzf.BGZFile(options.output, "wb")
+        else:
+            writer = open(options.output, "w")
+    nodes = defaultdict(lambda: [-1, -1])
+    with timers("read_gfa"):
+        read_gfa(options.gfa, nodes)
+    with timers("total_sort"):
+        sort(options.gaf, nodes, writer)
+    writer.close()
+
+
+def sort(gaf, nodes, writer):
+    logger.info("\n##### Parsing GAF file and sorting it #####")
+    if is_file_gzipped(gaf):
+        reader = gzip.open(gaf, "rt")
+    else:
+        reader = open(gaf, "r")
+
+    Alignment = namedtuple("Alignment", ["offset", "BO", "NO", "start"])
+    gaf_alignments = []
+
+    # First pass: Store all the alignment lines as minimally. Just storing line offset and alignment string.
+    with timers("read_gaf"):
+        logger.debug("Processing and Storing GAF Alignments...")
+        while True:
+            offset = reader.tell()
+            line = reader.readline()
+            if not line:
+                break
+            line = line.split("\t")
+            bo, no, start = process_alignment(line, nodes, offset)
+            gaf_alignments.append(Alignment(offset=offset, BO=bo, NO=no, start=start))
+
+    # Sorting the alignments based on BO and NO tag
+    with timers("sort_gaf"):
+        logger.debug("Sorting the alignments...")
+        gaf_alignments.sort(key=functools.cmp_to_key(compare_gaf))
+
+    # Writing the sorted file
+    with timers("write_gaf"):
+        logger.debug("Writing Output File...")
+        for alignment in gaf_alignments:
+            off = alignment.offset
+            reader.seek(off)
+            line = reader.readline()
+            write_to_file(line, writer)
+
+    reader.close()
+
+
+def read_gfa(gfa, node):
+    logger.info("\n##### Parsing GFA file and reading sort key information #####")
+    if is_file_gzipped(gfa):
+        reader = gzip.open(gfa, "rt")
+    else:
+        reader = open(gfa, "r")
+
+    total_nodes = 0
+    tagged_nodes = 0
+    while True:
+        line = reader.readline()
+        if not line:
+            break
+        if line[0] != "S":
+            continue
+        total_nodes += 1
+        fields = line.split("\t")
+        for f in fields:
+            if f.startswith("BO:i:"):
+                node[fields[1]][0] = int(f[5:])
+                tagged_nodes += 1
+            if f.startswith("NO:i:"):
+                node[fields[1]][1] = int(f[5:])
+
+    logger.info("Total Nodes Processed: %d" % (total_nodes))
+    logger.info("Nodes with tags: %d" % (tagged_nodes))
+    reader.close()
+
+
+def process_alignment(line, nodes, offset):
+    path = list(filter(None, re.split("(>)|(<)", line[5])))
+    orient = None
+    # If there is no scaffold node present in the alignment, then assuming that it is in the correct orientation.
+    # TODO: Need to find a way to deal with such alignments.
+    rv = False
+    bo = None
+    no = None
+    start = None
+    for n in path:
+        if n in [">", "<"]:
+            orient = n
+            continue
+        if nodes[n][0] == -1 or nodes[n][1] == -1:
+            logger.debug("[ERR]\tOF:i:%d\tND:Z:%s" % (offset, n))
+            # raise RuntimeError("Found a node which was not in VCF.")
+            # These nodes exist.
+            continue
+        if nodes[n][0] % 2 == 1 and nodes[n][1] == 0:
+            # Here we assume that the orientation of the first scaffold node found is the orientation for all the scaffold nodes. (Here we will have to keep in mind that some scaffold nodes are inside the bubbles also and their orientation can be anything there.)
+            # TODO: Have to check for these scaffold nodes that can be present inside bubbles (Cannot trust them)
+            # TODO: Have given the NO tag of 1 to these scaffold nodes to distinguish them from normal scaffold nodes which are tagged with 0.
+            if orient == ">":
+                break
+            else:
+                rv = True
+                break
+    # TODO: What to do when the start node that we have is an untagged node?
+    # One argument is to just sort them to the end and completely ignore them since they dont belong to any vcf bubble.
+    # That works for my tools but not ideal for general purpose.
+    if rv:
+        l = int(line[6])
+        e = int(line[8])
+        start = l - e
+        n = path[-1]
+        bo = nodes[n][0]
+        no = nodes[n][1]
+    else:
+        start = int(line[7])
+        n = path[1]
+        bo = nodes[n][0]
+        no = nodes[n][1]
+    # print(bo, no, start, sep='\t')
+    return bo, no, start
+
+
+def compare_gaf(al1, al2):
+    # Since we are sorting in ascending order, al1 is above al2 if it comes before al2.
+    # Have to consider the case when the start node is untagged and has BO and NO has -1. These should be sorted to the end and not the start
+    if al1.BO == -1 and al2.BO == -1:
+        if al1.offset < al2.offset:
+            return -1
+        else:
+            return 1
+    elif al1.BO == -1:
+        return 1
+    elif al2.BO == -1:
+        return 1
+
+    # Comparing BO tags
+    if al1.BO < al2.BO:
+        return -1
+    if al1.BO > al2.BO:
+        return 1
+
+    # Comparing NO tags
+    if al1.NO < al2.NO:
+        return -1
+    if al1.BO > al2.BO:
+        return 1
+
+    # Comparing start position in node
+    if al1.start < al2.start:
+        return -1
+    if al1.start > al2.start:
+        return 1
+
+    # This will be execulted only when two reads start at the exact same position at the same node. Then we use offset values. So the read which comes first in the GAF file will come higher up.
+    if al1.offset < al2.offset:
+        return -1
+    if al1.offset > al2.offset:
+        return 1
+
+
+def write_to_file(line, writer):
+    try:
+        writer.write(line)
+    except TypeError:
+        writer.write(str.encode(line))
+
+
+def is_file_gzipped(src):
+    with open(src, "rb") as inp:
+        return inp.read(2) == b"\x1f\x8b"
+
+
+def validate_arguments(options):
+    if options.bgzip and not options.output:
+        raise RuntimeError(
+            "--bgzip flag has been specified but not output path has been defined. Please define the output path."
+        )
+
+
+if __name__ == "__main__":
+    main()

gaftools-1.0.0/.attic/bicc_networkx.py

@@ -0,0 +1,165 @@
+def biccs_nx(graph):
+    def edge_stack_to_set(edge_stack):
+        out_set = set()
+        for es in edge_stack:
+            for n in es:
+                out_set.add(n)
+        return out_set
+
+    def next_child(stack_item):
+        if not stack_item[3]:
+            return None
+        if stack_item[2] >= len(stack_item[3]):
+            return None
+        else:
+            stack_item[2] += 1
+            return stack_item[3][stack_item[2] - 1]
+
+    # depth-first search algorithm to generate articulation points
+    # and biconnected components
+    visited = set()
+
+    for n in graph.nodes:
+        if n in visited:
+            continue
+        discovery = {n: 0}
+        low = {n: 0}
+        root_children = 0
+        artic_points = []
+        components = []
+        visited.add(n)
+        edge_stack = []
+        # stack = [(start, start, iter(G[start]))]
+        neighbors = graph[n].neighbors()
+        stack = [[n, n, 0, neighbors]]
+        while stack:
+            parent = stack[-1][0]
+            child = stack[-1][1]
+            nn = next_child(stack[-1])
+
+            # pdb.set_trace()
+            if nn:
+                if nn == parent:
+                    continue
+                if nn in visited:
+                    if discovery[nn] <= discovery[child]:
+                        edge_stack.append((child, nn))
+                        low[child] = min(low[child], discovery[nn])
+                else:
+                    low[nn] = len(discovery)
+                    discovery[nn] = low[nn]
+                    visited.add(nn)
+                    stack.append([child, nn, 0, graph[nn].neighbors()])
+                    edge_stack.append((child, nn))
+            elif nn is None:
+                stack.pop()
+                if len(stack) > 1:
+                    if low[child] >= discovery[parent]:
+                        artic_points.append(parent)
+                        cut_point = edge_stack.index((parent, child))
+                        comp = edge_stack_to_set(edge_stack[cut_point:])
+                        components.append(comp)
+                        edge_stack = edge_stack[:cut_point]
+                        # components.append(edge_stack_to_set(edge_stack[edge_stack.index((parent, child)):]))
+                    low[parent] = min(low[parent], low[child])
+                elif stack:
+                    root_children += 1
+                    components.append(
+                        edge_stack_to_set(edge_stack[edge_stack.index((parent, child)) :])
+                    )
+
+        if root_children > 1:
+            artic_points.append[n]
+
+    return components, artic_points
+
+    def bi_cc_rec(
+        self, n_id, parent, low, disc, stack, node_ids, all_bi_cc, artic_points, disc_time
+    ):
+        # Count of children in current node
+        u = node_ids[n_id]
+        children = 0
+        # Initialize discovery time and low value
+        disc[u] = disc_time[0]
+        low[u] = disc_time[0]
+        disc_time[0] += 1
+
+        # Recur for all the vertices adjacent to this vertex
+        for neighbor_id in self.nodes[n_id].neighbors():
+            v = node_ids[neighbor_id]
+            # If v is not visited yet, then make it a child of u
+            # in DFS tree and recur for it
+            if disc[v] == -1:
+                parent[v] = n_id
+                children += 1
+                stack.append((n_id, neighbor_id))  # store the edge in stack
+                self.bi_cc_rec(
+                    neighbor_id,
+                    parent,
+                    low,
+                    disc,
+                    stack,
+                    node_ids,
+                    all_bi_cc,
+                    artic_points,
+                    disc_time,
+                )
+
+                # Check if the subtree rooted with v has a connection to
+                # one of the ancestors of u
+                # Case 1 -- per Strongly Connected Components Article
+                low[u] = min(low[u], low[v])
+
+                # If u is an articulation point, pop
+                # all edges from stack until (u, v)
+                if parent[u] == -1 and children > 1 or parent[u] != -1 and low[v] >= disc[u]:
+                    artic_points.append(n_id)
+                    w = -1
+                    one_bi_cc = set()
+                    while w != (n_id, neighbor_id):
+                        w = stack.pop()
+                        for n in w:
+                            one_bi_cc.add(n)
+                    all_bi_cc.append(one_bi_cc)
+
+            elif neighbor_id != parent[u] and low[u] > disc[v]:
+                low[u] = min(low[u], disc[v])
+                stack.append((n_id, neighbor_id))
+
+    def bicc(self):
+        """
+        find biconnected components and returns a list of these components in terms of node ids
+        """
+        disc_time = [0]
+        all_bi_cc = list()
+        node_ids = dict()
+        artic_points = []
+        # node_ids_list = [0] * len(self)
+        for idx, n_id in enumerate(list(self.nodes.keys())):
+            node_ids[n_id] = idx
+            # node_ids_list[idx] = n_id
+
+        # Initialize disc and low, and parent arrays
+        n_vertices = len(self)
+        disc = [-1] * n_vertices
+        low = [-1] * n_vertices
+        parent = [-1] * n_vertices
+        stack = []
+        # Call the recursive helper function to
+        # find articulation points
+        # in DFS tree rooted with vertex 'i'
+        for n_id in self.nodes.keys():
+            i = node_ids[n_id]
+            if disc[i] == -1:
+                self.bi_cc_rec(
+                    n_id, parent, low, disc, stack, node_ids, all_bi_cc, artic_points, disc_time
+                )
+            # If stack is not empty, pop all edges from stack
+            if stack:
+                one_bi_cc = set()
+                while stack:
+                    w = stack.pop()
+                    for n in w:
+                        one_bi_cc.add(n)
+                all_bi_cc.append(one_bi_cc)
+        return all_bi_cc, artic_points