ArraySplitter 1.3.0__py3-none-any.whl

ArraySplitter/core_functions/tools/fs_tree.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 02.02.2024
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+
+from collections import deque
+
+import heapq
+
+class WeightedValueHeap:
+    def __init__(self):
+        self.heap = []
+    
+    def insert(self, weight, value):
+        heapq.heappush(self.heap, (weight, value))
+    
+    def pop(self):
+        if self.heap:
+            return heapq.heappop(self.heap)
+        return None
+    
+    def peek(self):
+        if self.heap:
+            return self.heap[0]
+        return None
+
+def update(
+    fs_tree,
+    queue,
+    nucl,
+    fs_x,
+    fs_xp,
+    current_cid,
+    cid,
+    cutoff,
+):
+    if len(fs_x) > cutoff:
+        fs_tree[cid] = (
+            cid,
+            [nucl],
+            fs_x,
+            fs_xp,
+            None,
+            [],
+        )
+        fs_tree[current_cid][5].append(cid)
+        queue.append(cid)
+        cid += 1
+    return cid
+
+
+def build_fs_tree_from_sequence(array, starting_seq_, names_, positions_, cutoff):
+    fs_tree = {}
+    queue = []
+    cid = 0
+    seq = starting_seq_
+    names = names_[::]
+    positions = positions_[::]
+    children = []
+    fs = (cid, [seq], names, positions, None, children)
+    queue.append(cid)
+    fs_tree[cid] = fs
+    cid += 1
+    cutoff = cutoff
+
+    while queue:
+        qcid = queue.pop()
+
+        fs = fs_tree[qcid]
+
+        fs_a, fs_c, fs_g, fs_t = [], [], [], []
+        fs_ap, fs_cp, fs_gp, fs_tp = [], [], [], []
+
+        for ii, pos in enumerate(fs[3]):
+            current_cid = fs[0]
+            seq = fs[1]
+            name = fs[2][ii]
+
+            if pos + 1 == len(array):
+                continue
+
+            nucl = array[pos + 1]
+            if nucl == "A":
+                fs_a.append(name)
+                fs_ap.append(pos + 1)
+            elif nucl == "C":
+                fs_c.append(name)
+                fs_cp.append(pos + 1)
+            elif nucl == "G":
+                fs_g.append(name)
+                fs_gp.append(pos + 1)
+            elif nucl == "T":
+                fs_t.append(name)
+                fs_tp.append(pos + 1)
+            
+        cid = update(
+            fs_tree,
+            queue,
+            "A",
+            fs_a,
+            fs_ap,
+            current_cid,
+            cid,
+            cutoff,
+        )
+        cid = update(
+            fs_tree,
+            queue,
+            "C",
+            fs_c,
+            fs_cp,
+            current_cid,
+            cid,
+            cutoff,
+        )
+        cid = update(
+            fs_tree,
+            queue,
+            "G",
+            fs_g,
+            fs_gp,
+            current_cid,
+            cid,
+            cutoff,
+        )
+        cid = update(
+            fs_tree,
+            queue,
+            "T",
+            fs_t,
+            fs_tp,
+            current_cid,
+            cid,
+            cutoff,
+        )
+
+    return fs_tree
+
+
+def iter_fs_tree_from_sequence(array, starting_seq_, names_, positions_, cutoff):
+    heap = []
+    names = names_[::]
+    positions = positions_[::]
+    fs = (1, names, positions)
+    heapq.heappush(heap, (1, fs))
+    cutoff = cutoff
+    while heap:
+        L, fs = heapq.heappop(heap)
+
+        fs_a, fs_c, fs_g, fs_t = [], [], [], []
+        fs_ap, fs_cp, fs_gp, fs_tp = [], [], [], []
+
+        for ii, pos in enumerate(fs[2]):
+            
+            name = fs[1][ii]
+
+            if pos + 1 == len(array):
+                continue
+
+            nucl = array[pos + 1]
+            if nucl == "A":
+                fs_a.append(name)
+                fs_ap.append(pos + 1)
+            elif nucl == "C":
+                fs_c.append(name)
+                fs_cp.append(pos + 1)
+            elif nucl == "G":
+                fs_g.append(name)
+                fs_gp.append(pos + 1)
+            elif nucl == "T":
+                fs_t.append(name)
+                fs_tp.append(pos + 1)
+            
+        if len(fs_a) > cutoff:
+            fs = (
+                L+1,
+                fs_a,
+                fs_ap,
+            )
+            heapq.heappush(heap, (L+1, fs))
+            yield fs
+
+        if len(fs_c) > cutoff:
+            fs = (
+                L+1,
+                fs_c,
+                fs_cp,
+            )
+            heapq.heappush(heap, (L+1, fs))
+            yield fs
+
+        if len(fs_g) > cutoff:
+            fs = (
+                L+1,
+                fs_g,
+                fs_gp,
+            )
+            heapq.heappush(heap, (L+1, fs))
+            yield fs
+
+        if len(fs_t) > cutoff:
+            fs = (
+                L+1,
+                fs_t,
+                fs_tp,
+            )
+            heapq.heappush(heap, (L+1, fs))
+            yield fs

ArraySplitter/core_functions/tools/parsers.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2007-2009 Aleksey Komissarov ( ad3002@gmail.com )
+# All rights reserved.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution.
+"""
+Function for various parsing tasks.
+
+- parse_fasta_head(fa_head) -> (P1,P2,P3)
+- parse_chromosome_name(head) -> string
+- trf_parse_line(line) -> [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '0', '0']
+- trf_parse_param(line) -> string
+- trf_parse_head(line) -> string  
+"""
+import re
+
+
+def parse_fasta_head(fa_head):
+    """Parsing fasta head.
+
+    - head -> (...) -> head parameters
+    - Head parsing <= fasta subtypes
+
+    #TODO: Write other variants
+
+    Valid examples of head format:
+
+    - >gi|20928815|ref|NW_003237.1|MmUn_WIFeb01_12612 Mus musculus chromosome Un genomic contig
+    - Short variant >(\d*)\t(\S*)
+    - >134124-14124\n
+    - >134124\n
+    - >probe|misat|ref|CAAA01154094|start|991|end|1019
+    - Repbase: >TC5A    Mariner/Tc1    Caenorhabditis elegans (split by \t <=== settings)
+    - >lcl|HmaUn_WGA106_1 Hydra magnipapillata genomic contig, re...
+    - plants: >gi|293886233|dbj|BABO01423189.1|
+    """
+
+    head_regexp = re.compile("^>?gi\|(\d+)\|(?:ref|dbj)\|([\w.]+)\|(.*)")
+    regexp_obj = head_regexp.search(fa_head)
+
+    head_regexp_lcl = re.compile("^>?lcl\|(.*?) (.*)")
+    regexp_obj_lcl = head_regexp_lcl.search(fa_head)
+
+    head_regexp_psu = re.compile("^>?psu\|(.*?) (.*)")
+    regexp_obj_psu = head_regexp_psu.search(fa_head)
+
+    head_regexp_short = re.compile("^>(\d+)\t(\S+)")
+    regexp_obj_short = head_regexp_short.search(fa_head)
+
+    head_regexp_comp = re.compile("^>(\d+)-(\d+)")
+    regexp_obj_comp = head_regexp_comp.search(fa_head)
+
+    head_regexp_number = re.compile("^>(\d+)")
+    regexp_obj_number = head_regexp_number.search(fa_head)
+
+    head_regexp_probe_index = re.compile(
+        "^>probe\|(.*?)\|ref\|(.*?)\|start\|(\d*?)\|end\|(\d*)"
+    )
+    regexp_obj_probe_index = head_regexp_probe_index.search(fa_head)
+
+    head_wgs = re.compile("^>?gi\|(\d+)\|\w+?\|([\w.]+)\|(.*)")
+    regexp_obj_wgs = head_wgs.search(fa_head)
+
+    head_trace = re.compile("^>gnl\|ti\|(\d+) (.*)")
+    regexp_obj_trace = head_trace.search(fa_head)
+
+    if regexp_obj:
+        match = regexp_obj.groups()
+        return list(match)
+    elif regexp_obj_probe_index:
+        match_l = []
+
+        match = regexp_obj_probe_index.groups()
+
+        gi = "%s_%s_%s" % (match[1], match[2], match[3])
+        desc = "%s_%s_%s_%s" % (match[1], match[2], match[3], match[0])
+
+        match_l.append(gi)
+        match_l.append("None")
+        match_l.append(desc)
+        return list(match_l)
+    elif regexp_obj_lcl:
+        match_l = []
+        match = regexp_obj_lcl.groups()
+        match_l.append(match[0])
+        match_l.append(match[0])
+        match_l.append(match[1])
+        return list(match_l)
+    elif regexp_obj_psu:
+        match_l = []
+        match = regexp_obj_psu.groups()
+        match_l.append(match[0])
+        match_l.append(match[0])
+        match_l.append(match[1])
+        return list(match_l)
+    elif regexp_obj_short:
+        match = regexp_obj_short.groups()
+        match = list(match)
+        match.append("Unknown")
+        return list(match)
+    elif regexp_obj_comp:
+        match = regexp_obj_comp.groups()
+        match = list(match)
+        match.append("Unknown")
+        return list(match)
+    elif regexp_obj_number:
+        match = regexp_obj_number.groups()
+        match = list(match)
+        match.append("Unknown")
+        match.append("Unknown")
+        return list(match)
+    elif regexp_obj_wgs:
+        match = regexp_obj_wgs.groups()
+        return list(match)
+    elif regexp_obj_trace:
+        match = list(regexp_obj_trace.groups())
+        match.append(match[-1])
+        return list(match)
+    else:
+        match = ("Unknown", "Unknown", "Unknown")
+        # print "Failed parse sequence head: %s" % (fa_head)
+        return list(match)
+
+
+def parse_chromosome_name(head):
+    """Parse chromosome name in ncbi fasta head"""
+    # Head -> (...) -> Chromosome name or ""
+    # TODO: write parse_chromosome_name function
+    try:
+        chr0 = re.compile("chromosome ([^, ]+)").findall(head)
+        chr1 = re.compile("chromosome (\S+?),").findall(head)
+        chr2 = re.compile("chromosome (\S+?),?").findall(head)
+        chr3 = re.compile("chr(\S+?) ").findall(head)
+        mit = re.compile(" (mitochon\S+?) ").findall(head)
+        if chr0:
+            return chr0[0]
+        if chr1:
+            return chr1[0]
+        if chr2:
+            return chr2[0]
+        if chr3:
+            return chr3[0]
+        if mit:
+            return "MT"
+        return "?"
+    except:
+        return "?"
+
+
+def trf_parse_line(line):
+    """Parse TRF data line"""
+    line = line.strip()
+    groups = re.split("\s", line)
+
+    if groups and len(groups) == 15:
+        return list(groups)
+    else:
+        print(("Failed parse ta: %s" % (line)))
+        return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "0", "0"]
+
+
+def trf_parse_param(line):
+    """TRF parameters line"""
+    try:
+        res = re.compile("Parameters: ([\d ]*)", re.S).findall(line)[0]
+        return res
+    except:
+        res = "Unknown"
+        print(("Failed parse param: %s" % (line)))
+        return res
+
+
+def trf_parse_head(line):
+    """Parse TRF head"""
+    try:
+        res = re.compile("Sequence: (.*?)\n", re.S).findall(line)
+        res2 = re.compile("Sequence: (.*)", re.S).findall(line)
+        if res:
+            return res[0]
+        if res2:
+            return res2[0]
+    except:
+        res = "Unknown"
+        print(("Failed parse head: %s" % (line)))
+        return res
+
+
+def get_wgs_prefix_from_ref(ref):
+    """Function parse WGS prefix from Genbank ref."""
+    reg_exp = "([A-Z]+)"
+    res = re.search(reg_exp, ref)
+    if res:
+        return res.group(0)
+    else:
+        return "UNKN"
+
+
+def get_wgs_prefix_from_head(head):
+    """Function parse WGS prefix from fasta head."""
+    reg_exp = "ref.([A-Z]{4,})"
+    res = re.search(reg_exp, head)
+    if res:
+        return res.group(1)
+    else:
+        reg_exp = "gb.([A-Z]{4,})"
+        res = re.search(reg_exp, head)
+        if res:
+            return res.group(1)
+        else:
+            return None
+
+
+def refine_name(i, trf_obj):
+    """Refine TRF name
+    :param trf_obj: TRF object
+    :return: TRF object with refined name
+    """
+    name = trf_obj.trf_head.split()
+    if len(name):
+        name = name[0]
+    else:
+        name = name
+    trf_obj.trf_id = f"{name}_{trf_obj.trf_l_ind}_{trf_obj.trf_r_ind}"
+    trf_obj.id = f"AGT{(i+1) * 100:013d}"
+    trf_obj.trf_consensus = trf_obj.trf_consensus.upper()
+    trf_obj.trf_array = trf_obj.trf_array.upper()
+    return trf_obj

ArraySplitter/core_functions/tools/rotation.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 20.02.2024
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+
+from .sequences import get_revcomp
+from collections import Counter
+from tqdm import tqdm
+
+def best_kmer_for_start(arrays, k=5):
+    k = 5
+    kmers = Counter()
+    for array in arrays:
+        repeat = "".join(array)
+        prev_kmer = None
+        for i in range(len(repeat)-k+1):
+            kmer = repeat[i:i+k]
+            if kmer == prev_kmer:
+                continue
+            prev_kmer = kmer
+            kmers[kmer] += 1
+
+    best_kmer = None
+    max_f1 = 0
+    for kmer, _ in tqdm(kmers.most_common(100)):
+        copy_number = []
+        for array in arrays:
+            for repeat in array:
+                copy_number.append(repeat.count(kmer))
+
+        n = len(copy_number)
+        r = len([x for x in copy_number if x > 0])/n
+        p = len([x for x in copy_number if x == 1])/n
+        f1 = 2 * (r * p) / (r + p)
+        if f1 < 0.5:
+            continue
+        if f1 > max_f1:
+            best_kmer = (r, p, f1, kmer)
+            max_f1 = f1
+    return max_f1, best_kmer
+
+def rotate_arrays(arrays_, starting_kmer=None):
+
+    if not starting_kmer:
+        arrays = []
+        for array in arrays_:
+            feature1 = array.count("C") < array.count("G")
+            feature2 = array.count("A") < array.count("T")
+            if feature1 and feature2 or (not feature1 and not feature2):
+                if feature1 and feature2:
+                    arrays.append(array.split())
+                else:
+                    arrays.append(get_revcomp(array).split())
+            else:
+                if feature1 and not feature2:
+                    arrays.append(get_revcomp(array).split())
+                else:
+                    arrays.append(array.split())
+        max_f1, (r, p, f1, best_kmer) = best_kmer_for_start(arrays)
+    else:
+        best_kmer = starting_kmer
+        best_rev_kmer = get_revcomp(best_kmer)
+        arrays = []
+        for array in arrays_:
+            forward_feature = array.count(best_rev_kmer) < array.count(best_kmer)
+            if forward_feature:
+                arrays.append(array.split())
+            else:
+                arrays.append(get_revcomp(array).split())
+        
+    new_arrays = []
+    for array in arrays:
+        new_array = []
+        prev_repeat = ""
+        queue = array[::]
+        while queue:
+            repeat = prev_repeat + queue.pop(0)
+            pos = repeat.find(best_kmer, 1)
+            if pos == -1:
+                prev_repeat = repeat
+                continue
+            new_array.append(repeat[:pos])
+            prev_repeat = repeat[pos:]
+        if prev_repeat:
+            while True:
+                pos = prev_repeat.find(best_kmer, 1)
+                if pos == -1:
+                    new_array.append(prev_repeat)
+                    break
+                new_array.append(prev_repeat[:pos])
+                prev_repeat = prev_repeat[pos:]
+            
+        assert "".join(array) == "".join(new_array)
+        
+        new_arrays.append(new_array)
+
+    return new_arrays

ArraySplitter/core_functions/tools/sequences.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 20.02.2024
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+
+
+REVCOMP_DICTIONARY = dict(list(zip("ATCGNatcgn~[] ", "TAGCNtagcn~][ ")))
+
+def get_revcomp(sequence):
+    """Return reverse complementary sequence.
+
+    >>> complementary('AT CG')
+    'CGAT'
+
+    """
+    return "".join(
+        REVCOMP_DICTIONARY.get(nucleotide, "") for nucleotide in reversed(sequence)
+    )