ArraySplitter 1.3.0__py3-none-any.whl

ArraySplitter/core_functions/io/fasta_reader.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 05.06.2011
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+
+import gzip
+
+
+def sc_iter_fasta_file(file_name, inmem=False, lower=False):
+    """Iter over fasta file."""
+
+    header = None
+    seq = []
+    if file_name.endswith(".gz"):
+        opener = gzip.open
+        decoder = lambda x: x.decode("utf8")
+    else:
+        opener = open
+        decoder = lambda x: x
+
+    with opener(file_name) as fh:
+        if inmem:
+            data = [decoder(x) for x in fh.readlines()]
+            decoder = lambda x: x
+        else:
+            data = fh
+        for line in data:
+            line = decoder(line)
+            if line.startswith(">"):
+                if seq or header:
+                    sequence = "".join(seq)
+                    if lower:
+                        sequence = sequence.lower()
+                    yield header, sequence
+                header = line.strip()[1:]
+                seq = []
+                continue
+            seq.append(line.strip())
+        if seq or header:
+            sequence = "".join(seq)
+            if lower:
+                sequence = sequence.lower()
+            yield header, sequence
+
+
+def sc_iter_arrays_fasta_file(fasta_file):
+    """Iter over fasta file and yield sequences."""
+    for header, sequence in sc_iter_fasta_file(fasta_file):
+        yield sequence

ArraySplitter/core_functions/io/satellome_reader.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 20.02.2024
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+
+from .tab_file import sc_iter_tab_file
+from ..models.trf_model import TRModel
+
+
+def sc_iter_arrays_satellome_file(trf_file):
+    """Iter over satellome file."""
+    for trf_obj in sc_iter_tab_file(trf_file, TRModel):
+        yield trf_obj.trf_array
+
+def sc_iter_satellome_file(trf_file):
+    """Iter over satellome file."""
+    for trf_obj in sc_iter_tab_file(trf_file, TRModel):
+        yield trf_obj.trf_id, trf_obj.trf_array

ArraySplitter/core_functions/io/tab_file.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 05.06.2011
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+"""
+Classes:
+    
+- TabDelimitedFileIO(AbstractFileIO)
+    
+"""
+import csv
+import os
+import tempfile
+
+from PyExp import AbstractFileIO
+
+csv.field_size_limit(1000000000)
+
+
+class TabDelimitedFileIO(AbstractFileIO):
+    """Working with tab delimited file.
+
+    Public methods:
+
+    - sort(self, sort_func, reversed=False)
+
+    Inherited public properties:
+
+    - data  - iterable data, each item is tuple (head, body)
+    - N     - a number of items in data
+
+    Overrided public methods:
+
+    - __init__(self, skip_first=False, format_func=None, delimeter="\\t", skip_startswith=None)
+    - write_to_file(self, output_file)
+    - read_from_file(self, input_file)
+    - read_online(self, input_file) ~> item
+
+    Inherited public methods:
+
+    - read_from_db(self, db_cursor)
+    - write_to_db(self, db_cursor)
+    - read_as_iter(self, source)
+    - iterate(self) ~> item of data
+    - do(self, cf, args) -> result
+    - process(self, cf, args)
+    - clear(self)
+    - do_with_iter(self, cf, args) -> [result,]
+    - process_with_iter(self, cf, args)
+
+    """
+
+    def __init__(
+        self, skip_first=False, format_func=None, delimeter="\t", skip_startswith="#"
+    ):
+        """Overrided. Set token velue.
+
+        Keyword arguments:
+
+        - skip_first   -- skip first line in file
+        - format_funcs -- list of functions that format corresponding item in tab delimited line
+        - delimeter    -- line delimeter
+        - skip_startswith -- skip lines starts with this value
+        """
+        super(TabDelimitedFileIO, self).__init__()
+
+        self.skip_first = skip_first
+        self.format_func = format_func
+        self.delimeter = delimeter
+        self.skip_startswith = skip_startswith
+
+    def read_from_file(self, input_file):
+        """OVerrided. Read data from tab delimeted input_file."""
+        with open(input_file) as fh:
+            self._data = fh.readlines()
+        if self.skip_first:
+            self._data.pop(0)
+        if self.skip_startswith:
+            self._data = [
+                line for line in self._data if not line.startswith(self.skip_startswith)
+            ]
+        self._data = [self._process_tab_delimeited_line(line) for line in self._data]
+
+    def read_online(self, input_file):
+        """Overrided. Yield items online from data from input_file."""
+        with open(input_file) as fh:
+            for i, line in enumerate(fh):
+                if self.skip_first and i == 0:
+                    continue
+                if self.skip_startswith and line.startswith(self.skip_startswith):
+                    continue
+                yield self._process_tab_delimeited_line(line)
+
+    def _process_tab_delimeited_line(self, line):
+        """Format line with format_func."""
+
+        line = line.strip().split(self.delimeter)
+        if self.format_func:
+            assert hasattr(self.format_func, "__call__")
+            line = self.format_func(line)
+        return line
+
+    def _all_str(self, line):
+        """Convert to string all items in line."""
+        return [str(x) for x in line]
+
+    def write_to_file(self, output_file):
+        """Overrided. Write data to tab delimited output_file."""
+        self._data = ["\t".join(self._all_str(line)) for line in self._data]
+        with open(output_file, "w") as fh:
+            fh.writelines(self._data)
+
+
+def sc_iter_tab_file(
+    input_file,
+    data_type,
+    skip_starts_with="#",
+    remove_starts_with=None,
+    preprocess_function=None,
+    check_function=None,
+):
+    """Iter over tab file, yield an object of given data_type."""
+
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+    temp_file_name = temp_file.name
+
+    if remove_starts_with:
+        with open(input_file, "r") as fh:
+            data = fh.readlines()
+        data = [x for x in data if not x.startswith(remove_starts_with)]
+        with open(temp_file_name, "w") as fh:
+            fh.writelines(data)
+        input_file = temp_file_name
+    if preprocess_function:
+        with open(input_file, "r") as fh:
+            data = fh.readlines()
+        data = [preprocess_function(x) for x in data]
+        with open(temp_file_name, "w") as fh:
+            fh.writelines(data)
+        input_file = temp_file_name
+    if check_function:
+        with open(input_file, "r") as fh:
+            data = fh.readlines()
+        data = [x for x in data if check_function(x)]
+        with open(temp_file_name, "w") as fh:
+            fh.writelines(data)
+        input_file = temp_file_name
+    with open(input_file) as fh:
+        fields = data_type().dumpable_attributes
+        for data in csv.DictReader(
+            fh, fieldnames=fields, delimiter="\t", quoting=csv.QUOTE_NONE
+        ):
+            if skip_starts_with:
+                if data[fields[0]].startswith(skip_starts_with):
+                    continue
+            obj = data_type()
+            obj.set_with_dict(data)
+            yield obj
+    if os.path.isfile(temp_file_name):
+        os.unlink(temp_file_name)
+
+
+def sc_iter_simple_tab_file(input_file):
+    """Iter tab file, yield a list."""
+    with open(input_file) as fh:
+        for data in csv.reader(fh, delimiter="\t", quoting=csv.QUOTE_NONE):
+            yield data
+
+
+def sc_read_dictionary(dict_file, value_func=None):
+    """Read file of tab-dilimited pairs.
+    key\tvalue
+    """
+    result = {}
+    with open(dict_file) as fh:
+        for data in csv.reader(fh, delimiter="\t", quoting=csv.QUOTE_NONE):
+            if hasattr(value_func, "__call__"):
+                data[1] = value_func(data[1])
+            result[data[0]] = data[1]
+    return result
+
+
+def sc_write_model_to_tab_file(output_file, objs):
+    """Write model obj to tab-delimited file."""
+    with open(output_file, "w") as fh:
+        for obj in objs:
+            fh.write(str(obj))
+
+
+def sc_read_simple_tab_file(input_file, skip_first=False):
+    """Iter tab file, yield a list."""
+    result = []
+    with open(input_file) as fh:
+        if skip_first:
+            fh.readline()
+        for data in csv.reader(fh, delimiter="\t", quoting=csv.QUOTE_NONE):
+            result.append(data)
+    return result

ArraySplitter/core_functions/io/trf_reader.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 05.06.2011
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+
+"""
+Classes:
+
+- TRFFileIO(AbstractBlockFileIO)
+
+"""
+
+from .block_file import AbstractBlockFileIO
+from ..models.trf_model import TRModel
+
+
+class TRFFileIO(AbstractBlockFileIO):
+    """Working with raw output from TRF."""
+
+    def __init__(self):
+        """Overridden. Hardcoded start token."""
+        token = "Sequence:"
+        super(TRFFileIO, self).__init__(token)
+
+    def iter_parse(self, trf_file, filter=True):
+        """Iterate over raw trf data and yield TRFObjs.
+
+        Args:
+            trf_file (str): The path to the TRF file.
+            filter (bool, optional): Whether to filter the TRF objects. Defaults to True.
+
+        Yields:
+            TRModel: The parsed TRF object.
+
+        """
+        for ii, (head, body, start, next) in enumerate(self.read_online(trf_file)):
+            head = head.replace("\t", " ")
+            obj_set = []
+            n = body.count("\n")
+            for i, line in enumerate(self._gen_data_line(body)):
+                trf_obj = TRModel()
+                trf_obj.set_raw_trf(head, None, line)
+                obj_set.append(trf_obj)
+            yield from obj_set
+
+    def _gen_data_line(self, data):
+        """Generate lines of data from the raw TRF data.
+
+        Args:
+            data (str): The raw TRF data.
+
+        Yields:
+            str: A line of data.
+
+        """
+        for line in data.split("\n"):
+            line = line.strip()
+            if line.startswith("Sequence"):
+                continue
+            if line.startswith("Parameters"):
+                continue
+            if not line:
+                continue
+            yield line
+
+
+def sc_iter_arrays_trf_file(file_name):
+    """Iter over trf file."""
+    reader = TRFFileIO()
+    for trf_obj in reader.iter_parse(file_name):
+        yield trf_obj.trf_array
+
+def sc_iter_trf_file(file_name):
+    """Iter over trf file."""
+    reader = TRFFileIO()
+    for trf_obj in reader.iter_parse(file_name):
+        yield trf_obj.trf_id, trf_obj.trf_array

ArraySplitter/core_functions/models/trf_model.py

@@ -0,0 +1,332 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# @created: 14.06.2011
+# @author: Aleksey Komissarov
+# @contact: ad3002@gmail.com
+
+import re
+
+from PyExp import AbstractModel
+
+from ..tools.parsers import (parse_chromosome_name,
+                                                        parse_fasta_head,
+                                                        trf_parse_head,
+                                                        trf_parse_line)
+
+
+def get_gc(sequence):
+    """Count GC content."""
+    length = len(sequence)
+    if not length:
+        return 0
+    count_c = sequence.count("c") + sequence.count("C")
+    count_g = sequence.count("g") + sequence.count("G")
+    gc = float(count_c + count_g) / float(length)
+    return float(gc)
+
+
+def clear_sequence(sequence):
+    """Clear sequence (full alphabet):
+
+    - lower case
+    - \s -> ""
+    - [^actgn] -> ""
+    """
+    sequence = sequence.strip().upper()
+    sequence = re.sub("\s+", "", sequence)
+    return re.sub("[^actgnuwsmkrybdhvACTGNUWSMKRYBDHV\-]", "", sequence)
+
+
+class TRModel(AbstractModel):
+    """Class for tandem repeat wrapping
+
+    Public properties:
+
+    Indexes:
+
+    - id,
+    - giid,
+    - trf_id,
+    - project,
+    - trf_gi
+
+    Coordinates:
+
+    - trf_l_ind,
+    - trf_r_indel
+
+    TR:
+
+    - trf_period,
+    - trf_n_copy,
+    - trf_pmatch,
+    - trf_pvar,
+    - trf_entropy,
+    - trf_array_length,
+    - trf_joined,
+    - trf_chr
+
+    Sequence:
+
+    - trf_consensus,
+    - trf_array
+
+    GC%:
+
+    - trf_array_gc,
+    - trf_consensus_gc
+
+    Other:
+
+    - trf_head,
+    - trf_params
+
+    Annotation:
+
+    - trf_family,
+    - trf_subfamily,
+    - trf_subsubfamily,
+    - trf_hor,
+    - trf_n_chrun,
+    - trf_n_refgenome,
+    - trf_ref_annotation,
+    - trf_bands_refgenome,
+    - trf_repbase,
+    - trf_strand
+
+    Dumpable attributes:
+
+    - "project",
+    - "id" (int),
+    - "trf_id" (int),
+    - "trf_type",
+    - "trf_family",
+    - "trf_family_prob",
+    - "trf_l_ind" (int),
+    - "trf_r_ind" (int),
+    - "trf_period" (int),
+    - "trf_n_copy" (float),
+    - "trf_pmatch" (float),
+    - "trf_pvar" (float),
+    - "trf_consensus",
+    - "trf_array",
+    - "trf_array_gc" (float),
+    - "trf_consensus_gc" (float),
+    - "trf_gi",
+    - "trf_head",
+    - "trf_param",
+    - "trf_array_length" (int),
+    - "trf_chr",
+    - "trf_joined" (int),
+    - "trf_superfamily",
+    - "trf_superfamily_ref",
+    - "trf_superfamily_self",
+    - "trf_subfamily",
+    - "trf_subsubfamily",
+    - "trf_family_network",
+    - "trf_family_self",
+    - "trf_family_ref",
+    - "trf_hor" (int),
+    - "trf_n_chrun" (int),
+    - "trf_ref_annotation",
+    - "trf_bands_refgenome",
+    - "trf_repbase",
+    - "trf_strand",
+
+    """
+
+    dumpable_attributes = [
+        "project",
+        "id",
+        "trf_id",
+        "trf_type",
+        "trf_family",
+        "trf_family_prob",
+        "trf_l_ind",
+        "trf_r_ind",
+        "trf_period",
+        "trf_n_copy",
+        "trf_pmatch",
+        "trf_pvar",
+        "trf_entropy",
+        "trf_consensus",
+        "trf_array",
+        "trf_array_gc",
+        "trf_consensus_gc",
+        "trf_gi",
+        "trf_head",
+        "trf_param",
+        "trf_array_length",
+        "trf_chr",
+        "trf_joined",
+        "trf_superfamily",
+        "trf_superfamily_ref",
+        "trf_superfamily_self",
+        "trf_subfamily",
+        "trf_subsubfamily",
+        "trf_family_network",
+        "trf_family_self",
+        "trf_family_ref",
+        "trf_hor",
+        "trf_n_chrun",
+        "trf_ref_annotation",
+        "trf_bands_refgenome",
+        "trf_repbase",
+        "trf_strand",
+    ]
+
+    int_attributes = [
+        "trf_l_ind",
+        "trf_r_ind",
+        "trf_period",
+        "trf_array_length",
+        "trf_joined",
+        "trf_hor",
+        "trf_n_chrun",
+    ]
+
+    float_attributes = [
+        "trf_n_copy",
+        "trf_family_prob",
+        "trf_entropy",
+        "trf_pmatch",
+        "trf_pvar",
+        "trf_array_gc",
+        "trf_consensus_gc",
+    ]
+
+    def set_project_data(self, project):
+        """Add project data to self.project."""
+        self.project = project
+
+    def set_raw_trf(self, head, body, line):
+        """Init object with data from parsed trf ouput.
+
+        Parameters:
+
+        - trf_obj: TRFObj instance
+        - head: parsed trf head
+        - body: parsed trf body
+        - line: parsed trf line
+        """
+        self.trf_param = 0
+        self.trf_head = trf_parse_head(head).strip()
+        self.trf_gi = parse_fasta_head(self.trf_head)[0]
+        self.trf_chr = parse_chromosome_name(self.trf_head)
+
+        (
+            self.trf_l_ind,
+            self.trf_r_ind,
+            self.trf_period,
+            self.trf_n_copy,
+            self.trf_l_cons,
+            self.trf_pmatch,
+            self.trf_indels,
+            self.trf_score,
+            self.trf_n_a,
+            self.trf_n_c,
+            self.trf_n_g,
+            self.trf_n_t,
+            self.trf_entropy,
+            self.trf_consensus,
+            self.trf_array,
+        ) = trf_parse_line(line)
+
+        self.trf_pmatch = float(self.trf_pmatch)
+        self.trf_pvar = int(100 - float(self.trf_pmatch))
+
+        try:
+            self.trf_l_ind = int(self.trf_l_ind)
+        except:
+            print("Error:", self)
+
+        self.trf_r_ind = int(self.trf_r_ind)
+        self.trf_period = int(self.trf_period)
+        self.trf_n_copy = float(self.trf_n_copy)
+
+        self.trf_consensus = clear_sequence(self.trf_consensus)
+        self.trf_array = clear_sequence(self.trf_array)
+
+        self.trf_array_gc = get_gc(self.trf_array)
+        self.trf_consensus_gc = get_gc(self.trf_consensus)
+        self.trf_chr = parse_chromosome_name(self.trf_head)
+        self.trf_array_length = len(self.trf_array)
+
+    def get_header_string(self):
+        """Get header string for tsv file."""
+        data = "\t".join(self.dumpable_attributes)
+        return f"#{data}\n"
+
+    def get_fasta_repr(self, add_project=False):
+        """Get array fasta representation, head - trf_id."""
+        if add_project:
+            return ">%s_%s\n%s\n" % (self.trf_id, self.project, self.trf_array)
+        else:
+            return ">%s\n%s\n" % (self.trf_id, self.trf_array)
+
+    @property
+    def fasta(self):
+        return self.get_fasta_repr()
+
+    def get_gff3_string(
+        self,
+        chromosome=True,
+        trs_type="complex_tandem_repeat",
+        probability=1000,
+        tool="PySatDNA",
+        prefix=None,
+        properties=None,
+        force_header=False,
+    ):
+        """Return TR in gff format."""
+        if chromosome and self.trf_chr and self.trf_chr != "?":
+            seqid = self.trf_chr
+        elif self.trf_gi and self.trf_gi != "Unknown":
+            seqid = self.trf_gi
+        else:
+            seqid = self.trf_head
+        features = []
+        if not properties:
+            properties = {}
+        for name, attr in properties.items():
+            features.append("%s=%s" % (name, getattr(self, attr)))
+        features = ";".join(features)
+        if prefix:
+            seqid = prefix + seqid
+        if self.trf_l_ind < self.trf_r_ind:
+            strand = "+"
+        else:
+            strand = "-"
+            self.trf_l_ind, self.trf_r_ind = self.trf_r_ind, self.trf_l_ind
+
+        if force_header:
+            seqid = self.trf_head
+        d = (
+            seqid,
+            tool,
+            trs_type,
+            self.trf_l_ind,
+            self.trf_r_ind,
+            probability,
+            strand,
+            ".",
+            features,
+        )
+        return "%s\n" % "\t".join(map(str, d))
+
+    def get_bed_string(self):
+        """Return TR in bed format."""
+        if self.trf_l_ind < self.trf_r_ind:
+            strand = "+"
+        else:
+            strand = "-"
+            self.trf_l_ind, self.trf_r_ind = self.trf_r_ind, self.trf_l_ind
+
+        seqid = self.trf_head
+        d = (
+            seqid,
+            self.trf_l_ind,
+            self.trf_r_ind,
+        )
+        return "%s\n" % "\t".join(map(str, d))