phykit 2.1.0__py3-none-any.whl

phykit/services/alignment/pairwise_identity.py

@@ -0,0 +1,159 @@
+import itertools
+from typing import Dict, List, Tuple
+import numpy as np
+import multiprocessing as mp
+from functools import partial
+import sys
+
+from Bio.Align import MultipleSeqAlignment
+try:
+    from tqdm import tqdm
+except ImportError:
+    # Fallback if tqdm is not installed
+    def tqdm(iterable, *args, **kwargs):
+        return iterable
+
+from .base import Alignment
+from ...helpers.stats_summary import (
+    calculate_summary_statistics_from_dict,
+    print_summary_statistics,
+)
+
+
+class PairwiseIdentity(Alignment):
+    def __init__(self, args) -> None:
+        super().__init__(**self.process_args(args))
+
+    def run(self):
+        alignment, _, is_protein = self.get_alignment_and_format()
+
+        pair_ids, pairwise_identities, stats = \
+            self.calculate_pairwise_identities(
+                alignment, self.exclude_gaps
+            )
+
+        if self.verbose:
+            try:
+                for pair, identity in zip(
+                    pair_ids, pairwise_identities.values()
+                ):
+                    print(f"{pair[0]}\t{pair[1]}\t{round(identity, 4)}")
+            except BrokenPipeError:
+                pass
+        else:
+            print_summary_statistics(stats)
+
+    def process_args(self, args) -> Dict[str, str]:
+        return dict(
+            alignment_file_path=args.alignment,
+            verbose=args.verbose,
+            exclude_gaps=args.exclude_gaps,
+        )
+
+    def _calculate_identity_vectorized(self, seq_arr1, seq_arr2, gap_mask=None, exclude_gaps=False):
+        """Vectorized calculation of sequence identity."""
+        matches = (seq_arr1 == seq_arr2)
+
+        if exclude_gaps and gap_mask is not None:
+            # Match original behavior: count identities when at least one doesn't have a gap
+            # This matches the original "res_one not in gap_chars or res_two not in gap_chars"
+            valid_for_identity = ~gap_mask[0] | ~gap_mask[1]
+            identities = np.sum(matches & valid_for_identity)
+        else:
+            identities = np.sum(matches)
+
+        # Total compared is always the full length (matching original behavior)
+        total_compared = len(seq_arr1)
+
+        return identities / total_compared if total_compared > 0 else 0
+
+    def _process_pair_batch(self, alignment_data, pair_indices, exclude_gaps, gap_chars):
+        """Process a batch of sequence pairs."""
+        results = []
+        for idx1, idx2 in pair_indices:
+            seq_one = alignment_data[idx1]['seq']
+            seq_two = alignment_data[idx2]['seq']
+
+            if exclude_gaps:
+                # Create boolean masks for gap positions
+                gap_mask1 = np.isin(seq_one, list(gap_chars))
+                gap_mask2 = np.isin(seq_two, list(gap_chars))
+                identity = self._calculate_identity_vectorized(
+                    seq_one, seq_two, (gap_mask1, gap_mask2), exclude_gaps
+                )
+            else:
+                identity = self._calculate_identity_vectorized(seq_one, seq_two)
+
+            results.append({
+                'pair_id': [alignment_data[idx1]['id'], alignment_data[idx2]['id']],
+                'identity': identity
+            })
+        return results
+
+    def calculate_pairwise_identities(
+        self,
+        alignment: MultipleSeqAlignment,
+        exclude_gaps: bool,
+    ) -> Tuple[List[List[str]], Dict[str, float], Dict[str, float]]:
+        gap_chars = self.get_gap_chars()
+
+        # Convert sequences to numpy arrays for faster comparison
+        alignment_data = []
+        for record in alignment:
+            seq_array = np.array([c.upper() for c in str(record.seq)], dtype='U1')
+            alignment_data.append({
+                'id': record.id,
+                'seq': seq_array
+            })
+
+        # Generate all pairwise combinations
+        all_pairs = list(itertools.combinations(range(len(alignment)), 2))
+
+        pairwise_identities = {}
+        pair_ids = []
+
+        # For small datasets or when not using multiprocessing
+        if len(all_pairs) < 50:
+            # Process all pairs without multiprocessing
+            results = self._process_pair_batch(alignment_data, all_pairs, exclude_gaps, gap_chars)
+            for result in results:
+                pair_id = result['pair_id']
+                pair_ids.append(pair_id)
+                pairwise_identities["-".join(pair_id)] = result['identity']
+        else:
+            # Use multiprocessing for larger datasets
+            num_workers = min(mp.cpu_count(), 8)
+            chunk_size = max(1, len(all_pairs) // (num_workers * 4))
+            pair_chunks = [all_pairs[i:i + chunk_size] for i in range(0, len(all_pairs), chunk_size)]
+
+            # Create partial function
+            process_func = partial(
+                self._process_pair_batch,
+                alignment_data,
+                exclude_gaps=exclude_gaps,
+                gap_chars=gap_chars
+            )
+
+            # Process in parallel with progress bar
+            with mp.Pool(processes=num_workers) as pool:
+                # Only show progress bar if stderr is a tty (not redirected)
+                if sys.stderr.isatty():
+                    chunk_results = list(tqdm(
+                        pool.imap(process_func, pair_chunks),
+                        total=len(pair_chunks),
+                        desc="Calculating pairwise identities",
+                        unit="batch"
+                    ))
+                else:
+                    chunk_results = pool.map(process_func, pair_chunks)
+
+            # Combine results
+            for chunk_result in chunk_results:
+                for result in chunk_result:
+                    pair_id = result['pair_id']
+                    pair_ids.append(pair_id)
+                    pairwise_identities["-".join(pair_id)] = result['identity']
+
+        stats = calculate_summary_statistics_from_dict(pairwise_identities)
+
+        return pair_ids, pairwise_identities, stats

phykit/services/alignment/parsimony_informative_sites.py

@@ -0,0 +1,81 @@
+from collections import Counter
+from typing import Dict, Tuple
+import numpy as np
+
+from Bio.Align import MultipleSeqAlignment
+
+from .base import Alignment
+
+
+class ParsimonyInformative(Alignment):
+    def __init__(self, args) -> None:
+        super().__init__(**self.process_args(args))
+
+    def run(self):
+        alignment, _, is_protein = self.get_alignment_and_format()
+        pi_sites, aln_len, pi_sites_per = self.calculate_parsimony_informative_sites(
+            alignment
+        )
+
+        print(f"{pi_sites}\t{aln_len}\t{round(pi_sites_per, 4)}")
+
+    def process_args(self, args) -> Dict[str, str]:
+        return dict(alignment_file_path=args.alignment)
+
+    def get_number_of_occurrences_per_character(
+        self,
+        alignment: MultipleSeqAlignment,
+        idx: int
+    ) -> Counter:
+        gap_chars = self.get_gap_chars()
+        seq_at_position = alignment[:, idx].upper()
+        filtered_seq = filter(lambda c: c not in gap_chars, seq_at_position)
+
+        return Counter(filtered_seq)
+
+    def is_parsimony_informative(
+        self,
+        num_occurrences: Counter,
+    ) -> bool:
+        """
+        Check if a site is parsimony informative.
+        That is, the site has two characters that appear at least twice.
+        """
+        informative_char_count = sum(1 for count in num_occurrences.values() if count >= 2)
+        return informative_char_count >= 2
+
+    def calculate_parsimony_informative_sites(
+        self,
+        alignment: MultipleSeqAlignment,
+    ) -> Tuple[int, int, float]:
+        aln_len = alignment.get_alignment_length()
+        gap_chars = self.get_gap_chars()
+
+        # Convert alignment to numpy array for vectorized operations
+        alignment_array = np.array([
+            [c.upper() for c in str(record.seq)]
+            for record in alignment
+        ], dtype='U1')
+
+        pi_sites = 0
+
+        # Process each column more efficiently
+        for col_idx in range(aln_len):
+            column = alignment_array[:, col_idx]
+
+            # Filter out gaps
+            non_gap_mask = ~np.isin(column, list(gap_chars))
+            filtered_column = column[non_gap_mask]
+
+            if len(filtered_column) > 0:
+                # Count occurrences of each character
+                unique_chars, counts = np.unique(filtered_column, return_counts=True)
+
+                # Check if parsimony informative (at least 2 chars appearing >= 2 times)
+                chars_appearing_twice = np.sum(counts >= 2)
+                if chars_appearing_twice >= 2:
+                    pi_sites += 1
+
+        pi_sites_per = (pi_sites / aln_len) * 100
+
+        return pi_sites, aln_len, pi_sites_per

phykit/services/alignment/rcv.py

@@ -0,0 +1,14 @@
+from .base import Alignment
+
+
+class RelativeCompositionVariability(Alignment):
+    def __init__(self, args) -> None:
+        super().__init__(**self.process_args(args))
+
+    def run(self):
+        # calc rcv and print val
+        relative_composition_variability = self.calculate_rcv()
+        print(round(relative_composition_variability, 4))
+
+    def process_args(self, args):
+        return dict(alignment_file_path=args.alignment)

phykit/services/alignment/rcvt.py

@@ -0,0 +1,47 @@
+import numpy as np
+
+from .base import Alignment
+
+
+class RelativeCompositionVariabilityTaxon(Alignment):
+    def __init__(self, args) -> None:
+        super().__init__(**self.process_args(args))
+
+    def run(self):
+        alignment, _, _ = self.get_alignment_and_format()
+        aln_len = alignment.get_alignment_length()
+        num_records = len(alignment)
+
+        # Convert alignment to numpy array for faster operations
+        alignment_array = np.array([
+            list(str(record.seq)) for record in alignment
+        ], dtype='U1')
+
+        # Get all unique characters and create mapping
+        unique_chars = np.unique(alignment_array)
+        char_to_idx = {char: i for i, char in enumerate(unique_chars)}
+
+        # Create integer representation for faster counting
+        alignment_int = np.zeros_like(alignment_array, dtype=np.int8)
+        for char, idx in char_to_idx.items():
+            alignment_int[alignment_array == char] = idx
+
+        # Vectorized counting for all sequences and characters
+        count_matrix = np.zeros((num_records, len(unique_chars)), dtype=np.float32)
+        for i in range(len(unique_chars)):
+            count_matrix[:, i] = np.sum(alignment_int == i, axis=1)
+
+        # Calculate average counts per sequence (total counts / num_records)
+        average_counts = np.sum(count_matrix, axis=0) / num_records
+
+        # Vectorized RCV calculation for all sequences at once
+        deviations = np.abs(count_matrix - average_counts)
+        rcv_values = np.sum(deviations, axis=1) / (num_records * aln_len)
+
+        # Print results - convert to float64 for consistent rounding
+        for i, record in enumerate(alignment):
+            rcv_val = float(rcv_values[i])
+            print(f"{record.id}\t{round(rcv_val, 4)}")
+
+    def process_args(self, args):
+        return dict(alignment_file_path=args.alignment)

phykit/services/alignment/rename_fasta_entries.py

@@ -0,0 +1,53 @@
+import sys
+from typing import Dict
+
+from Bio import SeqIO
+from Bio.SeqIO.FastaIO import FastaIterator
+
+from .base import Alignment
+
+
+class RenameFastaEntries(Alignment):
+    def __init__(self, args) -> None:
+        super().__init__(**self.process_args(args))
+
+    def run(self):
+        try:
+            records = SeqIO.parse(self.fasta, "fasta")
+        except FileNotFoundError:
+            print("FASTA file path corresponds to no such file. Please check the path.")
+            sys.exit(2)
+
+        idmap = self.load_idmap(self.idmap)
+
+        self.replace_ids_and_write(self.output_file_path, records, idmap)
+
+    def process_args(self, args) -> Dict[str, str]:
+        output_file_path = f"{args.output or args.fasta}.renamed.fa"
+        return dict(
+            fasta=args.fasta,
+            idmap=args.idmap,
+            output_file_path=output_file_path,
+        )
+
+    def load_idmap(self, idmap_file: str) -> Dict[str, str]:
+        try:
+            with open(idmap_file) as f:
+                return dict(line.split() for line in f)
+        except FileNotFoundError:
+            print("Idmap path corresponds to no such file. Please check the path.")
+            sys.exit(2)
+
+    def replace_ids_and_write(
+        self,
+        output_file_path: str,
+        records: FastaIterator,
+        idmap: Dict[str, str]
+    ) -> None:
+        print(records)
+        with open(output_file_path, "w") as output_file:
+            for record in records:
+                if record.id in idmap:
+                    record.id = idmap[record.id]
+                    record.description = ""
+                SeqIO.write(record, output_file, "fasta")

phykit/services/alignment/sum_of_pairs_score.py

@@ -0,0 +1,157 @@
+import itertools
+from typing import Dict, List, Tuple
+import numpy as np
+import multiprocessing as mp
+from functools import partial
+
+from Bio import SeqIO
+from Bio.SeqRecord import SeqRecord
+
+from .base import Alignment
+
+
+class SumOfPairsScore(Alignment):
+    def __init__(self, args) -> None:
+        super().__init__(**self.process_args(args))
+
+    def run(self):
+        query_records = SeqIO.to_dict(SeqIO.parse(self.fasta, "fasta"))
+        reference_records = SeqIO.to_dict(SeqIO.parse(self.reference, "fasta"))
+
+        record_id_pairs = list(
+            itertools.combinations(reference_records.keys(), 2)
+        )
+
+        number_of_matches, number_of_total_pairs = \
+            self.determine_number_of_matches_and_total_pairs(
+                record_id_pairs, reference_records, query_records
+            )
+
+        print(round(number_of_matches / number_of_total_pairs, 4))
+
+    def process_args(self, args) -> Dict[str, str]:
+        return dict(fasta=args.fasta, reference=args.reference)
+
+    @staticmethod
+    def _process_pair_batch(
+        pair_batch: List[Tuple[str, str]],
+        reference_records: Dict[str, SeqRecord],
+        query_records: Dict[str, SeqRecord],
+    ) -> Tuple[int, int]:
+        """Process a batch of sequence pairs in parallel."""
+        batch_matches = 0
+        batch_total = 0
+
+        # Pre-convert sequences to numpy arrays for the batch
+        seq_arrays = {}
+        for pair in pair_batch:
+            for seq_id in [pair[0], pair[1]]:
+                if seq_id not in seq_arrays:
+                    ref_seq = str(reference_records[seq_id].seq)
+                    query_seq = str(query_records[seq_id].seq)
+                    seq_arrays[seq_id] = {
+                        'ref': ref_seq,
+                        'query': query_seq,
+                        'ref_array': np.array(list(ref_seq), dtype='U1'),
+                        'query_array': np.array(list(query_seq), dtype='U1')
+                    }
+
+        for first_in_pair, second_in_pair in pair_batch:
+            ref_seq1 = seq_arrays[first_in_pair]['ref_array']
+            ref_seq2 = seq_arrays[second_in_pair]['ref_array']
+            query_seq1 = seq_arrays[first_in_pair]['query_array']
+            query_seq2 = seq_arrays[second_in_pair]['query_array']
+
+            # Check if all sequences have the same length
+            if (len(ref_seq1) == len(query_seq1) and
+                len(ref_seq2) == len(query_seq2) and
+                len(ref_seq1) == len(ref_seq2)):
+                # Use vectorized comparison
+                matches = (ref_seq1 == query_seq1) & (ref_seq2 == query_seq2)
+                batch_matches += np.sum(matches)
+                batch_total += len(ref_seq1)
+            else:
+                # Fall back to optimized comparison for mismatched lengths
+                ref_seq1_str = seq_arrays[first_in_pair]['ref']
+                ref_seq2_str = seq_arrays[second_in_pair]['ref']
+                query_seq1_str = seq_arrays[first_in_pair]['query']
+                query_seq2_str = seq_arrays[second_in_pair]['query']
+
+                min_len = min(len(ref_seq1_str), len(ref_seq2_str),
+                             len(query_seq1_str), len(query_seq2_str))
+
+                # Vectorize the mismatched comparison when possible
+                if min_len > 0:
+                    ref1_trimmed = np.array(list(ref_seq1_str[:min_len]), dtype='U1')
+                    ref2_trimmed = np.array(list(ref_seq2_str[:min_len]), dtype='U1')
+                    query1_trimmed = np.array(list(query_seq1_str[:min_len]), dtype='U1')
+                    query2_trimmed = np.array(list(query_seq2_str[:min_len]), dtype='U1')
+
+                    matches = (ref1_trimmed == query1_trimmed) & (ref2_trimmed == query2_trimmed)
+                    batch_matches += np.sum(matches)
+                    batch_total += min_len
+
+        return int(batch_matches), batch_total
+
+    def determine_number_of_matches_and_total_pairs(
+        self,
+        record_id_pairs: List[Tuple[str, str]],
+        reference_records: Dict[str, SeqRecord],
+        query_records: Dict[str, SeqRecord],
+    ) -> Tuple[int, int]:
+        # For small datasets, use sequential processing
+        if len(record_id_pairs) < 50:
+            number_of_matches = 0
+            number_of_total_pairs = 0
+
+            for first_in_pair, second_in_pair in record_id_pairs:
+                ref_seq1_str = str(reference_records[first_in_pair].seq)
+                ref_seq2_str = str(reference_records[second_in_pair].seq)
+                query_seq1_str = str(query_records[first_in_pair].seq)
+                query_seq2_str = str(query_records[second_in_pair].seq)
+
+                if (len(ref_seq1_str) == len(query_seq1_str) and
+                    len(ref_seq2_str) == len(query_seq2_str) and
+                    len(ref_seq1_str) == len(ref_seq2_str)):
+                    # Use vectorized comparison
+                    ref_seq1 = np.array(list(ref_seq1_str), dtype='U1')
+                    ref_seq2 = np.array(list(ref_seq2_str), dtype='U1')
+                    query_seq1 = np.array(list(query_seq1_str), dtype='U1')
+                    query_seq2 = np.array(list(query_seq2_str), dtype='U1')
+
+                    matches = (ref_seq1 == query_seq1) & (ref_seq2 == query_seq2)
+                    number_of_matches += np.sum(matches)
+                    number_of_total_pairs += len(ref_seq1)
+                else:
+                    min_len = min(len(ref_seq1_str), len(ref_seq2_str),
+                                 len(query_seq1_str), len(query_seq2_str))
+
+                    for i in range(min_len):
+                        if (ref_seq1_str[i] == query_seq1_str[i] and
+                            ref_seq2_str[i] == query_seq2_str[i]):
+                            number_of_matches += 1
+                        number_of_total_pairs += 1
+
+            return int(number_of_matches), number_of_total_pairs
+
+        # Use multiprocessing for larger datasets
+        num_workers = min(mp.cpu_count(), 8)
+        batch_size = max(10, len(record_id_pairs) // (num_workers * 4))
+
+        # Create batches
+        pair_batches = [record_id_pairs[i:i + batch_size]
+                       for i in range(0, len(record_id_pairs), batch_size)]
+
+        # Process batches in parallel
+        process_func = partial(self._process_pair_batch,
+                              reference_records=reference_records,
+                              query_records=query_records)
+
+        with mp.Pool(processes=num_workers) as pool:
+            batch_results = pool.map(process_func, pair_batches)
+
+        # Aggregate results
+        total_matches = sum(matches for matches, _ in batch_results)
+        total_pairs = sum(pairs for _, pairs in batch_results)
+
+        return int(total_matches), total_pairs

phykit/services/alignment/variable_sites.py

@@ -0,0 +1,54 @@
+from typing import Dict, Tuple
+import numpy as np
+
+from Bio.Align import MultipleSeqAlignment
+
+from .base import Alignment
+
+
+class VariableSites(Alignment):
+    def __init__(self, args) -> None:
+        super().__init__(**self.process_args(args))
+
+    def run(self):
+        alignment, _, is_protein = self.get_alignment_and_format()
+        var_sites, aln_len, var_sites_per = \
+            self.calculate_variable_sites(alignment)
+
+        print(f"{var_sites}\t{aln_len}\t{round(var_sites_per, 4)}")
+
+    def process_args(self, args) -> Dict[str, str]:
+        return dict(alignment_file_path=args.alignment)
+
+    def calculate_variable_sites(
+        self,
+        alignment: MultipleSeqAlignment
+    ) -> Tuple[int, int, float]:
+        aln_len = alignment.get_alignment_length()
+        gap_chars = self.get_gap_chars()
+
+        # Convert alignment to numpy array for vectorized operations
+        alignment_array = np.array([
+            [c.upper() for c in str(record.seq)]
+            for record in alignment
+        ], dtype='U1')
+
+        var_sites = 0
+
+        # Process each column
+        for col in range(aln_len):
+            column = alignment_array[:, col]
+
+            # Filter out gap characters
+            non_gap_mask = ~np.isin(column, list(gap_chars))
+            filtered_column = column[non_gap_mask]
+
+            # Check if variable (more than one unique character)
+            if len(filtered_column) > 0:
+                unique_chars = np.unique(filtered_column)
+                if len(unique_chars) > 1:
+                    var_sites += 1
+
+        var_sites_per = (var_sites / aln_len) * 100
+
+        return var_sites, aln_len, var_sites_per

phykit/services/base.py

@@ -0,0 +1,9 @@
+class BaseService(object):
+    def __init__(self):
+        pass
+
+    def process_args(self, args):
+        raise NotImplementedError()
+
+    def run(self):
+        raise NotImplementedError()

phykit/services/tree/__init__.py

@@ -0,0 +1,29 @@
+from .bipartition_support_stats import BipartitionSupportStats
+from .branch_length_multiplier import BranchLengthMultiplier
+from .covarying_evolutionary_rates import CovaryingEvolutionaryRates
+from .dvmc import DVMC
+from .evolutionary_rate import EvolutionaryRate
+from .hidden_paralogy_check import HiddenParalogyCheck
+from .internal_branch_stats import InternalBranchStats
+from .internode_labeler import InternodeLabeler
+from .last_common_ancestor_subtree import LastCommonAncestorSubtree
+from .lb_score import LBScore
+from .monophyly_check import MonophylyCheck
+from .nearest_neighbor_interchange import NearestNeighborInterchange
+from .patristic_distances import PatristicDistances
+from .polytomy_test import PolytomyTest
+from .print_tree import PrintTree
+from .prune_tree import PruneTree
+from .rename_tree_tips import RenameTreeTips
+from .root_tree import RootTree
+from .rf_distance import RobinsonFouldsDistance
+from .saturation import Saturation
+from .spurious_sequence import SpuriousSequence
+from .treeness import Treeness
+from .treeness_over_rcv import TreenessOverRCV
+from .terminal_branch_stats import TerminalBranchStats
+from .tip_labels import TipLabels
+from .tip_to_tip_distance import TipToTipDistance
+from .tip_to_tip_node_distance import TipToTipNodeDistance
+from .total_tree_length import TotalTreeLength
+from .collapse_branches import CollapseBranches