ArraySplitter 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

ArraySplitter/__init__.py

@@ -5,6 +5,6 @@
 # @author: Aleksey Komissarov
 # @contact: ad3002@gmail.com
 
-__version__ = "1.3.0"
+__version__ = "1.4.0"
 
 # __all__ = []

ArraySplitter/indel_chromosome_analysis.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Indel Chromosome Specificity Analysis.
+
+Analyzes how chromosome-specific indels are in MSA alignments.
+For each indel, calculates:
+- Which chromosomes contain sequences with this indel
+- What fraction of sequences from each chromosome have the indel
+- Overall specificity score
+"""
+
+import argparse
+from collections import defaultdict
+from typing import List, Tuple, Dict, Set
+from dataclasses import dataclass
+import re
+import json
+
+
+@dataclass
+class Indel:
+    start: int
+    end: int
+    length: int
+
+    def __hash__(self):
+        return hash((self.start, self.end))
+
+    def __eq__(self, other):
+        return self.start == other.start and self.end == other.end
+
+
+def parse_fasta(filepath: str) -> List[Tuple[str, str]]:
+    """Parse FASTA file."""
+    sequences = []
+    with open(filepath) as f:
+        name = ""
+        seq = ""
+        for line in f:
+            line = line.strip()
+            if line.startswith(">"):
+                if seq:
+                    sequences.append((name, seq))
+                name = line[1:].split()[0]
+                seq = ""
+            else:
+                seq += line
+        if seq:
+            sequences.append((name, seq))
+    return sequences
+
+
+def extract_chromosome(name: str) -> str:
+    """Extract chromosome name from sequence header."""
+    # Pattern: chr1_mat:123-456|... or chr1A_pat:...
+    match = re.match(r'(chr[^:]+)', name)
+    if match:
+        return match.group(1)
+    return "unknown"
+
+
+def find_indels_in_sequence(seq: str) -> List[Indel]:
+    """Find all indel regions in a sequence."""
+    indels = []
+    in_gap = False
+    gap_start = 0
+
+    for i, base in enumerate(seq):
+        if base == '-':
+            if not in_gap:
+                in_gap = True
+                gap_start = i
+        else:
+            if in_gap:
+                indels.append(Indel(start=gap_start, end=i, length=i - gap_start))
+                in_gap = False
+
+    if in_gap:
+        indels.append(Indel(start=gap_start, end=len(seq), length=len(seq) - gap_start))
+
+    return indels
+
+
+def analyze_chromosome_specificity(input_fasta: str, output_tsv: str, output_json: str,
+                                    min_count: int = 3, verbose: bool = False) -> Dict:
+    """
+    Analyze chromosome specificity of indels.
+
+    Args:
+        input_fasta: Aligned FASTA file
+        output_tsv: Output TSV with detailed analysis
+        output_json: Output JSON for HTML report
+        min_count: Minimum sequence count for indel to be included
+        verbose: Print progress
+    """
+    sequences = parse_fasta(input_fasta)
+    n_seqs = len(sequences)
+
+    if verbose:
+        print(f"Loaded {n_seqs} sequences")
+
+    # Count sequences per chromosome
+    chr_seq_counts: Dict[str, int] = defaultdict(int)
+    for name, _ in sequences:
+        chrom = extract_chromosome(name)
+        chr_seq_counts[chrom] += 1
+
+    if verbose:
+        print(f"Found {len(chr_seq_counts)} chromosomes")
+
+    # Map indels to chromosomes and sequences
+    indel_to_chr_seqs: Dict[Indel, Dict[str, List[str]]] = defaultdict(lambda: defaultdict(list))
+
+    for name, seq in sequences:
+        chrom = extract_chromosome(name)
+        indels = find_indels_in_sequence(seq)
+        for indel in indels:
+            indel_to_chr_seqs[indel][chrom].append(name)
+
+    # Analyze each indel
+    results = []
+
+    for indel, chr_seqs in indel_to_chr_seqs.items():
+        total_count = sum(len(seqs) for seqs in chr_seqs.values())
+
+        # Skip rare indels
+        if total_count < min_count:
+            continue
+
+        # Calculate per-chromosome statistics
+        chr_stats = []
+        for chrom, seq_names in chr_seqs.items():
+            count_in_chr = len(seq_names)
+            total_in_chr = chr_seq_counts[chrom]
+            fraction = count_in_chr / total_in_chr if total_in_chr > 0 else 0
+            chr_stats.append({
+                "chr": chrom,
+                "count": count_in_chr,
+                "total": total_in_chr,
+                "fraction": fraction
+            })
+
+        # Sort by fraction descending
+        chr_stats.sort(key=lambda x: -x["fraction"])
+
+        # Calculate specificity metrics
+        n_chromosomes = len(chr_stats)
+        max_fraction = chr_stats[0]["fraction"] if chr_stats else 0
+
+        # Specificity score: high if indel is concentrated in few chromosomes
+        # If indel appears in all sequences of 1-2 chromosomes, it's highly specific
+        top_chr_contribution = sum(s["count"] for s in chr_stats[:2]) / total_count if total_count > 0 else 0
+
+        # Is this indel chromosome-specific?
+        # Criteria: >80% of occurrences from top 2 chromosomes AND top chr has >50% of its sequences with indel
+        is_chr_specific = top_chr_contribution > 0.8 and max_fraction > 0.5
+
+        results.append({
+            "start": indel.start,
+            "end": indel.end,
+            "length": indel.length,
+            "total_count": total_count,
+            "frequency": total_count / n_seqs,
+            "n_chromosomes": n_chromosomes,
+            "max_fraction": max_fraction,
+            "top_chr_contribution": top_chr_contribution,
+            "is_chr_specific": is_chr_specific,
+            "chr_stats": chr_stats
+        })
+
+    # Sort by total count descending
+    results.sort(key=lambda x: -x["total_count"])
+
+    # Summary statistics
+    n_total_indels = len(results)
+    n_chr_specific = sum(1 for r in results if r["is_chr_specific"])
+
+    # Write TSV
+    with open(output_tsv, 'w') as f:
+        f.write("# Indel Chromosome Specificity Analysis\n")
+        f.write(f"# Input: {input_fasta}\n")
+        f.write(f"# Total sequences: {n_seqs}\n")
+        f.write(f"# Chromosomes: {len(chr_seq_counts)}\n")
+        f.write(f"# Indels analyzed (count >= {min_count}): {n_total_indels}\n")
+        f.write(f"# Chromosome-specific indels: {n_chr_specific} ({n_chr_specific/n_total_indels*100:.1f}%)\n")
+        f.write("#\n")
+        f.write("start\tend\tlength\ttotal_count\tfrequency\tn_chromosomes\tmax_chr_fraction\ttop2_contribution\tis_specific\ttop_chromosomes\n")
+
+        for r in results:
+            top_chrs = ", ".join(f"{s['chr']}({s['count']}/{s['total']}={s['fraction']*100:.0f}%)"
+                                  for s in r["chr_stats"][:5])
+            f.write(f"{r['start']}\t{r['end']}\t{r['length']}\t{r['total_count']}\t"
+                    f"{r['frequency']*100:.2f}%\t{r['n_chromosomes']}\t{r['max_fraction']*100:.1f}%\t"
+                    f"{r['top_chr_contribution']*100:.1f}%\t{'YES' if r['is_chr_specific'] else 'no'}\t{top_chrs}\n")
+
+    # Write JSON for HTML report
+    json_data = {
+        "summary": {
+            "total_sequences": n_seqs,
+            "n_chromosomes": len(chr_seq_counts),
+            "n_indels_analyzed": n_total_indels,
+            "n_chr_specific": n_chr_specific,
+            "pct_chr_specific": round(n_chr_specific / n_total_indels * 100, 1) if n_total_indels > 0 else 0
+        },
+        "chromosome_counts": dict(chr_seq_counts),
+        "indels": results[:50]  # Top 50 for HTML
+    }
+
+    with open(output_json, 'w') as f:
+        json.dump(json_data, f, indent=2)
+
+    if verbose:
+        print(f"\nResults:")
+        print(f"  Indels analyzed: {n_total_indels}")
+        print(f"  Chromosome-specific: {n_chr_specific} ({n_chr_specific/n_total_indels*100:.1f}%)")
+        print(f"\nOutput: {output_tsv}, {output_json}")
+
+    return json_data
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Analyze chromosome specificity of indels")
+    parser.add_argument("-i", "--input", required=True, help="Aligned FASTA file")
+    parser.add_argument("-o", "--output", required=True, help="Output prefix")
+    parser.add_argument("-m", "--min-count", type=int, default=3,
+                        help="Minimum count for indel to be included (default: 3)")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
+
+    args = parser.parse_args()
+
+    analyze_chromosome_specificity(
+        input_fasta=args.input,
+        output_tsv=f"{args.output}.chr_specificity.tsv",
+        output_json=f"{args.output}.chr_specificity.json",
+        min_count=args.min_count,
+        verbose=args.verbose
+    )
+
+
+if __name__ == "__main__":
+    main()

ArraySplitter/main.py

@@ -31,6 +31,12 @@ Examples:
   # Extract unique monomers
   arraysplitter extract -i decomposed.fa -o monomers
 
+  # Refine decomposition (split merged, join fragments)
+  arraysplitter refine -i decomposed.fa -o refined.fa
+
+  # Refine MSA by removing sequences with rare indels
+  arraysplitter msa_refine -i aligned.fa -o cleaned
+
 For help on specific commands:
   arraysplitter <command> -h
 """
@@ -122,7 +128,51 @@ For help on specific commands:
         type=int,
         default=1
     )
-    
+
+    # Refine command
+    parser_refine = subparsers.add_parser(
+        'refine',
+        help='Refine decomposition: split merged monomers, join fragments'
+    )
+    parser_refine.add_argument("-i", "--input", help="Input .monomers.tsv file", required=True)
+    parser_refine.add_argument("-o", "--output", help="Output .monomers.tsv file (refined)", required=True)
+    parser_refine.add_argument(
+        "-l", "--length",
+        help="Expected monomer length (auto-detected if not specified)",
+        type=int,
+        default=None
+    )
+    parser_refine.add_argument(
+        "-m", "--mismatches",
+        help="Maximum mismatches for cut sequence matching (default: 3)",
+        type=int,
+        default=3
+    )
+    parser_refine.add_argument(
+        "-v", "--verbose",
+        help="Verbose output",
+        action="store_true"
+    )
+
+    # MSA Refine command
+    parser_msa_refine = subparsers.add_parser(
+        'msa_refine',
+        help='Remove sequences with rare indels from MSA'
+    )
+    parser_msa_refine.add_argument("-i", "--input", help="Input aligned FASTA file", required=True)
+    parser_msa_refine.add_argument("-o", "--output", help="Output prefix", required=True)
+    parser_msa_refine.add_argument(
+        "-f", "--min-freq",
+        help="Minimum indel frequency threshold (0-1, default: 0.05 = 5%%)",
+        type=float,
+        default=0.05
+    )
+    parser_msa_refine.add_argument(
+        "-v", "--verbose",
+        help="Verbose output",
+        action="store_true"
+    )
+
     # Parse arguments
     args = parser.parse_args()
     
@@ -178,9 +228,50 @@ For help on specific commands:
     
     elif args.command == 'extract':
         from .extract import main as extract_main
-        
+
         extract_main(args.fasta, args.output, min_tf=args.min)
-    
+
+    elif args.command == 'refine':
+        from .refine import main as refine_main
+        import os
+
+        if not os.path.isfile(args.input):
+            print(f"Error: Input file {args.input} not found")
+            sys.exit(1)
+
+        refine_main(
+            input_tsv=args.input,
+            output_tsv=args.output,
+            expected_length=args.length,
+            max_mismatches=args.mismatches,
+            verbose=args.verbose
+        )
+
+    elif args.command == 'msa_refine':
+        from .msa_refine import refine_msa
+        import os
+
+        if not os.path.isfile(args.input):
+            print(f"Error: Input file {args.input} not found")
+            sys.exit(1)
+
+        output_clean = f"{args.output}.clean.fasta"
+        output_removed = f"{args.output}.removed.fasta"
+        output_stats = f"{args.output}.indel_stats.tsv"
+
+        stats = refine_msa(
+            input_fasta=args.input,
+            output_clean=output_clean,
+            output_removed=output_removed,
+            output_stats=output_stats,
+            min_freq=args.min_freq,
+            verbose=args.verbose
+        )
+
+        print(f"\nRefinement complete:")
+        print(f"  Total: {stats['total']} -> Clean: {stats['clean']}, Removed: {stats['removed']}")
+        print(f"  Indels: {stats['total_indels']} total, {stats['rare_indels']} rare (< {stats['min_freq']*100:.1f}%)")
+
     else:
         print(f"Error: Unknown command '{args.command}'")
         parser.print_help()

ArraySplitter/monomer_rotation.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ArraySplitter Monomer Rotation - Rotate monomers to common starting position.
+
+This module finds the optimal rotation k-mer and rotates all monomers to start
+from that position. Important for tandem repeats: the sequence is duplicated
+before searching to handle cases where the k-mer spans the boundary.
+
+Algorithm:
+1. Find k-mer that appears exactly ONCE in the maximum number of monomers
+2. Duplicate each sequence (seq+seq) to handle boundary cases
+3. Find best match position using edit distance
+4. Rotate sequence to start from that position
+5. Ensure all sequences are on the same strand
+"""
+
+import argparse
+from collections import Counter
+from typing import List, Tuple, Dict, Optional
+import editdistance as ed
+
+
+def get_revcomp(seq: str) -> str:
+    """Get reverse complement of sequence."""
+    complement = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C', 'N': 'N'}
+    return ''.join(complement.get(b, b) for b in reversed(seq))
+
+
+def find_best_rotation_kmer(sequences: List[str], k: int = 16) -> Tuple[Optional[str], float]:
+    """
+    Find k-mer that appears exactly once in the maximum number of sequences.
+
+    Args:
+        sequences: List of DNA sequences (monomers)
+        k: K-mer length (default 16)
+
+    Returns:
+        (best_kmer, fraction_unique) - k-mer and fraction of sequences with unique occurrence
+    """
+    kmer_unique_count: Counter = Counter()
+
+    for seq in sequences:
+        # Count k-mers in this sequence
+        seq_kmers: Counter = Counter()
+        for i in range(len(seq) - k + 1):
+            kmer = seq[i:i+k]
+            seq_kmers[kmer] += 1
+
+        # Record k-mers that appear exactly once
+        for kmer, count in seq_kmers.items():
+            if count == 1:
+                kmer_unique_count[kmer] += 1
+
+    if not kmer_unique_count:
+        return None, 0.0
+
+    best_kmer, best_count = kmer_unique_count.most_common(1)[0]
+    fraction = best_count / len(sequences)
+
+    return best_kmer, fraction
+
+
+def find_kmer_position_in_doubled(seq: str, kmer: str, max_ed: int = 5) -> Tuple[int, int, str]:
+    """
+    Find best position of k-mer in doubled sequence (seq+seq) to handle boundary cases.
+    Checks both strands.
+
+    Args:
+        seq: Original sequence
+        kmer: K-mer to find
+        max_ed: Maximum edit distance allowed
+
+    Returns:
+        (position, edit_distance, strand) where position is in original [0, len(seq))
+    """
+    k = len(kmer)
+    seq_len = len(seq)
+
+    # Double the sequence to handle boundary
+    doubled_fwd = seq + seq
+    doubled_rev = get_revcomp(seq) + get_revcomp(seq)
+
+    best_pos = -1
+    best_dist = max_ed + 1
+    best_strand = "unk"
+
+    # Search in forward strand (doubled)
+    for i in range(seq_len):  # Only search first copy positions
+        window = doubled_fwd[i:i+k]
+        dist = ed.eval(kmer, window)
+        if dist < best_dist:
+            best_dist = dist
+            best_pos = i
+            best_strand = "fwd"
+
+    # Search in reverse complement (doubled)
+    for i in range(seq_len):
+        window = doubled_rev[i:i+k]
+        dist = ed.eval(kmer, window)
+        if dist < best_dist:
+            best_dist = dist
+            best_pos = i
+            best_strand = "rev"
+
+    if best_dist <= max_ed:
+        return best_pos, best_dist, best_strand
+
+    return -1, best_dist, "unk"
+
+
+def rotate_sequence(seq: str, position: int, strand: str) -> str:
+    """
+    Rotate sequence to start from given position.
+
+    Args:
+        seq: Original sequence
+        position: Position to start from
+        strand: "fwd" or "rev" - which strand to use
+
+    Returns:
+        Rotated sequence
+    """
+    if strand == "rev":
+        seq = get_revcomp(seq)
+
+    return seq[position:] + seq[:position]
+
+
+def rotate_monomers(input_fasta: str, output_fasta: str,
+                    kmer: Optional[str] = None, k: int = 16,
+                    max_ed: int = 5, verbose: bool = False) -> Dict:
+    """
+    Main rotation function.
+
+    Args:
+        input_fasta: Input FASTA file with monomers
+        output_fasta: Output FASTA file with rotated monomers
+        kmer: Optional rotation k-mer (auto-detected if not provided)
+        k: K-mer length for auto-detection
+        max_ed: Maximum edit distance for rotation
+        verbose: Print progress
+
+    Returns:
+        Statistics dictionary
+    """
+    # Read sequences
+    sequences = []
+    headers = []
+
+    with open(input_fasta) as f:
+        seq = ""
+        header = ""
+        for line in f:
+            if line.startswith(">"):
+                if seq:
+                    sequences.append(seq)
+                    headers.append(header)
+                seq = ""
+                header = line.strip()
+            else:
+                seq += line.strip()
+        if seq:
+            sequences.append(seq)
+            headers.append(header)
+
+    if verbose:
+        print(f"Loaded {len(sequences)} sequences")
+
+    # Auto-detect rotation k-mer if not provided
+    if kmer is None:
+        # Filter to normal-length monomers for k-mer detection
+        normal_seqs = [s for s in sequences if 600 <= len(s) <= 800]
+        if not normal_seqs:
+            normal_seqs = sequences
+
+        kmer, fraction = find_best_rotation_kmer(normal_seqs, k)
+
+        if verbose:
+            print(f"Auto-detected rotation k-mer: {kmer}")
+            print(f"  Unique in {fraction*100:.1f}% of monomers")
+
+    if kmer is None:
+        raise ValueError("Could not detect rotation k-mer")
+
+    # Rotate all sequences
+    rotated = []
+    stats = {
+        "total": len(sequences),
+        "fwd": 0,
+        "rev": 0,
+        "failed": 0,
+        "ed_distribution": Counter()
+    }
+
+    for header, seq in zip(headers, sequences):
+        pos, dist, strand = find_kmer_position_in_doubled(seq, kmer, max_ed)
+
+        if pos >= 0:
+            rotated_seq = rotate_sequence(seq, pos, strand)
+            stats[strand] += 1
+            stats["ed_distribution"][dist] += 1
+        else:
+            # Keep original if rotation failed
+            rotated_seq = seq
+            strand = "failed"
+            stats["failed"] += 1
+
+        rotated.append((header, rotated_seq, strand, dist))
+
+    # Write output
+    with open(output_fasta, 'w') as f:
+        for header, seq, strand, dist in rotated:
+            new_header = f"{header}|rot_{strand}|ed{dist}"
+            f.write(f"{new_header}\n{seq}\n")
+
+    if verbose:
+        print(f"\nRotation results:")
+        print(f"  Forward: {stats['fwd']}")
+        print(f"  Reverse: {stats['rev']}")
+        print(f"  Failed: {stats['failed']}")
+        print(f"\nEdit distance distribution:")
+        for d, c in sorted(stats["ed_distribution"].items()):
+            print(f"  ED={d}: {c}")
+
+    stats["kmer"] = kmer
+    return stats
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Rotate monomers to common starting position"
+    )
+    parser.add_argument("-i", "--input", required=True,
+                        help="Input FASTA file with monomers")
+    parser.add_argument("-o", "--output", required=True,
+                        help="Output FASTA file with rotated monomers")
+    parser.add_argument("-k", "--kmer", default=None,
+                        help="Rotation k-mer (auto-detected if not provided)")
+    parser.add_argument("-l", "--kmer-length", type=int, default=16,
+                        help="K-mer length for auto-detection (default: 16)")
+    parser.add_argument("-e", "--max-ed", type=int, default=5,
+                        help="Maximum edit distance for rotation (default: 5)")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Verbose output")
+
+    args = parser.parse_args()
+
+    stats = rotate_monomers(
+        input_fasta=args.input,
+        output_fasta=args.output,
+        kmer=args.kmer,
+        k=args.kmer_length,
+        max_ed=args.max_ed,
+        verbose=args.verbose
+    )
+
+    print(f"Rotation complete. K-mer used: {stats['kmer']}")
+
+
+if __name__ == "__main__":
+    main()

ArraySplitter/msa_refine.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+MSA Refine - Remove sequences with rare indels from multiple sequence alignment.
+
+This module analyzes indel patterns in MSA and removes sequences containing
+rare indels (below frequency threshold). This helps clean up alignments
+before final consensus generation.
+
+Algorithm:
+1. Parse aligned FASTA and identify all indel patterns (contiguous gaps)
+2. Calculate frequency of each indel pattern across all sequences
+3. Mark sequences containing rare indels (frequency < threshold)
+4. Output: cleaned sequences, removed sequences, statistics report
+"""
+
+import argparse
+from collections import Counter, defaultdict
+from typing import List, Tuple, Dict, Set
+from dataclasses import dataclass
+import re
+
+
+@dataclass
+class Indel:
+    """Represents an indel (gap region) in the alignment."""
+    start: int
+    end: int
+    length: int
+
+    def __hash__(self):
+        return hash((self.start, self.end))
+
+    def __eq__(self, other):
+        return self.start == other.start and self.end == other.end
+
+
+@dataclass
+class IndelStats:
+    """Statistics for an indel pattern."""
+    indel: Indel
+    count: int
+    frequency: float
+    sequences: List[str]  # sequence names containing this indel
+
+
+def parse_fasta(filepath: str) -> List[Tuple[str, str]]:
+    """Parse FASTA file, return list of (name, sequence) tuples."""
+    sequences = []
+    with open(filepath) as f:
+        name = ""
+        seq = ""
+        for line in f:
+            line = line.strip()
+            if line.startswith(">"):
+                if seq:
+                    sequences.append((name, seq))
+                name = line[1:].split()[0]  # Take first word as name
+                seq = ""
+            else:
+                seq += line
+        if seq:
+            sequences.append((name, seq))
+    return sequences
+
+
+def find_indels_in_sequence(seq: str) -> List[Indel]:
+    """Find all indel (gap) regions in a sequence."""
+    indels = []
+    in_gap = False
+    gap_start = 0
+
+    for i, base in enumerate(seq):
+        if base == '-':
+            if not in_gap:
+                in_gap = True
+                gap_start = i
+        else:
+            if in_gap:
+                indels.append(Indel(start=gap_start, end=i, length=i - gap_start))
+                in_gap = False
+
+    # Handle gap at end
+    if in_gap:
+        indels.append(Indel(start=gap_start, end=len(seq), length=len(seq) - gap_start))
+
+    return indels
+
+
+def analyze_indels(sequences: List[Tuple[str, str]]) -> Dict[Indel, IndelStats]:
+    """Analyze all indels across sequences and compute statistics."""
+    indel_to_seqs: Dict[Indel, List[str]] = defaultdict(list)
+    n_seqs = len(sequences)
+
+    for name, seq in sequences:
+        indels = find_indels_in_sequence(seq)
+        for indel in indels:
+            indel_to_seqs[indel].append(name)
+
+    # Compute statistics
+    stats = {}
+    for indel, seq_names in indel_to_seqs.items():
+        count = len(seq_names)
+        stats[indel] = IndelStats(
+            indel=indel,
+            count=count,
+            frequency=count / n_seqs,
+            sequences=seq_names
+        )
+
+    return stats
+
+
+def refine_msa(
+    input_fasta: str,
+    output_clean: str,
+    output_removed: str,
+    output_stats: str,
+    min_freq: float = 0.05,
+    verbose: bool = False
+) -> Dict:
+    """
+    Refine MSA by removing sequences with rare indels.
+
+    Args:
+        input_fasta: Input aligned FASTA file
+        output_clean: Output FASTA with cleaned sequences
+        output_removed: Output FASTA with removed sequences
+        output_stats: Output TSV with indel statistics
+        min_freq: Minimum frequency threshold for indels (default 5%)
+        verbose: Print progress
+
+    Returns:
+        Statistics dictionary
+    """
+    # Parse sequences
+    sequences = parse_fasta(input_fasta)
+    n_total = len(sequences)
+
+    if verbose:
+        print(f"Loaded {n_total} aligned sequences")
+
+    # Analyze indels
+    indel_stats = analyze_indels(sequences)
+
+    if verbose:
+        print(f"Found {len(indel_stats)} unique indel patterns")
+
+    # Find rare indels and sequences to remove
+    rare_indels = {indel: stats for indel, stats in indel_stats.items()
+                   if stats.frequency < min_freq}
+
+    # Collect sequences to remove (those with ANY rare indel)
+    seqs_to_remove: Set[str] = set()
+    for stats in rare_indels.values():
+        seqs_to_remove.update(stats.sequences)
+
+    if verbose:
+        print(f"Found {len(rare_indels)} rare indels (freq < {min_freq*100:.1f}%)")
+        print(f"Sequences to remove: {len(seqs_to_remove)}")
+
+    # Split sequences
+    clean_seqs = []
+    removed_seqs = []
+
+    for name, seq in sequences:
+        if name in seqs_to_remove:
+            removed_seqs.append((name, seq))
+        else:
+            clean_seqs.append((name, seq))
+
+    # Write clean sequences (remove columns that are now all gaps)
+    if clean_seqs:
+        # Find columns that have at least one non-gap
+        align_len = len(clean_seqs[0][1])
+        non_gap_cols = []
+        for i in range(align_len):
+            has_base = any(seq[i] != '-' for _, seq in clean_seqs)
+            if has_base:
+                non_gap_cols.append(i)
+
+        # Write cleaned sequences with gap-only columns removed
+        with open(output_clean, 'w') as f:
+            for name, seq in clean_seqs:
+                cleaned_seq = ''.join(seq[i] for i in non_gap_cols)
+                f.write(f">{name}\n{cleaned_seq}\n")
+
+        cols_removed = align_len - len(non_gap_cols)
+        if verbose and cols_removed > 0:
+            print(f"Removed {cols_removed} gap-only columns")
+
+    # Write removed sequences
+    with open(output_removed, 'w') as f:
+        for name, seq in removed_seqs:
+            f.write(f">{name}\n{seq}\n")
+
+    # Write statistics
+    with open(output_stats, 'w') as f:
+        f.write("# MSA Refinement Statistics\n")
+        f.write(f"# Input: {input_fasta}\n")
+        f.write(f"# Total sequences: {n_total}\n")
+        f.write(f"# Clean sequences: {len(clean_seqs)}\n")
+        f.write(f"# Removed sequences: {len(removed_seqs)}\n")
+        f.write(f"# Min frequency threshold: {min_freq*100:.1f}%\n")
+        f.write(f"# Unique indel patterns: {len(indel_stats)}\n")
+        f.write(f"# Rare indels (below threshold): {len(rare_indels)}\n")
+        f.write("#\n")
+        f.write("# Indel details (sorted by frequency):\n")
+        f.write("start\tend\tlength\tcount\tfrequency\tis_rare\texample_sequences\n")
+
+        # Sort by frequency descending
+        sorted_indels = sorted(indel_stats.items(),
+                               key=lambda x: (-x[1].frequency, x[0].start))
+
+        for indel, stats in sorted_indels:
+            is_rare = "RARE" if stats.frequency < min_freq else ""
+            example_seqs = ",".join(stats.sequences[:3])
+            if len(stats.sequences) > 3:
+                example_seqs += f"... (+{len(stats.sequences)-3} more)"
+            f.write(f"{indel.start}\t{indel.end}\t{indel.length}\t"
+                    f"{stats.count}\t{stats.frequency*100:.2f}%\t{is_rare}\t{example_seqs}\n")
+
+    if verbose:
+        print(f"\nOutput files:")
+        print(f"  Clean sequences: {output_clean}")
+        print(f"  Removed sequences: {output_removed}")
+        print(f"  Statistics: {output_stats}")
+
+    return {
+        "total": n_total,
+        "clean": len(clean_seqs),
+        "removed": len(removed_seqs),
+        "total_indels": len(indel_stats),
+        "rare_indels": len(rare_indels),
+        "min_freq": min_freq
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Refine MSA by removing sequences with rare indels"
+    )
+    parser.add_argument("-i", "--input", required=True,
+                        help="Input aligned FASTA file")
+    parser.add_argument("-o", "--output", required=True,
+                        help="Output prefix for cleaned files")
+    parser.add_argument("-f", "--min-freq", type=float, default=0.05,
+                        help="Minimum indel frequency threshold (default: 0.05 = 5%%)")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Verbose output")
+
+    args = parser.parse_args()
+
+    output_clean = f"{args.output}.clean.fasta"
+    output_removed = f"{args.output}.removed.fasta"
+    output_stats = f"{args.output}.indel_stats.tsv"
+
+    stats = refine_msa(
+        input_fasta=args.input,
+        output_clean=output_clean,
+        output_removed=output_removed,
+        output_stats=output_stats,
+        min_freq=args.min_freq,
+        verbose=args.verbose
+    )
+
+    print(f"\nRefinement complete:")
+    print(f"  Total: {stats['total']} → Clean: {stats['clean']}, Removed: {stats['removed']}")
+    print(f"  Indels: {stats['total_indels']} total, {stats['rare_indels']} rare (< {stats['min_freq']*100:.1f}%)")
+
+
+if __name__ == "__main__":
+    main()

ArraySplitter/refine.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ArraySplitter Refine - Post-processing of decomposed monomers.
+
+This module refines the .monomers.tsv file by:
+1. Splitting merged monomers (doubled, tripled, etc.) that have mutations in cut sites
+2. Merging fragmented half-monomers with their neighbors
+
+Input/Output: .monomers.tsv format
+"""
+
+import argparse
+from collections import Counter
+from typing import List, Dict, Optional, Tuple
+import editdistance as ed
+
+
+def read_monomers_tsv(filepath: str) -> Tuple[List[str], List[Dict]]:
+    """
+    Read .monomers.tsv file.
+    Returns (header_fields, list of row dicts).
+    """
+    rows = []
+    header_fields = []
+
+    with open(filepath, 'r') as f:
+        header = f.readline().strip()
+        header_fields = header.split('\t')
+
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split('\t')
+            row = {}
+            for i, field in enumerate(header_fields):
+                if i < len(parts):
+                    row[field] = parts[i]
+                else:
+                    row[field] = ''
+
+            # Convert numeric fields
+            if 'length' in row:
+                row['length'] = int(row['length']) if row['length'] else 0
+            if 'index' in row:
+                row['index'] = int(row['index']) if row['index'] else 0
+
+            rows.append(row)
+
+    return header_fields, rows
+
+
+def write_monomers_tsv(filepath: str, header_fields: List[str], rows: List[Dict]):
+    """Write rows to .monomers.tsv file."""
+    with open(filepath, 'w') as f:
+        f.write('\t'.join(header_fields) + '\n')
+        for row in rows:
+            values = []
+            for field in header_fields:
+                val = row.get(field, '')
+                values.append(str(val))
+            f.write('\t'.join(values) + '\n')
+
+
+def find_cut_sequence(rows: List[Dict], kmer_len: int = 16) -> Optional[str]:
+    """
+    Discover the most common starting k-mer from monomers.
+    """
+    kmer_counts: Counter = Counter()
+
+    for row in rows:
+        if row.get('type') == 'MONOMER':
+            seq = row.get('sequence', '')
+            if len(seq) >= kmer_len:
+                kmer = seq[:kmer_len]
+                kmer_counts[kmer] += 1
+
+    if not kmer_counts:
+        return None
+
+    return kmer_counts.most_common(1)[0][0]
+
+
+def get_expected_lengths(rows: List[Dict]) -> Tuple[List[int], int]:
+    """
+    Analyze length distribution to find expected monomer lengths.
+    Returns (common_lengths, modal_length).
+    """
+    lengths = []
+    for row in rows:
+        if row.get('type') == 'MONOMER':
+            lengths.append(row.get('length', 0))
+
+    if not lengths:
+        return [], 700
+
+    length_counts = Counter(lengths)
+    common_lengths = [l for l, c in length_counts.most_common(10) if c >= 5]
+
+    if not common_lengths:
+        common_lengths = [length_counts.most_common(1)[0][0]]
+
+    modal_length = common_lengths[0]
+    return common_lengths, modal_length
+
+
+def find_approximate_cut_positions(sequence: str, cut_seq: str,
+                                    max_mismatches: int = 3) -> List[int]:
+    """
+    Find positions where cut_seq appears with up to max_mismatches.
+    """
+    positions = []
+    cut_len = len(cut_seq)
+
+    for i in range(len(sequence) - cut_len + 1):
+        window = sequence[i:i + cut_len]
+        dist = ed.eval(cut_seq, window)
+        if dist <= max_mismatches:
+            positions.append(i)
+
+    return positions
+
+
+def split_sequence(sequence: str, cut_seq: str, expected_length: int,
+                   max_mismatches: int = 3) -> List[str]:
+    """
+    Split a merged sequence into individual monomers.
+    """
+    seq_len = len(sequence)
+    num_monomers = round(seq_len / expected_length)
+
+    if num_monomers < 2:
+        return [sequence]
+
+    cut_positions = find_approximate_cut_positions(sequence, cut_seq, max_mismatches)
+
+    if not cut_positions:
+        return [sequence]
+
+    # Filter positions - need minimum distance between cuts
+    min_distance = expected_length * 0.6
+    filtered_positions = []
+    last_pos = 0
+
+    for pos in sorted(cut_positions):
+        if pos > 0 and pos - last_pos >= min_distance:
+            filtered_positions.append(pos)
+            last_pos = pos
+
+    if not filtered_positions:
+        return [sequence]
+
+    # Split at found positions
+    result = []
+    prev_pos = 0
+
+    for pos in filtered_positions:
+        part = sequence[prev_pos:pos]
+        if part:
+            result.append(part)
+        prev_pos = pos
+
+    last_part = sequence[prev_pos:]
+    if last_part:
+        result.append(last_part)
+
+    return result
+
+
+def refine_monomers_tsv(input_tsv: str, output_tsv: str,
+                        expected_length: Optional[int] = None,
+                        max_mismatches: int = 3,
+                        min_length_ratio: float = 0.5,
+                        max_length_ratio: float = 1.5,
+                        verbose: bool = False) -> Dict:
+    """
+    Main refinement function for TSV files.
+    """
+    # Read input
+    header_fields, rows = read_monomers_tsv(input_tsv)
+
+    if verbose:
+        print(f"Read {len(rows)} rows from {input_tsv}")
+
+    # Find cut sequence
+    cut_seq = find_cut_sequence(rows)
+    if cut_seq is None:
+        print("Warning: Could not detect cut sequence")
+    if verbose:
+        print(f"Detected cut sequence: {cut_seq}")
+
+    # Analyze length distribution
+    common_lengths, modal_length = get_expected_lengths(rows)
+
+    if expected_length is None:
+        expected_length = modal_length
+
+    if verbose:
+        print(f"Expected monomer length: {expected_length} bp")
+        print(f"Common lengths: {common_lengths[:5]}")
+
+    # Define thresholds
+    min_length = int(expected_length * min_length_ratio)
+    max_length = int(expected_length * max_length_ratio)
+
+    if verbose:
+        print(f"Normal range: {min_length}-{max_length} bp")
+
+    # Process rows
+    refined_rows = []
+    stats = {
+        'total_input': len(rows),
+        'normal': 0,
+        'split': 0,
+        'merged': 0,
+        'flanks': 0,
+        'split_into': 0,
+        'merged_from': 0,
+    }
+
+    i = 0
+    while i < len(rows):
+        row = rows[i]
+        row_type = row.get('type', '')
+        seq = row.get('sequence', '')
+        seq_len = row.get('length', len(seq))
+
+        # Skip flanks - keep as is
+        if 'FLANK' in row_type:
+            refined_rows.append(row)
+            stats['flanks'] += 1
+            i += 1
+            continue
+
+        # Normal length - keep as is
+        if min_length <= seq_len <= max_length:
+            refined_rows.append(row)
+            stats['normal'] += 1
+            i += 1
+            continue
+
+        # Merged monomer (too long) - try to split
+        if seq_len > max_length:
+            if cut_seq is not None:
+                parts = split_sequence(seq, cut_seq, expected_length, max_mismatches)
+
+                if len(parts) > 1:
+                    stats['split'] += 1
+                    stats['split_into'] += len(parts)
+
+                    if verbose:
+                        print(f"  Split {seq_len}bp -> {[len(p) for p in parts]}")
+
+                    # Create new rows for each part
+                    for j, part in enumerate(parts):
+                        new_row = row.copy()
+                        new_row['sequence'] = part
+                        new_row['length'] = len(part)
+                        new_row['type'] = f"MONOMER|split_{j+1}of{len(parts)}"
+                        refined_rows.append(new_row)
+
+                    i += 1
+                    continue
+
+            # Could not split
+            row_copy = row.copy()
+            row_copy['type'] = row_type + '|unsplit'
+            refined_rows.append(row_copy)
+            i += 1
+            continue
+
+        # Short monomer - try to merge with neighbor
+        if seq_len < min_length:
+            merged = False
+
+            # Look at next row
+            if i + 1 < len(rows):
+                next_row = rows[i + 1]
+                next_type = next_row.get('type', '')
+                next_seq = next_row.get('sequence', '')
+                next_len = next_row.get('length', len(next_seq))
+
+                # Check if same array and both are short monomers
+                if (row.get('sequence_id') == next_row.get('sequence_id') and
+                    'FLANK' not in next_type and
+                    next_len < min_length):
+
+                    combined_len = seq_len + next_len
+                    # Check if combined is reasonable
+                    if min_length <= combined_len <= max_length * 1.1:
+                        # Merge
+                        merged_seq = seq + next_seq
+                        merged_row = row.copy()
+                        merged_row['sequence'] = merged_seq
+                        merged_row['length'] = len(merged_seq)
+                        merged_row['type'] = 'MONOMER|merged'
+                        refined_rows.append(merged_row)
+
+                        stats['merged'] += 1
+                        stats['merged_from'] += 2
+
+                        if verbose:
+                            print(f"  Merged {seq_len}bp + {next_len}bp = {len(merged_seq)}bp")
+
+                        i += 2
+                        merged = True
+
+            if not merged:
+                # Keep as is with tag
+                row_copy = row.copy()
+                row_copy['type'] = row_type + '|short'
+                refined_rows.append(row_copy)
+                i += 1
+            continue
+
+        # Fallback
+        refined_rows.append(row)
+        i += 1
+
+    stats['total_output'] = len(refined_rows)
+
+    # Renumber indices within each sequence_id
+    current_seq_id = None
+    current_idx = 0
+    for row in refined_rows:
+        seq_id = row.get('sequence_id')
+        if seq_id != current_seq_id:
+            current_seq_id = seq_id
+            current_idx = 0
+        row['index'] = current_idx
+        current_idx += 1
+
+    # Write output
+    write_monomers_tsv(output_tsv, header_fields, refined_rows)
+
+    if verbose:
+        print(f"\nRefinement statistics:")
+        print(f"  Input rows: {stats['total_input']}")
+        print(f"  Output rows: {stats['total_output']}")
+        print(f"  Normal (unchanged): {stats['normal']}")
+        print(f"  Split merged: {stats['split']} -> {stats['split_into']} parts")
+        print(f"  Merged short: {stats['merged']} (from {stats['merged_from']} fragments)")
+        print(f"  Flanks (unchanged): {stats['flanks']}")
+
+    return stats
+
+
+def main(input_tsv: str, output_tsv: str,
+         expected_length: Optional[int] = None,
+         max_mismatches: int = 3,
+         verbose: bool = False) -> Dict:
+    """Main entry point for refine command."""
+
+    stats = refine_monomers_tsv(
+        input_tsv=input_tsv,
+        output_tsv=output_tsv,
+        expected_length=expected_length,
+        max_mismatches=max_mismatches,
+        verbose=verbose
+    )
+
+    return stats
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Refine ArraySplitter .monomers.tsv by splitting merged monomers and joining fragments"
+    )
+    parser.add_argument("-i", "--input", required=True,
+                        help="Input .monomers.tsv file")
+    parser.add_argument("-o", "--output", required=True,
+                        help="Output .monomers.tsv file (refined)")
+    parser.add_argument("-l", "--length", type=int, default=None,
+                        help="Expected monomer length (auto-detected if not specified)")
+    parser.add_argument("-m", "--mismatches", type=int, default=3,
+                        help="Maximum mismatches for cut sequence matching (default: 3)")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Verbose output")
+
+    args = parser.parse_args()
+
+    main(
+        input_tsv=args.input,
+        output_tsv=args.output,
+        expected_length=args.length,
+        max_mismatches=args.mismatches,
+        verbose=args.verbose
+    )

{arraysplitter-1.3.0.dist-info → arraysplitter-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ArraySplitter
-Version: 1.3.0
+Version: 1.4.0
 Summary: De Novo Decomposition of Satellite DNA Arrays into Monomers within Telomere-to-Telomere Assemblies
 Home-page: https://github.com/aglabx/ArraySplitter
 Author: Aleksey Komissarov

{arraysplitter-1.3.0.dist-info → arraysplitter-1.4.0.dist-info}/RECORD

@@ -1,8 +1,12 @@
-ArraySplitter/__init__.py,sha256=h8Fz1eD4MXGpCgKpn1RqfA_t7dfCg6O8Vx0ysxCMdHE,169
+ArraySplitter/__init__.py,sha256=FzGKjGO2V7GyNG6KBBUhvlc9IeGFcEMJofm6vKxgGEg,169
 ArraySplitter/classify.py,sha256=9OkmGecGvEa40mmIK4BPblVpuRBfZFS7oWbkETJ2idQ,21023
 ArraySplitter/decompose.py,sha256=Voqcwgquy6-lSnVjusjPrKiszxG5x0SIhmjrtGCTVvo,70530
 ArraySplitter/extract.py,sha256=bqlC8E6MjndMCBa44Zvv23gXmH2yr7IdQBXnc5foqhM,2300
-ArraySplitter/main.py,sha256=GTOI6Ca38-Ws_a127MbLtPnSzKPBKrDFHS48_1Tey54,5709
+ArraySplitter/indel_chromosome_analysis.py,sha256=xSgJqY1Pmeie47hnuc2tKZvyuB0RWpfTAwYB_K3cJ5w,8283
+ArraySplitter/main.py,sha256=hoFw4OMMF27nmz0eyg1-gMtpQ43sXBkqQAyads5Lmrw,8799
+ArraySplitter/monomer_rotation.py,sha256=tdpqmNJYJQxatsEZ6v6ieEUizZQTS66ggrDKosQk1FM,8023
+ArraySplitter/msa_refine.py,sha256=AHbUzRv7uCpxCJ9AFAWDEEwahz57wRiYT3XaFu9cpuw,8886
+ArraySplitter/refine.py,sha256=CmXuUViCbbGo5ZtojfAzqyVaV2DA7RlIITFMoKN8ou4,12069
 ArraySplitter/rotate.py,sha256=ZvH2WDnRlfeqbXtUDmrgqE2CQEGe5qKDVR-my_uKkLA,1231
 ArraySplitter/core_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ArraySplitter/core_functions/io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -19,9 +23,9 @@ ArraySplitter/core_functions/tools/fs_tree.py,sha256=OHvamldtHW_7VEiWg7XiJ7Xwml2
 ArraySplitter/core_functions/tools/parsers.py,sha256=J_rtt384ycfgrWlgrazb3qj7Sr0waWj7yShlril5rqM,6766
 ArraySplitter/core_functions/tools/rotation.py,sha256=v_-3gkx8AR10RDCNMO82Pok1ZbqT2Zy9asyWZ6Ib_v8,3109
 ArraySplitter/core_functions/tools/sequences.py,sha256=v74uHp1nTF_UvMRefYmJfMbnzdurRn995C06hugoSa4,441
-arraysplitter-1.3.0.dist-info/licenses/LICENSE,sha256=m2dY1BxIk3gKB3jSa8kDtlSnNfW68VumXAQzGacCtvc,1063
-arraysplitter-1.3.0.dist-info/METADATA,sha256=E-rQhLl9Cn-xcF9hYuuhjyXxRURf2ca8trtgd8cAa_E,10177
-arraysplitter-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-arraysplitter-1.3.0.dist-info/entry_points.txt,sha256=H8EDFoNSqnM6J8m2w97H8HKjzAqs-iNTrIHk87-kfhY,60
-arraysplitter-1.3.0.dist-info/top_level.txt,sha256=zk1yzRnjygCl8ZF8rx3w4WSWdW8FUw1-MqSk-s3NOHY,14
-arraysplitter-1.3.0.dist-info/RECORD,,
+arraysplitter-1.4.0.dist-info/licenses/LICENSE,sha256=m2dY1BxIk3gKB3jSa8kDtlSnNfW68VumXAQzGacCtvc,1063
+arraysplitter-1.4.0.dist-info/METADATA,sha256=tyZZZNL2E6gqOWVVlmAEbevdHD2QmX8O_-ropuYoWvQ,10177
+arraysplitter-1.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+arraysplitter-1.4.0.dist-info/entry_points.txt,sha256=H8EDFoNSqnM6J8m2w97H8HKjzAqs-iNTrIHk87-kfhY,60
+arraysplitter-1.4.0.dist-info/top_level.txt,sha256=zk1yzRnjygCl8ZF8rx3w4WSWdW8FUw1-MqSk-s3NOHY,14
+arraysplitter-1.4.0.dist-info/RECORD,,