ArraySplitter 1.3.0__tar.gz → 1.4.0__tar.gz

{arraysplitter-1.3.0/src/ArraySplitter.egg-info → arraysplitter-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ArraySplitter
-Version: 1.3.0
+Version: 1.4.0
 Summary: De Novo Decomposition of Satellite DNA Arrays into Monomers within Telomere-to-Telomere Assemblies
 Home-page: https://github.com/aglabx/ArraySplitter
 Author: Aleksey Komissarov

{arraysplitter-1.3.0 → arraysplitter-1.4.0}/src/ArraySplitter/__init__.py

@@ -5,6 +5,6 @@
 # @author: Aleksey Komissarov
 # @contact: ad3002@gmail.com
 
-__version__ = "1.3.0"
+__version__ = "1.4.0"
 
 # __all__ = []

arraysplitter-1.4.0/src/ArraySplitter/indel_chromosome_analysis.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Indel Chromosome Specificity Analysis.
+
+Analyzes how chromosome-specific indels are in MSA alignments.
+For each indel, calculates:
+- Which chromosomes contain sequences with this indel
+- What fraction of sequences from each chromosome have the indel
+- Overall specificity score
+"""
+
+import argparse
+from collections import defaultdict
+from typing import List, Tuple, Dict, Set
+from dataclasses import dataclass
+import re
+import json
+
+
+@dataclass
+class Indel:
+    start: int
+    end: int
+    length: int
+
+    def __hash__(self):
+        return hash((self.start, self.end))
+
+    def __eq__(self, other):
+        return self.start == other.start and self.end == other.end
+
+
+def parse_fasta(filepath: str) -> List[Tuple[str, str]]:
+    """Parse FASTA file."""
+    sequences = []
+    with open(filepath) as f:
+        name = ""
+        seq = ""
+        for line in f:
+            line = line.strip()
+            if line.startswith(">"):
+                if seq:
+                    sequences.append((name, seq))
+                name = line[1:].split()[0]
+                seq = ""
+            else:
+                seq += line
+        if seq:
+            sequences.append((name, seq))
+    return sequences
+
+
+def extract_chromosome(name: str) -> str:
+    """Extract chromosome name from sequence header."""
+    # Pattern: chr1_mat:123-456|... or chr1A_pat:...
+    match = re.match(r'(chr[^:]+)', name)
+    if match:
+        return match.group(1)
+    return "unknown"
+
+
+def find_indels_in_sequence(seq: str) -> List[Indel]:
+    """Find all indel regions in a sequence."""
+    indels = []
+    in_gap = False
+    gap_start = 0
+
+    for i, base in enumerate(seq):
+        if base == '-':
+            if not in_gap:
+                in_gap = True
+                gap_start = i
+        else:
+            if in_gap:
+                indels.append(Indel(start=gap_start, end=i, length=i - gap_start))
+                in_gap = False
+
+    if in_gap:
+        indels.append(Indel(start=gap_start, end=len(seq), length=len(seq) - gap_start))
+
+    return indels
+
+
+def analyze_chromosome_specificity(input_fasta: str, output_tsv: str, output_json: str,
+                                    min_count: int = 3, verbose: bool = False) -> Dict:
+    """
+    Analyze chromosome specificity of indels.
+
+    Args:
+        input_fasta: Aligned FASTA file
+        output_tsv: Output TSV with detailed analysis
+        output_json: Output JSON for HTML report
+        min_count: Minimum sequence count for indel to be included
+        verbose: Print progress
+    """
+    sequences = parse_fasta(input_fasta)
+    n_seqs = len(sequences)
+
+    if verbose:
+        print(f"Loaded {n_seqs} sequences")
+
+    # Count sequences per chromosome
+    chr_seq_counts: Dict[str, int] = defaultdict(int)
+    for name, _ in sequences:
+        chrom = extract_chromosome(name)
+        chr_seq_counts[chrom] += 1
+
+    if verbose:
+        print(f"Found {len(chr_seq_counts)} chromosomes")
+
+    # Map indels to chromosomes and sequences
+    indel_to_chr_seqs: Dict[Indel, Dict[str, List[str]]] = defaultdict(lambda: defaultdict(list))
+
+    for name, seq in sequences:
+        chrom = extract_chromosome(name)
+        indels = find_indels_in_sequence(seq)
+        for indel in indels:
+            indel_to_chr_seqs[indel][chrom].append(name)
+
+    # Analyze each indel
+    results = []
+
+    for indel, chr_seqs in indel_to_chr_seqs.items():
+        total_count = sum(len(seqs) for seqs in chr_seqs.values())
+
+        # Skip rare indels
+        if total_count < min_count:
+            continue
+
+        # Calculate per-chromosome statistics
+        chr_stats = []
+        for chrom, seq_names in chr_seqs.items():
+            count_in_chr = len(seq_names)
+            total_in_chr = chr_seq_counts[chrom]
+            fraction = count_in_chr / total_in_chr if total_in_chr > 0 else 0
+            chr_stats.append({
+                "chr": chrom,
+                "count": count_in_chr,
+                "total": total_in_chr,
+                "fraction": fraction
+            })
+
+        # Sort by fraction descending
+        chr_stats.sort(key=lambda x: -x["fraction"])
+
+        # Calculate specificity metrics
+        n_chromosomes = len(chr_stats)
+        max_fraction = chr_stats[0]["fraction"] if chr_stats else 0
+
+        # Specificity score: high if indel is concentrated in few chromosomes
+        # If indel appears in all sequences of 1-2 chromosomes, it's highly specific
+        top_chr_contribution = sum(s["count"] for s in chr_stats[:2]) / total_count if total_count > 0 else 0
+
+        # Is this indel chromosome-specific?
+        # Criteria: >80% of occurrences from top 2 chromosomes AND top chr has >50% of its sequences with indel
+        is_chr_specific = top_chr_contribution > 0.8 and max_fraction > 0.5
+
+        results.append({
+            "start": indel.start,
+            "end": indel.end,
+            "length": indel.length,
+            "total_count": total_count,
+            "frequency": total_count / n_seqs,
+            "n_chromosomes": n_chromosomes,
+            "max_fraction": max_fraction,
+            "top_chr_contribution": top_chr_contribution,
+            "is_chr_specific": is_chr_specific,
+            "chr_stats": chr_stats
+        })
+
+    # Sort by total count descending
+    results.sort(key=lambda x: -x["total_count"])
+
+    # Summary statistics
+    n_total_indels = len(results)
+    n_chr_specific = sum(1 for r in results if r["is_chr_specific"])
+
+    # Write TSV
+    with open(output_tsv, 'w') as f:
+        f.write("# Indel Chromosome Specificity Analysis\n")
+        f.write(f"# Input: {input_fasta}\n")
+        f.write(f"# Total sequences: {n_seqs}\n")
+        f.write(f"# Chromosomes: {len(chr_seq_counts)}\n")
+        f.write(f"# Indels analyzed (count >= {min_count}): {n_total_indels}\n")
+        f.write(f"# Chromosome-specific indels: {n_chr_specific} ({n_chr_specific/n_total_indels*100:.1f}%)\n")
+        f.write("#\n")
+        f.write("start\tend\tlength\ttotal_count\tfrequency\tn_chromosomes\tmax_chr_fraction\ttop2_contribution\tis_specific\ttop_chromosomes\n")
+
+        for r in results:
+            top_chrs = ", ".join(f"{s['chr']}({s['count']}/{s['total']}={s['fraction']*100:.0f}%)"
+                                  for s in r["chr_stats"][:5])
+            f.write(f"{r['start']}\t{r['end']}\t{r['length']}\t{r['total_count']}\t"
+                    f"{r['frequency']*100:.2f}%\t{r['n_chromosomes']}\t{r['max_fraction']*100:.1f}%\t"
+                    f"{r['top_chr_contribution']*100:.1f}%\t{'YES' if r['is_chr_specific'] else 'no'}\t{top_chrs}\n")
+
+    # Write JSON for HTML report
+    json_data = {
+        "summary": {
+            "total_sequences": n_seqs,
+            "n_chromosomes": len(chr_seq_counts),
+            "n_indels_analyzed": n_total_indels,
+            "n_chr_specific": n_chr_specific,
+            "pct_chr_specific": round(n_chr_specific / n_total_indels * 100, 1) if n_total_indels > 0 else 0
+        },
+        "chromosome_counts": dict(chr_seq_counts),
+        "indels": results[:50]  # Top 50 for HTML
+    }
+
+    with open(output_json, 'w') as f:
+        json.dump(json_data, f, indent=2)
+
+    if verbose:
+        print(f"\nResults:")
+        print(f"  Indels analyzed: {n_total_indels}")
+        print(f"  Chromosome-specific: {n_chr_specific} ({n_chr_specific/n_total_indels*100:.1f}%)")
+        print(f"\nOutput: {output_tsv}, {output_json}")
+
+    return json_data
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Analyze chromosome specificity of indels")
+    parser.add_argument("-i", "--input", required=True, help="Aligned FASTA file")
+    parser.add_argument("-o", "--output", required=True, help="Output prefix")
+    parser.add_argument("-m", "--min-count", type=int, default=3,
+                        help="Minimum count for indel to be included (default: 3)")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
+
+    args = parser.parse_args()
+
+    analyze_chromosome_specificity(
+        input_fasta=args.input,
+        output_tsv=f"{args.output}.chr_specificity.tsv",
+        output_json=f"{args.output}.chr_specificity.json",
+        min_count=args.min_count,
+        verbose=args.verbose
+    )
+
+
+if __name__ == "__main__":
+    main()

{arraysplitter-1.3.0 → arraysplitter-1.4.0}/src/ArraySplitter/main.py

@@ -31,6 +31,12 @@ Examples:
   # Extract unique monomers
   arraysplitter extract -i decomposed.fa -o monomers
 
+  # Refine decomposition (split merged, join fragments)
+  arraysplitter refine -i decomposed.fa -o refined.fa
+
+  # Refine MSA by removing sequences with rare indels
+  arraysplitter msa_refine -i aligned.fa -o cleaned
+
 For help on specific commands:
   arraysplitter <command> -h
 """
@@ -122,7 +128,51 @@ For help on specific commands:
         type=int,
         default=1
     )
-    
+
+    # Refine command
+    parser_refine = subparsers.add_parser(
+        'refine',
+        help='Refine decomposition: split merged monomers, join fragments'
+    )
+    parser_refine.add_argument("-i", "--input", help="Input .monomers.tsv file", required=True)
+    parser_refine.add_argument("-o", "--output", help="Output .monomers.tsv file (refined)", required=True)
+    parser_refine.add_argument(
+        "-l", "--length",
+        help="Expected monomer length (auto-detected if not specified)",
+        type=int,
+        default=None
+    )
+    parser_refine.add_argument(
+        "-m", "--mismatches",
+        help="Maximum mismatches for cut sequence matching (default: 3)",
+        type=int,
+        default=3
+    )
+    parser_refine.add_argument(
+        "-v", "--verbose",
+        help="Verbose output",
+        action="store_true"
+    )
+
+    # MSA Refine command
+    parser_msa_refine = subparsers.add_parser(
+        'msa_refine',
+        help='Remove sequences with rare indels from MSA'
+    )
+    parser_msa_refine.add_argument("-i", "--input", help="Input aligned FASTA file", required=True)
+    parser_msa_refine.add_argument("-o", "--output", help="Output prefix", required=True)
+    parser_msa_refine.add_argument(
+        "-f", "--min-freq",
+        help="Minimum indel frequency threshold (0-1, default: 0.05 = 5%%)",
+        type=float,
+        default=0.05
+    )
+    parser_msa_refine.add_argument(
+        "-v", "--verbose",
+        help="Verbose output",
+        action="store_true"
+    )
+
     # Parse arguments
     args = parser.parse_args()
     
@@ -178,9 +228,50 @@ For help on specific commands:
     
     elif args.command == 'extract':
         from .extract import main as extract_main
-        
+
         extract_main(args.fasta, args.output, min_tf=args.min)
-    
+
+    elif args.command == 'refine':
+        from .refine import main as refine_main
+        import os
+
+        if not os.path.isfile(args.input):
+            print(f"Error: Input file {args.input} not found")
+            sys.exit(1)
+
+        refine_main(
+            input_tsv=args.input,
+            output_tsv=args.output,
+            expected_length=args.length,
+            max_mismatches=args.mismatches,
+            verbose=args.verbose
+        )
+
+    elif args.command == 'msa_refine':
+        from .msa_refine import refine_msa
+        import os
+
+        if not os.path.isfile(args.input):
+            print(f"Error: Input file {args.input} not found")
+            sys.exit(1)
+
+        output_clean = f"{args.output}.clean.fasta"
+        output_removed = f"{args.output}.removed.fasta"
+        output_stats = f"{args.output}.indel_stats.tsv"
+
+        stats = refine_msa(
+            input_fasta=args.input,
+            output_clean=output_clean,
+            output_removed=output_removed,
+            output_stats=output_stats,
+            min_freq=args.min_freq,
+            verbose=args.verbose
+        )
+
+        print(f"\nRefinement complete:")
+        print(f"  Total: {stats['total']} -> Clean: {stats['clean']}, Removed: {stats['removed']}")
+        print(f"  Indels: {stats['total_indels']} total, {stats['rare_indels']} rare (< {stats['min_freq']*100:.1f}%)")
+
     else:
         print(f"Error: Unknown command '{args.command}'")
         parser.print_help()

arraysplitter-1.4.0/src/ArraySplitter/monomer_rotation.py

@@ -0,0 +1,262 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ArraySplitter Monomer Rotation - Rotate monomers to common starting position.
+
+This module finds the optimal rotation k-mer and rotates all monomers to start
+from that position. Important for tandem repeats: the sequence is duplicated
+before searching to handle cases where the k-mer spans the boundary.
+
+Algorithm:
+1. Find k-mer that appears exactly ONCE in the maximum number of monomers
+2. Duplicate each sequence (seq+seq) to handle boundary cases
+3. Find best match position using edit distance
+4. Rotate sequence to start from that position
+5. Ensure all sequences are on the same strand
+"""
+
+import argparse
+from collections import Counter
+from typing import List, Tuple, Dict, Optional
+import editdistance as ed
+
+
+def get_revcomp(seq: str) -> str:
+    """Get reverse complement of sequence."""
+    complement = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C', 'N': 'N'}
+    return ''.join(complement.get(b, b) for b in reversed(seq))
+
+
+def find_best_rotation_kmer(sequences: List[str], k: int = 16) -> Tuple[Optional[str], float]:
+    """
+    Find k-mer that appears exactly once in the maximum number of sequences.
+
+    Args:
+        sequences: List of DNA sequences (monomers)
+        k: K-mer length (default 16)
+
+    Returns:
+        (best_kmer, fraction_unique) - k-mer and fraction of sequences with unique occurrence
+    """
+    kmer_unique_count: Counter = Counter()
+
+    for seq in sequences:
+        # Count k-mers in this sequence
+        seq_kmers: Counter = Counter()
+        for i in range(len(seq) - k + 1):
+            kmer = seq[i:i+k]
+            seq_kmers[kmer] += 1
+
+        # Record k-mers that appear exactly once
+        for kmer, count in seq_kmers.items():
+            if count == 1:
+                kmer_unique_count[kmer] += 1
+
+    if not kmer_unique_count:
+        return None, 0.0
+
+    best_kmer, best_count = kmer_unique_count.most_common(1)[0]
+    fraction = best_count / len(sequences)
+
+    return best_kmer, fraction
+
+
+def find_kmer_position_in_doubled(seq: str, kmer: str, max_ed: int = 5) -> Tuple[int, int, str]:
+    """
+    Find best position of k-mer in doubled sequence (seq+seq) to handle boundary cases.
+    Checks both strands.
+
+    Args:
+        seq: Original sequence
+        kmer: K-mer to find
+        max_ed: Maximum edit distance allowed
+
+    Returns:
+        (position, edit_distance, strand) where position is in original [0, len(seq))
+    """
+    k = len(kmer)
+    seq_len = len(seq)
+
+    # Double the sequence to handle boundary
+    doubled_fwd = seq + seq
+    doubled_rev = get_revcomp(seq) + get_revcomp(seq)
+
+    best_pos = -1
+    best_dist = max_ed + 1
+    best_strand = "unk"
+
+    # Search in forward strand (doubled)
+    for i in range(seq_len):  # Only search first copy positions
+        window = doubled_fwd[i:i+k]
+        dist = ed.eval(kmer, window)
+        if dist < best_dist:
+            best_dist = dist
+            best_pos = i
+            best_strand = "fwd"
+
+    # Search in reverse complement (doubled)
+    for i in range(seq_len):
+        window = doubled_rev[i:i+k]
+        dist = ed.eval(kmer, window)
+        if dist < best_dist:
+            best_dist = dist
+            best_pos = i
+            best_strand = "rev"
+
+    if best_dist <= max_ed:
+        return best_pos, best_dist, best_strand
+
+    return -1, best_dist, "unk"
+
+
+def rotate_sequence(seq: str, position: int, strand: str) -> str:
+    """
+    Rotate sequence to start from given position.
+
+    Args:
+        seq: Original sequence
+        position: Position to start from
+        strand: "fwd" or "rev" - which strand to use
+
+    Returns:
+        Rotated sequence
+    """
+    if strand == "rev":
+        seq = get_revcomp(seq)
+
+    return seq[position:] + seq[:position]
+
+
+def rotate_monomers(input_fasta: str, output_fasta: str,
+                    kmer: Optional[str] = None, k: int = 16,
+                    max_ed: int = 5, verbose: bool = False) -> Dict:
+    """
+    Main rotation function.
+
+    Args:
+        input_fasta: Input FASTA file with monomers
+        output_fasta: Output FASTA file with rotated monomers
+        kmer: Optional rotation k-mer (auto-detected if not provided)
+        k: K-mer length for auto-detection
+        max_ed: Maximum edit distance for rotation
+        verbose: Print progress
+
+    Returns:
+        Statistics dictionary
+    """
+    # Read sequences
+    sequences = []
+    headers = []
+
+    with open(input_fasta) as f:
+        seq = ""
+        header = ""
+        for line in f:
+            if line.startswith(">"):
+                if seq:
+                    sequences.append(seq)
+                    headers.append(header)
+                seq = ""
+                header = line.strip()
+            else:
+                seq += line.strip()
+        if seq:
+            sequences.append(seq)
+            headers.append(header)
+
+    if verbose:
+        print(f"Loaded {len(sequences)} sequences")
+
+    # Auto-detect rotation k-mer if not provided
+    if kmer is None:
+        # Filter to normal-length monomers for k-mer detection
+        normal_seqs = [s for s in sequences if 600 <= len(s) <= 800]
+        if not normal_seqs:
+            normal_seqs = sequences
+
+        kmer, fraction = find_best_rotation_kmer(normal_seqs, k)
+
+        if verbose:
+            print(f"Auto-detected rotation k-mer: {kmer}")
+            print(f"  Unique in {fraction*100:.1f}% of monomers")
+
+    if kmer is None:
+        raise ValueError("Could not detect rotation k-mer")
+
+    # Rotate all sequences
+    rotated = []
+    stats = {
+        "total": len(sequences),
+        "fwd": 0,
+        "rev": 0,
+        "failed": 0,
+        "ed_distribution": Counter()
+    }
+
+    for header, seq in zip(headers, sequences):
+        pos, dist, strand = find_kmer_position_in_doubled(seq, kmer, max_ed)
+
+        if pos >= 0:
+            rotated_seq = rotate_sequence(seq, pos, strand)
+            stats[strand] += 1
+            stats["ed_distribution"][dist] += 1
+        else:
+            # Keep original if rotation failed
+            rotated_seq = seq
+            strand = "failed"
+            stats["failed"] += 1
+
+        rotated.append((header, rotated_seq, strand, dist))
+
+    # Write output
+    with open(output_fasta, 'w') as f:
+        for header, seq, strand, dist in rotated:
+            new_header = f"{header}|rot_{strand}|ed{dist}"
+            f.write(f"{new_header}\n{seq}\n")
+
+    if verbose:
+        print(f"\nRotation results:")
+        print(f"  Forward: {stats['fwd']}")
+        print(f"  Reverse: {stats['rev']}")
+        print(f"  Failed: {stats['failed']}")
+        print(f"\nEdit distance distribution:")
+        for d, c in sorted(stats["ed_distribution"].items()):
+            print(f"  ED={d}: {c}")
+
+    stats["kmer"] = kmer
+    return stats
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Rotate monomers to common starting position"
+    )
+    parser.add_argument("-i", "--input", required=True,
+                        help="Input FASTA file with monomers")
+    parser.add_argument("-o", "--output", required=True,
+                        help="Output FASTA file with rotated monomers")
+    parser.add_argument("-k", "--kmer", default=None,
+                        help="Rotation k-mer (auto-detected if not provided)")
+    parser.add_argument("-l", "--kmer-length", type=int, default=16,
+                        help="K-mer length for auto-detection (default: 16)")
+    parser.add_argument("-e", "--max-ed", type=int, default=5,
+                        help="Maximum edit distance for rotation (default: 5)")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Verbose output")
+
+    args = parser.parse_args()
+
+    stats = rotate_monomers(
+        input_fasta=args.input,
+        output_fasta=args.output,
+        kmer=args.kmer,
+        k=args.kmer_length,
+        max_ed=args.max_ed,
+        verbose=args.verbose
+    )
+
+    print(f"Rotation complete. K-mer used: {stats['kmer']}")
+
+
+if __name__ == "__main__":
+    main()