telometer 1.1__tar.gz → 2.0.1__tar.gz

telometer-2.0.1/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: telometer
+Version: 2.0.1
+Summary: Quantitative digital telomere measurement from nanopore long-reads
+Author-email: "Santiago E. Sanchez" <santy.esanchez@gmail.com>
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: numpy
+
+A simple tool for measuring chromosome-specific telomeres from long-read alignments.
+
+[Telometer Github](https://github.com/santiago-es/Telometer) 

telometer-2.0.1/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["setuptools>=77.0.3", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "telometer"
+version = "2.0.1"
+description = "Quantitative digital telomere measurement from nanopore long-reads"
+readme = "README.md"
+requires-python = ">=3.7"
+license = "MIT"
+license-files = ["LICENSE"]
+
+authors = [
+    { name = "Santiago E. Sanchez", email = "santy.esanchez@gmail.com" }
+]
+
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent"
+]
+
+dependencies = [
+    "requests",
+    "numpy"
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["telometer*"]
+exclude = ["build*", "dist*"]

telometer-2.0.1/telometer/telometer.py

@@ -0,0 +1,503 @@
+#!/usr/bin/env python3
+# Telometer v2.0: eight-feature baseline with optional variant detection and enrichment
+# Baseline changes retained: 01, 03, 07, 11, 12, 14, 15, 16
+# Created by: Santiago E Sanchez
+# Artandi Lab, Stanford University, 2024
+# Measures telomeres from ONT or PacBio long reads aligned to a T2T genome assembly
+# Simple Usage: telometer -b sorted_t2t.bam -o output.tsv
+# Optional enrichment: add -e to your existing Telometer command.
+# -e adds the sample-level telomeres_per_gb column and console totals. Each QNAME
+# contributes one consistent recorded full length, including soft/hard clips,
+# before any Telometer filter. Missing/conflicting lengths yield NA, not a
+# partial-denominator rate. This measures yield represented by the input BAM;
+# reads/bases removed upstream without retained metadata cannot be recovered.
+# Memory for -e scales with unique read IDs; -l is not a total-RSS limit.
+
+__version__ = "2.0"
+
+import pysam
+from pathlib import Path
+from variant_support_v2 import (BASE_COLUMNS, VARIANT_COLUMNS, TRACT_COLUMNS,
+    validate_variant_options, strict_runs, coverage_prefix, combined_window_density, summarize_variant)
+import regex as re
+import pandas as pd
+import time
+import argparse
+import sys
+from multiprocessing import Pool, cpu_count
+import numpy as np
+from scipy.signal import savgol_filter
+from telometer_variant_support import alignment_fields, cached_findall, note, reverse_complement_iupac, stable_record_key
+
+def reverse_complement(seq):
+    complement = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A', 'N': 'N'}
+    return "".join(complement[base] for base in reversed(seq))
+
+def get_telomere_repeats():
+    telomere_repeats = ['CCCTAA', 'TTAGGG']
+    telomere_repeats_rc = [reverse_complement(repeat) for repeat in telomere_repeats]
+    return telomere_repeats + telomere_repeats_rc
+
+def identify_telomere_regions(seq, base_qualities, telomere_motifs, window_size=120, step_size=12,
+                              density_threshold=0.1, max_gap_length=20, quality_threshold=15, variant_motif=None, min_variant_units=2):
+    variant_prefix = coverage_prefix(len(seq), strict_runs(seq, variant_motif, min_variant_units)) if variant_motif is not None else None
+    telomere_regions = []
+    current_start = None
+    gap_length = 0
+    has_large_gap = False
+    densities = []
+
+    combined_motif_pattern = '|'.join(f'({motif})' for motif in telomere_motifs)
+
+    for i in range(0, len(seq) - window_size + 1, step_size):
+        window_seq = seq[i:i + window_size]
+        motif_count = len(cached_findall(combined_motif_pattern, window_seq))
+        density = motif_count / (window_size / len(telomere_motifs[0]))
+        if variant_prefix is not None:
+            density = combined_window_density(window_seq, i, combined_motif_pattern, motif_count, variant_prefix)
+        densities.append(density)
+
+        if density >= density_threshold:
+            if current_start is None:
+                current_start = i
+            gap_length = 0
+        else:
+            if current_start is not None:
+                gap_length += step_size
+
+                gap_start = i - gap_length
+                gap_end = i
+                gap_quality = base_qualities[gap_start:gap_end]
+
+                if len(gap_quality) > 0:
+                    avg_gap_quality = sum(gap_quality) / len(gap_quality)
+                else:
+                    avg_gap_quality = 0
+
+                if avg_gap_quality < quality_threshold:
+                    gap_length = 0
+                    continue
+
+                if gap_length > max_gap_length:
+                    mismatch_motif_count = len(cached_findall(f"({combined_motif_pattern}){{e<=1}}", window_seq))
+                    mismatch_density = mismatch_motif_count / (window_size / len(telomere_motifs[0]))
+
+                    if mismatch_density >= density_threshold:
+                        gap_length = 0
+                    else:
+                        has_large_gap = True
+                        telomere_regions.append((current_start, i - gap_length + step_size))
+                        current_start = None
+                        gap_length = 0
+
+    if current_start is not None:
+        telomere_regions.append((current_start, len(seq)))
+
+    return telomere_regions, densities, has_large_gap, combined_motif_pattern
+
+def detect_discontinuities(base_qualities, window_length=11, polyorder=2, gradient_threshold=-1):
+    if len(base_qualities) < window_length:
+        window_length = len(base_qualities) if len(base_qualities) % 2 == 1 else len(base_qualities) - 1
+
+    if len(base_qualities) > polyorder:
+        smoothed_qualities = savgol_filter(base_qualities, window_length=window_length, polyorder=polyorder)
+    else:
+        smoothed_qualities = base_qualities
+
+    gradient = np.diff(smoothed_qualities)
+    discontinuities = np.where(gradient < gradient_threshold)[0]
+
+    return discontinuities, smoothed_qualities
+
+def check_gap(seq, base_qualities, telomere_regions, gradient_threshold=-1, quality_threshold=15):
+    if not telomere_regions:
+        return []
+    merged_regions = []
+    current_start, current_end = telomere_regions[0]
+    for start, end in telomere_regions[1:]:
+        gap_quality = base_qualities[current_end:start]
+        discontinuities, smoothed_qualities = detect_discontinuities(gap_quality, gradient_threshold=gradient_threshold)
+        avg_gap_quality = sum(gap_quality) / len(gap_quality) if len(gap_quality) > 0 else 0
+        if avg_gap_quality < quality_threshold or len(discontinuities) == 0:
+            current_end = max(current_end, end)
+        else:
+            merged_regions.append((current_start, current_end))
+            current_start, current_end = start, end
+    merged_regions.append((current_start, current_end))
+    return merged_regions
+
+def measure_telomere_length(seq, telomere_motifs, base_qualities, window_size=120, step_size=12,
+                                         density_threshold=0.1, max_gap_length=20, quality_threshold=15,
+                                         variant_motif=None, min_variant_units=2):
+
+    telomere_regions, densities, has_large_gap, combined_motif_pattern = identify_telomere_regions(
+        seq, base_qualities, telomere_motifs, window_size, step_size, density_threshold, max_gap_length, quality_threshold,
+        variant_motif=variant_motif, min_variant_units=min_variant_units)
+
+    if not telomere_regions:
+        return 0, 0, 0, densities, has_large_gap, combined_motif_pattern
+
+
+    if not any(density >= 0.75 for density in densities):
+        return 0, 0, 0, densities, has_large_gap, combined_motif_pattern
+
+
+    merged_telomere_regions = check_gap(seq, base_qualities, telomere_regions, gradient_threshold=-1, quality_threshold=quality_threshold)
+
+    terminus_regions = []
+    for start, end in merged_telomere_regions:
+        if start < 100 or end > len(seq) - 100:
+            terminus_regions.append((start, end))
+
+    if not terminus_regions:
+        return 0, 0, 0, densities, has_large_gap, combined_motif_pattern
+
+
+    telomere_start, telomere_end = terminus_regions[0]
+    if not any(density >= 0.75 and telomere_start <= j * step_size and
+               j * step_size + window_size <= telomere_end
+               for j, density in enumerate(densities)):
+        note('candidate_local_dense_failed')
+        return 0, 0, 0, densities, has_large_gap, combined_motif_pattern
+    telomere_length = telomere_end - telomere_start
+
+    return telomere_start, telomere_end, telomere_length, densities, has_large_gap, combined_motif_pattern
+
+
+def process_read(read_data, telomere_motifs, max_gap_length, min_read_len, variant_motif=None, min_variant_units=2):
+    if read_data['is_unmapped'] or read_data['reference_name'] == 'chrM':
+        return None
+
+    if read_data['reference_start'] > 30000 and read_data['reference_end'] < read_data['reference_length'] - 30000:
+        return None
+
+    seq = read_data['query_sequence']
+    base_qualities = read_data['query_qualities']
+    if seq is None or len(seq) < min_read_len:
+        return None
+    if read_data['is_reverse']:
+        seq = reverse_complement_iupac(seq)
+        base_qualities = base_qualities[::-1]
+
+    alignment_start = read_data['reference_start']
+    alignment_end = read_data['reference_end']
+    reference_genome_length = read_data['reference_length']
+
+    near_p = alignment_start <= 30000
+    near_q = alignment_end >= reference_genome_length - 30000
+    if near_p and not near_q:
+        arm = "p"
+    elif near_q and not near_p:
+        arm = "q"
+    elif near_p and near_q:
+        arm = "ambiguous"
+    else:
+        arm = "unassigned"
+
+    direction = "rev" if read_data['is_reverse'] else "fwd"
+
+    telomere_start, telomere_end, telomere_length, densities, has_large_gap, combined_motif_pattern = measure_telomere_length(
+        seq, telomere_motifs, base_qualities, max_gap_length=max_gap_length,
+        variant_motif=variant_motif, min_variant_units=min_variant_units)
+
+    if telomere_length < 1:
+        return None
+
+    if (telomere_length + 50) > len(seq):
+        return None
+
+    result = {
+        'chromosome': read_data['reference_name'],
+        'reference_start': alignment_start,
+        'reference_end': alignment_end,
+        'telomere_length': telomere_length,
+        'read_id': read_data['query_name'],
+        'mapping_quality': read_data['mapping_quality'],
+        'read_length': len(seq),
+        'arm': arm,
+        'direction': direction,
+        '_selection_key': stable_record_key(read_data['_ranking_fields'])
+    }
+    if variant_motif is not None:
+        fields, tracts, details = summarize_variant(seq, base_qualities, telomere_start, telomere_end,
+                                                  variant_motif, min_variant_units, read_data)
+        result.update(fields)
+        result['_variant_tracts'] = tracts
+        result['_variant_details'] = details
+    return result
+
+def process_read_wrapper(args):
+    return process_read(*args)
+
+def estimate_memory_usage(reads_list):
+    """
+    Estimate the memory usage of the reads in bytes.
+    Approximate size based on the size of sequences, base qualities, and metadata.
+    """
+    memory_usage = 0
+    for read in reads_list:
+        seq_len = len(read['query_sequence']) if read['query_sequence'] else 0
+        quality_len = len(read['query_qualities']) if read['query_qualities'] else 0
+        memory_usage += sys.getsizeof(read['query_name']) + seq_len + quality_len + sys.getsizeof(read)
+    return memory_usage
+
+def recorded_full_read_length(read):
+    """Return the positive full length recorded by SEQ/CIGAR, or None if unknown.
+
+    Includes soft clips and reported hard clips, but never reconstructs bases.
+    Malformed or mutually inconsistent length metadata raises ValueError.
+    """
+    sequence_length = read.query_length
+    if sequence_length is None:
+        sequence_length = 0
+    if (isinstance(sequence_length, bool) or not isinstance(sequence_length, int)
+            or sequence_length < 0):
+        raise ValueError('Invalid stored query length')
+    cigar = read.cigartuples or []
+    if not cigar:
+        return sequence_length or None
+
+    if not isinstance(cigar, (list, tuple)) or any(
+            not isinstance(item, (tuple, list)) or len(item) != 2 for item in cigar):
+        raise ValueError('Malformed CIGAR operations')
+    query_span, hard_clips = 0, 0
+    first_inner = 1 if cigar[0][0] == 5 else 0
+    last_inner = len(cigar)-2 if cigar[-1][0] == 5 else len(cigar)-1
+    for i, item in enumerate(cigar):
+        if not isinstance(item, (tuple, list)) or len(item) != 2:
+            raise ValueError('Malformed CIGAR operation')
+        op, length = item
+        if (isinstance(op, bool) or not isinstance(op, int) or op not in range(9)
+                or isinstance(length, bool) or not isinstance(length, int) or length <= 0):
+            raise ValueError('Invalid CIGAR operation or length')
+        if op == 5:
+            if i not in (0, len(cigar)-1):
+                raise ValueError('Internal hard clip')
+            hard_clips += length
+        elif op == 4 and i not in (first_inner, last_inner):
+            raise ValueError('Internal soft clip')
+        if op in (0, 1, 4, 7, 8):
+            query_span += length
+    if sequence_length and sequence_length != query_span:
+        raise ValueError('SEQ length disagrees with query-consuming CIGAR span')
+    return query_span + hard_clips or None
+
+
+class EnrichmentCounter:
+    """One length/status per QNAME; no sequences or worker-local counters.
+
+    Memory scales with distinct read IDs, independently of the -l batch target.
+    This estimates only yield represented by the BAM, not reads/bases removed
+    upstream. Distinct IDs count separately even for duplicate-marked reads.
+    """
+    def __init__(self):
+        self._lengths = {}  # None: unresolved; -1: conflicting/invalid; >0: known.
+        self._missing_id_records = 0
+        self._paired_records = 0
+
+    def add(self, read):
+        if read.is_paired:
+            self._paired_records += 1
+        read_id = read.query_name
+        if not isinstance(read_id, str) or not read_id.strip() or read_id == '*':
+            self._missing_id_records += 1
+            return
+        previous = self._lengths.setdefault(read_id, None)
+        if previous == -1:
+            return
+        try:
+            length = recorded_full_read_length(read)
+        except (ValueError, TypeError, IndexError):
+            self._lengths[read_id] = -1
+            return
+        if length is None:
+            return  # Another alignment can still establish this read's length.
+        if previous is not None and previous != length:
+            self._lengths[read_id] = -1
+        else:
+            self._lengths[read_id] = length
+
+    def summarize(self, measured_read_ids):
+        unknown = sum(length is None for length in self._lengths.values())
+        invalid = sum(length == -1 for length in self._lengths.values())
+        reasons = []
+        if unknown:
+            reasons.append(f'{unknown} read IDs without a usable recorded full length')
+        if invalid:
+            reasons.append(f'{invalid} read IDs with conflicting or invalid length metadata')
+        if self._missing_id_records:
+            reasons.append(f'{self._missing_id_records} records without a usable read ID')
+        if self._paired_records:
+            reasons.append(f'{self._paired_records} paired-end records (unpaired long reads required)')
+        total_bases = None if reasons else sum(self._lengths.values())
+        total_gb = None if total_bases is None else total_bases / 1_000_000_000
+        rate = measured_read_ids * 1_000_000_000 / total_bases if total_bases else None
+        if total_bases == 0:
+            reasons.append('zero total recorded bases in the input BAM')
+        return {'unique_read_ids': len(self._lengths), 'total_bases': total_bases,
+                'total_gb': total_gb, 'telomeres_per_gb': rate,
+                'warning': 'Enrichment unavailable: ' + '; '.join(reasons) if reasons else None}
+
+
+def process_bam_file(bam_file_path, output_file_path, max_gap_length=20, min_read_len=1000, num_processes=8, memory_limit_gb=8,
+                     variant_motif=None, min_variant_units=None, enrichment=False):
+    variant_motif, min_variant_units = validate_variant_options(variant_motif, min_variant_units)
+    enrichment_counter = EnrichmentCounter() if enrichment else None
+    start_time = time.time()
+    bam_file = pysam.AlignmentFile(bam_file_path, "rb")
+    telomere_motifs = get_telomere_repeats()
+
+    results = []
+    read_results = {}
+    total_reads = 0
+    memory_limit_bytes = memory_limit_gb * (1024 ** 3)
+
+    batch = []
+    batch_memory = 0
+    pool = Pool(processes=num_processes)
+
+    def process_batch(batch_data):
+        nonlocal total_reads, read_results
+
+        for result in pool.imap_unordered(process_read_wrapper, [(rd, telomere_motifs, max_gap_length, min_read_len, variant_motif, min_variant_units) for rd in batch_data]):
+            if result:
+                total_reads += 1
+                existing_result = read_results.get(result['read_id'])
+                if existing_result:
+                    if (result['mapping_quality'] > existing_result['mapping_quality'] or
+                        (result['mapping_quality'] == existing_result['mapping_quality'] and
+                         result['telomere_length'] > existing_result['telomere_length']) or
+                        (result['mapping_quality'] == existing_result['mapping_quality'] and
+                         result['telomere_length'] == existing_result['telomere_length'] and
+                         result['_selection_key'] < existing_result['_selection_key'])):
+                        read_results[result['read_id']] = result
+                else:
+                    read_results[result['read_id']] = result
+
+        batch_data.clear()
+
+    for read in bam_file:
+        if enrichment_counter is not None:
+            enrichment_counter.add(read)
+        if read.reference_name is not None and read.reference_name != 'chrM':
+            if read.is_unmapped:
+                continue
+            if read.reference_start > 30000 and read.reference_end < bam_file.get_reference_length(read.reference_name) - 30000:
+                continue
+            if read.query_sequence is None or len(read.query_sequence) < min_read_len:
+                continue
+            base_qualities = read.query_qualities
+
+            if base_qualities is not None and len(base_qualities) > 0:
+                avg_phred_score = sum(base_qualities) / len(base_qualities)
+            else:
+                avg_phred_score = 0
+                #print("no phred score")
+
+            if avg_phred_score > 9:
+                read_data = {
+                    'query_name': read.query_name,
+                    'is_unmapped': read.is_unmapped,
+                    'is_reverse': read.is_reverse,
+                    'reference_start': read.reference_start,
+                    'reference_end': read.reference_end,
+                    'reference_name': read.reference_name,
+                    'mapping_quality': read.mapping_quality,
+                    'query_sequence': read.query_sequence,
+                    'query_qualities': base_qualities,
+                    '_ranking_fields': alignment_fields(read),
+                    'reference_length': bam_file.get_reference_length(read.reference_name) if read.reference_name is not None else None
+                }
+
+                if variant_motif is not None:
+                    read_data['_variant_cigar'] = read.cigartuples or []
+                read_memory = estimate_memory_usage([read_data])
+                batch_memory += read_memory
+                batch.append(read_data)
+
+                if batch_memory >= memory_limit_bytes:
+                    print(f"Processing batch of {len(batch)} reads, approx memory used: {batch_memory / (1024 ** 3):.2f} GB")
+                    process_batch(batch)
+                    batch_memory = 0
+                    batch = []
+            #else:
+                #print("low phred")
+
+    if batch:
+        print(f"Processing final batch of {len(batch)} reads")
+        process_batch(batch)
+
+    pool.close()
+    pool.join()
+    bam_file.close()
+
+    enrichment_summary = enrichment_counter.summarize(len(read_results)) if enrichment_counter is not None else None
+    results_df = pd.DataFrame(list(read_results.values()), columns=BASE_COLUMNS + (VARIANT_COLUMNS if variant_motif is not None else []))
+    if enrichment_summary is not None:
+        rate = enrichment_summary['telomeres_per_gb']
+        results_df['telomeres_per_gb'] = rate if rate is not None else np.nan
+    results_df.to_csv(output_file_path, sep='\t', index=False)
+    if variant_motif is not None:
+        tract_rows = [tract for result in read_results.values() for tract in result['_variant_tracts']]
+        tract_path = Path(output_file_path).with_name(Path(output_file_path).stem + '_variant_tracts.tsv')
+        pd.DataFrame(tract_rows, columns=TRACT_COLUMNS).to_csv(tract_path, sep='\t', index=False)
+
+    print(f"Telometer completed successfully. Total telomeres measured: {len(read_results)}")
+    print(f"Total processing time: {time.time() - start_time:.2f} seconds")
+    if enrichment_summary is not None:
+        for key, label in [('total_bases', 'Nonredundant recorded bases in BAM'),
+                           ('total_gb', 'Recorded sequencing yield in BAM (Gb)'),
+                           ('telomeres_per_gb', 'Telomeres measured per Gb')]:
+            value = enrichment_summary[key]
+            print(f"{label}: {value if value is not None else 'NA'}")
+        if enrichment_summary['warning']:
+            print(f"Warning: {enrichment_summary['warning']}", file=sys.stderr)
+    # After the final print statements
+    if len(read_results) > 0:
+        # Convert read results to a DataFrame for easy statistical computation
+        results_df = pd.DataFrame(list(read_results.values()))
+
+        # Calculate summary statistics for the 'telomere_length' column
+        telomere_lengths = results_df['telomere_length']
+
+        min_length = telomere_lengths.min()
+        percentile_25 = telomere_lengths.quantile(0.25)
+        median_length = telomere_lengths.median()
+        mean_length = telomere_lengths.mean()
+        percentile_75 = telomere_lengths.quantile(0.75)
+        max_length = telomere_lengths.max()
+
+        # Print summary statistics
+        print("\nTelomere Length Summary Statistics:")
+        print(f"Min: {min_length:.2f}")
+        print(f"25th Percentile: {percentile_25:.2f}")
+        print(f"Median: {median_length:.2f}")
+        print(f"Mean: {mean_length:.2f}")
+        print(f"75th Percentile: {percentile_75:.2f}")
+        print(f"Max: {max_length:.2f}")
+    else:
+        print("No telomere measurements to summarize.")
+
+
+def run_telometer():
+    parser = argparse.ArgumentParser(description=f'Telometer v{__version__}: telomere length with optional exact 6-bp variant-repeat tracts and enrichment.')
+    parser.add_argument('--version', action='version', version=f'Telometer v{__version__}')
+    parser.add_argument('-b', '--bam', help='The path to the sorted BAM file.', required=True)
+    parser.add_argument('-o', '--output', help='The path to the output file.', required=True)
+    parser.add_argument('-m', '--minreadlen', default=1000, type=int, help='Minimum read length to consider (Default: 1000 for telomere capture, use 4000 for WGS). Optional', required=False)
+    parser.add_argument('-g', '--maxgaplen', default=20, type=int, help='Maximum allowed gap length between telomere regions. Optional', required=False)
+    parser.add_argument('-t', '--threads', default=cpu_count(), type=int, help='Number of processing threads to use. Optional', required=False)
+    parser.add_argument('-l', '--memlimit', default=8, type=int, help="Maximum amount of memory to commit per batch of reads while processing. Optional, default = 8 Gb", required=False)
+    parser.add_argument('-e', '--enrichment', action='store_true', help='Append telomeres_per_gb and print sample totals using one recorded full length per read ID across the entire BAM. Optional; off by default.')
+    parser.add_argument('--variant-motif', default=None, help='Optional noncanonical 6-bp DNA motif; rotations and reverse complements are included.')
+    parser.add_argument('--min-variant-units', default=None, type=int, help='Minimum non-overlapping exact 6-bp units (effective default: 2; requires --variant-motif).')
+    args = parser.parse_args()
+    try:
+        validate_variant_options(args.variant_motif, args.min_variant_units, emit_warnings=False)
+    except ValueError as error:
+        parser.error(str(error))
+    process_bam_file(args.bam, args.output, max_gap_length=args.maxgaplen, min_read_len=args.minreadlen, num_processes=args.threads, memory_limit_gb=args.memlimit,
+                     variant_motif=args.variant_motif, min_variant_units=args.min_variant_units, enrichment=args.enrichment)
+
+if __name__ == "__main__":
+    run_telometer()