pytransdecoder 0.2.1__py3-none-any.whl

pytransdecoder/genome.py

@@ -0,0 +1,338 @@
+"""
+Propagate ORFs predicted on transcript sequences to genome coordinates through the
+transcripts' genome structures, producing gene / mRNA / exon / CDS / UTR models.
+
+Transcript structures can come from
+  - GTF (exon features with gene_id / transcript_id: StringTie, LRAA, Cufflinks, GENCODE ...),
+  - alignment GFF3 (cDNA_match / match lines with Target=<transcript> <start> <end>: GMAP,
+    PASA, util/gtf_to_alignment_gff3.pl),
+  - SAM / BAM spliced alignments (minimap2 -ax splice; primary alignments; BAM read via samtools).
+
+For GTF and alignment GFF3 the result is the same as Perl TransDecoder's
+util/cdna_alignment_orf_to_genome_orf.pl, except that each mRNA is named by its own ORF.
+Transcripts of unknown strand ('.', e.g. StringTie single-exon transcripts) are taken to be
+on the strand their sequence was extracted from: '+' for gffread (default), '-' for
+TransDecoder's util/gtf_genome_to_cdna_fasta.pl (unstranded_as='-'). Alignment blocks carry transcript coordinates, so
+transcripts with indels or clipped ends are mapped base-accurately; an ORF whose CDS is not
+entirely within the aligned part of its transcript is not propagated.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import subprocess
+from collections import OrderedDict, defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Tuple
+
+from pytransdecoder.core.fasta import read_fasta
+from pytransdecoder.core.gene_model import CodingGene, CodingModel
+
+logger = logging.getLogger(__name__)
+
+# (genome lend, genome rend, transcript start, transcript end); transcript coordinates run in
+# the transcribed direction: on '-' structures they decrease with genome position
+Block = Tuple[int, int, int, int]
+
+# alignment blocks separated by at most this many genome bases (an alignment deletion) belong
+# to the same exon; larger gaps are introns
+MAX_INDEL = 20
+
+
+@dataclass
+class TranscriptStructure:
+    transcript_id: str
+    seqid: str
+    strand: str                      # orientation of the transcript sequence on the genome
+    blocks: List[Block] = field(default_factory=list)
+    gene_id: str = ""
+    exons: List[Tuple[int, int]] = field(default_factory=list)   # merged across indels
+
+    def finalize(self, transcript_length: Optional[int] = None) -> None:
+        """Order blocks; assign transcript coordinates when the source has none (GTF)."""
+        self.blocks.sort()
+        if self.blocks and self.blocks[0][2] == 0:
+            # contiguous transcript: cumulative exon lengths in the transcribed direction
+            ordered = self.blocks if self.strand != "-" else self.blocks[::-1]
+            pos, blocks = 1, []
+            for g0, g1, _, _ in ordered:
+                n = g1 - g0 + 1
+                blocks.append((g0, g1, pos, pos + n - 1) if self.strand != "-" else (g0, g1, pos + n - 1, pos))
+                pos += n
+            self.blocks = sorted(blocks)
+        if not self.exons:
+            self.exons = _merge(((g0, g1) for g0, g1, _, _ in self.blocks), max_gap=MAX_INDEL)
+        self.exons = _merge(self.exons)
+
+    def to_genome(self, tpos: int) -> Optional[int]:
+        """Genome position of transcript position tpos (1-based), None if not aligned."""
+        for g0, g1, t0, t1 in self.blocks:
+            lo, hi = min(t0, t1), max(t0, t1)
+            if lo <= tpos <= hi:
+                return g0 + (tpos - t0) if t1 >= t0 else g0 + (t0 - tpos)
+        return None
+
+
+def _merge(segments: Iterable[Tuple[int, int]], max_gap: int = 0) -> List[Tuple[int, int]]:
+    merged: List[List[int]] = []
+    for a, b in sorted(segments):
+        if merged and a <= merged[-1][1] + 1 + max_gap:
+            merged[-1][1] = max(merged[-1][1], b)
+        else:
+            merged.append([a, b])
+    return [(a, b) for a, b in merged]
+
+
+# --------------------------------------------------------------------------------------
+# readers
+# --------------------------------------------------------------------------------------
+
+def _gtf_attrs(text: str) -> Dict[str, str]:
+    attrs = {}
+    for part in text.strip().split(";"):
+        part = part.strip()
+        if not part:
+            continue
+        key, _, val = part.partition(" ")
+        attrs.setdefault(key, val.strip().strip('"'))
+    return attrs
+
+
+def read_gtf(path: Path, unstranded_as: str = "+") -> Dict[str, TranscriptStructure]:
+    structs: Dict[str, TranscriptStructure] = OrderedDict()
+    with open(path) as fh:
+        for line in fh:
+            if line.startswith("#"):
+                continue
+            x = line.rstrip("\n").split("\t")
+            if len(x) < 9 or x[2] != "exon":
+                continue
+            a = _gtf_attrs(x[8])
+            tid = a.get("transcript_id")
+            if not tid:
+                raise ValueError(f"GTF exon without transcript_id: {line.strip()}")
+            s = structs.get(tid)
+            if s is None:
+                strand = x[6] if x[6] in ("+", "-") else unstranded_as
+                s = structs[tid] = TranscriptStructure(tid, x[0], strand, gene_id=a.get("gene_id", ""))
+            s.blocks.append((int(x[3]), int(x[4]), 0, 0))
+    return structs
+
+
+def read_alignment_gff3(path: Path) -> Dict[str, TranscriptStructure]:
+    structs: Dict[str, TranscriptStructure] = OrderedDict()
+    target_re = re.compile(r"Target=([^;\s]+)(?:\s+(\d+)\s+(\d+))?")
+    with open(path) as fh:
+        for line in fh:
+            if line.startswith("#") or not line.strip():
+                continue
+            x = line.rstrip("\n").split("\t")
+            if len(x) < 9:
+                continue
+            m = target_re.search(x[8])
+            if not m:
+                raise ValueError(f"alignment GFF3 line without Target=: {line.strip()}")
+            tid, gene_id = m.group(1), ""
+            gt = re.match(r"GENE\^(\S+),TRANS\^(\S+)", tid)   # util/gtf_to_alignment_gff3.pl
+            if gt:
+                gene_id, tid = gt.group(1), gt.group(2)
+            s = structs.get(tid)
+            if s is None:
+                s = structs[tid] = TranscriptStructure(tid, x[0], x[6], gene_id=gene_id)
+            g0, g1 = int(x[3]), int(x[4])
+            if m.group(2):
+                t0, t1 = int(m.group(2)), int(m.group(3))
+                t0, t1 = (min(t0, t1), max(t0, t1)) if x[6] != "-" else (max(t0, t1), min(t0, t1))
+                s.blocks.append((g0, g1, t0, t1))
+            else:
+                s.blocks.append((g0, g1, 0, 0))
+    return structs
+
+
+_CIGAR_RE = re.compile(r"(\d+)([MIDNSHP=X])")
+
+
+def read_sam(path: Path) -> Dict[str, TranscriptStructure]:
+    """Primary spliced alignments; BAM input is read through `samtools view`."""
+    structs: Dict[str, TranscriptStructure] = OrderedDict()
+    if str(path).endswith(".bam"):
+        proc = subprocess.Popen(["samtools", "view", "-F", "0x904", str(path)],
+                                stdout=subprocess.PIPE, text=True)
+        lines = proc.stdout
+    else:
+        proc, lines = None, open(path)
+    try:
+        for line in lines:
+            if line.startswith("@"):
+                continue
+            x = line.split("\t", 11)
+            flag = int(x[1])
+            if flag & 0x904 or x[2] == "*":
+                continue
+            tid, seqid, gpos, cigar = x[0], x[2], int(x[3]), x[5]
+            ops = [(int(n), op) for n, op in _CIGAR_RE.findall(cigar)]
+            qlen = sum(n for n, op in ops if op in "MIS=XH")
+            minus = bool(flag & 16)
+            strand = "-" if minus else "+"
+            q, g = 0, gpos
+            blocks, exons, exon_start = [], [], gpos
+            for n, op in ops:
+                if op in "M=X":
+                    t0, t1 = (q + 1, q + n) if not minus else (qlen - q, qlen - q - n + 1)
+                    blocks.append((g, g + n - 1, t0, t1))
+                    q += n
+                    g += n
+                elif op in "ISH":
+                    q += n
+                elif op == "D":
+                    g += n
+                elif op == "N":
+                    exons.append((exon_start, g - 1))
+                    g += n
+                    exon_start = g
+            exons.append((exon_start, g - 1))
+            s = TranscriptStructure(tid, seqid, strand, blocks=blocks, exons=exons)
+            structs[tid] = s
+    finally:
+        if proc is not None:
+            proc.stdout.close()
+            if proc.wait() != 0:
+                raise RuntimeError(f"samtools view {path} failed")
+        else:
+            lines.close()
+    return structs
+
+
+def read_structures(path: Path, fmt: Optional[str] = None,
+                    unstranded_as: str = "+") -> Dict[str, TranscriptStructure]:
+    path = Path(path)
+    if fmt is None:
+        name = path.name.lower()
+        if name.endswith((".bam", ".sam")):
+            fmt = "sam"
+        elif name.endswith(".gtf"):
+            fmt = "gtf"
+        else:
+            fmt = "gff3"
+    structs = read_gtf(path, unstranded_as) if fmt == "gtf" else \
+        {"gff3": read_alignment_gff3, "sam": read_sam}[fmt](path)
+    for s in structs.values():
+        s.finalize()
+    return structs
+
+
+@dataclass
+class TranscriptOrf:
+    transcript_id: str
+    model_id: str
+    gene_id: str
+    name: str
+    strand: str
+    cds_lend: int
+    cds_rend: int
+
+
+def read_orfs(td_gff3: Path) -> "OrderedDict[str, List[TranscriptOrf]]":
+    """ORFs of a TransDecoder transcript-level GFF3 (Perl or PyTransDecoder), by transcript."""
+    gene_names: Dict[str, str] = {}
+    mrnas: "OrderedDict[str, dict]" = OrderedDict()
+    cds: Dict[str, List[int]] = defaultdict(list)
+    with open(td_gff3) as fh:
+        for line in fh:
+            x = line.rstrip("\n").split("\t")
+            if line.startswith("#") or len(x) != 9:
+                continue
+            attrs = dict(kv.split("=", 1) for kv in x[8].split(";") if "=" in kv)
+            if x[2] == "gene":
+                gene_names[attrs["ID"]] = attrs.get("Name", "").strip('"')
+            elif x[2] == "mRNA":
+                mrnas[attrs["ID"]] = {"seqid": x[0], "gene": attrs.get("Parent", ""), "strand": x[6]}
+            elif x[2] == "CDS":
+                cds[attrs["Parent"]] += [int(x[3]), int(x[4])]
+    orfs: "OrderedDict[str, List[TranscriptOrf]]" = OrderedDict()
+    for model_id, m in mrnas.items():
+        if not cds.get(model_id):
+            continue
+        orf = TranscriptOrf(m["seqid"], model_id, m["gene"], gene_names.get(m["gene"], ""),
+                            m["strand"], min(cds[model_id]), max(cds[model_id]))
+        orfs.setdefault(m["seqid"], []).append(orf)
+    return orfs
+
+
+def read_fasta_lengths(path: Path) -> Dict[str, int]:
+    return {record.id: len(record.seq) for record in read_fasta(path)}
+
+
+# --------------------------------------------------------------------------------------
+# propagation
+# --------------------------------------------------------------------------------------
+
+def place_orf(orf: TranscriptOrf, struct: TranscriptStructure, transcript_length: int) -> Optional[CodingModel]:
+    """Genome model of one ORF, or None (logged) if it cannot be placed."""
+    cds_lend, cds_rend = orf.cds_lend, min(orf.cds_rend, transcript_length)
+    tstrand = struct.strand if struct.strand in "+-" else "+"
+    if orf.strand == "+":
+        strand = tstrand
+    else:
+        if len(struct.exons) > 1:
+            logger.warning("%s: ORF on the opposite strand of spliced transcript %s; skipped",
+                           orf.model_id, struct.transcript_id)
+            return None
+        strand = "-" if tstrand == "+" else "+"   # single exon: orient by the ORF
+    g_a, g_b = struct.to_genome(cds_lend), struct.to_genome(cds_rend)
+    if g_a is None or g_b is None:
+        logger.warning("%s: CDS %d-%d of %s is not entirely in its aligned part; skipped",
+                       orf.model_id, cds_lend, cds_rend, struct.transcript_id)
+        return None
+    return CodingModel(seqid=struct.seqid, strand=strand, exons=list(struct.exons),
+                       cds_lend=min(g_a, g_b), cds_rend=max(g_a, g_b), model_id=orf.model_id, name=orf.name)
+
+
+def propagate(orfs_by_transcript, structures: Dict[str, TranscriptStructure],
+              transcript_lengths: Dict[str, int]) -> Tuple[List[CodingGene], int, int]:
+    """Returns (genes sorted by id, number propagated, total ORFs)."""
+    genes: Dict[str, CodingGene] = {}
+    n_done = n_total = 0
+    for tid in sorted(orfs_by_transcript):
+        for orf in orfs_by_transcript[tid]:
+            n_total += 1
+            struct = structures.get(tid)
+            if struct is None:
+                logger.warning("%s has no genome structure; skipped", tid)
+                continue
+            if tid not in transcript_lengths:
+                raise ValueError(f"no sequence length for {tid}: provide the transcript FASTA")
+            model = place_orf(orf, struct, transcript_lengths[tid])
+            if model is None:
+                continue
+            base_gene = struct.gene_id or re.split(r"~~|::", orf.gene_id)[0]
+            # gene ids can repeat at different loci or strands
+            gene_key = f"{base_gene}^{model.seqid}^{model.strand}"
+            genes.setdefault(gene_key, CodingGene(gene_key)).models.append(model)
+            n_done += 1
+    return [genes[k] for k in sorted(genes)], n_done, n_total
+
+
+def run(td_gff3: Path, structures_file: Path, transcripts_fasta: Path, out_gff3: Path,
+        structures_format: Optional[str] = None, source: str = "transdecoder",
+        unstranded_as: str = "+", out_bed: Optional[Path] = None) -> Tuple[int, int]:
+    """Propagate transcript ORFs to the genome; writes GFF3 and BED12 (default: the GFF3 path
+    with .gff3 replaced by .bed, as Perl TransDecoder's genome mode)."""
+    structures = read_structures(structures_file, structures_format, unstranded_as)
+    genes, n_done, n_total = propagate(read_orfs(td_gff3), structures, read_fasta_lengths(transcripts_fasta))
+    with open(out_gff3, "w") as out:
+        for gene in genes:
+            out.write(gene.gff3_text(source))
+    if out_bed is None:
+        out_bed = Path(str(out_gff3).removesuffix(".gff3") + ".bed")
+    with open(out_bed, "w") as out:
+        for gene in genes:
+            # name as Perl's Gene_obj::to_BED_format: ID=<transcript>;<gene>;<gene Name>
+            gene_name = (gene.name or gene.models[0].name).strip("'\"")
+            for model in gene.models:
+                name = f"ID={model.model_id};{gene.gene_id};{gene_name or model.model_id}"
+                out.write(model.bed12(name.replace(" ", "_")) + "\n")
+    logger.info("%d / %d transcript ORFs propagated to the genome", n_done, n_total)
+    return n_done, n_total

pytransdecoder/longorfs.py

@@ -0,0 +1,180 @@
+"""
+TransDecoder.LongOrfs - Extract long ORFs from transcript sequences
+
+Phase 1 of TransDecoder pipeline.
+"""
+
+import sys
+from pathlib import Path
+from typing import Optional, Dict
+from collections import defaultdict
+from .core.fasta import read_fasta
+from tqdm import tqdm
+
+from .core.orf_finder import ORFFinder
+from .core.models import ORF
+from .core.writers import GFF3Writer, FastaWriter
+from .core.base_freqs import compute_base_counts, write_base_frequencies
+
+# Minimum ORF length (aa). Predict's decoy q-values (stratified by ORF length) decide how much coding
+# evidence short ORFs need.
+DEFAULT_MIN_PROTEIN_LENGTH = 50
+
+
+def run_longorfs(
+    transcripts_file: Path,
+    min_protein_length: int = DEFAULT_MIN_PROTEIN_LENGTH,
+    genetic_code: str = "universal",
+    strand_specific: bool = False,
+    output_dir: Optional[Path] = None,
+    gene_trans_map_file: Optional[Path] = None,
+    complete_orfs_only: bool = False,
+    verbose: bool = False
+):
+    """
+    Extract long ORFs from transcripts (Phase 1).
+    
+    Args:
+        transcripts_file: Input FASTA file
+        min_protein_length: Minimum protein length in amino acids
+        genetic_code: Genetic code name
+        strand_specific: Only analyze top strand
+        output_dir: Output directory
+        gene_trans_map_file: Gene to transcript mapping file
+        complete_orfs_only: Only output complete ORFs
+        verbose: Verbose output
+    """
+    
+    # Set up output directory
+    if output_dir is None:
+        output_dir = Path.cwd()
+    output_dir = Path(output_dir)
+    
+    # Create transdecoder_dir
+    workdir = output_dir / f"{transcripts_file.name}.transdecoder_dir"
+    workdir.mkdir(parents=True, exist_ok=True)
+    
+    if verbose:
+        print(f"Output directory: {workdir}", file=sys.stderr)
+    
+    # Load gene-transcript mapping if provided
+    gene_trans_map = {}
+    if gene_trans_map_file:
+        with open(gene_trans_map_file) as f:
+            for line in f:
+                parts = line.strip().split('\t')
+                if len(parts) >= 2:
+                    gene_id, trans_id = parts[0], parts[1]
+                    gene_trans_map[trans_id] = gene_id
+    
+    # LongOrfs settings, for Predict's decoy ORFs (same finder settings on shuffled transcripts)
+    import json
+    with open(workdir / "longorfs.params.json", "w") as fh:
+        json.dump({"min_protein_length": min_protein_length, "genetic_code": genetic_code,
+                   "strand_specific": bool(strand_specific),
+                   "complete_orfs_only": bool(complete_orfs_only)}, fh, indent=1)
+
+    # Step 1: Compute base frequencies
+    if verbose:
+        print("\n\n-first extracting base frequencies, we'll need them later.\n", file=sys.stderr)
+    
+    base_freqs_file = workdir / "base_freqs.dat"
+    base_counts = compute_base_counts(str(transcripts_file), strand_specific)
+    write_base_frequencies(base_counts, str(base_freqs_file))
+    
+    # Step 2: Extract ORFs
+    if verbose:
+        print("\n\n- extracting ORFs from transcripts.\n", file=sys.stderr)
+    
+    # Initialize ORF finder
+    orf_finder = ORFFinder(
+        min_protein_length=min_protein_length,
+        allow_5prime_partial=not complete_orfs_only,
+        allow_3prime_partial=not complete_orfs_only,
+        forward_strand=True,
+        reverse_strand=not strand_specific,
+        genetic_code=genetic_code,
+        complete_orfs_only=complete_orfs_only
+    )
+    
+    # Output files
+    prefix = workdir / "longest_orfs"
+    cds_file = prefix.with_suffix('.cds')
+    gff3_file = prefix.with_suffix('.gff3')
+    pep_file = prefix.with_suffix('.pep')
+    
+    # Count total transcripts for progress bar
+    total_transcripts = sum(1 for _ in read_fasta(transcripts_file))
+    
+    if verbose:
+        print(f"-total transcripts to examine: {total_transcripts}\n", file=sys.stderr)
+    
+    # Process transcripts
+    model_counter = 0
+    trans_counter = 0
+    seen_prot_ids = set()
+    
+    with GFF3Writer(gff3_file) as gff_writer, \
+         FastaWriter(cds_file) as cds_writer, \
+         FastaWriter(pep_file) as pep_writer:
+        
+        progress = tqdm(
+            read_fasta(transcripts_file),
+            total=total_transcripts,
+            desc="Processing transcripts",
+            disable=not verbose,
+            file=sys.stderr
+        )
+        
+        for record in progress:
+            trans_counter += 1
+            
+            acc = record.id
+            sequence = str(record.seq)
+            
+            # Find all ORFs
+            orfs = orf_finder.find_all_orfs(sequence, acc)
+            
+            # Process each ORF
+            for orf in orfs:
+                model_counter += 1
+                
+                # Generate unique model ID
+                pcounter = 1
+                model_id = f"{acc}.p{pcounter}"
+                while model_id in seen_prot_ids:
+                    pcounter += 1
+                    model_id = f"{acc}.p{pcounter}"
+                seen_prot_ids.add(model_id)
+                
+                # Generate gene ID
+                if acc in gene_trans_map:
+                    gene_id = gene_trans_map[acc]
+                else:
+                    gene_id = f"GENE.{acc}"
+                
+                gene_id = f"{gene_id}~~{model_id}"
+                
+                # Update ORF with IDs
+                orf.model_id = model_id
+                orf.gene_id = gene_id
+                orf.description = f"ORF type:{orf.orf_type} ({orf.strand})"
+                
+                # Write outputs
+                gff_writer.write_orf(orf)
+                
+                # Write CDS
+                cds_header = f">{model_id} type:{orf.orf_type} {acc}:{orf.start}-{orf.end}({orf.strand})"
+                cds_writer.write_sequence(cds_header, orf.sequence)
+                
+                # Write protein
+                pep_header = f">{model_id} type:{orf.orf_type} gc:{genetic_code} {acc}:{orf.start}-{orf.end}({orf.strand})"
+                pep_writer.write_sequence(pep_header, orf.protein)
+    
+    # Print summary
+    if verbose:
+        print(f"\n\n#################################", file=sys.stderr)
+        print(f"### Done preparing long ORFs  ###", file=sys.stderr)
+        print(f"##################################", file=sys.stderr)
+        print(f"\n\tUse file: {pep_file}  for Pfam and/or BlastP searches to enable homology-based coding region identification.\n\n", file=sys.stderr)
+        print(f"\tThen, run TransDecoder.Predict\n\n", file=sys.stderr)