strkit 0.11.8__tar.gz → 0.12.0__tar.gz

{strkit-0.11.8/strkit.egg-info → strkit-0.12.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: strkit
-Version: 0.11.8
+Version: 0.12.0
 Summary: A toolkit for analyzing variation in short(ish) tandem repeats.
 Home-page: https://github.com/davidlougheed/strkit
 Author: David Lougheed
@@ -16,6 +16,7 @@ Requires-Dist: Flask<3.1,>=2.2.5
 Requires-Dist: pysam<0.23,>=0.19
 Requires-Dist: numpy<1.27,>=1.23.4
 Requires-Dist: parasail<1.4,>=1.2.4
+Requires-Dist: pyfamsa<0.4,>=0.3.0
 Requires-Dist: scikit-learn<1.4,>=1.2.1
 Requires-Dist: scipy<1.12,>=1.10
 Requires-Dist: statsmodels<0.15,>=0.14.0
@@ -115,6 +116,8 @@ long read data should still work.
   * Whole-genome and targeted genotyping modes to adjust this re-weighting.
 * Incorporation of single-nucleotide variation (SNVs) for better and faster calling plus 
   additional downstream analysis possibilities.
+  * Recommended for **HiFi data only**. In my testing, this worsens runtime and call quality for 
+    ONT-UL data, but speeds up the tool and improves call quality for HiFi data. 
 * Parallelized for faster computing on clusters and for ad-hoc fast analysis of single samples.
 * 95% confidence intervals on calls via a user-configurable optional parametric bootstrapping process.
 
@@ -128,10 +131,11 @@ strkit call \
   --realign \  # If using PacBio HiFi reads, enable this to enable local realignment / read recovery
   --ref path/to/reference.fa.gz \  # [REQUIRED] Indexed FASTA-formatted reference genome
   --loci path/to/loci.bed \  # [REQUIRED] TRF-formatted (or 4-col, with motif as last column) list of loci to genotype
-  --incorporate-snvs path/to/dbsnp.vcf.gz \   # If you want SNV calls to help phase STRs & speed up STRkit, include this
+  --incorporate-snvs path/to/dbsnp.vcf.gz \   # [RECOMMENDED FOR HIFI ONLY] If you want SNV calls to help phase STRs & speed up calling
   --min-reads 4 \  # Minimum number of supporting reads needed to make a call
   --min-allele-reads 2 \  # Minimum number of supporting reads needed to call a specific allele size 
   --flank-size 70 \  # Size of the flanking region to use on either side of a region to properly anchor reads
+  --consensus \  # If this flag is set, consensus sequences are calculated for alleles. This increases runtime! 
   --seed 183 \  # Fixed random number generator seed for replicability
   --processes 10  # Number of parallel processes to use; DEFAULT: 1
 ```
@@ -160,6 +164,9 @@ specify a path to output a more detailed JSON document to. This document contain
 labels, and some other information that isn't included in the normal TSV file. If you want this
 file to be indented and human-readable, use the `--indent-json` flag in addition to `--json ...`.
 
+If you want to output a VCF file (currently with STR sequences only, not with phased SNVs), use the
+`--vcf ...` argument.
+
 See the '[Caller catalog format & choosing a catalog](./docs/caller_catalog.md)' page for more on
 how to format a locus catalog or choose from existing available catalogs.
 

{strkit-0.11.8 → strkit-0.12.0}/README.md

@@ -90,6 +90,8 @@ long read data should still work.
   * Whole-genome and targeted genotyping modes to adjust this re-weighting.
 * Incorporation of single-nucleotide variation (SNVs) for better and faster calling plus 
   additional downstream analysis possibilities.
+  * Recommended for **HiFi data only**. In my testing, this worsens runtime and call quality for 
+    ONT-UL data, but speeds up the tool and improves call quality for HiFi data. 
 * Parallelized for faster computing on clusters and for ad-hoc fast analysis of single samples.
 * 95% confidence intervals on calls via a user-configurable optional parametric bootstrapping process.
 
@@ -103,10 +105,11 @@ strkit call \
   --realign \  # If using PacBio HiFi reads, enable this to enable local realignment / read recovery
   --ref path/to/reference.fa.gz \  # [REQUIRED] Indexed FASTA-formatted reference genome
   --loci path/to/loci.bed \  # [REQUIRED] TRF-formatted (or 4-col, with motif as last column) list of loci to genotype
-  --incorporate-snvs path/to/dbsnp.vcf.gz \   # If you want SNV calls to help phase STRs & speed up STRkit, include this
+  --incorporate-snvs path/to/dbsnp.vcf.gz \   # [RECOMMENDED FOR HIFI ONLY] If you want SNV calls to help phase STRs & speed up calling
   --min-reads 4 \  # Minimum number of supporting reads needed to make a call
   --min-allele-reads 2 \  # Minimum number of supporting reads needed to call a specific allele size 
   --flank-size 70 \  # Size of the flanking region to use on either side of a region to properly anchor reads
+  --consensus \  # If this flag is set, consensus sequences are calculated for alleles. This increases runtime! 
   --seed 183 \  # Fixed random number generator seed for replicability
   --processes 10  # Number of parallel processes to use; DEFAULT: 1
 ```
@@ -135,6 +138,9 @@ specify a path to output a more detailed JSON document to. This document contain
 labels, and some other information that isn't included in the normal TSV file. If you want this
 file to be indented and human-readable, use the `--indent-json` flag in addition to `--json ...`.
 
+If you want to output a VCF file (currently with STR sequences only, not with phased SNVs), use the
+`--vcf ...` argument.
+
 See the '[Caller catalog format & choosing a catalog](./docs/caller_catalog.md)' page for more on
 how to format a locus catalog or choose from existing available catalogs.
 

{strkit-0.11.8 → strkit-0.12.0}/setup.py

@@ -18,6 +18,7 @@ setup(
         "pysam>=0.19,<0.23",
         "numpy>=1.23.4,<1.27",
         "parasail>=1.2.4,<1.4",
+        "pyfamsa>=0.3.0,<0.4",
         "scikit-learn>=1.2.1,<1.4",
         "scipy>=1.10,<1.12",
         "statsmodels>=0.14.0,<0.15",

strkit-0.12.0/strkit/VERSION

@@ -0,0 +1 @@
+0.12.0

{strkit-0.11.8 → strkit-0.12.0}/strkit/call/call_locus.py

@@ -18,10 +18,11 @@ from sklearn.cluster import AgglomerativeClustering
 from numpy.typing import NDArray
 from typing import Iterable, Literal, Optional, Union
 
-from strkit.call.allele import CallDict, get_n_alleles, call_alleles
+from strkit.call.allele import CallDict, call_alleles
 from strkit.utils import apply_or_none
 
 from .align_matrix import match_score
+from .consensus import consensus_seq
 from .realign import realign_read
 from .repeats import get_repeat_count, get_ref_repeat_count
 from .snvs import (
@@ -109,6 +110,10 @@ def get_read_coords_from_matched_pairs(
     left_flank_start, _ = matched_pairs[lhs]
     # ------------------------------------------------------------------------------------------------------------------
 
+    # Binary search for left flank end ---------------------------------------------------------------------------------
+    # TODO
+    # ------------------------------------------------------------------------------------------------------------------
+
     for query_coord, ref_coord in matched_pairs[lhs+1:]:
         # Skip gaps on either side to find mapped flank indices
         if ref_coord < left_coord:
@@ -136,7 +141,7 @@ def get_read_coords_from_matched_pairs(
 
 
 def get_overlapping_segments_and_related_data(
-    bfs: tuple[pysam.AlignmentFile],
+    bfs: tuple[pysam.AlignmentFile, ...],
     read_contig: str,
     left_flank_coord: int,
     right_flank_coord: int,
@@ -551,6 +556,7 @@ def debug_log_flanking_seq(logger_: logging.Logger, locus_log_str: str, rn: str,
 def call_locus(
     t_idx: int,
     t: tuple,
+    n_alleles: int,
     bfs: tuple[AlignmentFile, ...],
     ref: FastaFile,
     min_reads: int,
@@ -561,7 +567,6 @@ def call_locus(
     seed: int,
     logger_: logging.Logger,
     sample_id: Optional[str] = None,
-    sex_chroms: Optional[str] = None,
     realign: bool = False,
     hq: bool = False,
     # incorporate_snvs: bool = False,
@@ -572,6 +577,7 @@ def call_locus(
     fractional: bool = False,
     respect_ref: bool = False,
     count_kmers: str = "none",  # "none" | "peak" | "read"
+    consensus: bool = False,
     log_level: int = logging.WARNING,
     read_file_has_chr: bool = True,
     ref_file_has_chr: bool = True,
@@ -599,10 +605,6 @@ def call_locus(
     ref_seq: str = ""
     raised: bool = False
 
-    n_alleles: Optional[int] = get_n_alleles(2, sex_chroms, contig)
-    if n_alleles is None:  # Sex chromosome, but we don't have a specified sex chromosome karyotype
-        return None
-
     # Currently, only support diploid use of SNVs. There's not much of a point with haploid loci,
     # and polyploidy is hard.
     # should_incorporate_snvs: bool = incorporate_snvs and n_alleles == 2
@@ -799,6 +801,7 @@ def call_locus(
         flank_len: int = len(flank_left_seq) + len(flank_right_seq)
         tr_len_w_flank: int = tr_len + flank_len
 
+        tr_read_seq = qs[left_flank_end:right_flank_start]
         tr_read_seq_wc = calculate_seq_with_wildcards(qs[left_flank_end:right_flank_start], qqs)
 
         if count_kmers != "none":
@@ -848,6 +851,7 @@ def call_locus(
         read_dict_extra[rn] = {
             "_ref_start": segment_start,
             "_ref_end": segment_end,
+            **({"_tr_seq": tr_read_seq} if consensus else {}),
         }
 
         # Reads can show up more than once - TODO - cache this information across loci
@@ -886,6 +890,7 @@ def call_locus(
         }),
         "motif": motif,
         "ref_cn": ref_cn,
+        **({"ref_seq": ref_seq} if consensus else {}),
         "reads": read_dict,
     }
 
@@ -1008,12 +1013,13 @@ def call_locus(
     call_stdevs = call_data.get("peak_stdevs")
     call_modal_n = call_data.get("modal_n_peaks")
 
-    # Assign reads to peaks and compute peak k-mers --------------------------------------------------------------------
+    # Assign reads to peaks and compute peak k-mers (and optionally consensus sequences) -------------------------------
 
     # We cannot call read-level cluster labels with >2 peaks using distance alone;
     # don't know how re-sampling has occurred.
     call_peak_n_reads: list[int] = []
     peak_kmers: list[Counter] = [Counter() for _ in range(call_modal_n or 0)]
+    call_seqs: list[str] = []
     if read_peaks_called := call_modal_n and call_modal_n <= 2:
         peaks: NDArray[np.float_] = call_peaks[:call_modal_n]
         stdevs: NDArray[np.float_] = call_stdevs[:call_modal_n]
@@ -1059,6 +1065,11 @@ def call_locus(
 
         call_peak_n_reads = list(map(len, allele_reads))
 
+        if consensus:
+            call_seqs = list(
+                map(lambda a: consensus_seq(map(lambda rr: read_dict_extra[rr]["_tr_seq"], a)), allele_reads)
+            )
+
     peak_data = {
         "means": call_peaks.tolist(),  # from np.ndarray
         "weights": call_weights.tolist(),  # from np.ndarray
@@ -1066,6 +1077,7 @@ def call_locus(
         "modal_n": call_modal_n,
         "n_reads": call_peak_n_reads,
         **({"kmers": list(map(dict, peak_kmers))} if count_kmers in ("peak", "both") else {}),
+        **({"seqs": call_seqs} if consensus else {}),
     } if call_data else None
 
     # Calculate call time ----------------------------------------------------------------------------------------------

{strkit-0.11.8 → strkit-0.12.0}/strkit/call/call_sample.py

@@ -9,22 +9,19 @@ import pathlib
 import numpy as np
 import os
 import re
-import sys
 import time
 
 from datetime import datetime
 from multiprocessing.synchronize import Event as EventClass  # For type hinting
 from pysam import AlignmentFile
 
-from typing import Literal, Optional, Union
-
-from strkit import __version__
-
-from strkit.json import dumps_indented, json
+from typing import Literal, Optional
 from strkit.logger import logger
 
+from .allele import get_n_alleles
 from .call_locus import call_locus
 from .non_daemonic_pool import NonDaemonicPool
+from .output import output_json_report, output_tsv as output_tsv_fn, output_vcf
 from .utils import get_new_seed
 
 __all__ = [
@@ -48,7 +45,6 @@ def locus_worker(
     num_bootstrap: int,
     flank_size: int,
     sample_id: Optional[str],
-    sex_chroms: Optional[str],
     realign: bool,
     hq: bool,
     # incorporate_snvs: bool,
@@ -57,6 +53,7 @@ def locus_worker(
     fractional: bool,
     respect_ref: bool,
     count_kmers: str,
+    consensus: bool,
     log_level: int,
     locus_queue: mp.Queue,
     is_single_processed: bool,
@@ -102,9 +99,9 @@ def locus_worker(
         if td is None:  # Kill signal
             break
 
-        t_idx, t, locus_seed = td
+        t_idx, t, n_alleles, locus_seed = td
         res = call_locus(
-            t_idx, t, bfs, ref,
+            t_idx, t, n_alleles, bfs, ref,
             min_reads=min_reads,
             min_allele_reads=min_allele_reads,
             min_avg_phred=min_avg_phred,
@@ -113,7 +110,6 @@ def locus_worker(
             seed=locus_seed,
             logger_=lg,
             sample_id=sample_id,
-            sex_chroms=sex_chroms,
             realign=realign,
             hq=hq,
             # incorporate_snvs=incorporate_snvs,
@@ -124,6 +120,7 @@ def locus_worker(
             fractional=fractional,
             respect_ref=respect_ref,
             count_kmers=count_kmers,
+            consensus=consensus,
             log_level=log_level,
             read_file_has_chr=read_file_has_chr,
             ref_file_has_chr=ref_file_has_chr,
@@ -182,10 +179,6 @@ def parse_loci_bed(loci_file: str):
         yield from (tuple(line.split("\t")) for line in (s.strip() for s in tf) if line and not line.startswith("#"))
 
 
-def _cn_to_str(cn: Union[int, float]) -> str:
-    return f"{cn:.1f}" if isinstance(cn, float) else str(cn)
-
-
 def call_sample(
     read_files: tuple[str, ...],
     reference_file: str,
@@ -205,8 +198,10 @@ def call_sample(
     fractional: bool = False,
     respect_ref: bool = False,
     count_kmers: str = "none",  # "none" | "peak" | "read"
+    consensus: bool = False,
     log_level: int = logging.WARNING,
     json_path: Optional[str] = None,
+    vcf_path: Optional[str] = None,
     indent_json: bool = False,
     output_tsv: bool = True,
     processes: int = 1,
@@ -253,11 +248,17 @@ def call_sample(
     # Keep track of all contigs we are processing to speed up downstream Mendelian inheritance analysis.
     contig_set: set[str] = set()
     for t_idx, t in enumerate(parse_loci_bed(loci_file), 1):
+        contig = t[0]
+
+        n_alleles: Optional[int] = get_n_alleles(2, sex_chroms, contig)
+        if n_alleles is None:  # Sex chromosome, but we don't have a specified sex chromosome karyotype
+            continue  # --> skip the locus
+
         # We use locus-specific random seeds for replicability, no matter which order
         # the loci are yanked out of the queue / how many processes we have.
         # Tuple of (1-indexed locus index, locus data, locus-specific random seed)
-        locus_queue.put((t_idx, t, get_new_seed(rng)))
-        contig_set.add(t[0])
+        locus_queue.put((t_idx, t, n_alleles, get_new_seed(rng)))
+        contig_set.add(contig)
         num_loci += 1
 
     # At the end of the queue, add a None value (* the # of processes).
@@ -275,7 +276,6 @@ def call_sample(
         "num_bootstrap": num_bootstrap,
         "flank_size": flank_size,
         "sample_id": sample_id_final,
-        "sex_chroms": sex_chroms,
         "realign": realign,
         "hq": hq,
         # "incorporate_snvs": incorporate_snvs,
@@ -284,6 +284,7 @@ def call_sample(
         "fractional": fractional,
         "respect_ref": respect_ref,
         "count_kmers": count_kmers,
+        "consensus": consensus,
         "log_level": log_level,
     }
 
@@ -317,54 +318,19 @@ def call_sample(
     time_taken = datetime.now() - start_time
 
     if output_tsv:
-        for res in results:
-            has_call = res["call"] is not None
-            # n_peaks = res["peaks"]["modal_n"]
-
-            sys.stdout.write("\t".join((
-                res["contig"],
-                str(res["start"]),
-                str(res["end"]),
-                res["motif"],
-                _cn_to_str(res["ref_cn"]),
-                ",".join(map(_cn_to_str, sorted(r["cn"] for r in res["reads"].values()))) if res["reads"] else ".",
-                "|".join(map(_cn_to_str, res["call"])) if has_call else ".",
-                ("|".join("-".join(map(_cn_to_str, gc)) for gc in res["call_95_cis"]) if has_call else "."),
-                # *((res["assign_method"] if has_call else ".",) if incorporate_snvs else ()),
-                *((res["assign_method"] if has_call else ".",) if snv_vcf is not None else ()),
-
-                # ("|".join(map(lambda x: f"{x:.5f}", res["peaks"]["means"][:n_peaks]))
-                #  if has_call and n_peaks <= 2 else "."),
-                # ("|".join(map(lambda x: f"{x:.5f}", res["peaks"]["weights"][:n_peaks]))
-                #  if has_call and n_peaks <= 2 else "."),
-                # ("|".join(map(lambda x: f"{x:.5f}", res["peaks"]["stdevs"][:n_peaks]))
-                #  if has_call and n_peaks <= 2 else "."),
-            )) + "\n")
+        output_tsv_fn(results, has_snv_vcf=snv_vcf is not None)
 
     if json_path:
-        json_report = {
-            "sample_id": sample_id_final,
-            "caller": {
-                "name": "strkit",
-                "version": __version__,
-            },
-            "parameters": {
-                **job_params,
-                "processes": processes,
-            },
-            "runtime": time_taken.total_seconds(),
-            "contigs": tuple(contig_set),
-            "results": results,
-        }
-
-        dfn = dumps_indented if indent_json else json.dumps
-        report_data = dfn(json_report)
-
-        if json_path == "stdout":
-            sys.stdout.buffer.write(report_data)
-            sys.stdout.write("\n")
-            sys.stdout.flush()
-        else:
-            with open(json_path, "wb") as jf:
-                jf.write(report_data)
-                jf.write(b"\n")
+        output_json_report(
+            sample_id_final,
+            job_params,
+            processes,
+            time_taken,
+            contig_set,
+            results,
+            json_path,
+            indent_json,
+        )
+
+    if vcf_path:
+        output_vcf(sample_id_final, reference_file, results, vcf_path)

strkit-0.12.0/strkit/call/consensus.py

@@ -0,0 +1,26 @@
+import numpy as np
+import pyfamsa
+from scipy.stats import mode
+from typing import Iterable
+
+__all__ = [
+    "consensus_seq",
+]
+
+GAP_ORD = ord("-")
+
+aligner = pyfamsa.Aligner(threads=1, guide_tree="upgma", keep_duplicates=True)
+join_str = "".join
+
+
+def _not_gap(x: int) -> bool:
+    return x != GAP_ORD
+
+
+def consensus_seq(seqs: Iterable[str]) -> str:
+    # Align all the allele tandem repeat sequences together into a multiple sequence alignment
+    msa = aligner.align(pyfamsa.Sequence(f"s{i}".encode("ascii"), seq.encode("ascii")) for i, seq in enumerate(seqs))
+    # Turn the MSA into a numpy matrix as setup for calculating the modal value at each position
+    msa_grid = np.array(list(map(lambda aseq: list(map(int, aseq.sequence)), msa)))
+    # Return a stringified, gapless version of the column-wise mode for the MSA
+    return join_str(map(chr, filter(_not_gap, getattr(mode(msa_grid), "mode", ()))))

strkit-0.12.0/strkit/call/output/__init__.py

@@ -0,0 +1,9 @@
+from .json_report import output_json_report
+from .tsv import output_tsv
+from .vcf import output_vcf
+
+__all__ = [
+    "output_json_report",
+    "output_tsv",
+    "output_vcf",
+]

strkit-0.12.0/strkit/call/output/json_report.py

@@ -0,0 +1,46 @@
+import sys
+from datetime import timedelta
+from typing import Optional
+
+from strkit import __version__
+from strkit.json import dumps_indented, json
+
+__all__ = ["output_json_report"]
+
+
+def output_json_report(
+    sample_id: Optional[str],
+    job_params: dict,
+    processes: int,
+    time_taken: timedelta,
+    contig_set: set[str],
+    results: tuple[dict, ...],
+    json_path: str,
+    indent_json: bool,
+):
+    json_report = {
+        "sample_id": sample_id,
+        "caller": {
+            "name": "strkit",
+            "version": __version__,
+        },
+        "parameters": {
+            **job_params,
+            "processes": processes,
+        },
+        "runtime": time_taken.total_seconds(),
+        "contigs": tuple(contig_set),
+        "results": results,
+    }
+
+    dfn = dumps_indented if indent_json else json.dumps
+    report_data = dfn(json_report)
+
+    if json_path == "stdout":
+        sys.stdout.buffer.write(report_data)
+        sys.stdout.write("\n")
+        sys.stdout.flush()
+    else:
+        with open(json_path, "wb") as jf:
+            jf.write(report_data)
+            jf.write(b"\n")

strkit-0.12.0/strkit/call/output/tsv.py

@@ -0,0 +1,34 @@
+import sys
+from typing import Union
+
+__all__ = ["output_tsv"]
+
+
+def _cn_to_str(cn: Union[int, float]) -> str:
+    return f"{cn:.1f}" if isinstance(cn, float) else str(cn)
+
+
+def output_tsv(results: tuple[dict, ...], has_snv_vcf: bool):
+    for res in results:
+        has_call = res["call"] is not None
+        # n_peaks = res["peaks"]["modal_n"]
+
+        sys.stdout.write("\t".join((
+            res["contig"],
+            str(res["start"]),
+            str(res["end"]),
+            res["motif"],
+            _cn_to_str(res["ref_cn"]),
+            ",".join(map(_cn_to_str, sorted(r["cn"] for r in res["reads"].values()))) if res["reads"] else ".",
+            "|".join(map(_cn_to_str, res["call"])) if has_call else ".",
+            ("|".join("-".join(map(_cn_to_str, gc)) for gc in res["call_95_cis"]) if has_call else "."),
+            # *((res["assign_method"] if has_call else ".",) if incorporate_snvs else ()),
+            *((res["assign_method"] if has_call else ".",) if has_snv_vcf else ()),
+
+            # ("|".join(map(lambda x: f"{x:.5f}", res["peaks"]["means"][:n_peaks]))
+            #  if has_call and n_peaks <= 2 else "."),
+            # ("|".join(map(lambda x: f"{x:.5f}", res["peaks"]["weights"][:n_peaks]))
+            #  if has_call and n_peaks <= 2 else "."),
+            # ("|".join(map(lambda x: f"{x:.5f}", res["peaks"]["stdevs"][:n_peaks]))
+            #  if has_call and n_peaks <= 2 else "."),
+        )) + "\n")