mpath-pseudotime 0.2.0__py3-none-any.whl

mpath/__init__.py

@@ -0,0 +1,3 @@
+"""MPATH: methylation pseudotime analysis for Oxford Nanopore long reads."""
+
+__version__ = "0.2.0"  # x-release-please-version

mpath/cli.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+"""Command-line interface for MPATH.
+
+Subcommands::
+
+    mpath metrics    compute read-level metrics from modkit calls + WGBS bed
+    mpath pca fit    fit a PCA on labelled nascent/mature metric tables
+    mpath pca apply  project an unlabelled metric table into a fitted PCA space
+"""
+
+import argparse
+
+from . import __version__
+
+
+def _add_metrics_parser(subparsers):
+    p = subparsers.add_parser(
+        "metrics",
+        help="compute read-level methylation metrics",
+        description="Compute read-level metrics from a modkit read-calls TSV + WGBS reference, "
+        "or from a pre-merged bed.",
+    )
+    # Single-dash long options preserved for continuity with the original scripts.
+    # Mode A: merge calls + WGBS on the fly. Mode B: a pre-merged bed (escape hatch).
+    p.add_argument("-path_calls", default=None, help="modkit read-calls TSV (optionally .gz)")
+    p.add_argument("-path_wgbs", default=None, help="WGBS reference bed (optionally .gz)")
+    p.add_argument(
+        "-path_input_bed",
+        default=None,
+        help="pre-merged 7-col bed (chrom,start,stop,strand,read_id,meth,wgbs); skips the WGBS merge",
+    )
+    p.add_argument("-path_output_csv", required=True, help="output wide-format metrics CSV")
+    p.add_argument("-wgbs_column", default=3, type=int, help="0-based column of the WGBS ratio (default 3)")
+    p.add_argument("-min_cpgs", default=3, type=int, help="minimum CpGs on a read to compute metrics")
+    p.add_argument("-bin_limits", default="0,100,1000,5000,10000", help="comma-separated distance-bin limits")
+    p.add_argument("-batch_size", default=int(1e8), type=int, help="approx CpGs per batch (RAM control)")
+    p.add_argument("-p", default=1, type=int, help="number of parallel processes")
+    p.add_argument("--use_full_matrix", action="store_true", help="use the full CpG-pair matrix")
+    p.add_argument("--include-hydroxy", action="store_true", help="count 5hmC ('h') calls as methylated")
+    p.add_argument(
+        "--keep-unmatched-wgbs",
+        action="store_true",
+        help="keep CpGs with no WGBS entry (NaN ratio) instead of dropping them",
+    )
+    # WGBS reference ratio scale (explicit -- 0-1 vs 0-100 can't be auto-detected).
+    p.add_argument(
+        "--wgbs-scale",
+        choices=["0-1", "0-100"],
+        default="0-1",
+        help="scale of the WGBS ratio column: 0-1 (default) or 0-100 (percentages)",
+    )
+    # Coordinate-convention handling. These are auto-probed by default (the data
+    # tells us which offset/strand mode matches); override only to force one.
+    p.add_argument(
+        "--wgbs-offset",
+        choices=["auto", "-1", "0", "1"],
+        default="auto",
+        help="global coordinate offset for the CpG<->WGBS join (default: auto-probe)",
+    )
+    p.add_argument(
+        "--wgbs-collapse",
+        choices=["auto", "on", "off"],
+        default="auto",
+        help="map -strand CpGs to the + dyad anchor: on/off, or auto-probe (default)",
+    )
+    p.set_defaults(func=_run_metrics)
+
+
+def _run_metrics(args):
+    from . import io as mpath_io
+    from . import metrics
+
+    methyl_codes = ("m", "h") if args.include_hydroxy else mpath_io.DEFAULT_METHYL_CODES
+    metrics.run_metrics(
+        path_calls=args.path_calls,
+        path_wgbs=args.path_wgbs,
+        path_input_bed=args.path_input_bed,
+        path_output_csv=args.path_output_csv,
+        wgbs_column=args.wgbs_column,
+        min_cpgs=args.min_cpgs,
+        bin_limits=args.bin_limits,
+        batch_size=args.batch_size,
+        processes=args.p,
+        use_full_matrix=args.use_full_matrix,
+        methyl_codes=methyl_codes,
+        drop_unmatched_wgbs=not args.keep_unmatched_wgbs,
+        wgbs_scale=args.wgbs_scale,
+        wgbs_offset=args.wgbs_offset,
+        wgbs_collapse=args.wgbs_collapse,
+    )
+
+
+def _add_pca_parser(subparsers):
+    p = subparsers.add_parser("pca", help="PCA of read-level metrics")
+    pca_sub = p.add_subparsers(dest="pca_command", required=True)
+
+    fit = pca_sub.add_parser("fit", help="fit a PCA on labelled nascent + mature tables")
+    fit.add_argument("--nascent", required=True, help="nascent metrics CSV")
+    fit.add_argument("--mature", required=True, help="mature metrics CSV")
+    fit.add_argument("--out-dir", required=True, help="output directory for scores, model and figures")
+    fit.add_argument("--columns", default=None, help="comma-separated metric columns to use (default: auto)")
+    fit.add_argument("--standardize", action="store_true", help="z-score metrics before PCA (default: off)")
+    fit.add_argument("--n-components", default=None, type=int, help="number of PCs to keep (default: all)")
+    fit.set_defaults(func=_run_pca_fit)
+
+    ap = pca_sub.add_parser("apply", help="project an unlabelled table into a fitted PCA space")
+    ap.add_argument("--model", required=True, help="pca_model.npz produced by `mpath pca fit`")
+    ap.add_argument("--input", required=True, help="unlabelled metrics CSV")
+    ap.add_argument("--out", required=True, help="output CSV with appended PCA scores")
+    ap.set_defaults(func=_run_pca_apply)
+
+
+def _run_pca_fit(args):
+    from . import pca
+
+    columns = [c.strip() for c in args.columns.split(",")] if args.columns else None
+    pca.fit(
+        path_nascent=args.nascent,
+        path_mature=args.mature,
+        out_dir=args.out_dir,
+        columns=columns,
+        standardize=args.standardize,
+        n_components=args.n_components,
+    )
+
+
+def _run_pca_apply(args):
+    from . import pca
+
+    pca.apply(path_model=args.model, path_input=args.input, path_output=args.out)
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(prog="mpath", description="MPATH: methylation pseudotime analysis")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    _add_metrics_parser(subparsers)
+    _add_pca_parser(subparsers)
+    return parser
+
+
+def main(argv=None):
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        args.func(args)
+    except (ValueError, FileNotFoundError) as e:
+        # Present input/usage errors cleanly instead of as a raw traceback.
+        parser.error(str(e))
+
+
+if __name__ == "__main__":
+    main()

mpath/io.py

@@ -0,0 +1,417 @@
+#!/usr/bin/env python3
+
+"""
+Native merge of ``modkit extract calls`` output with a WGBS reference bed.
+
+This replaces the old workflow of hand-building a 7-column ``input.bed`` before
+running the metrics. The metrics CLI now consumes the modkit calls TSV and the
+WGBS bed directly:
+
+  * the modkit calls TSV gives, per CpG call: ``read_id``, ``chrom``,
+    ``ref_position`` and ``call_code`` (``-`` canonical, ``m`` 5mC, ``h`` 5hmC),
+  * the WGBS bed gives the expected methylation ratio for each CpG of the cell
+    type, in a user-specified column.
+
+The two are joined on ``(chrom, ref_position)``. modkit emits all rows for a
+given read contiguously, so reads are streamed in groups with
+``itertools.groupby`` and yielded in CpG-bounded batches -- whole-genome files
+never have to be held in memory at once. The WGBS reference is stored as
+compact per-chromosome sorted ``int32``/``float32`` arrays and looked up with a
+vectorised ``np.searchsorted`` per read.
+"""
+
+import gzip
+import itertools
+
+import numpy as np
+import polars as pl
+
+# Column names emitted by `modkit extract --read-calls` (modkit >= 0.4) and the
+# equivalent `modkit extract calls`. We resolve them from the header so the code
+# tolerates added/reordered columns across modkit versions.
+COL_READ_ID = "read_id"
+COL_REF_POSITION = "ref_position"
+COL_CHROM = "chrom"
+COL_CALL_CODE = "call_code"
+COL_FAIL = "fail"
+COL_REF_STRAND = "ref_strand"
+
+# Default modification codes counted as "methylated". 5hmC (`h`) is excluded by
+# default; pass methyl_codes=("m", "h") to include it.
+DEFAULT_METHYL_CODES = ("m",)
+
+
+def load_wgbs_reference(path_wgbs, wgbs_column, chrom_column=0, position_column=1, scale="0-1"):
+    """Load a WGBS bed into per-chromosome sorted (positions, ratios) arrays.
+
+    The expected input is a simple tab-separated bed (no header), e.g. the common
+    ``chrom, start, end, ratio`` 4-column form (ratio at the default
+    ``wgbs_column=3``).
+
+    Parameters
+    ----------
+    path_wgbs : str
+        Path to the WGBS bed (optionally gzipped).
+    wgbs_column : int
+        0-based column index holding the expected methylation ratio.
+    chrom_column, position_column : int
+        0-based column indices of the chromosome and (0-based) start position.
+    scale : str
+        ``"0-1"`` (ratios already in [0, 1]) or ``"0-100"`` (percentages; divided
+        by 100). No auto-detection -- declare the scale explicitly.
+
+    Returns
+    -------
+    dict[str, tuple[np.ndarray, np.ndarray]]
+        Mapping ``chrom -> (positions_sorted_int64, ratios_float64)``.
+    """
+    if scale not in ("0-1", "0-100"):
+        raise ValueError(f"scale must be '0-1' or '0-100', got {scale!r}")
+
+    needed = sorted({chrom_column, position_column, wgbs_column})
+    # polars reads gzip transparently and parses far faster than the csv module.
+    frame = pl.read_csv(
+        path_wgbs,
+        separator="\t",
+        has_header=False,
+        columns=needed,
+        infer_schema_length=0,  # read everything as str, we cast explicitly below
+    )
+    cols = frame.columns  # named like "column_<idx>" in selection order
+    name_for = {idx: cols[i] for i, idx in enumerate(needed)}
+
+    frame = frame.select(
+        pl.col(name_for[chrom_column]).alias("chrom"),
+        pl.col(name_for[position_column]).cast(pl.Int64).alias("pos"),
+        pl.col(name_for[wgbs_column]).cast(pl.Float64, strict=False).alias("ratio"),
+    ).sort(["chrom", "pos"])
+
+    divisor = 100.0 if scale == "0-100" else 1.0
+    reference = {}
+    for chrom, sub in frame.group_by("chrom", maintain_order=True):
+        chrom = chrom[0] if isinstance(chrom, tuple) else chrom
+        reference[str(chrom)] = (
+            sub["pos"].to_numpy(),
+            sub["ratio"].to_numpy() / divisor,
+        )
+
+    # A wrong scale silently corrupts read_wgbs_distance (and the position-only
+    # intersection QC would not catch it), so flag ratios that look like percents.
+    if scale == "0-1":
+        ref_max = max((r.max() for _, r in reference.values() if len(r)), default=0.0)
+        if ref_max > 1.5:
+            print(
+                f"WARNING: WGBS ratios reach {ref_max:.1f} (> 1) -- they look like 0-100 "
+                "percentages. Pass --wgbs-scale 0-100 if so."
+            )
+    return reference
+
+
+def lookup_wgbs(reference, chrom, positions):
+    """Vectorised lookup of WGBS ratios for one read's CpG positions.
+
+    Returns an array aligned with ``positions``; entries with no matching CpG in
+    the reference are ``np.nan``.
+    """
+    arrays = reference.get(str(chrom))
+    if arrays is None:
+        return np.full(len(positions), np.nan, dtype=np.float64)
+    ref_pos, ref_ratio = arrays
+    idx = np.searchsorted(ref_pos, positions)
+    idx_clipped = np.clip(idx, 0, len(ref_pos) - 1)
+    matched = ref_pos[idx_clipped] == positions
+    ratios = np.where(matched, ref_ratio[idx_clipped], np.nan).astype(np.float64)
+    return ratios
+
+
+def _open_text(path):
+    """Open a possibly-gzipped text file."""
+    if str(path).endswith(".gz"):
+        return gzip.open(path, "rt")
+    return open(path)
+
+
+def _resolve_columns(header_line):
+    """Map required modkit column names to indices from the header line."""
+    names = header_line.rstrip("\n").split("\t")
+    index = {name: i for i, name in enumerate(names)}
+    missing = [c for c in (COL_READ_ID, COL_REF_POSITION, COL_CHROM, COL_CALL_CODE) if c not in index]
+    if missing:
+        raise ValueError(
+            f"modkit calls file is missing required column(s) {missing}. "
+            f"Found columns: {names}. Generate it with `modkit extract --read-calls`."
+        )
+    return index
+
+
+def new_stats():
+    """Fresh intersection-QC accumulator (see :func:`iter_read_batches`)."""
+    return {
+        "reads_total": 0,
+        "reads_dropped": 0,  # reads with zero WGBS-matched CpGs
+        "cpgs_total": 0,
+        "cpgs_matched": 0,
+        "calls_chroms": set(),
+    }
+
+
+def iter_read_batches(
+    path_calls,
+    wgbs_reference,
+    batch_size,
+    methyl_codes=DEFAULT_METHYL_CODES,
+    drop_unmatched_wgbs=True,
+    collapse_strands=True,
+    position_offset=0,
+    stats=None,
+):
+    """Stream reads from a modkit calls file in CpG-bounded batches.
+
+    Each yielded batch is a list of read records with the keys the metrics code
+    expects: ``read_id``, ``meth_values`` (0/1), ``positions``, ``wgbs_values``.
+
+    Parameters
+    ----------
+    path_calls : str
+        Path to the modkit read-calls TSV (optionally gzipped).
+    wgbs_reference : dict
+        Output of :func:`load_wgbs_reference`.
+    batch_size : int
+        Approximate maximum number of CpGs per batch (reads are never split).
+    methyl_codes : tuple[str, ...]
+        ``call_code`` values counted as methylated (others are unmethylated).
+    drop_unmatched_wgbs : bool
+        If True (default), drop CpGs with no WGBS reference entry (inner join);
+        otherwise keep them with a NaN WGBS value.
+    collapse_strands : bool
+        If True (default) and a ``ref_strand`` column is present, a cytosine on
+        the reference ``-`` strand (``ref_position`` = ``p``) is matched to the
+        ``+`` strand anchor of its CpG dyad (``p - 1``). This is what lets a
+        combined/``+``-strand WGBS reference match ``-`` strand read CpGs;
+        without it roughly half of all CpGs silently fail to join. Distances
+        between CpGs are unaffected (a uniform shift), so only the lookup uses
+        the anchor.
+    stats : dict, optional
+        If given (see :func:`new_stats`), intersection-quality counters are
+        accumulated in place for the caller to report.
+    """
+    methyl_set = set(methyl_codes)
+    with _open_text(path_calls) as fh:
+        header = fh.readline()
+        if not header:
+            return
+        col = _resolve_columns(header)
+        i_read = col[COL_READ_ID]
+        i_pos = col[COL_REF_POSITION]
+        i_chrom = col[COL_CHROM]
+        i_code = col[COL_CALL_CODE]
+        i_fail = col.get(COL_FAIL)
+        i_strand = col.get(COL_REF_STRAND)
+
+        rows = (line.rstrip("\n").split("\t") for line in fh)
+
+        batch = []
+        n_cpgs = 0
+        for read_id, group in itertools.groupby(rows, key=lambda r: r[i_read]):
+            meth = []
+            positions = []
+            chrom = None
+            ref_strand = None
+            for r in group:
+                # Skip calls that failed modkit's confidence threshold.
+                if i_fail is not None and r[i_fail].lower() == "true":
+                    continue
+                chrom = r[i_chrom]
+                if i_strand is not None:
+                    ref_strand = r[i_strand]
+                positions.append(int(r[i_pos]))
+                meth.append(1 if r[i_code] in methyl_set else 0)
+
+            if not positions:
+                continue
+
+            positions = np.asarray(positions, dtype=np.int64)
+            meth = np.asarray(meth, dtype=np.float64)
+
+            # Compute the WGBS lookup anchor: a global coordinate offset
+            # (0-/1-based etc., usually chosen by probe_alignment) plus, when
+            # collapsing strands, the -1 shift that maps a -strand C onto its
+            # + strand dyad anchor. A read aligns to one strand, so ref_strand is
+            # constant. Distances between CpGs are unaffected (uniform shift).
+            anchors = positions + position_offset
+            if collapse_strands and ref_strand == "-":
+                anchors = anchors - 1
+            wgbs = lookup_wgbs(wgbs_reference, chrom, anchors)
+
+            if stats is not None:
+                stats["reads_total"] += 1
+                stats["cpgs_total"] += positions.size
+                matched = int(np.count_nonzero(~np.isnan(wgbs)))
+                stats["cpgs_matched"] += matched
+                stats["calls_chroms"].add(chrom)
+                if matched == 0:
+                    stats["reads_dropped"] += 1
+
+            if drop_unmatched_wgbs:
+                keep = ~np.isnan(wgbs)
+                positions = positions[keep]
+                meth = meth[keep]
+                wgbs = wgbs[keep]
+                if positions.size == 0:
+                    continue
+
+            batch.append(
+                {
+                    "read_id": read_id,
+                    "meth_values": meth,
+                    "positions": positions,
+                    "wgbs_values": wgbs,
+                }
+            )
+            n_cpgs += positions.size
+
+            if n_cpgs >= batch_size:
+                yield batch
+                batch = []
+                n_cpgs = 0
+
+        if batch:
+            yield batch
+
+
+def report_intersection(stats, wgbs_reference, warn_threshold=0.5):
+    """Print WGBS-intersection QC and warn (not fail) on a poor overlap.
+
+    Returns the matched fraction in [0, 1] (0.0 if no CpGs were seen).
+    """
+    total = stats["cpgs_total"]
+    matched = stats["cpgs_matched"]
+    frac = matched / total if total else 0.0
+    print(
+        f"WGBS intersection: matched {matched}/{total} CpGs ({frac:.1%}); "
+        f"{stats['reads_dropped']}/{stats['reads_total']} reads had no WGBS overlap."
+    )
+    if total and frac < warn_threshold:
+        calls_chroms = stats["calls_chroms"]
+        ref_chroms = set(wgbs_reference)
+        print(f"WARNING: only {frac:.1%} of CpGs matched the WGBS reference. Likely causes:")
+        if calls_chroms and not (calls_chroms & ref_chroms):
+            print(
+                f"  - chromosome naming mismatch: calls use e.g. {sorted(calls_chroms)[:3]} "
+                f"but the WGBS reference has e.g. {sorted(ref_chroms)[:3]}"
+            )
+        print("  - wrong ratio column or scale: check -wgbs_column / --wgbs-scale")
+        print("  - genome build / coordinate-system difference between calls and WGBS")
+        print("  - if you forced --wgbs-offset / --wgbs-collapse, try 'auto'")
+    return frac
+
+
+# Candidate global coordinate offsets the aligner probe tries (0-/1-based and
+# off-by-one dyad/strand-anchor conventions). 0 first so ties prefer no shift.
+CANDIDATE_OFFSETS = (0, -1, 1)
+
+
+def probe_alignment(
+    path_calls,
+    wgbs_reference,
+    offsets=CANDIDATE_OFFSETS,
+    collapse_options=(True, False),
+    sample_size=20000,
+):
+    """Empirically detect the coordinate convention of a WGBS reference.
+
+    WGBS beds vary in convention (0- vs 1-based, which base of the CpG dyad/strand
+    they anchor on) and you usually can't tell from the file. So rather than
+    requiring the user to know, we read a sample of CpGs from the head of the
+    calls file and measure the match rate for each ``(offset, collapse_strands)``
+    combination -- the correct convention reveals itself as a sharp jump in the
+    matched fraction. Self-verifying, unlike the ratio scale, which is why this is
+    probed while ``--wgbs-scale`` stays an explicit flag.
+
+    Returns
+    -------
+    list[tuple[int, bool, float]]
+        ``(offset, collapse_strands, matched_fraction)`` sorted best-first.
+    """
+    sample = {}  # chrom -> ([positions], [is_minus])
+    n = 0
+    with _open_text(path_calls) as fh:
+        header = fh.readline()
+        if not header:
+            return []
+        col = _resolve_columns(header)
+        i_pos = col[COL_REF_POSITION]
+        i_chrom = col[COL_CHROM]
+        i_strand = col.get(COL_REF_STRAND)
+        for line in fh:
+            r = line.rstrip("\n").split("\t")
+            chrom = r[i_chrom]
+            pos_list, minus_list = sample.setdefault(chrom, ([], []))
+            pos_list.append(int(r[i_pos]))
+            minus_list.append(i_strand is not None and r[i_strand] == "-")
+            n += 1
+            if n >= sample_size:
+                break
+
+    sample = {
+        chrom: (np.asarray(p, dtype=np.int64), np.asarray(m, dtype=bool))
+        for chrom, (p, m) in sample.items()
+    }
+
+    results = []
+    for offset in offsets:
+        for collapse in collapse_options:
+            matched = total = 0
+            for chrom, (positions, minus) in sample.items():
+                anchors = positions + offset
+                if collapse:
+                    anchors = anchors - minus.astype(np.int64)
+                w = lookup_wgbs(wgbs_reference, chrom, anchors)
+                matched += int(np.count_nonzero(~np.isnan(w)))
+                total += positions.size
+            results.append((offset, collapse, matched / total if total else 0.0))
+    results.sort(key=lambda t: t[2], reverse=True)
+    return results
+
+
+# Column layout of a pre-merged 7-column bed (the escape hatch / legacy format):
+# chrom, start, stop, strand, read_id, methylation(0/1), wgbs_ratio.
+PREMERGED_READ_ID_COLUMN = 4
+PREMERGED_POSITION_COLUMN = 1
+PREMERGED_METH_COLUMN = 5
+PREMERGED_WGBS_COLUMN = 6
+
+
+def iter_premerged_batches(path_bed, batch_size):
+    """Stream a pre-merged 7-column bed (chrom, start, stop, strand, read_id,
+    methylation, wgbs) in CpG-bounded batches.
+
+    This is the escape hatch for users who do the CpG<->WGBS intersection
+    themselves (e.g. with bedtools in a pipeline): MPATH does no merge/probe and
+    just computes metrics. Rows must be grouped by ``read_id`` (no header).
+    """
+    with _open_text(path_bed) as fh:
+        rows = (line.rstrip("\n").split("\t") for line in fh)
+        batch = []
+        n_cpgs = 0
+        for read_id, group in itertools.groupby(rows, key=lambda r: r[PREMERGED_READ_ID_COLUMN]):
+            positions, meth, wgbs = [], [], []
+            for r in group:
+                positions.append(int(r[PREMERGED_POSITION_COLUMN]))
+                meth.append(float(r[PREMERGED_METH_COLUMN]))
+                wgbs.append(float(r[PREMERGED_WGBS_COLUMN]))
+            batch.append(
+                {
+                    "read_id": read_id,
+                    "meth_values": np.asarray(meth, dtype=np.float64),
+                    "positions": np.asarray(positions, dtype=np.int64),
+                    "wgbs_values": np.asarray(wgbs, dtype=np.float64),
+                }
+            )
+            n_cpgs += len(positions)
+            if n_cpgs >= batch_size:
+                yield batch
+                batch = []
+                n_cpgs = 0
+        if batch:
+            yield batch