telinker 0.1.0__py3-none-any.whl

telinker/__init__.py

@@ -0,0 +1,7 @@
+import sys
+
+__version__ = "0.1.0"
+
+
+def log(msg):
+    print(f"[telinker] {msg}", file=sys.stderr)

telinker/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

telinker/cli.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+import argparse
+import sys
+from collections import Counter
+
+import numpy as np
+
+from . import __version__, log
+from .impute import impute
+from .inputs import read_bed, read_plink, read_raw, read_vcf, sample_name, sniff
+from .match import match
+from .output import PRESENT, write_tsv, write_vcf
+from .panel import BUILDS, PANEL_DIR, load_panel
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(
+        prog="telinker", description="Genotype polymorphic TEs from VCF, PLINK, array, BED "
+                                    "or BAM/CRAM files.")
+    ap.add_argument("--version", action="version", version=__version__)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    g = sub.add_parser("genotype", help="genotype catalog polyTEs")
+    g.add_argument("input", nargs="+",
+                   help="VCF, PLINK prefix, 23andMe/Ancestry file, BED of TE calls or BAM/CRAM; "
+                        "several inputs are merged by sample name")
+    g.add_argument("-o", "--out", help="output file (default: stdout)")
+    g.add_argument("--build", choices=("auto",) + BUILDS, default="auto",
+                   help="genome build of the inputs (default: from the header)")
+    g.add_argument("--min-r2", type=float, default=0.5,
+                   help="minimum model cv r2 for imputed TEs (default: 0.5)")
+    g.add_argument("--model", choices=("mixed", "ridge", "tag"), default="mixed",
+                   help="imputation model: best available (default), ridge only, or single tag SNP")
+    g.add_argument("--vcf", action="store_true", help="write VCF (GT:DS) instead of TSV")
+    g.add_argument("--reference", help="reference FASTA for CRAM")
+    g.add_argument("--include-del-contractions", action="store_true",
+                   help="also report DEL loci that are in-element contractions")
+
+    sub.add_parser("info", help="show the reference panel summary")
+    a = ap.parse_args(argv)
+
+    if a.cmd == "info":
+        print((PANEL_DIR / "PANEL.md").read_text(), end="")
+        return
+    genotype(a)
+
+
+def genotype(a):
+    jobs = [inspect(path, a.reference) for path in a.input]
+    build = pick_build(jobs, a.build)
+    for j in jobs:
+        j["direct"] = j["kind"] in ("vcf_sv", "bed") or \
+            (j["kind"] == "reads" and j["long"] and build == "t2t")
+    panel = load_panel(build, models=not all(j["direct"] for j in jobs))
+
+    results = []
+    for j in jobs:
+        if len(jobs) > 1:
+            log(f"--- {j['path']}")
+        results.append(run(j, panel, a))
+    samples, calls, clashes, disagree = merge(results)
+    shared = [s for s, n in Counter(s for names, _ in results for s in set(names)).items() if n > 1]
+    if shared:
+        log(f"merged {len(shared):,} sample name(s) found in more than one input "
+            f"(e.g. {', '.join(shared[:3])}); rename files that hold different people")
+    if clashes:
+        log(f"{clashes:,} genotypes called by more than one input ({disagree:,} disagreed); "
+            "kept the first")
+
+    stats = Counter()
+    if not a.include_del_contractions:
+        keep = [c for c in calls if panel.loci["del_ok"][c[0]]]
+        skipped = len(calls) - len(keep)
+        calls = keep
+    if a.vcf:
+        placed = [c for c in calls if panel.te_pos()[c[0]] > 0]
+        if len(placed) < len(calls):
+            log(f"{len(calls) - len(placed):,} TEs have no {build} position; left out of the VCF")
+        calls = placed
+    stats.update(m.rstrip("0123456789") for c in calls for m in c[1].split(","))
+
+    out = open(a.out, "w") if a.out else sys.stdout
+    (write_vcf if a.vcf else write_tsv)(out, panel, samples, calls)
+    if a.out:
+        out.close()
+    log(f"{len(calls):,} TEs written for {len(samples):,} sample(s) ("
+        + ", ".join(f"{k} {v:,}" for k, v in stats.most_common()) + ")")
+    if not a.include_del_contractions and skipped:
+        log(f"{skipped:,} DEL contractions skipped (--include-del-contractions to keep)")
+
+
+def inspect(path, reference):
+    kind, build = sniff(path)
+    job = dict(path=path, kind=kind, build=build, long=False)
+    if kind == "reads":
+        from . import reads
+        with reads.open_bam(path, reference) as bam:
+            job["samples"], job["rg"], job["build"], job["long"] = reads.describe(bam, path)
+    return job
+
+
+def pick_build(jobs, forced):
+    if forced != "auto":
+        clash = [j for j in jobs if j["build"] not in (None, forced)]
+        if clash:
+            sys.exit(f"--build {forced} conflicts with the header of " + ", ".join(
+                f"{j['path']} ({j['build']})" for j in clash))
+        log(f"build {forced}")
+        return forced
+    for j in jobs:
+        if j["build"] is None and j["kind"] == "raw":
+            j["build"] = "hg19"
+            log(f"{j['path']}: no build in header, assuming hg19")
+    unknown = [j["path"] for j in jobs if j["build"] is None]
+    if unknown:
+        sys.exit(f"Unknown genome build for {', '.join(unknown)}; pass --build t2t|hg38|hg19")
+    builds = {j["build"] for j in jobs}
+    if len(builds) > 1:
+        sys.exit("Inputs are on different builds: " + ", ".join(
+            f"{j['path']} ({j['build']})" for j in jobs))
+    build = builds.pop()
+    log(f"build {build}")
+    return build
+
+
+def run(j, panel, a):
+    log(f"input: {j['kind']}")
+    kind, path, calls = j["kind"], j["path"], []
+    if kind == "bed":
+        samples, svs = [sample_name(path)], read_bed(path)
+        calls = matched(panel, svs, np.full((len(svs), 1), PRESENT), "bed")
+    elif kind in ("vcf_snp", "vcf_sv"):
+        samples, G, svs, dose = read_vcf(path, panel)
+        if svs or j["direct"]:
+            calls = matched(panel, svs, dose, "sv")
+    elif kind == "reads":
+        from . import reads
+        samples = j["samples"]
+        with reads.open_bam(path, a.reference) as bam:
+            if j["direct"]:
+                log("long reads: genotyping from CIGAR indels")
+                rows = [i for i in range(len(panel))
+                        if not reads.in_chry_par(panel.loci["chrom"][i], panel.loci["pos"][i])]
+                log(f"{len(panel) - len(rows):,} chrY PAR loci skipped (called on chrX)")
+                res = reads.genotype_long(bam, panel, rows, samples, j["rg"])
+                calls = [(i, "reads", None, d) for i, d in res.items()]
+            else:
+                log("pileup at panel SNPs, then imputing")
+                G = reads.pileup_sites(bam, panel, samples, j["rg"])
+    else:
+        samples, G = {"plink": read_plink, "raw": read_raw}[kind](path, panel)
+    if not j["direct"]:
+        log(f"{len(G):,} of {len(panel.sites):,} panel SNPs found")
+        done = {c[0] for c in calls}
+        res = impute(panel, G, len(samples), a.min_r2, a.model)
+        calls += [(i, *r) for i, r in res.items() if i not in done]
+    return samples, calls
+
+
+def matched(panel, svs, dose, method):
+    pairs = match(panel, svs)
+    log(f"{len(pairs):,} of {len(svs):,} input calls matched a catalog TE")
+    return [(i, method, None, dose[k]) for k, i in pairs]
+
+
+def merge(results):
+    if len(results) == 1:
+        samples, calls = results[0]
+        return samples, sorted(calls, key=lambda c: c[0]), 0, 0
+    samples = list(dict.fromkeys(s for names, _ in results for s in names))
+    col = {s: k for k, s in enumerate(samples)}
+    rows, clashes, disagree = {}, 0, 0
+    for names, calls in results:
+        idx = np.array([col[s] for s in names])
+        for i, method, r2, doses in calls:
+            methods, r2s, d = rows.setdefault(i, ([], [], np.full(len(samples), np.nan)))
+            methods += [m for m in method.split(",") if m not in methods]
+            if r2 is not None:
+                r2s.append(r2)
+            have, new = ~np.isnan(d[idx]), ~np.isnan(doses)
+            both = have & new
+            clashes += both.sum()
+            disagree += (d[idx[both]] != doses[both]).sum()
+            d[idx[new & ~have]] = doses[new & ~have]
+    calls = [(i, ",".join(m), min(r) if r else None, d) for i, (m, r, d) in sorted(rows.items())]
+    return samples, calls, clashes, disagree
+
+
+if __name__ == "__main__":
+    main()

telinker/impute.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+from itertools import groupby
+
+import numpy as np
+
+COMPLEMENT = str.maketrans("ACGT", "TGCA")
+MIN_OBSERVED = 0.8
+
+
+def orient(ref, alt, panel_ref, panel_alt):
+    for r, a in ((ref, alt), (ref.translate(COMPLEMENT), alt.translate(COMPLEMENT))):
+        if (r, a) == (panel_ref, panel_alt):
+            return 1
+        if (r, a) == (panel_alt, panel_ref):
+            return -1
+        if is_palindromic(panel_ref, panel_alt):
+            return 0
+    return 0
+
+
+def is_palindromic(ref, alt):
+    return {ref, alt} in ({"A", "T"}, {"C", "G"})
+
+
+def impute(panel, G, n_samples, min_r2=0.5, kind="mixed"):
+    out = {}
+    for te, models in groupby(panel.models, key=lambda m: m.te):
+        dose, used = np.full(n_samples, np.nan), []
+        for m in models:
+            if m.cv_r2 < min_r2:
+                break
+            if kind != "mixed" and (m.name == "ridge") != (kind == "ridge"):
+                continue
+            ok = [s in G for s in m.sites]
+            need = MIN_OBSERVED if m.name == "ridge" else 1.0
+            if np.mean(ok) < need:
+                continue
+            X = np.full((len(m.sites), n_samples), np.nan)
+            for j in np.flatnonzero(ok):
+                X[j] = G[m.sites[j]]
+            fill = np.isnan(dose) & ((~np.isnan(X)).mean(0) >= need)
+            if not fill.any():
+                continue
+            X = np.where(np.isnan(X), m.means[:, None], X)
+            dose[fill] = np.clip(m.intercept + m.weights @ X[:, fill], 0, 2)
+            used.append(m)
+            if not np.isnan(dose).any():
+                break
+        if used:
+            out[te] = (",".join(m.name for m in used), min(m.cv_r2 for m in used), dose)
+    return out

telinker/inputs.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+import gzip
+import io
+import re
+import zipfile
+from pathlib import Path
+
+import numpy as np
+
+from . import log
+from .impute import COMPLEMENT, is_palindromic, orient
+from .match import catalog_positions, has_near
+from .panel import norm_chrom
+
+CHR1_LENGTH = {248387328: "t2t", 248956422: "hg38", 249250621: "hg19"}
+SV_TYPES = re.compile(r"(?:^|;)SVTYPE=([^;]+)")
+SV_LEN = re.compile(r"(?:^|;)SVLEN=(-?\d+)")
+INS_TYPES = re.compile(r"INS|ALU|LINE|L1|SVA|HERV|MEI|ME:", re.I)
+
+
+def sample_name(path):
+    return Path(path).name.split(".")[0]
+
+
+def open_text(path):
+    path = Path(path)
+    with open(path, "rb") as f:
+        magic = f.read(4)
+    if magic[:2] == b"\x1f\x8b":
+        return gzip.open(path, "rt")
+    if magic == b"PK\x03\x04":
+        z = zipfile.ZipFile(path)
+        return io.TextIOWrapper(z.open(z.namelist()[0]))
+    return open(path)
+
+
+def plink_prefix(path):
+    s = str(path)
+    return s[:-4] if s.lower().endswith((".bed", ".bim", ".fam")) else s
+
+
+def sniff(path):
+    name = Path(path).name.lower()
+    if name.endswith((".bam", ".cram")):
+        return "reads", None
+    if name.endswith(".bcf"):
+        raise SystemExit("BCF is not supported; convert with: bcftools view -Oz -o x.vcf.gz x.bcf")
+    prefix = plink_prefix(path)
+    if Path(prefix + ".bim").exists() and Path(prefix + ".bed").exists():
+        return "plink", None
+    if not Path(path).exists():
+        raise SystemExit(f"No such file: {path}")
+    if name.endswith((".bed", ".bed.gz")):
+        return "bed", None
+    with open_text(path) as f:
+        first = next(f, "")
+        if first.startswith("##fileformat=VCF"):
+            return sniff_vcf(f)
+        head = first + "".join(next(f, "") for _ in range(30))
+    text = head.lower()
+    if "rsid" in text:
+        build = "hg19" if re.search(r"build 37|grch37", text) else \
+                "hg38" if re.search(r"build 38|grch38", text) else None
+        return "raw", build
+    raise SystemExit(f"{path}: unrecognized format (expected VCF, PLINK, BED, BAM/CRAM or 23andMe)")
+
+
+def sniff_vcf(f, n=2000):
+    build = None
+    for line in f:
+        if line.startswith("##contig=") and re.search(r"[<,]ID=(chr)?1[,>]", line):
+            m = re.search(r"length=(\d+)", line)
+            build = CHR1_LENGTH.get(int(m.group(1))) if m else None
+        if line.startswith("#CHROM"):
+            break
+    snv = sv = 0
+    for i, line in enumerate(f):
+        ref, alt, _, _, info = line.split("\t", 8)[3:8]
+        if len(ref) == 1 and len(alt) == 1 and alt in "ACGT":
+            snv += 1
+        elif sv_kind(ref, alt, info):
+            sv += 1
+        if i >= n:
+            break
+    return ("vcf_sv" if sv > snv else "vcf_snp"), build
+
+
+def vcf_records(path):
+    with open_text(path) as f:
+        for line in f:
+            if line.startswith("#CHROM"):
+                yield line.rstrip("\n").split("\t")[9:]
+                break
+        for line in f:
+            yield line
+
+
+def gt_dosage(samples_fields):
+    v = np.array(samples_fields, dtype="S3").view(np.uint8).reshape(-1, 3)
+    a, b = v[:, 0], v[:, 2]
+    dose = (a == ord("1")).astype(np.float32) + (b == ord("1"))
+    dose[(a == ord(".")) | (b == ord(".")) | (v[:, 1] == 0)] = np.nan
+    return dose
+
+
+def sample_dosage(fields):
+    fmt = fields[8].split(":")
+    if "DS" in fmt:
+        i, k, n = fmt.index("DS"), len(fmt), len(fields) - 9
+        values = ":".join(fields[9:]).split(":")
+        if len(values) == k * n:
+            try:
+                return np.array(values[i::k], dtype=np.float32)
+            except ValueError:
+                pass
+        return np.array([_float(s.split(":")[i]) if s.count(":") >= i else np.nan
+                         for s in fields[9:]], dtype=np.float32)
+    if fmt[0] != "GT":
+        return np.full(len(fields) - 9, np.nan, dtype=np.float32)
+    return gt_dosage(fields[9:])
+
+
+def _float(s):
+    try:
+        return float(s)
+    except ValueError:
+        return np.nan
+
+
+def read_vcf(path, panel):
+    near = catalog_positions(panel)
+    recs = vcf_records(path)
+    samples = next(recs)
+    G, calls, dose = {}, [], []
+    for line in recs:
+        chrom, pos, _ = line.split("\t", 2)
+        chrom, pos = norm_chrom(chrom), int(pos)
+        hits = panel.site_index.get((chrom, pos))
+        ix = near.get(chrom)
+        sv_near = ix is not None and has_near(ix, pos)
+        if not (hits or sv_near):
+            continue
+        fields = line.rstrip("\n").split("\t")
+        ref, alt = fields[3], fields[4]
+        if "," in alt:
+            continue
+        d = None
+        for s in hits or ():
+            sign = orient(ref, alt, *panel.sites[s][2:])
+            if sign:
+                d = sample_dosage(fields) if d is None else d
+                G[s] = d if sign > 0 else 2 - d
+        kind = sv_near and sv_kind(ref, alt, fields[7])
+        if kind:
+            d = gt_dosage([s.split(":")[0] for s in fields[9:]]) if fields[8].startswith("GT") \
+                else np.full(len(samples), np.nan, np.float32)
+            calls.append((chrom, pos, kind, sv_length(ref, alt, fields[7])))
+            dose.append(d if kind == "INS" else 2 - d)
+    return samples, G, calls, np.array(dose, dtype=np.float32).reshape(len(calls), len(samples))
+
+
+def sv_kind(ref, alt, info):
+    m = SV_TYPES.search(info)
+    t = m.group(1) if m else (alt.strip("<>") if alt.startswith("<") else "")
+    if t.upper().startswith("DEL"):
+        return "DEL"
+    if INS_TYPES.search(t):
+        return "INS"
+    if not t and abs(len(alt) - len(ref)) >= 50:
+        return "INS" if len(alt) > len(ref) else "DEL"
+    return None
+
+
+def sv_length(ref, alt, info):
+    m = SV_LEN.search(info)
+    if m and m.group(1) != "0":
+        return abs(int(m.group(1)))
+    if not alt.startswith("<"):
+        return abs(len(alt) - len(ref))
+    return None
+
+
+def read_bed(path):
+    calls = []
+    with open_text(path) as f:
+        for line in f:
+            if line.startswith(("#", "track", "browser")) or not line.strip():
+                continue
+            c, s, e = line.split("\t")[:3]
+            s, e = int(s), int(e)
+            calls.append((norm_chrom(c), s + 1, "INS", e - s if e - s >= 50 else None))
+    return calls
+
+
+def read_plink(path, panel):
+    prefix = plink_prefix(path)
+    samples = [l.split()[1] for l in open(prefix + ".fam")]
+    n = len(samples)
+    bpr = (n + 3) // 4
+    bed = np.memmap(prefix + ".bed", dtype=np.uint8, mode="r")
+    if bed[:3].tolist() != [0x6C, 0x1B, 0x01]:
+        raise SystemExit(f"{prefix}.bed: not a SNP-major PLINK 1 file")
+    a1_count = np.array([2, np.nan, 1, 0], dtype=np.float32)
+    G, strand = {}, Strand()
+    with open(prefix + ".bim") as f:
+        for i, line in enumerate(f):
+            chrom, _, _, pos, a1, a2 = line.split()
+            hits = panel.site_index.get((norm_chrom(chrom), int(pos)))
+            if not hits:
+                continue
+            raw = bed[3 + i * bpr: 3 + (i + 1) * bpr]
+            codes = ((raw[:, None] >> np.array([0, 2, 4, 6], np.uint8)) & 3).ravel()[:n]
+            dose = a1_count[codes]
+            for s in hits:
+                sign = orient(a2, a1, *panel.sites[s][2:])
+                if sign:
+                    G[s] = dose if sign > 0 else 2 - dose
+                    strand.add(s, {a1, a2}, panel)
+    strand.drop_ambiguous(G, path)
+    return samples, G
+
+
+def read_raw(path, panel):
+    G, strand = {}, Strand()
+    with open_text(path) as f:
+        for line in f:
+            if line.startswith("#") or line.lower().startswith("rsid"):
+                continue
+            parts = line.replace(",", "\t").split()
+            if len(parts) < 4:
+                continue
+            hits = panel.site_index.get((norm_chrom(parts[1]), int(parts[2])))
+            if not hits:
+                continue
+            gt = "".join(parts[3:5]).upper()
+            for s in hits:
+                ref, alt = panel.sites[s][2:]
+                d = genotype_count(gt, ref, alt)
+                if d == d:
+                    G[s] = np.array([d], dtype=np.float32)
+                    strand.add(s, set(gt), panel)
+    strand.drop_ambiguous(G, path)
+    return [sample_name(path)], G
+
+
+def genotype_count(gt, ref, alt):
+    if len(gt) != 2 or not set(gt) <= set("ACGT"):
+        return np.nan
+    if set(gt) <= {ref, alt}:
+        return float(gt.count(alt))
+    if {ref, alt} not in ({"A", "T"}, {"C", "G"}):
+        flipped = gt.translate(COMPLEMENT)
+        if set(flipped) <= {ref, alt}:
+            return float(flipped.count(alt))
+    return np.nan
+
+
+class Strand:
+    def __init__(self):
+        self.matched, self.flipped, self.palindromic = 0, 0, []
+
+    def add(self, site, alleles, panel):
+        ref, alt = panel.sites[site][2:]
+        self.matched += 1
+        if is_palindromic(ref, alt):
+            self.palindromic.append(site)
+        elif not alleles <= {ref, alt}:
+            self.flipped += 1
+
+    def drop_ambiguous(self, G, path):
+        if self.flipped <= 0.01 * self.matched:
+            return
+        for s in self.palindromic:
+            G.pop(s, None)
+        log(f"{path}: not strand-aligned, dropped {len(self.palindromic):,} A/T and C/G SNPs")

telinker/match.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+import numpy as np
+
+POS_TOL = 100
+LEN_LO, LEN_HI = 0.80, 1.25
+
+
+def catalog_positions(panel):
+    pos = panel.te_pos()
+    out = {}
+    for i in np.flatnonzero(pos > 0):
+        out.setdefault(panel.loci["chrom"][i], []).append(pos[i])
+    return {c: np.sort(p) for c, p in out.items()}
+
+
+def has_near(sorted_pos, p):
+    k = np.searchsorted(sorted_pos, p - POS_TOL)
+    return k < len(sorted_pos) and sorted_pos[k] <= p + POS_TOL
+
+
+def match(panel, calls):
+    pos = panel.te_pos()
+    by_chrom = {}
+    for i in np.flatnonzero(pos > 0):
+        by_chrom.setdefault(panel.loci["chrom"][i], []).append(i)
+    by_chrom = {c: np.array(sorted(ix, key=lambda i: pos[i])) for c, ix in by_chrom.items()}
+
+    cands = []
+    same_ref = panel.build == "t2t"
+    for k, (chrom, p, kind, length) in enumerate(calls):
+        ix = by_chrom.get(chrom)
+        if ix is None:
+            continue
+        lo, hi = np.searchsorted(pos[ix], [p - POS_TOL, p + POS_TOL + 1])
+        for i in ix[lo:hi]:
+            if same_ref and kind and kind != panel.loci["type"][i]:
+                continue
+            ratio = length / panel.loci["len"][i] if length else 1.0
+            if LEN_LO <= ratio <= LEN_HI:
+                cands.append((abs(np.log(ratio)), abs(int(pos[i]) - p), k, i))
+    used_call, used_te, out = set(), set(), []
+    for _, _, k, i in sorted(cands):
+        if k not in used_call and i not in used_te:
+            used_call.add(k)
+            used_te.add(i)
+            out.append((k, i))
+    return out

telinker/output.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+import sys
+
+COLUMNS = ["te_id", "chrom", "pos", "type", "subfamily", "method", "r2"]
+PRESENT = -1.0
+
+
+def fmt(d):
+    if d != d:
+        return "."
+    if d == PRESENT:
+        return "present"
+    return str(int(d)) if float(d).is_integer() else f"{d:.2f}"
+
+
+def write_tsv(out, panel, samples, calls):
+    pos = panel.te_pos()
+    print("\t".join(COLUMNS + list(samples)), file=out)
+    for i, method, r2, doses in calls:
+        L = panel.loci
+        row = [L["te_id"][i], L["chrom"][i], pos[i] if pos[i] > 0 else ".", L["type"][i],
+               L["subfamily"][i],
+               method, "" if r2 is None else f"{r2:.2f}"]
+        print("\t".join(map(str, row + list(map(fmt, doses.tolist())))), file=out)
+
+
+def write_vcf(out, panel, samples, calls):
+    pos = panel.te_pos()
+    print("##fileformat=VCFv4.2", file=out)
+    print(f"##source=telinker ({' '.join(sys.argv[1:])})", file=out)
+    print(f"##reference={panel.build}", file=out)
+    print('##ALT=<ID=TE,Description="Polymorphic TE present (TE-presence allele)">', file=out)
+    print('##INFO=<ID=SUBFAM,Number=1,Type=String,Description="TE subfamily">', file=out)
+    print('##INFO=<ID=CATTYPE,Number=1,Type=String,Description="Catalog type vs T2T-CHM13 (INS/DEL)">', file=out)
+    print('##INFO=<ID=METHOD,Number=1,Type=String,Description="ridge, tag1-10, sv, bed or reads">', file=out)
+    print('##INFO=<ID=R2,Number=1,Type=Float,Description="Panel out-of-fold r2 (imputed calls)">', file=out)
+    print('##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">', file=out)
+    print('##FORMAT=<ID=DS,Number=1,Type=Float,Description="TE-presence dosage">', file=out)
+    print("\t".join(["#CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT"]
+                    + list(samples)), file=out)
+    rows = sorted(calls, key=lambda c: (chrom_key(panel.loci["chrom"][c[0]]), pos[c[0]]))
+    for i, method, r2, doses in rows:
+        L = panel.loci
+        info = f"SUBFAM={L['subfamily'][i]};CATTYPE={L['type'][i]};METHOD={method}"
+        if r2 is not None:
+            info += f";R2={r2:.3f}"
+        gts = []
+        for d in doses.tolist():
+            if d != d or d == PRESENT:
+                gts.append("1/.:." if d == PRESENT else "./.:.")
+            else:
+                g = int(round(d))
+                gts.append(f"{['0/0', '0/1', '1/1'][g]}:{d:.2f}")
+        print("\t".join(map(str, ["chr" + L["chrom"][i], pos[i], L["te_id"][i], "N", "<TE>",
+                                  ".", "PASS", info, "GT:DS"] + gts)), file=out)
+
+
+def chrom_key(c):
+    return (0, int(c)) if c.isdigit() else (1, c)

telinker/panel/PANEL.md

@@ -0,0 +1,14 @@
+# teLinker reference panel
+
+Built 2026-10-21 by `dev/build_panel.py` from 213 HPRC individuals.
+
+- Catalog loci: 30,257 (17,260 INS, 12,997 DEL; 7,002 DELs fail the contraction gate `del_gate_r50`)
+- TEs with at least one SNP model (cv r² ≥ 0.2): 21,749
+  - best cv r² ≥ 0.5: 18,783;  ≥ 0.8: 14,647
+- Ridge vs best single tag, same TEs (n=19,291): median cv r² 0.914 vs 0.904; ridge better on 40.2%
+- Median best cv r² by TE MAF: <1% 0.48, 1–5% 0.91, 5–20% 0.96, ≥20% 0.97
+- Median cv r² within AFR samples (best model): 0.891 (tags and windows learned mostly on non-AFR haplotypes)
+- hg38 lift of top tags agrees with the 1.6.1 tag table on 99.32%
+
+cv r² is out-of-fold with SNP selection repeated inside each fold.
+Dosages are TE-PRESENCE (0-2) for both INS and DEL loci.

telinker/panel.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+import csv
+import gzip
+import os
+from dataclasses import dataclass, field
+from itertools import groupby
+from pathlib import Path
+
+import numpy as np
+
+PANEL_DIR = Path(os.environ.get("TELINKER_PANEL", Path(__file__).parent / "panel"))
+BUILDS = ("t2t", "hg38", "hg19")
+POS_COL = {"t2t": "pos", "hg38": "pos_hg38", "hg19": "pos_hg19"}
+
+
+def norm_chrom(c: str) -> str:
+    c = c[3:] if c.lower().startswith("chr") else c
+    return "M" if c == "MT" else c
+
+
+def read_tsv(path):
+    with gzip.open(path, "rt") as f:
+        return list(csv.DictReader(f, delimiter="\t"))
+
+
+@dataclass
+class Model:
+    te: int
+    name: str
+    cv_r2: float
+    intercept: float
+    sites: np.ndarray
+    weights: np.ndarray
+    means: np.ndarray
+
+
+@dataclass
+class Panel:
+    build: str
+    loci: dict
+    sites: list = field(default_factory=list)
+    site_index: dict = field(default_factory=dict)
+    models: list = field(default_factory=list)
+
+    def __len__(self):
+        return len(self.loci["te_id"])
+
+    def te_pos(self):
+        return self.loci[POS_COL[self.build]]
+
+
+def load_panel(build: str, panel_dir: Path = PANEL_DIR, models: bool = True) -> Panel:
+    rows = read_tsv(panel_dir / "loci.tsv.gz")
+    loci = {k: np.array([r[k] for r in rows]) for k in rows[0]}
+    for k in ("pos", "len", "pos_hg38", "pos_hg19"):
+        loci[k] = np.array([int(v) if v else -1 for v in loci[k]])
+    loci["del_ok"] = loci["del_ok"] == "1"
+    loci["chrom"] = np.array([norm_chrom(c) for c in loci["chrom"]])
+    panel = Panel(build, loci)
+    if models:
+        load_models(panel, panel_dir / "models.tsv.gz")
+    return panel
+
+
+def load_models(panel, path):
+    te_row = {t: i for i, t in enumerate(panel.loci["te_id"])}
+    site_row = {}
+    with gzip.open(path, "rt") as f:
+        r = csv.reader(f, delimiter="\t")
+        col = {k: i for i, k in enumerate(next(r))}
+        te, name, pos = col["te_id"], col["model"], col[POS_COL[panel.build]]
+        for (te_id, model), terms in groupby(r, key=lambda t: (t[te], t[name])):
+            terms = list(terms)
+            sites = []
+            for t in terms:
+                if not t[pos]:
+                    sites.append(-1)
+                    continue
+                snp = (norm_chrom(t[col["chrom"]]), int(t[pos]), t[col["ref"]], t[col["alt"]])
+                if snp not in site_row:
+                    site_row[snp] = len(panel.sites)
+                    panel.site_index.setdefault(snp[:2], []).append(len(panel.sites))
+                    panel.sites.append(snp)
+                sites.append(site_row[snp])
+            panel.models.append(Model(
+                te_row[te_id], model, float(terms[0][col["cv_r2"]]),
+                float(terms[0][col["intercept"]]), np.array(sites),
+                np.array([float(t[col["weight"]]) for t in terms]),
+                np.array([float(t[col["mean"]]) for t in terms])))
+    panel.models.sort(key=lambda m: (m.te, -m.cv_r2))

telinker/reads.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import numpy as np
+
+from .inputs import CHR1_LENGTH, sample_name
+from .panel import norm_chrom
+
+FLANK = 200
+WIN = 300
+ANCHOR = 1000
+FRAC = 0.5
+MIN_DEPTH = 10
+MIN_SNP_DEPTH = 5
+MIN_MAPQ = 20
+BLOCK = 100_000
+BASE = {b: i for i, b in enumerate("ACGT")}
+CHRY_PAR = ((0, 2_458_320), (62_122_809, 62_460_029))
+
+
+def open_bam(path, reference=None):
+    try:
+        import pysam
+    except ImportError:
+        raise SystemExit("BAM/CRAM input needs pysam: pip install 'telinker[reads]'")
+    return pysam.AlignmentFile(str(path), reference_filename=reference)
+
+
+def describe(bam, path):
+    header = bam.header.to_dict()
+    rg = {r["ID"]: r["SM"] for r in header.get("RG", []) if r.get("SM")}
+    names = list(dict.fromkeys(rg.values()))
+    samples, rg = (names, rg) if len(names) > 1 else ([sample_name(path)], {})
+    build = None
+    for sq in header.get("SQ", []):
+        if norm_chrom(sq["SN"]) == "1":
+            build = CHR1_LENGTH.get(sq["LN"])
+    lengths = []
+    for rd in bam.fetch(until_eof=True):
+        if not (rd.is_secondary or rd.is_supplementary or rd.is_unmapped):
+            lengths.append(rd.query_length)
+        if len(lengths) >= 2000:
+            break
+    return samples, rg, build, bool(lengths) and float(np.median(lengths)) > 2000
+
+
+def contig_names(bam):
+    return {norm_chrom(c): c for c in bam.references}
+
+
+def usable(rd):
+    return not (rd.is_secondary or rd.is_supplementary or rd.is_unmapped)
+
+
+def sample_of(rd, rg, col):
+    if not rg:
+        return 0
+    return col.get(rg.get(rd.get_tag("RG"))) if rd.has_tag("RG") else None
+
+
+def in_chry_par(chrom, pos):
+    return chrom == "Y" and any(lo <= pos < hi for lo, hi in CHRY_PAR)
+
+
+def read_evidence(cigar, start, pos, length, is_ins):
+    end = pos if is_ins else pos + length
+    left = (pos - WIN - ANCHOR, pos - WIN)
+    right = (end + WIN, end + WIN + ANCHOR)
+    need = FRAC * length
+    r, left_bp, right_bp, alt, other = start, 0, 0, False, False
+    for op, n in cigar:
+        if op == 1 or op == 2:
+            if abs(r - pos) <= WIN:
+                if n >= need and (op == 1) == is_ins:
+                    alt = True
+                elif n >= max(50, need):
+                    other = True
+            r += n if op == 2 else 0
+        elif op == 3:
+            r += n
+        elif op in (0, 7, 8):
+            left_bp += max(0, min(r + n, left[1]) - max(r, left[0]))
+            right_bp += max(0, min(r + n, right[1]) - max(r, right[0]))
+            r += n
+    if left_bp < FLANK or right_bp < FLANK:
+        return None
+    return "alt" if alt else "other" if other else "ref"
+
+
+def call(n_alt, n_ref):
+    n = n_alt + n_ref
+    if n < MIN_DEPTH:
+        return None
+    f = n_alt / n
+    if f >= 0.90:
+        return 2
+    if f <= 0.05:
+        return 0
+    if 0.20 <= f <= 0.80:
+        return 1
+    return None
+
+
+def genotype_long(bam, panel, rows, samples, rg):
+    names = contig_names(bam)
+    col = {s: k for k, s in enumerate(samples)}
+    out = {}
+    for i in rows:
+        chrom = names.get(panel.loci["chrom"][i])
+        if chrom is None:
+            continue
+        pos, length = int(panel.loci["pos"][i]), int(panel.loci["len"][i])
+        is_ins = panel.loci["type"][i] == "INS"
+        n_alt, n_ref = [0] * len(samples), [0] * len(samples)
+        for rd in bam.fetch(chrom, max(0, pos - 3000), pos + (0 if is_ins else length) + 3000):
+            k = sample_of(rd, rg, col) if usable(rd) and rd.cigartuples else None
+            if k is None:
+                continue
+            ev = read_evidence(rd.cigartuples, rd.reference_start, pos, length, is_ins)
+            n_alt[k] += ev == "alt"
+            n_ref[k] += ev == "ref"
+        gs = [call(a, r) for a, r in zip(n_alt, n_ref)]
+        if any(g is not None for g in gs):
+            out[i] = np.array([np.nan if g is None else g if is_ins else 2 - g
+                               for g in gs])
+    return out
+
+
+def pileup_sites(bam, panel, samples, rg):
+    names = contig_names(bam)
+    col = {s: k for k, s in enumerate(samples)}
+    filters = [lambda rd, k=k: usable(rd) and not (rd.is_duplicate or rd.is_qcfail)
+               and rd.mapping_quality >= MIN_MAPQ and sample_of(rd, rg, col) == k
+               for k in range(len(samples))]
+    by_chrom = {}
+    for s, (chrom, pos, _, _) in enumerate(panel.sites):
+        if chrom in names:
+            by_chrom.setdefault(chrom, []).append((pos, s))
+
+    G = {}
+    for chrom, sites in by_chrom.items():
+        sites.sort()
+        i = 0
+        while i < len(sites):
+            start = sites[i][0] - 1
+            j = i
+            while j < len(sites) and sites[j][0] <= start + BLOCK:
+                j += 1
+            block, end = sites[i:j], sites[j - 1][0]
+            g = np.full((len(block), len(samples)), np.nan, dtype=np.float32)
+            for k, keep in enumerate(filters):
+                counts = np.array(bam.count_coverage(names[chrom], start, end, quality_threshold=20,
+                                                     read_callback=keep))
+                for b, (pos, s) in enumerate(block):
+                    ref, alt = panel.sites[s][2:]
+                    n_ref, n_alt = counts[BASE[ref], pos - 1 - start], counts[BASE[alt], pos - 1 - start]
+                    if n_ref + n_alt >= MIN_SNP_DEPTH:
+                        f = n_alt / (n_ref + n_alt)
+                        g[b, k] = 0 if f < 0.15 else 2 if f > 0.85 else 1
+            for b, (_, s) in enumerate(block):
+                if not np.isnan(g[b]).all():
+                    G[s] = g[b]
+            i = j
+    return G

telinker-0.1.0.dist-info/METADATA

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: telinker
+Version: 0.1.0
+Summary: Genotype polymorphic transposable elements from VCF, PLINK, array, BED or BAM/CRAM files
+License-Expression: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.20
+Provides-Extra: reads
+Requires-Dist: pysam>=0.19; extra == "reads"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
+
+# teLinker
+
+Genotype polymorphic transposable elements in your own data.
+
+```
+pip install telinker                 # numpy is the only dependency
+pip install 'telinker[reads]'        # + pysam, only for BAM/CRAM input
+
+telinker genotype cohort.vcf.gz -o tes.tsv
+```
+
+## Report bugs
+
+https://github.com/jaleesr/teLinker
+
+## Inputs
+
+The input type and genome build are detected automatically. Pass `--build t2t|hg38|hg19`
+if the build information is not present in the file header.
+
+| Input | How TEs are called | `method` |
+|---|---|---|
+| SNP VCF (WGS, or imputed with `DS`) | Imputed from nearby SNPs | `ridge`, `tagN` |
+| PLINK `.bed/.bim/.fam` | Imputed | `ridge`, `tagN` |
+| 23andMe / AncestryDNA raw file (`.txt` or `.zip`) | Imputed (mostly single tags: arrays are sparse) | `tagN`, `ridge` |
+| SV VCF (Sniffles, SVAN, MELT, PAV, …) | Matched to the catalog | `sv` |
+| BED of TE insertion calls | Matched; presence only | `bed` |
+| Long-read BAM/CRAM on T2T-CHM13 | Read directly from indels in the alignments | `reads` |
+| Short-read BAM/CRAM | SNPs piled up, then imputed | `ridge`, `tagN` |
+
+BCF is not read directly; convert it with `bcftools view -Oz`.
+
+### Several samples
+
+**Multi-sample files.** VCF and PLINK inputs give one column per sample. A BAM/CRAM whose
+read groups carry several `SM` tags is split by read group into one column per `SM`
+
+**Several files.** These are merged into one table:
+
+```
+telinker genotype donor1.bam donor2.bam donor3.bam -o cohort.tsv
+telinker genotype cohort.chr*.vcf.gz -o cohort.tsv     # per-chromosome splits
+```
+
+- Every input must be on the same build. 
+- Sample names are matched across files.
+- If two files genotype the same TE in the same sample, the first file wins.
+- Single-sample files (a one-sample BAM, a 23andMe file, a BED) are named after the file,
+  up to the first dot. So `donor1.sorted.bam` becomes `donor1`.
+- If the same sample name comes from more than one file, telinker merges them and warns. That
+  is right for a cohort split by chromosome. If the files are different people rename them.
+
+In long-read mode, chrY pseudoautosomal loci are skipped. Their reads are shared with chrX,
+so a chrY call would just repeat the chrX genotype.
+
+## Output
+
+One TSV. Each row is a TE, and each sample column is a **TE-presence dosage** (0–2):
+
+```
+te_id                   chrom  pos        type  subfamily  method  r2    S1    S2
+10_102771350_INS_306_0  10     103646104  INS   AluYa5     tag1    1.00  0     1
+10_100722374_INS_226_0  10     .          INS   SVA_F      tag1    0.70  0.92  1.78
+```
+
+- **Dosage cells:** dosage always counts copies of the TE, for both INS and DEL loci.
+  Imputed calls are continuous. `.` means the TE could not be called in that sample.
+- **`r2`:** the panel's held-out accuracy for that TE's model. It is empty for direct calls.
+- **`pos`:** the position in your input's build. `.` means the locus does not lift to that build.
+- **`--vcf`:** writes the same calls as a VCF with `GT:DS`, ready for plink2, regenie
+  or tensorQTL. Its ALT allele is `<TE>` (TE present).
+
+
+## Layout
+
+```
+telinker/         the installed package (numpy only; pysam imported lazily for BAM/CRAM)
+  cli.py          argument parsing, input detection, dispatch, summary log
+  inputs.py       format sniffing + readers: VCF (SNPs and SVs), PLINK, 23andMe/Ancestry, BED
+  impute.py       allele matching and the linear-model prediction / fallback
+  match.py        SV and BED calls -> catalog loci
+  reads.py        BAM/CRAM: long-read CIGAR genotyper, short-read SNP pileup
+  output.py       TSV and VCF writers
+  panel.py        loads telinker/panel/ (loci.tsv.gz, models.tsv.gz, PANEL.md)
+tests/            pytest suite on a hand-built four-TE panel
+```
+
+## Tests
+
+```
+pip install '.[test,reads]'
+pytest tests
+```

telinker-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+telinker/__init__.py,sha256=Rurx7kofeNw1aaIhfCXiO0rrrzBYy3lxoodkpL0pHDI,98
+telinker/__main__.py,sha256=bYt9eEaoRQWdejEHFD8REx9jxVEdZptECFsV7F49Ink,30
+telinker/cli.py,sha256=csgbYBPa5TM_Ep2qP_vKUED2m0e-LHbOHF8_zTl4G_8,8104
+telinker/impute.py,sha256=0HgZZMVc1jivWmRXcmI6Rv_t6xZcSfr86iSK_kjiGJ4,1725
+telinker/inputs.py,sha256=wl8RCN93cISKvQZwJk_e9WRcdwauv5c-KDHpNO3_hJ4,9179
+telinker/match.py,sha256=inETMnXRbe2qveVXS9dRGkjy1KDdroZb5SzUQvFw554,1542
+telinker/output.py,sha256=NsP92QoKVjr66Bb6LUVcWtjjsJ21eoCksVd0WGVB3H8,2630
+telinker/panel.py,sha256=0kqMCPoBziuhiW4UEuPP26Lq6Zb2YG_FW8_HqL13Pk0,2952
+telinker/reads.py,sha256=PuF38tQBp5rtssO1gjZKn3fQxsMIkHzy-yTgmA02ffI,5586
+telinker/panel/PANEL.md,sha256=qtC7aVkMstvUWCJ0yeN5Q7sx6BjFqLcplBp7skIjZDI,792
+telinker/panel/loci.tsv.gz,sha256=mca5raf-w2sMHRfb2RgQJpfy-m48l3RyWkfi3zKj4Dk,724295
+telinker/panel/models.tsv.gz,sha256=8L462PnIQphs2s13rf-VpuOAmf8jatVHlYaeEKMuuOk,23664759
+telinker-0.1.0.dist-info/licenses/LICENSE,sha256=o5sgzIYQ4mOG2B74mfzd8q-qq9AgSdYT1TIGtHwKVUM,1134
+telinker-0.1.0.dist-info/METADATA,sha256=e-jj0mpy05Ek40Iy86JIGfP51meW9zKfqXkdKm6gIz0,4231
+telinker-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+telinker-0.1.0.dist-info/entry_points.txt,sha256=Pppwl8-u4gXhy1Z_o6yXC7CxHQX7VDi0JotDArofqsQ,47
+telinker-0.1.0.dist-info/top_level.txt,sha256=qDQ3iES0tUcK7g3awDycyDkadN4PqaNV-WxmHY2lKOw,9
+telinker-0.1.0.dist-info/RECORD,,

telinker-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

telinker-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+telinker = telinker.cli:main

telinker-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mark Sanborn, Jalees Rehman, Board of Trustees of the University of Illinois.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

telinker-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+telinker