biopipen 0.21.0__py3-none-any.whl → 0.34.26__py3-none-any.whl

biopipen/scripts/snp/Plink2GTMat.py

@@ -0,0 +1,148 @@
+
+from os import path
+from glob import glob
+from biopipen.utils.misc import run_command, logger
+
+indir: str = {{in.indir | quote}}  # noqa: E999 # pyright: ignore
+outfile: str = {{out.outfile | quote}}  # pyright: ignore
+plink: str = {{envs.plink | quote}}  # pyright: ignore
+ncores: int = {{envs.ncores | repr}}  # pyright: ignore
+transpose: bool = {{envs.transpose | repr}}  # pyright: ignore
+samid: str = {{envs.samid | repr}}  # pyright: ignore
+varid: str = {{envs.varid | repr}}  # pyright: ignore
+trans_chr: dict = {{envs.trans_chr | repr}}  # pyright: ignore
+missing_id: str = {{envs.missing_id | repr}}  # pyright: ignore
+gtcoding: str = {{envs.gtcoding | repr}}  # pyright: ignore
+trans_chr = trans_chr or {}
+
+bedfile = glob(path.join(indir, '*.bed'))
+if len(bedfile) == 0:
+    raise FileNotFoundError(f"No .bed file found in `in.indir`")
+elif len(bedfile) > 1:
+    logger.warning(f"Multiple .bed files found in `in.indir`, using the first one.")
+
+bedfile = bedfile[0]
+input   = path.splitext(bedfile)[0]
+output  = path.splitext(outfile)[0]
+
+cmd = [
+    plink,
+    "--bfile", input,
+    "--out", output,
+    "--threads", ncores,
+    "--keep-allele-order",
+    "--recode", "A-transpose" if not transpose else "A",
+]
+# if transpose:
+#     cmd += ["tabx"]
+
+run_command(cmd, fg=True, env={"cwd": path.dirname(outfile)})
+
+
+def _vcf_gtcoding(gt):
+    try:
+        return str(2 - int(gt))
+    except (ValueError, TypeError):
+        return "NA"
+
+
+if not transpose:  # rows are variants, columns are samples
+    # .traw file is created, tab-separated, with the following columns:
+    trawfile = output + ".traw"
+    # CHR     Chromosome code
+    # SNP     Variant identifier
+    # (C)M	  Position in morgans or centimorgans
+    # POS     Base-pair coordinate
+    # COUNTED Counted allele (defaults to A1), the actual alternative allele
+    #           with --keep-allele-order
+    # ALT     Other allele(s), comma-separated, the actual reference allele
+    # <FID>_<IID>... Allelic dosages
+    #   (0/1/2/'NA' for diploid variants, 0/2/'NA' for haploid)
+    with open(trawfile, 'r') as fin:
+        with open(outfile, 'w') as fout:
+            samples = fin.readline().strip().split('\t')[6:]
+            header = ["Variant"]
+            for sam in samples:
+                try:
+                    fid, iid = sam.split('_')
+                except ValueError:
+                    raise ValueError(
+                        f"Can't determine FID and IID from sample ID: {sam}, "
+                        f"extra underscore (_) detected."
+                    ) from None
+                sam = samid.replace('{fid}', fid).replace('{iid}', iid)
+                header.append(sam)
+            fout.write('\t'.join(header) + '\n')
+
+            for line in fin:
+                line = line.strip().split('\t')
+                chrom = trans_chr.get(line[0], line[0])
+                var = line[1]
+                if var == "." or var == "":
+                    var = missing_id
+                pos = line[3]
+                ref = line[5]
+                alt = line[4]
+                variant = (
+                    varid
+                    .replace('{chr}', chrom)
+                    .replace('{varid}', var)
+                    .replace('{pos}', pos)
+                    .replace('{ref}', ref)
+                    .replace('{alt}', alt)
+                )
+                if gtcoding == "plink":
+                    record = [variant] + line[6:]
+                else:  # vcf
+                    record = [variant] + [_vcf_gtcoding(x) for x in line[6:]]
+                fout.write('\t'.join(record) + '\n')
+
+else:
+    # .raw file is created, tab-separated, with the following columns:
+    rawfile = output + ".raw"
+    # FID       Family ID
+    # IID       Individual ID
+    # PAT       Paternal ID
+    # MAT       Maternal ID
+    # SEX       Sex (1 = male, 2 = female, 0 = unknown)
+    # PHENOTYPE Main phenotype value
+    # <VariantID>... Allelic dosage (0/1/2/NA for diploid variants, 0/2/NA for haploid)
+    #
+    # Variant information may not be included in <VariantID>
+    # We use the .bim file to get the variant information
+    bimfile = input + ".bim"
+    with open(rawfile, 'r') as fin:
+        with open(outfile, 'w') as fout:
+            header = ["Sample"]
+            with open(bimfile, 'r') as fbim:
+                for line in fbim:
+                    line = line.strip().split('\t')
+                    chrom = trans_chr.get(line[0], line[0])
+                    var = line[1]
+                    if var == "." or var == "":
+                        var = missing_id
+                    pos = line[3]
+                    ref = line[5]
+                    alt = line[4]
+                    variant = (
+                        varid
+                        .replace('{chr}', chrom)
+                        .replace('{varid}', var)
+                        .replace('{pos}', pos)
+                        .replace('{ref}', ref)
+                        .replace('{alt}', alt)
+                    )
+                    header.append(variant)
+            fout.write('\t'.join(header) + '\n')
+
+            next(fin)  # skip header
+            for line in fin:
+                line = line.strip().split('\t')
+                fid = line[0]
+                iid = line[1]
+                sam = samid.replace('{fid}', fid).replace('{iid}', iid)
+                if gtcoding == "plink":
+                    record = [sam] + line[6:]
+                else:  # vcf
+                    record = [sam] + [_vcf_gtcoding(x) for x in line[6:]]
+                fout.write('\t'.join(record) + '\n')

biopipen/scripts/snp/PlinkCallRate.R

@@ -0,0 +1,199 @@
+library(plotthis)
+library(biopipen.utils)
+
+indir <- {{in.indir | r}}
+outdir <- {{out.outdir | r}}
+plink <- {{envs.plink | r}}
+ncores <- {{envs.ncores | r}}
+doplot <- {{envs.plot | r}}
+devpars <- {{envs.devpars | r}}
+samplecr <- {{envs.samplecr | r}}
+varcr <- {{envs.varcr | r}}
+max_iter <- {{envs.max_iter | r}}
+
+log <- get_logger()
+
+bedfile = Sys.glob(file.path(indir, '*.bed'))
+if (length(bedfile) == 0)
+    stop("No bed files found in the input directory.")
+if (length(bedfile) > 1) {
+    log$warn("Multiple bed files found in the input directory. Using the first one.")
+    bedfile <- bedfile[1]
+}
+input <- tools::file_path_sans_ext(bedfile)
+output <- file.path(outdir, basename(input))
+
+all_smiss_file = paste0(output, '.smiss')
+all_vmiss_file = paste0(output, '.vmiss')
+all_samplecr_fail_file = paste0(output, '.samplecr.fail')
+all_varcr_fail_file = paste0(output, '.varcr.fail')
+if (file.exists(all_smiss_file)) invisible(file.remove(all_smiss_file))
+if (file.exists(all_vmiss_file)) invisible(file.remove(all_vmiss_file))
+for (i in 1:max_iter) {
+    log$info("Iteration {i} ...")
+    # iter_out <- paste0(output, "-", i)
+    iter_dir <- file.path(outdir, paste0("iter", i))
+    dir.create(iter_dir, showWarnings = FALSE)
+    iter_out <- file.path(iter_dir, basename(output))
+    cmd <- c(
+        plink,
+        "--threads", ncores,
+        "--bfile", input,
+        "--missing",
+        "--out", iter_out
+    )
+    run_command(cmd, fg = TRUE)
+
+    smissfile <- paste0(iter_out, '.smiss')
+    smiss <- read.table(
+        smissfile,
+        header = TRUE,
+        row.names = NULL,
+        check.names = FALSE,
+        comment.char = ""
+    )
+    smiss$Iteration <- i
+    # append it to all_smiss_file
+    write.table(
+        smiss,
+        all_smiss_file,
+        append = i > 1,
+        col.names = !file.exists(all_smiss_file),
+        row.names = FALSE,
+        sep = "\t",
+        quote = FALSE
+    )
+    callrate.sample <- data.frame(Callrate = 1 - smiss$F_MISS)
+    rownames(callrate.sample) <- paste(smiss$FID, smiss$IID, sep = "\t")
+    callrate.sample.fail = rownames(callrate.sample[
+        callrate.sample$Callrate < samplecr, , drop = FALSE
+    ])
+    writeLines(callrate.sample.fail, con = file(paste0(iter_out, '.samplecr.fail')))
+    # append it to all_samplecr_fail_file
+    write(
+        paste0(sapply(
+            callrate.sample.fail,
+            function(x){ paste0(x, "\n") }
+        ), collapse = ""),
+        file = file(all_samplecr_fail_file),
+        append = i > 1
+    )
+
+    vmiss <- read.table(
+        paste0(iter_out, '.vmiss'),
+        header = TRUE,
+        row.names = NULL,
+        check.names = FALSE,
+        comment.char = ""
+    )
+    vmiss$Iteration <- i
+    # append it to all_vmiss_file
+    write.table(
+        vmiss,
+        all_vmiss_file,
+        append = i > 1,
+        col.names = !file.exists(all_vmiss_file),
+        row.names = FALSE,
+        sep = "\t",
+        quote = FALSE
+    )
+    vmiss$Callrate <- 1 - vmiss$F_MISS
+    callrate.var.fail <- vmiss[which(vmiss$Callrate < varcr), 'ID', drop = TRUE]
+    writeLines(callrate.var.fail, con = file(paste0(iter_out, '.varcr.fail')))
+    # append it to all_varcr_fail_file
+    write(
+        paste0(sapply(
+            callrate.var.fail,
+            function(x){ paste0(x, "\n") }
+        ), collapse = ""),
+        file = file(all_varcr_fail_file),
+        append = i > 1
+    )
+
+    if (length(callrate.sample.fail) == 0 && length(callrate.var.fail) == 0) {
+        # make symbolic links to output from input .bed, .bim and .fam files
+        file.symlink(paste0(input, '.bed'), paste0(output, '.bed'))
+        file.symlink(paste0(input, '.bim'), paste0(output, '.bim'))
+        file.symlink(paste0(input, '.fam'), paste0(output, '.fam'))
+        break
+    }
+
+    # remove samples in iter_out.samplecr.fail and variants in iter_out.varcr.fail
+    cmd <- c(
+        plink,
+        "--threads", ncores,
+        "--bfile", input,
+        "--remove", paste0(iter_out, '.samplecr.fail'),
+        "--exclude", paste0(iter_out, '.varcr.fail'),
+        "--make-bed",
+        "--out", iter_out
+    )
+    run_command(cmd, fg = TRUE)
+    input <- iter_out
+}
+
+smiss <- read.table(
+    smissfile,
+    header = TRUE,
+    row.names = NULL,
+    check.names = FALSE,
+    comment.char = ""
+)
+callrate.sample <- data.frame(Callrate = 1 - smiss$F_MISS)
+rownames(callrate.sample) <- paste(smiss$FID, smiss$IID, sep = "\t")
+
+vmiss <- read.table(
+    paste0(iter_out, '.vmiss'),
+    header = TRUE,
+    row.names = NULL,
+    check.names = FALSE,
+    comment.char = ""
+)
+vmiss$Callrate <- 1 - vmiss$F_MISS
+
+if (doplot) {
+    log$info("Plotting ...")
+    callrate.sample$Status <- "Pass"
+    callrate.sample[callrate.sample.fail, "Status"] <- "Fail"
+    callrate.sample$Status <- factor(callrate.sample$Status, levels = c("Fail", "Pass"))
+
+    p_callrate_file <- paste0(output, '.samplecr.png')
+    p_callrate <- Histogram(
+        callrate.sample,
+        x = "Callrate",
+        group_by = "Status",
+        xlab = "Sample Call Rate",
+        ylab = "Count",
+        palette = "Set1",
+        alpha = 0.8,
+        bins = 50
+    )
+    res <- 70
+    height <- attr(p_callrate, "height") * res
+    width <- attr(p_callrate, "width") * res
+    png(p_callrate_file, width = width, height = height, res = res)
+    print(p_callrate)
+    dev.off()
+
+    vmiss$Status <- "Pass"
+    vmiss[which(vmiss$Callrate < varcr), "Status"] <- "Fail"
+    vmiss$Status <- factor(vmiss$Status, levels = c("Fail", "Pass"))
+
+    p_varcr_file <- paste0(output, '.varcr.png')
+    p_varcr <- Histogram(
+        vmiss,
+        x = "Callrate",
+        group_by = "Status",
+        xlab = "Variant Call Rate",
+        ylab = "Count",
+        palette = "Set1",
+        alpha = 0.8,
+        bins = 50
+    )
+    res <- 70
+    height <- attr(p_varcr, "height") * res
+    width <- attr(p_varcr, "width") * res
+    png(p_varcr_file, width = width, height = height, res = res)
+    print(p_varcr)
+    dev.off()
+}

biopipen/scripts/snp/PlinkFilter.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from pathlib import Path
+from biopipen.utils.misc import run_command, dict_to_cli_args, logger
+
+indir: str = {{in.indir | quote}}  # pyright: ignore # noqa: #999
+samples_file = {{in.samples_file | quote}}  # pyright: ignore
+variants_file = {{in.variants_file | quote}}  # pyright: ignore
+outdir: str = {{out.outdir | quote}}  # pyright: ignore
+
+plink = {{envs.plink | repr}}  # pyright: ignore
+ncores = {{envs.ncores | repr}}  # pyright: ignore
+samples: list[str] | str = {{envs.samples | repr}}  # pyright: ignore
+variants: list[str] | str = {{envs.variants | repr}}  # pyright: ignore
+e_samples_file = {{envs.samples_file | repr}}  # pyright: ignore
+e_variants_file = {{envs.variants_file | repr}}  # pyright: ignore
+keep = {{envs.keep | repr}}  # pyright: ignore
+vfile_type = {{envs.vfile_type | repr}}  # pyright: ignore
+chr = {{envs.chr | repr}}  # pyright: ignore
+not_chr = {{envs.not_chr | repr}}  # pyright: ignore
+autosome = {{envs.autosome | repr}}  # pyright: ignore
+autosome_xy = {{envs.autosome_xy | repr}}  # pyright: ignore
+snps_only = {{envs.snps_only | repr}}  # pyright: ignore
+
+samples_file = samples_file or e_samples_file
+if not samples_file and samples:
+    samples_file = Path(outdir) / "_samples.txt"
+    if isinstance(samples, str):
+        samples = [s.strip() for s in samples.split(",")]
+
+    with open(samples_file, "w") as fh:
+        fh.writelines(
+            [
+                line.replace("/", "\t") + "\n"
+                if "/" in line
+                else line + "\t" + line + "\n"
+                for line in samples
+            ]
+        )
+
+variants_file = variants_file or e_variants_file
+if not variants_file and variants:
+    if vfile_type != "id":
+        logger.warning(
+            "envs.vfile_type should be 'id' if only envs.variants is provided."
+        )
+        vfile_type = "id"
+
+    variants_file = Path(outdir) / "_variants.txt"
+    if isinstance(variants, str):
+        variants = [v.strip() for v in variants.split(",")]
+
+    with open(variants_file, "w") as fh:
+        fh.writelines([line + "\n" for line in variants])
+
+bedfile = list(Path(indir).glob("*.bed"))
+if len(bedfile) == 0:
+    raise FileNotFoundError(f"No .bed file found in `in.indir`")
+elif len(bedfile) > 1:
+    logger.warning(f"Multiple .bed files found in `in.indir`, using the first one.")
+
+bedfile = bedfile[0]
+input = bedfile.with_suffix("")
+output = Path(outdir) / bedfile.stem
+
+args = {
+    "": [plink],
+    "bfile": input,
+    "out": output,
+    "threads": ncores,
+    "make-bed": True,
+}
+
+if keep:
+    if samples_file:
+        args["keep"] = samples_file
+    if variants_file:
+        args["extract"] = (
+            variants_file if vfile_type == "id" else [vfile_type, variants_file]
+        )
+else:
+    if samples_file:
+        args["remove"] = samples_file
+    if variants_file:
+        args["exclude"] = (
+            variants_file if vfile_type == "id" else [vfile_type, variants_file]
+        )
+
+if chr:
+    args["chr"] = chr
+if not_chr:
+    args["not_chr"] = not_chr
+if autosome:
+    args["autosome"] = True
+if autosome_xy:
+    args["autosome"] = True
+if snps_only:
+    args["snps_only"] = snps_only
+
+run_command(dict_to_cli_args(args, dashify=True, dup_key=False), fg=True)