biopipen 0.27.7__tar.gz → 0.27.9__tar.gz

{biopipen-0.27.7 → biopipen-0.27.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: biopipen
-Version: 0.27.7
+Version: 0.27.9
 Summary: Bioinformatics processes/pipelines that can be run from `pipen run`
 License: MIT
 Author: pwwang

biopipen-0.27.9/biopipen/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.27.9"

{biopipen-0.27.7 → biopipen-0.27.9}/biopipen/ns/scrna.py

@@ -1769,13 +1769,18 @@ class SeuratMap2Ref(Proc):
         sobjfile: The seurat object
 
     Output:
-        outfile: The rds file of seurat object with cell type annotated
+        outfile: The rds file of seurat object with cell type annotated.
+            Note that the reduction name will be `ref.umap` for the mapping.
+            To visualize the mapping, you should use `ref.umap` as the reduction name.
 
     Envs:
         ncores (type=int;order=-100): Number of cores to use.
-            Used in `future::plan(strategy = "multicore", workers = <ncores>)`
+            When `split_by` is used, this will be the number of cores for each object to map to the reference.
+            When `split_by` is not used, this is used in `future::plan(strategy = "multicore", workers = <ncores>)`
             to parallelize some Seurat procedures.
-            See also: <https://satijalab.org/seurat/articles/future_vignette.html>
+            See also: <https://satijalab.org/seurat/archive/v3.0/future_vignette.html>
+        mutaters (type=json): The mutaters to mutate the metadata.
+            This is helpful when we want to create new columns for `split_by`.
         use: A column name of metadata from the reference
             (e.g. `celltype.l1`, `celltype.l2`) to transfer to the query as the
             cell types (ident) for downstream analysis. This field is required.
@@ -1787,16 +1792,29 @@ class SeuratMap2Ref(Proc):
             `Seurat::LoadH5Seurat()`.
             The file type is determined by the extension. `.rds` or `.RDS` for
             RDS file, `.h5seurat` or `.h5` for h5seurat file.
+        refnorm (choice): Normalization method the reference used. The same method will be used for the query.
+            - NormalizeData: Using [`NormalizeData`](https://satijalab.org/seurat/reference/normalizedata).
+            - SCTransform: Using [`SCTransform`](https://satijalab.org/seurat/reference/sctransform).
+            - auto: Automatically detect the normalization method.
+                If the default assay of reference is `SCT`, then `SCTransform` will be used.
+        split_by: The column name in metadata to split the query into multiple objects.
+            This helps when the original query is too large to process.
         SCTransform (ns): Arguments for [`SCTransform()`](https://satijalab.org/seurat/reference/sctransform)
             - do-correct-umi (flag): Place corrected UMI matrix in assay counts layer?
             - do-scale (flag): Whether to scale residuals to have unit variance?
             - do-center (flag): Whether to center residuals to have mean zero?
             - <more>: See <https://satijalab.org/seurat/reference/sctransform>.
                 Note that the hyphen (`-`) will be transformed into `.` for the keys.
+        NormalizeData (ns): Arguments for [`NormalizeData()`](https://satijalab.org/seurat/reference/normalizedata)
+            - normalization-method: Normalization method.
+            - <more>: See <https://satijalab.org/seurat/reference/normalizedata>.
+                Note that the hyphen (`-`) will be transformed into `.` for the keys.
         FindTransferAnchors (ns): Arguments for [`FindTransferAnchors()`](https://satijalab.org/seurat/reference/findtransferanchors)
             - normalization-method (choice): Name of normalization method used.
                 - LogNormalize: Log-normalize the data matrix
                 - SCT: Scale data using the SCTransform method
+                - auto: Automatically detect the normalization method.
+                    See `envs.refnorm`.
             - reference-reduction: Name of dimensional reduction to use from the reference if running the pcaproject workflow.
                 Optionally enables reuse of precomputed reference dimensional reduction.
             - <more>: See <https://satijalab.org/seurat/reference/findtransferanchors>.
@@ -1822,14 +1840,19 @@ class SeuratMap2Ref(Proc):
         "ncores": config.misc.ncores,
         "use": None,
         "ident": "seurat_clusters",
+        "mutaters": {},
         "ref": None,
+        "refnorm": "auto",
+        "split_by": None,
         "SCTransform": {
             "do-correct-umi": False,
             "do-scale": False,
             "do-center": True,
         },
+        "NormalizeData": {
+            "normalization-method": "LogNormalize",
+        },
         "FindTransferAnchors": {
-            "normalization-method": "SCT",
             "reference-reduction": "spca",
         },
         "MapQuery": {

{biopipen-0.27.7 → biopipen-0.27.9}/biopipen/ns/tcr.py

@@ -1310,6 +1310,10 @@ class TCRClusterStats(Proc):
                 numbers on the heatmap.
             - heatmap_meta (list): The columns of metadata to show on the
                 heatmap.
+            - cluster_rows (flag): Whether to cluster the rows on the heatmap.
+            - sample_order: The order of the samples on the heatmap.
+                Either a string separated by `,` or a list of sample names.
+                This only works for columns if `cluster_rows` is `True`.
             - grouping: The groups to investigate the shared clusters.
                 If specified, venn diagrams will be drawn instead of heatmaps.
                 In such case, `numbers_on_heatmap` and `heatmap_meta` will be
@@ -1373,6 +1377,9 @@ class TCRClusterStats(Proc):
         "shared_clusters": {
             "numbers_on_heatmap": True,
             "heatmap_meta": [],
+            "cluster_rows": True,
+            "sample_order": None,
+            "cluster_rows": True,
             "grouping": None,
             "devpars": {"width": 1000, "height": 1000, "res": 100},
             "cases": {},

{biopipen-0.27.7 → biopipen-0.27.9}/biopipen/scripts/scrna/SeuratClusterStats-stats.R

@@ -38,7 +38,7 @@ do_one_stats = function(name) {
         df_cells = df_cells %>% filter(!!rlang::parse_expr(case$subset))
     }
 
-    select_cols = c(case$ident, case$group.by, case$split.by)
+    select_cols = unique(c(case$ident, case$group.by, case$split.by))
     if (!is.null(case$split.by)) {
         plot_df = do_call(rbind, lapply(group_split(
             df_cells %>% select(all_of(select_cols)),

biopipen-0.27.9/biopipen/scripts/scrna/SeuratMap2Ref.R

@@ -0,0 +1,302 @@
+source("{{biopipen_dir}}/utils/misc.R")
+
+library(parallel)
+library(Seurat)
+library(SeuratDisk)
+library(rlang)
+library(dplyr)
+
+set.seed(8525)
+
+sobjfile = {{in.sobjfile | r}}
+outfile = {{out.outfile | r}}
+use = {{envs.use | r}}
+ident = {{envs.ident | r}}
+ref = {{envs.ref | r}}
+refnorm = {{envs.refnorm | r}}
+ncores = {{envs.ncores | r}}
+split_by = {{envs.split_by | r}}
+mutaters = {{envs.mutaters | r}}
+sctransform_args = {{envs.SCTransform | r: todot="-"}}
+normalizedata_args = {{envs.NormalizeData | r: todot="-"}}
+findtransferanchors_args = {{envs.FindTransferAnchors | r: todot="-"}}
+mappingscore_args = {{envs.MappingScore | r: todot="-"}}
+mapquery_args = {{envs.MapQuery | r: todot="-"}}
+
+# See if we have a reference
+if (is.null(ref)) {
+    stop("No reference provided (envs.ref)")
+}
+
+if (is.null(use)) {
+    stop("No use provided (envs.use), don't know which column to transfer as cluster")
+}
+
+if (is.null(mapquery_args$refdata) || length(mapquery_args$refdata) == 0) {
+    mapquery_args$refdata = list()
+}
+
+mapquery_args$refdata[[use]] = use
+
+outdir = dirname(outfile)
+if (is.null(split_by)) {
+    options(future.globals.maxSize = 80000 * 1024^2)
+    future::plan(strategy = "multicore", workers = ncores)
+}
+
+.expand_dims = function(args, name = "dims") {
+    # Expand dims from 30 to 1:30
+    if (is.numeric(args[[name]]) && length(args[[name]] == 1)) {
+        args[[name]] = 1:args[[name]]
+    }
+    args
+}
+findtransferanchors_args = .expand_dims(findtransferanchors_args)
+
+# Load reference
+log_info("- Loading reference")
+if (endsWith(ref, ".rds") || endsWith(ref, ".RDS")) {
+    reference = readRDS(ref)
+} else if (endsWith(ref, ".h5ad") || endsWith(ref, ".H5AD")) {
+    reference = ReadH5AD(ref)
+} else {
+    reference = LoadH5Seurat(ref)
+}
+
+if (refnorm == "auto" && DefaultAssay(reference) == "SCT") {
+    refnorm = "SCTransform"
+}
+log_info("  Normalization method used: {refnorm}")
+if (refnorm == "SCTransform") {
+    findtransferanchors_args$normalization.method = "SCT"
+} else if (refnorm == "NormalizeData") {
+    findtransferanchors_args$normalization.method = "LogNormalize"
+} else {
+    stop("Unknown normalization method: {refnorm}")
+}
+
+# Load Seurat object
+log_info("- Loading Seurat object")
+sobj = readRDS(sobjfile)
+
+if (!is.null(mutaters) && length(mutaters) > 0) {
+    log_info("- Applying mutaters")
+    sobj@meta.data <- sobj@meta.data %>% mutate(!!!lapply(mutaters, parse_expr))
+}
+
+if (!is.null(split_by)) {
+    # check if each split has more than 100 cells
+    cellno = table(sobj@meta.data[[split_by]])
+    cellno = cellno[cellno < 100]
+    if (length(cellno) > 0) {
+        # stop and print the splits with # cells
+        stop(paste0(
+            "The following splits have less than 100 cells: \n",
+            paste0("- ", names(cellno), ": ", cellno, collapse = "\n"),
+            "\n\n",
+            "You can use `envs.mutaters` to merge these splits and use `newsplit` as `envs.split_by`: \n",
+            "> mutaters = {\n",
+            ">   newsplit = \"if_else(oldsplit %in% c('split1', 'split2'), 'mergedsplit', oldsplit)\"\n",
+            "> }\n"
+        ))
+    }
+    sobj = SplitObject(sobj, split.by = split_by)
+}
+
+# Normalize data
+log_info("- Normalizing data")
+if (refnorm == "SCTransform") {
+    log_info("  Using SCTransform normalization")
+    sctransform_args$residual.features = rownames(x = reference)
+    if (is.null(split_by)) {
+        sctransform_args$object = sobj
+        query = do_call(SCTransform, sctransform_args)
+    } else {
+        query = mclapply(
+            X = sobj,
+            FUN = function(x) {
+                sctransform_args$object = x
+                do_call(SCTransform, sctransform_args)
+            },
+            mc.cores = ncores
+        )
+        if (any(unlist(lapply(query, class)) == "try-error")) {
+            stop(paste0("\nmclapply (SCTransform) error:", query))
+        }
+    }
+} else {
+    log_info("  Using NormalizeData normalization")
+    if (is.null(split_by)) {
+        normalizedata_args$object = sobj
+        query = do_call(NormalizeData, normalizedata_args)
+    } else {
+        query = mclapply(
+            X = sobj,
+            FUN = function(x) {
+                normalizedata_args$object = x
+                do_call(NormalizeData, normalizedata_args)
+            },
+            mc.cores = ncores
+        )
+        if (any(unlist(lapply(query, class)) == "try-error")) {
+            stop(paste0("\nmclapply (NormalizeData) error:", query))
+        }
+    }
+}
+
+# Find anchors between query and reference
+log_info("- Finding anchors")
+findtransferanchors_args$reference = reference
+if (is.null(split_by)) {
+    findtransferanchors_args$query = query
+    anchors = do_call(FindTransferAnchors, findtransferanchors_args)
+} else {
+    anchors = mclapply(
+        X = query,
+        FUN = function(x) {
+            findtransferanchors_args$query = x
+            do_call(FindTransferAnchors, findtransferanchors_args)
+        },
+        mc.cores = ncores
+    )
+    if (any(unlist(lapply(anchors, class)) == "try-error")) {
+        stop(paste0("\nmclapply (FindTransferAnchors) error:", anchors))
+    }
+}
+
+# Map query to reference
+log_info("- Mapping query to reference")
+mapquery_args$reference = reference
+if (is.null(split_by)) {
+    mapquery_args$query = query
+    mapquery_args$anchorset = anchors
+    query = do_call(MapQuery, mapquery_args)
+} else {
+    query = mclapply(
+        X = seq_along(query),
+        FUN = function(i) {
+            mapquery_args$query = query[[i]]
+            mapquery_args$anchorset = anchors[[i]]
+            do_call(MapQuery, mapquery_args)
+        },
+        mc.cores = ncores
+    )
+    if (any(unlist(lapply(query, class)) == "try-error")) {
+        stop(paste0("\nmclapply (MapQuery) error:", query))
+    }
+}
+
+# Calculating mapping score
+log_info("- Calculating mapping score")
+mappingscore_sob_msg = paste0(
+    "While calculating mapping score, the following error was encountered: \n",
+    "subscript out of bounds.  \n\n",
+    "You may want to try a smaller `ndim` (default: 50) in `envs.MappingScore`."
+)
+if (is.null(split_by)) {
+    mappingscore_args$anchors = anchors
+    mappingscore = tryCatch({
+        do_call(MappingScore, mappingscore_args)
+    }, error = function(e) {
+        if (e$message == "subscript out of bounds") stop(mappingscore_sob_msg)
+        stop(e)
+    })
+} else {
+    mappingscore = mclapply(
+        X = seq_along(query),
+        FUN = function(i) {
+            mappingscore_args$anchors = anchors[[i]]
+            tryCatch({
+                do_call(MappingScore, mappingscore_args)
+            }, error = function(e) {
+                if (e$message == "subscript out of bounds") stop(mappingscore_sob_msg)
+                stop(e)
+            })
+        },
+        mc.cores = ncores
+    )
+    if (any(unlist(lapply(mappingscore, class)) == "try-error")) {
+        stop(paste0("\nmclapply (MappingScore) error:", mappingscore))
+    }
+}
+
+# Calculate mapping score and add to metadata
+log_info("- Adding mapping score to metadata")
+if (is.null(split_by)) {
+    query = AddMetaData(
+        object = query,
+        metadata = mappingscore,
+        col.name = "mapping.score"
+    )
+} else {
+    query = mclapply(
+        X = seq_along(query),
+        FUN = function(i) {
+            AddMetaData(
+                object = query[[i]],
+                metadata = mappingscore[[i]],
+                col.name = "mapping.score"
+            )
+        },
+        mc.cores = ncores
+    )
+    if (any(unlist(lapply(query, class)) == "try-error")) {
+        stop(paste0("\nmclapply (AddMetaData) error:", query))
+    }
+
+    # Combine the results
+    log_info("- Merging the results")
+    query = merge(query[[1]], query[2:length(query)], merge.dr = "ref.umap")
+}
+
+# Add the alias to the metadata for the clusters
+log_info("- Adding ident to metadata and set as ident")
+query@meta.data = query@meta.data %>% mutate(
+    !!sym(ident) := as.factor(!!parse_expr(paste0("predicted.", use)))
+)
+Idents(query) = ident
+
+# Save
+log_info("- Saving result ...")
+saveRDS(query, file = outfile)
+
+
+# ############################
+# Some plots
+# ############################
+
+# # Plot the UMAP
+log_info("- Plotting for transferred data ...")
+ref.reduction = mapquery_args$reduction.model %||% "wnn.umap"
+for (qname in names(mapquery_args$refdata)) {
+    rname <- mapquery_args$refdata[[qname]]
+
+    if (grepl("Array", class(reference[[rname]])) && grepl("Array", class(query[[qname]]))) {
+        log_warn("  Skipping transferred array: {qname} -> {rname}")
+        next
+    }
+
+    log_info("  Plotting transferred data: {qname} -> {rname}")
+
+    ref_p <- DimPlot(
+        object = reference,
+        reduction = ref.reduction,
+        group.by = rname,
+        label = TRUE,
+        label.size = 3,
+        repel = TRUE,
+    ) + NoLegend()
+
+    query_p <- DimPlot(
+        object = query,
+        reduction = "ref.umap",
+        group.by = paste0("predicted.", qname),
+        label = TRUE,
+        label.size = 3,
+        repel = TRUE,
+    ) + NoLegend()
+
+    png(file.path(outdir, paste0("UMAPs.png")), width = 1400, height = 700, res = 100)
+    print(ref_p | query_p)
+    dev.off()
+}

{biopipen-0.27.7 → biopipen-0.27.9}/biopipen/scripts/tcr/TCRClusterStats.R

@@ -137,6 +137,26 @@ shared_clusters = function(name) {
         anno = do_call(ComplexHeatmap::HeatmapAnnotation, anno)
     }
 
+    if (!is.null(case$sample_order) && length(case$sample_order) > 0) {
+        if (length(case$sample_order) == 1) {
+            case$sample_order = trimws(strsplit(case$sample_order, ",")[[1]])
+        }
+        nonexisting = setdiff(case$sample_order, samples)
+        if (length(nonexisting) > 0) {
+            stop(paste("  The following samples do not exist in `sample_order`:", paste(nonexisting, collapse=", ")))
+        }
+        plotdata = plotdata[, case$sample_order, drop=FALSE]
+    }
+
+    cluster_rows = case$cluster_rows && nrow(plotdata) > 2
+    col_samples = colnames(plotdata)
+    if (!cluster_rows) {
+        plotdata = plotdata[col_samples, ]
+        row_samples = col_samples
+    } else {
+        row_samples = samples
+    }
+
     # Plot heatmap
     plotHeatmap(
         plotdata,
@@ -144,12 +164,12 @@ shared_clusters = function(name) {
             name = "Shared TCR Clusters",
             col = c("#ffe1e1", "red3"),
             cluster_columns = FALSE,
-            cluster_rows = nrow(plotdata) > 2,
+            cluster_rows = cluster_rows,
             top_annotation = anno,
             cell_fun = if (
                 is.null(case$numbers_on_heatmap) || !case$numbers_on_heatmap
             ) NULL else function(j, i, x, y, width, height, fill) {
-                grid.text(plotdata[samples[i], samples[j]], x, y, gp = gpar(fontsize = 10))
+                grid.text(row_samples[i], col_samples[j], x, y, gp = gpar(fontsize = 10))
             }
         ),
         devpars = case$devpars,

{biopipen-0.27.7 → biopipen-0.27.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "biopipen"
-version = "0.27.7"
+version = "0.27.9"
 description = "Bioinformatics processes/pipelines that can be run from `pipen run`"
 authors = ["pwwang <pwwang@pwwang.com>"]
 license = "MIT"

{biopipen-0.27.7 → biopipen-0.27.9}/setup.py

@@ -81,7 +81,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'biopipen',
-    'version': '0.27.7',
+    'version': '0.27.9',
     'description': 'Bioinformatics processes/pipelines that can be run from `pipen run`',
     'long_description': 'None',
     'author': 'pwwang',

biopipen-0.27.7/biopipen/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.27.7"

biopipen-0.27.7/biopipen/scripts/scrna/SeuratMap2Ref.R

@@ -1,156 +0,0 @@
-source("{{biopipen_dir}}/utils/misc.R")
-
-library(Seurat)
-library(SeuratDisk)
-library(rlang)
-library(dplyr)
-
-set.seed(8525)
-
-sobjfile = {{in.sobjfile | r}}
-outfile = {{out.outfile | r}}
-use = {{envs.use | r}}
-ident = {{envs.ident | r}}
-ref = {{envs.ref | r}}
-ncores = {{envs.ncores | r}}
-sctransform_args = {{envs.SCTransform | r: todot="-"}}
-findtransferanchors_args = {{envs.FindTransferAnchors | r: todot="-"}}
-mappingscore_args = {{envs.MappingScore | r: todot="-"}}
-mapquery_args = {{envs.MapQuery | r: todot="-"}}
-
-# See if we have a reference
-if (is.null(ref)) {
-    stop("No reference provided (envs.ref)")
-}
-
-if (is.null(use)) {
-    stop("No use provided (envs.use), don't know which column to transfer as cluster")
-}
-
-if (is.null(mapquery_args$refdata) || length(mapquery_args$refdata) == 0) {
-    mapquery_args$refdata = list()
-}
-
-mapquery_args$refdata[[use]] = use
-
-outdir = dirname(outfile)
-options(future.globals.maxSize = 80000 * 1024^2)
-plan(strategy = "multicore", workers = ncores)
-
-.expand_dims = function(args, name = "dims") {
-    # Expand dims from 30 to 1:30
-    if (is.numeric(args[[name]]) && length(args[[name]] == 1)) {
-        args[[name]] = 1:args[[name]]
-    }
-    args
-}
-findtransferanchors_args = .expand_dims(findtransferanchors_args)
-
-# Load reference
-log_info("- Loading reference")
-if (endsWith(ref, ".rds") || endsWith(ref, ".RDS")) {
-    reference = readRDS(ref)
-} else if (endsWith(ref, ".h5ad") || endsWith(ref, ".H5AD")) {
-    reference = ReadH5AD(ref)
-} else {
-    reference = LoadH5Seurat(ref)
-}
-
-# Load Seurat object
-log_info("- Loading Seurat object")
-sobj = readRDS(sobjfile)
-
-# Normalize data
-log_info("- Normalizing data")
-sctransform_args$object = sobj
-sctransform_args$residual.features = rownames(x = reference)
-query = do_call(SCTransform, sctransform_args)
-
-# Find anchors between query and reference
-log_info("- Finding anchors")
-findtransferanchors_args$reference = reference
-findtransferanchors_args$query = query
-anchors = do_call(FindTransferAnchors, findtransferanchors_args)
-
-# Map query to reference
-log_info("- Mapping query to reference")
-mapquery_args$reference = reference
-mapquery_args$query = query
-mapquery_args$anchorset = anchors
-query = do_call(MapQuery, mapquery_args)
-
-# Calculating mapping score
-log_info("- Calculating mapping score")
-mappingscore_args$anchors = anchors
-mappingscore = tryCatch({
-    do_call(MappingScore, mappingscore_args)
-}, error = function(e) {
-    if (e$message == "subscript out of bounds") {
-        stop(paste0(
-            "While calculating mapping score, the following error was encountered: \n",
-            "subscript out of bounds.  \n\n",
-            "You may want to try a smaller `ndim` (default: 50) in `envs.MappingScore`."
-        ))
-    }
-    stop(e)
-})
-
-# Calculate mapping score and add to metadata
-log_info("- Calculating mapping score")
-query = AddMetaData(
-  object = query,
-  metadata = mappingscore,
-  col.name = "mapping.score"
-)
-
-# Add the alias to the metadata for the clusters
-log_info("- Adding ident to metadata and set as ident")
-query@meta.data = query@meta.data %>% mutate(
-    !!sym(ident) := as.factor(!!parse_expr(paste0("predicted.", use)))
-)
-Idents(query) = ident
-
-# Save
-log_info("- Saving result ...")
-saveRDS(query, file = outfile)
-
-
-# ############################
-# Some plots
-# ############################
-
-# # Plot the UMAP
-log_info("- Plotting for transferred data ...")
-ref.reduction = mapquery_args$reduction.model %||% "wnn.umap"
-for (qname in names(mapquery_args$refdata)) {
-    rname <- mapquery_args$refdata[[qname]]
-
-    if (grepl("Array", class(reference[[rname]])) && grepl("Array", class(query[[qname]]))) {
-        log_warn("  Skipping transferred array: {qname} -> {rname}")
-        next
-    }
-
-    log_info("  Plotting transferred data: {qname} -> {rname}")
-
-    ref_p <- DimPlot(
-        object = reference,
-        reduction = ref.reduction,
-        group.by = rname,
-        label = TRUE,
-        label.size = 3,
-        repel = TRUE,
-    ) + NoLegend()
-
-    query_p <- DimPlot(
-        object = query,
-        reduction = "ref.umap",
-        group.by = paste0("predicted.", qname),
-        label = TRUE,
-        label.size = 3,
-        repel = TRUE,
-    ) + NoLegend()
-
-    png(file.path(outdir, paste0("UMAPs.png")), width = 1400, height = 700, res = 100)
-    print(ref_p | query_p)
-    dev.off()
-}