ugbio-featuremap 1.20.0__tar.gz

ugbio_featuremap-1.20.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: ugbio_featuremap
+Version: 1.20.0
+Summary: Ultima Genomics FeatureMap utils
+Author-email: Itai Rusinek <itai.rusinek@ultimagen.com>, Gat Krieger <gat.krieger@ultimagen.com>, Avigail Moldovan <avigail.moldovan@ultimagen.com>
+License: Apache-2.0
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: ugbio_core[ml,vcfbed]
+Requires-Dist: ugbio_ppmseq
+Requires-Dist: polars>=1.27.1
+
+# ugbio_featuremap
+
+This module includes featuremap python scripts and utils for bioinformatics pipelines.

ugbio_featuremap-1.20.0/README.featuremap.md

@@ -0,0 +1,3 @@
+# ugbio_featuremap
+
+This module includes featuremap python scripts and utils for bioinformatics pipelines.

ugbio_featuremap-1.20.0/pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "ugbio_featuremap"
+version = "1.20.0"
+requires-python = ">=3.11"
+dependencies = [
+    "ugbio_core[vcfbed,ml]",
+    "ugbio_ppmseq",
+    "polars>=1.27.1",
+]
+description = "Ultima Genomics FeatureMap utils"
+authors = [
+    { name = "Itai Rusinek", email = "itai.rusinek@ultimagen.com" },
+    { name = "Gat Krieger", email = "gat.krieger@ultimagen.com" },
+    { name = "Avigail Moldovan", email = "avigail.moldovan@ultimagen.com" },
+]
+readme = "README.featuremap.md"
+
+[project.license]
+text = "Apache-2.0"
+
+[project.scripts]
+run_tests = "pytest:main"
+featuremap_to_dataframe = "ugbio_featuremap.featuremap_to_dataframe:main"
+filter_featuremap = "ugbio_featuremap.filter_dataframe:main"
+add_aggregate_params_and_xgb_score_to_pileup_featuremap = "ugbio_featuremap.featuremap_xgb_prediction:main"
+create_somatic_featuremap = "ugbio_featuremap.create_somatic_featuremap:main"
+integrate_mpileup_to_sfm = "ugbio_featuremap.integrate_mpileup_to_sfm:main"
+somatic_featuremap_fields_transformation = "ugbio_featuremap.somatic_featuremap_fields_transformation:main"
+
+[tool.uv.sources.ugbio_core]
+workspace = true
+
+[tool.uv.sources.ugbio_ppmseq]
+workspace = true
+
+[tool.setuptools.package-data]
+ugbio_featuremap = [
+    "*.awk",
+]
+
+[build-system]
+requires = [
+    "setuptools>=61.0",
+]
+build-backend = "setuptools.build_meta"

ugbio_featuremap-1.20.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ugbio_featuremap-1.20.0/ugbio_featuremap/aggregate_lists.awk

@@ -0,0 +1,121 @@
+#!/usr/bin/awk -f
+# AWK script for computing aggregation metrics (mean, min, max, count, count_zero) for list format fields in VCF TSV output
+# Also supports expanding fixed-size columns (e.g., AD -> AD_0, AD_1)
+#
+# Usage:
+#   awk -v list_indices="3,4,5" -f aggregate_lists.awk input.tsv
+#   awk -v list_indices="3,4" -v expand_indices="5" -v expand_sizes="2" -f aggregate_lists.awk input.tsv
+
+BEGIN {
+    # Parse list_indices parameter (0-based column indices, convert to 1-based for AWK)
+    num_list_cols = split(list_indices, indices, ",")
+    for (i = 1; i <= num_list_cols; i++) {
+        col_idx = indices[i] + 1  # Convert to 1-based
+        list_cols[col_idx] = 1
+        ordered_list_cols[i] = col_idx
+    }
+
+    # Parse expand_indices and sizes (for expanding fixed-size columns)
+    if (expand_indices != "") {
+        num_expand_cols = split(expand_indices, expand_idx_list, ",")
+        split(expand_sizes, expand_size_list, ",")
+        for (i = 1; i <= num_expand_cols; i++) {
+            col_idx = expand_idx_list[i] + 1  # Convert to 1-based
+            expand_cols[col_idx] = expand_size_list[i]
+        }
+    }
+}
+
+function compute_aggregations(col_idx, values_str, values, n, i, val, sum, count, min_val, max_val, mean, count_zero) {
+    # Split the list (comma-separated values)
+    n = split(values_str, values, ",")
+
+    sum = 0
+    count = 0
+    count_zero = 0
+    min_val = ""
+    max_val = ""
+
+    # Process each value in the list
+    for (i = 1; i <= n; i++) {
+        # Trim whitespace and try to convert to number
+        val = values[i]
+        gsub(/^[ \t]+|[ \t]+$/, "", val)
+
+        # Skip empty values, ".", or non-numeric values
+        if (val == "" || val == "." || val !~ /^-?[0-9]+\.?[0-9]*$/) {
+            continue
+        }
+
+        val = val + 0  # Convert to number
+
+        if (val == 0) count_zero++
+
+        if (count == 0) {
+            min_val = val
+            max_val = val
+        } else {
+            if (val < min_val) min_val = val
+            if (val > max_val) max_val = val
+        }
+
+        sum += val
+        count++
+    }
+
+    # Return results as string: "mean\tmin\tmax\tcount\tcount_zero"
+    if (count == 0) {
+        return ".\t.\t.\t0\t0"
+    }
+
+    mean = sum / count
+    return sprintf("%.6f\t%.6f\t%.6f\t%d\t%d", mean, min_val, max_val, count, count_zero)
+}
+
+function expand_column(values_str, size, values, n, i, result) {
+    # Expand the column and output individual elements
+    n = split(values_str, values, ",")
+    result = ""
+
+    for (i = 1; i <= size; i++) {
+        if (i > 1) result = result "\t"
+        if (i <= n && values[i] != "" && values[i] != ".") {
+            result = result values[i]
+        } else {
+            result = result "."
+        }
+    }
+
+    return result
+}
+
+{
+    # Store the current row
+    for (i = 1; i <= NF; i++) {
+        row[i] = $i
+    }
+
+    # Output columns, replacing list columns with aggregated metrics or expanded columns
+    first = 1
+    for (j = 1; j <= NF; j++) {
+        if (j in list_cols) {
+            # Replace list column with mean, min, max, count, count_zero
+            if (!first) printf "\t"
+            aggs = compute_aggregations(j, row[j])
+            printf "%s", aggs
+            first = 0
+        } else if (j in expand_cols) {
+            # Expand fixed-size column into individual columns
+            if (!first) printf "\t"
+            expanded_vals = expand_column(row[j], expand_cols[j])
+            printf "%s", expanded_vals
+            first = 0
+        } else {
+            # Output non-list column as-is
+            if (!first) printf "\t"
+            printf "%s", row[j]
+            first = 0
+        }
+    }
+    printf "\n"
+}

ugbio_featuremap-1.20.0/ugbio_featuremap/annotate_featuremap.py

@@ -0,0 +1,88 @@
+#!/env/python
+# Copyright 2023 Ultima Genomics Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+# DESCRIPTION
+#    Add additional feature annotations to featuremap, to be used from single-read SNV qual recalibration
+# CHANGELOG in reverse chronological order
+import argparse
+import sys
+
+from ugbio_core.consts import DEFAULT_FLOW_ORDER
+
+from ugbio_featuremap.featuremap_utils import annotate_featuremap
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(prog="annotate featuremap", description=run.__doc__)
+    parser.add_argument(
+        "-i",
+        "--featuremap_path",
+        type=str,
+        required=True,
+        help="input featuremap file",
+    )
+    parser.add_argument("--ppmSeq_adapter_version", type=str, default=None, help="ppmSeq adapter version")
+    parser.add_argument(
+        "-o",
+        "--output_featuremap",
+        type=str,
+        required=True,
+        help="Path of annotated featuremap file",
+    )
+    parser.add_argument("-r", "--ref_fasta", type=str, required=True, help="Reference genome fasta file")
+    parser.add_argument(
+        "--flow_order",
+        type=str,
+        default=DEFAULT_FLOW_ORDER,
+    )
+    parser.add_argument(
+        "--motif_length_to_annotate",
+        type=int,
+        default=3,
+    )
+    parser.add_argument("--max_hmer_length", type=int, default=20)
+    parser.add_argument(
+        "-@",
+        "--process_number",
+        type=int,
+        default=0,
+        help="""Number of processes to use for parallelization.
+             If N < 1, use all-available - abs(N) cores. Default 0""",
+    )
+
+    return parser.parse_args(argv[1:])
+
+
+def run(argv):
+    """Add additional feature annotations to featuremap"""
+    args = parse_args(argv)
+
+    annotate_featuremap(
+        input_featuremap=args.featuremap_path,
+        output_featuremap=args.output_featuremap,
+        ref_fasta=args.ref_fasta,
+        ppmseq_adapter_version=args.ppmSeq_adapter_version,
+        flow_order=args.flow_order,
+        motif_length_to_annotate=args.motif_length_to_annotate,
+        max_hmer_length=args.max_hmer_length,
+        process_number=args.process_number,
+    )
+
+
+def main():
+    run(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

ugbio_featuremap-1.20.0/ugbio_featuremap/create_somatic_featuremap.py

@@ -0,0 +1,260 @@
+import argparse
+import logging
+import os
+import subprocess
+import sys
+from os.path import join as pjoin
+
+import pysam
+from ugbio_core.logger import logger
+from ugbio_core.vcf_utils import VcfUtils
+
+vu = VcfUtils()
+created_files = []
+
+
+def add_single_read_filter(input_vcf: str, output_directory: str, n_threads: int = 1) -> str:
+    """
+    Add SingleRead filter to a VCF file.
+
+    Parameters
+    ----------
+    input_vcf : str
+        Path to the input VCF file.
+    output_directory : str
+        Output directory for the filtered VCF file.
+    n_threads : int, optional
+        Number of threads to use (default is 1).
+
+    Returns
+    -------
+    str
+        Path to the output VCF file with SingleRead filter applied.
+    """
+    logger.info("Adding SingleRead filter to the tumor file")
+    out_add_filter_vcf = pjoin(
+        output_directory, os.path.basename(input_vcf).replace(".vcf.gz", "") + ".with_sr_filter.vcf.gz"
+    )
+    vu.filter_vcf(
+        input_vcf=input_vcf,
+        output_vcf=out_add_filter_vcf,
+        filter_name="SingleRead",
+        exclude_expression="sum(FMT/FILT)<2",
+        n_threads=n_threads,
+    )
+    vu.index_vcf(out_add_filter_vcf)
+    logger.info("Adding SingleRead filter to the tumor file: done")
+    created_files.append(out_add_filter_vcf)
+    created_files.append(f"{out_add_filter_vcf}.tbi")
+    return out_add_filter_vcf
+
+
+def merge_vcf_files(tumor_vcf, normal_vcf, out_merged_vcf, n_cpu: int | None = None):
+    """
+    Merge tumor and normal VCF files into a single VCF file.
+
+    Parameters
+    ----------
+    tumor_vcf : str
+        Path to the tumor VCF file with INFO fields moved to FORMAT.
+    normal_vcf : str
+        Path to the normal VCF file with INFO fields moved to FORMAT.
+    out_merged_vcf : str
+        Path to the output merged VCF file.
+    n_cpu: int
+        Number of CPU to use in merge and view
+
+    Returns
+    -------
+    str
+        Path to the output merged VCF file with tumor-PASS variants only.
+    """
+    if n_cpu is None:
+        n_cpu = os.cpu_count()
+
+    # Check if tumor and normal VCFs have the same sample name
+    with pysam.VariantFile(tumor_vcf) as tumor_vcf_in, pysam.VariantFile(normal_vcf) as normal_vcf_in:
+        tumor_sample_name = list(tumor_vcf_in.header.samples)[0]
+        normal_sample_name = list(normal_vcf_in.header.samples)[0]
+
+    # merging T-N VCF files - this results with records from both tumor and normal VCF files
+    out_merged_full_vcf = out_merged_vcf.replace(".vcf.gz", "") + ".full.vcf.gz"
+    cmd_merge = [
+        "bcftools",
+        "merge",
+        "--threads",
+        str(n_cpu),
+        "-m",
+        "none",
+        "-Oz",
+        "-o",
+        out_merged_full_vcf,
+        tumor_vcf,
+        normal_vcf,
+    ]
+    if tumor_sample_name == normal_sample_name:
+        logger.warning(
+            f"Tumor and normal VCFs have the same sample name ({tumor_sample_name}). "
+            "Using --force-samples to allow merging."
+        )
+        cmd_merge.insert(2, "--force-samples")
+    logger.debug(" ".join(cmd_merge))
+    subprocess.check_call(cmd_merge)
+    vu.index_vcf(out_merged_full_vcf)
+
+    created_files.append(out_merged_full_vcf)
+    created_files.append(out_merged_full_vcf + ".tbi")
+
+    # Keep only records that are present in the tumor VCF file
+    vu.view_vcf(
+        input_vcf=out_merged_full_vcf,
+        output_vcf=out_merged_vcf,
+        n_threads=n_cpu,
+        extra_args="-i 'COUNT(FORMAT/RL[0:0]) > 0'",
+    )
+    vu.index_vcf(out_merged_vcf)
+
+
+def __parse_args(argv: list[str]) -> argparse.Namespace:
+    """
+    Parse command line arguments.
+
+    Parameters
+    ----------
+    argv : list of str
+        Command line arguments.
+
+    Returns
+    -------
+    argparse.Namespace
+        Parsed arguments.
+    """
+    parser = argparse.ArgumentParser(
+        prog="create_somatic_pileup_featuremap.py",
+        description=run.__doc__,
+    )
+    parser.add_argument("--tumor_vcf", help="tumor vcf file", required=True, type=str)
+    parser.add_argument("--normal_vcf", help="normal vcf file", required=True, type=str)
+    parser.add_argument("--sample_name", help="sample_name", required=True, type=str)
+    parser.add_argument("--cpu", help="number of CPU to use", required=False, type=int, default=8)
+    parser.add_argument(
+        "--out_directory",
+        help="out directory where intermediate and output files will be saved."
+        " if not supplied all files will be written to current directory",
+        required=False,
+        type=str,
+        default=".",
+    )
+    parser.add_argument(
+        "--keep-non-pass-tumor-candidates",
+        help="If set, the output VCF will also contain non-PASS variants.",
+        action="store_true",
+        default=False,
+    )
+    return parser.parse_args(argv[1:])
+
+
+def run(argv):
+    """
+    Merge two VCF files (tumor and normal) into a single VCF file.
+
+    The output VCF file will have tumor records (filtered for SingleRead variants)
+    merged with corresponding normal records.
+    If the `--keep-non-pass-tumor-candidates` flag is set, non-PASS variants will also be included in the output.
+
+    Parameters
+    ----------
+    argv : list of str
+        Command line arguments.
+
+    Returns
+    -------
+    None
+    """
+    args = __parse_args(argv)
+    logger.setLevel(logging.DEBUG)
+    for handler in logger.handlers:
+        handler.setLevel(logging.DEBUG)
+
+    logger.info(f"Output directory: {args.out_directory}")
+
+    # Create output directory if it doesn't exist
+    if not os.path.exists(args.out_directory):
+        os.makedirs(args.out_directory)
+        logger.info(f"Created output directory: {args.out_directory}")
+
+    # Add SingleRead filter to the tumor VCF file
+    out_add_filter_vcf = add_single_read_filter(args.tumor_vcf, args.out_directory, args.cpu)
+
+    # Merge the tumor and normal VCF files into a single VCF file
+    if args.keep_non_pass_tumor_candidates:
+        logger.info("including non-PASS tumor variants.")
+        tumor_vcf = out_add_filter_vcf.replace(".with_sr_filter.vcf.gz", ".no_sr.vcf.gz")
+        vu.view_vcf(
+            input_vcf=out_add_filter_vcf,
+            output_vcf=tumor_vcf,
+            n_threads=args.cpu,
+            extra_args="-i 'FILTER!~\"SingleRead\"'",
+        )
+        vu.index_vcf(tumor_vcf)
+        logger.info("No filtering for tumor-PASS variants. Filtering only for SingleRead variants.")
+        created_files.append(tumor_vcf)
+        created_files.append(tumor_vcf + ".tbi")
+    else:
+        logger.info("Filtering for tumor-PASS variants only.")
+        tumor_vcf = out_add_filter_vcf.replace(".vcf.gz", ".tumor_PASS.vcf.gz")
+        vu.view_vcf(
+            input_vcf=out_add_filter_vcf,
+            output_vcf=tumor_vcf,
+            n_threads=args.cpu,
+            extra_args="-f PASS -i 'FILTER!~\"SingleRead\"'",
+        )
+        vu.index_vcf(tumor_vcf)
+        logger.info("Filtering for tumor-PASS variants only: done")
+        created_files.append(tumor_vcf)
+        created_files.append(tumor_vcf + ".tbi")
+
+    unfiltered_normal_vcf = pjoin(
+        args.out_directory, os.path.basename(args.normal_vcf).replace(".vcf.gz", ".unfiltered.vcf.gz")
+    )
+    vu.remove_filter_annotations(args.normal_vcf, unfiltered_normal_vcf, args.cpu)
+    vu.index_vcf(unfiltered_normal_vcf)
+    created_files.append(unfiltered_normal_vcf)
+    created_files.append(unfiltered_normal_vcf + ".tbi")
+
+    # Set up the output VCF file path
+    out_merged_tmp_vcf = pjoin(args.out_directory, f"{args.sample_name}.tumor_normal.merged.tmp.vcf.gz")
+    merge_vcf_files(tumor_vcf, unfiltered_normal_vcf, out_merged_tmp_vcf)
+    created_files.append(out_merged_tmp_vcf)
+    created_files.append(out_merged_tmp_vcf + ".tbi")
+
+    # Add tumor sample name to merged VCF header
+    with pysam.VariantFile(out_merged_tmp_vcf) as vcf_in:
+        tumor_sample_name = list(vcf_in.header.samples)[0]
+    # Add header line using pysam
+    out_merged_vcf = out_merged_tmp_vcf.replace(".tmp.vcf.gz", ".vcf.gz")
+    with pysam.VariantFile(out_merged_tmp_vcf) as vcf_in:
+        header = vcf_in.header.copy()
+        header.add_line(f"##tumor_sample={tumor_sample_name}")
+        with pysam.VariantFile(out_merged_vcf, "wz", header=header) as vcf_out:
+            for rec in vcf_in:
+                vcf_out.write(rec)
+    vu.index_vcf(out_merged_vcf)
+    logger.info(f"Added tumor sample header: ##tumor_sample={tumor_sample_name}")
+    logger.info(f"Output merged VCF file: {out_merged_vcf}")
+
+    # Clean up intermediate files
+    for f in created_files:
+        try:
+            os.remove(f)
+            logger.debug(f"Removed temporary file: {f}")
+        except Exception as e:
+            logger.warning(f"Could not remove temporary file {f}: {e}")
+
+
+def main():
+    run(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

ugbio_featuremap-1.20.0/ugbio_featuremap/explode_lists.awk

@@ -0,0 +1,54 @@
+#!/usr/bin/awk -f
+# AWK script for exploding list format fields in VCF TSV output
+# Usage: awk -v list_indices="3,4,5" -f explode_lists.awk input.tsv
+
+BEGIN {
+    # Parse list_indices parameter (0-based column indices, convert to 1-based for AWK)
+    split(list_indices, indices, ",")
+    for (i in indices) {
+        list_cols[indices[i] + 1] = 1  # Convert to 1-based
+    }
+}
+
+{
+    # Store the current row
+    for (i = 1; i <= NF; i++) {
+        row[i] = $i
+    }
+
+    # Find the maximum list length among all list columns
+    max_len = 1
+    for (col_idx in list_cols) {
+        if (col_idx <= NF) {
+            # Split the list (comma-separated values)
+            n = split(row[col_idx], values, ",")
+            if (n > max_len) {
+                max_len = n
+            }
+            # Store the split values
+            for (j = 1; j <= n; j++) {
+                lists[col_idx][j] = values[j]
+            }
+            # Fill missing values with "."
+            for (j = n + 1; j <= max_len; j++) {
+                lists[col_idx][j] = "."
+            }
+        }
+    }
+
+    # Output exploded rows
+    for (i = 1; i <= max_len; i++) {
+        for (j = 1; j <= NF; j++) {
+            if (j in list_cols) {
+                printf "%s", lists[j][i]
+            } else {
+                printf "%s", row[j]
+            }
+            if (j < NF) printf "\t"
+        }
+        printf "\n"
+    }
+
+    # Clear arrays for next row
+    delete lists
+}