phc-ingestion 0.8.31__py3-none-any.whl → 0.8.33__py3-none-any.whl

ingestion/nextgen/process.py

@@ -2,6 +2,7 @@ from lifeomic_logging import scoped_logger
 from typing import Any, TypedDict
 from ruamel.yaml import YAML
 
+from ingestion.nextgen.util.alteration_table import extract_variant_table_rows_and_hyperdiploidy
 from ingestion.nextgen.util.pre_filter_somatic_vcf import pre_filter_somatic_vcf
 from ingestion.nextgen.util.process_cnv import process_cnv
 from ingestion.nextgen.util.process_manifest import process_manifest
@@ -36,54 +37,61 @@ def process(
         "projectId": project_id,
         "archiveFileId": source_file_id,
         "caseId": case_id,
-        "ingestion_id": ingestion_id,
+        "ingestionId": ingestion_id,
     }
     with scoped_logger(__name__, log_context) as log:
+        (
+            short_variant_table_rows,
+            copy_number_variant_table_rows,
+            structural_variant_table_rows,
+            hyperdiploidy_chromosomes,
+        ) = extract_variant_table_rows_and_hyperdiploidy(vendor_files["xmlFile"], log)
         cnv_path_name = process_cnv(
-            xml_in_file=vendor_files["xmlFile"],
-            cnv_in_file=vendor_files["somaticCnvTxtFile"],
-            root_path=local_output_dir,
-            prefix=case_id,
-            log=log,
+            vendor_files["somaticCnvTxtFile"],
+            copy_number_variant_table_rows,
+            local_output_dir,
+            case_id,
+            log,
         )
         structural_path_name, translocations = process_structural(
-            xml_in_file=vendor_files["xmlFile"],
-            sv_in_file=vendor_files["somaticSvVcfFile"],
-            root_path=local_output_dir,
-            prefix=case_id,
-            log=log,
+            vendor_files["somaticSvVcfFile"],
+            structural_variant_table_rows,
+            local_output_dir,
+            case_id,
+            log,
         )
         manifest = process_manifest(
-            xml_in_file=vendor_files["xmlFile"],
-            source_file_id=source_file_id,
-            prefix=case_id,
-            include_copy_number=bool(cnv_path_name),
-            include_structural=bool(structural_path_name),
-            somatic_translocations=translocations,
-            log=log,
+            vendor_files["xmlFile"],
+            source_file_id,
+            case_id,
+            bool(cnv_path_name),
+            bool(structural_path_name),
+            translocations,
+            hyperdiploidy_chromosomes,
         )
         pre_filtered_somatic_vcf_path = pre_filter_somatic_vcf(
             vendor_files["somaticVcfFile"],
             vendor_files["somaticVcfSnvFile"],
             vendor_files["somaticVcfIndelFile"],
+            short_variant_table_rows,
             local_output_dir,
             log,
         )
         somatic_vcf_meta_data = process_vcf(
-            vcf_in_file=pre_filtered_somatic_vcf_path,
-            root_path=local_output_dir,
-            case_id=case_id,
-            sequence_type="somatic",
-            xml_in_file=vendor_files["xmlFile"],
+            pre_filtered_somatic_vcf_path,
+            local_output_dir,
+            case_id,
+            "somatic",
+            short_variant_table_rows,
             log=log,
         )
         germline_vcf_meta_data = process_vcf(
-            vcf_in_file=vendor_files["germlineVcfFile"],
-            root_path=local_output_dir,
-            case_id=case_id,
-            sequence_type="germline",
-            xml_in_file=vendor_files["xmlFile"],
-            log=log,
+            vendor_files["germlineVcfFile"],
+            local_output_dir,
+            case_id,
+            "germline",
+            short_variant_table_rows,
+            log,
         )
 
     manifest_path_name = f"{local_output_dir}/{case_id}.ga4gh.genomics.yml"

ingestion/nextgen/util/alteration_table.py

@@ -1,10 +1,41 @@
 from logging import Logger
-import pandas as pd
 import re
-from typing import cast, Literal, TypedDict
+from typing import TypedDict, Generic, TypeVar
 
 
-short_variant_types: list[str] = [
+T = TypeVar("T")
+
+
+class AlterationTableRow(Generic[T], TypedDict):
+    gene: T
+    type: str
+    description: str
+    vaf: str
+    info: str
+
+
+class ShortVariantGene(TypedDict):
+    chr: str
+    pos: int
+
+
+class CopyNumberVariantGene(TypedDict):
+    gene: str
+    chr: str
+    start: int
+    end: int
+
+
+class StructuralVariantGene(TypedDict):
+    gene1: str
+    chr1: str
+    pos1: int
+    gene2: str
+    chr2: str
+    pos2: int
+
+
+base_short_variant_types: list[str] = [
     "Missense",
     "Frameshift",
     "Stop gained",
@@ -20,12 +51,23 @@ short_variant_types: list[str] = [
 ]
 
 
+def get_short_variant_types() -> list[str]:
+    # For multi-word short variant types, sometimes the spaces are not included
+    short_variant_types: list[str] = []
+    for short_variant_type in base_short_variant_types:
+        short_variant_types.append(short_variant_type)
+        if " " in short_variant_type:
+            short_variant_types.append(short_variant_type.replace(" ", ""))
+
+    return short_variant_types
+
+
 def extract_all_table_lines(xml_in_file: str) -> list[str]:
     with open(xml_in_file, "r") as f:
         xml_lines = f.readlines()
 
     in_range_trigger = False
-    table_lines = []
+    table_lines: list[str] = []
     for line in xml_lines:
         if "Gene (Chr. Position, hg38)" in line:
             in_range_trigger = True
@@ -37,7 +79,7 @@ def extract_all_table_lines(xml_in_file: str) -> list[str]:
     return table_lines
 
 
-def extract_alteration_table(xml_in_file: str, log: Logger) -> pd.DataFrame:
+def extract_alteration_table_rows(xml_in_file: str, log: Logger) -> list[AlterationTableRow[str]]:
     table_lines = extract_all_table_lines(xml_in_file)
     # Remove completely empty lines
     table_lines = [line for line in table_lines if line.strip() != ""]
@@ -49,90 +91,94 @@ def extract_alteration_table(xml_in_file: str, log: Logger) -> pd.DataFrame:
             if current_row:
                 table_row_lines.append(current_row)
                 current_row = []
-        line = re.sub(r"<T.>", "", line)
-        line = re.sub(r"</T.>", "", line)
-        line = re.sub(r"<T./>", "", line)
-        if line.strip() not in ["", "p."]:
-            current_row.append(line.strip())
-
-    gene_column = []
-    type_column = []
-    description_column = []
-    vaf_column = []
-    info_column = []
-
-    for row in table_row_lines:
-        gene_column.append(row[0])
-        type_column.append(row[1])
-        description_column.append(row[2])
-        vaf_column.append(row[3])
-        # Sometimes the info column is empty, so we need to check if it actually exists
-        # So far, it seems like rows with empty "info" columns are generally not useful for us
-        # and the data in them will not be used anywhere, so we just fill in an empty string
-        if len(row) > 4:
-            info_column.append(row[4])
-        else:
-            info_column.append("")
-
-    # If the test is negative we will have a type column with only NA values
-    # We return an empty df which we check for later when scraping annotations
-    # Ignore the first row which is the header
-    if set(type_column[1:]) == {"NA"}:
-        log.info("Alteration table is empty")
-        return pd.DataFrame()
-
-    alteration_df = pd.DataFrame(
-        {
-            "gene": gene_column,
-            "type": type_column,
-            "description": description_column,
-            "vaf": vaf_column,
-            "info": info_column,
-        }
-    )
-
-    return alteration_df
-
-
-def extract_variant_table(
-    xml_in_file: str, variant_type: Literal["copy number", "structural", "short"], log: Logger
-) -> pd.DataFrame:
-    alteration_table = extract_alteration_table(xml_in_file, log)
-    if alteration_table.empty:
-        return alteration_table
-
-    # Drop by variant type
-    if variant_type == "copy number":
-        variant_df = alteration_table[alteration_table["type"] == "CNV"]
-    elif variant_type == "structural":
-        variant_df = alteration_table[alteration_table["type"] == "Translocation"]
-    elif variant_type == "short":
-        variant_df = alteration_table[alteration_table["type"].isin(short_variant_types)]
-
-    return variant_df
-
-
-class AlterationTableRow(TypedDict):
-    gene: str
-    type: str
-    description: str
-    vaf: str
-    info: str
-
-
-def extract_hyperdiploidy_row(xml_in_file: str, log: Logger) -> None | AlterationTableRow:
-    alteration_table = extract_alteration_table(xml_in_file, log)
-    if alteration_table.empty:
-        return None
-
-    hyperdiploidy_df = alteration_table[alteration_table["type"] == "Hyperdiploidy"]
-
-    if hyperdiploidy_df.empty:
-        return None
-    # We only expect one hyperdiploidy row. If we get more than 1, just fail the ingestion so we can investigate
-    if hyperdiploidy_df.shape[0] > 1:
-        raise ValueError("More than one hyperdiploidy row found")
-
-    hyperdiploidy_row = cast(AlterationTableRow, hyperdiploidy_df.iloc[0].to_dict())
-
-    return hyperdiploidy_row
+        line = re.sub(r"<\/?T.\/?>", "", line).strip()
+        if line and line != "p.":
+            current_row.append(line)
+
+    alteration_table_rows: list[AlterationTableRow[str]] = []
+
+    # Skip the first row which is the header
+    for row in table_row_lines[1:]:
+        # Sometimes the alteration table is "empty", in which case the `type` column will only contain "NA" values
+        if row[1] == "NA":
+            continue
+        alteration_table_rows.append(
+            {
+                "gene": row[0],
+                "type": row[1],
+                "description": row[2],
+                "vaf": row[3],
+                # Sometimes the info column is empty, so we need to check if it actually exists
+                # So far, it seems like rows with empty "info" columns are generally not useful for us
+                # and the data in them will not be used anywhere, so we just fill in an empty string
+                "info": row[4] if len(row) > 4 else "",
+            }
+        )
+
+    return alteration_table_rows
+
+
+def parse_short_variant_gene(gene: str) -> ShortVariantGene:
+    pattern = r"^.*\((?P<chr>chr\d+|chrX|chrY):(?P<pos>\d+).*\).*$"
+    match = re.match(pattern, gene)
+    if not match:
+        raise RuntimeError(f"Failed to parse gene field for short variant")
+    return {"chr": match.group("chr"), "pos": int(match.group("pos"))}
+
+
+def parse_copy_number_variant_gene(gene: str) -> CopyNumberVariantGene:
+    pattern = r"^(?P<gene>[A-Z1-9]*).*?\((?P<chr>chr\d+|chrX|chrY):(?P<start>\d+)_(?P<end>\d+)\).*$"
+    match = re.match(pattern, gene)
+    if not match:
+        raise RuntimeError(f"Failed to parse gene field for copy number variant")
+    return {
+        "gene": match.group("gene"),
+        "chr": match.group("chr"),
+        "start": int(match.group("start")),
+        "end": int(match.group("end")),
+    }
+
+
+def parse_structural_variant_gene(gene: str) -> StructuralVariantGene:
+    pattern = r"^(?P<gene1>[A-Z1-9]*)(-|\/)(?P<gene2>[A-Z1-9]*).*\(.*(?P<chr1>chr\d+|chrX|chrY):(?P<pos1>\d+).*;.*(?P<chr2>chr\d+|chrX|chrY):(?P<pos2>\d+).*\).*$"
+    match = re.match(pattern, gene)
+    if not match:
+        raise RuntimeError(f"Failed to parse gene field for structural variant")
+    return {
+        "gene1": match.group("gene1"),
+        "chr1": match.group("chr1"),
+        "pos1": int(match.group("pos1")),
+        "gene2": match.group("gene2"),
+        "chr2": match.group("chr2"),
+        "pos2": int(match.group("pos2")),
+    }
+
+
+def extract_variant_table_rows_and_hyperdiploidy(xml_in_file: str, log: Logger) -> tuple[
+    list[AlterationTableRow[ShortVariantGene]],
+    list[AlterationTableRow[CopyNumberVariantGene]],
+    list[AlterationTableRow[StructuralVariantGene]],
+    list[str] | None,
+]:
+    alteration_table_rows = extract_alteration_table_rows(xml_in_file, log)
+
+    short_variant_rows: list[AlterationTableRow[ShortVariantGene]] = []
+    copy_number_rows: list[AlterationTableRow[CopyNumberVariantGene]] = []
+    structural_variant_rows: list[AlterationTableRow[StructuralVariantGene]] = []
+    hyperdiploidy_chromosomes: list[str] | None = None
+
+    short_variant_types = get_short_variant_types()
+
+    for row in alteration_table_rows:
+        if row["type"] in short_variant_types:
+            short_variant_rows.append({**row, "gene": parse_short_variant_gene(row["gene"])})
+        elif row["type"] == "CNV":
+            copy_number_rows.append({**row, "gene": parse_copy_number_variant_gene(row["gene"])})
+        elif row["type"] == "Translocation":
+            structural_variant_rows.append(
+                {**row, "gene": parse_structural_variant_gene(row["gene"])}
+            )
+        elif row["type"] == "Hyperdiploidy":
+            hyperdiploidy_chromosomes = re.findall(r"\d+", row["gene"])
+
+    return short_variant_rows, copy_number_rows, structural_variant_rows, hyperdiploidy_chromosomes

ingestion/nextgen/util/manifest_helpers.py

@@ -1,5 +1,3 @@
-from ingestion.nextgen.util.alteration_table import extract_hyperdiploidy_row
-
 from logging import Logger
 import re
 
@@ -42,12 +40,3 @@ def parse_report_date(line: str) -> str:
     return parse_pattern(
         r"^.*Diagnostic Genomics Laboratory.*(\d{2}\/\d{2}\/\d{4}).*$", line, "report date"
     )
-
-
-def extract_hyperdiploidy_chromosomes(xml_in_file: str, log: Logger) -> list[str] | None:
-    hyperdiploidy_row_dict = extract_hyperdiploidy_row(xml_in_file, log)
-
-    if not hyperdiploidy_row_dict:
-        return None
-
-    return re.findall(r"\d+", hyperdiploidy_row_dict["gene"])

ingestion/nextgen/util/nextgen_specific_genes.py

@@ -14,7 +14,7 @@ nextgen_specific_genes_with_location: list[GeneWithLocation] = [
     {"gene": "CCND3", "chr": "chr6", "start": 41920534, "end": 42562008},
     {"gene": "MYC", "chr": "chr8", "start": 125309416, "end": 129673293},
     {"gene": "CCND1", "chr": "chr11", "start": 69090733, "end": 69656860},
-    {"gene": "IGH", "chr": "chr14", "start": 105578834, "end": 109902208},
+    {"gene": "IGH", "chr": "chr14", "start": 105516968, "end": 109902208},
     {"gene": "MAF", "chr": "chr16", "start": 78428398, "end": 79615096},
     {"gene": "MAFB", "chr": "chr20", "start": 39039005, "end": 40688948},
     {"gene": "IGL", "chr": "chr22", "start": 22012552, "end": 22965858},
@@ -22,7 +22,7 @@ nextgen_specific_genes_with_location: list[GeneWithLocation] = [
 nextgen_specific_genes: set[str] = {gene["gene"] for gene in nextgen_specific_genes_with_location}
 
 
-def maybe_get_matching_gene_for_location(chr: str, position: int) -> str | None:
+def maybe_get_nextgen_specific_gene(chr: str, position: int) -> str | None:
     for gene in nextgen_specific_genes_with_location:
         if gene["chr"] == chr and gene["start"] <= position <= gene["end"]:
             return gene["gene"]

ingestion/nextgen/util/pre_filter_somatic_vcf.py

@@ -1,5 +1,6 @@
 from logging import Logger
 
+from ingestion.nextgen.util.alteration_table import AlterationTableRow, ShortVariantGene
 from ingestion.shared_util.open_maybe_gzipped import open_maybe_gzipped
 
 
@@ -14,26 +15,54 @@ def extract_filter_from_vcf_line(line: str) -> str:
     return split_line[6]
 
 
-def replace_filter_in_vcf_line(line: str, new_filter: str) -> str:
+def replace_filter_in_line(line: str, new_filter: str) -> str:
     split_line = line.strip().split("\t")
     split_line[6] = new_filter
     return "\t".join(split_line) + "\n"
 
 
+def is_line_in_alteration_table(
+    line: str, short_variant_table_rows: list[AlterationTableRow[ShortVariantGene]]
+) -> bool:
+    """
+    Returns True if the line in the VCF appears in
+    the alteration table, False otherwise.
+
+    Matching in the alteration table is less strict than in the
+    VCF files; we only need to match chromosome and position.
+
+    Also position may differ by +1 or -1, as deletion and insertion positions
+    are represented differently in the VCF and the alteration table.
+    """
+    split_line = line.strip().split("\t")
+    chrom, pos = split_line[0], int(split_line[1])
+
+    for row in short_variant_table_rows:
+        ref_chrom, ref_pos = row["gene"]["chr"], row["gene"]["pos"]
+
+        if ref_chrom == chrom and (abs(ref_pos - pos) <= 1):
+            return True
+
+    return False
+
+
 def pre_filter_somatic_vcf(
     somatic_vcf_file: str,
     somatic_vcf_snv_file: str,
     somatic_vcf_indel_file: str,
+    short_variant_table_rows: list[AlterationTableRow[ShortVariantGene]],
     working_dir: str,
     log: Logger,
 ) -> str:
     """
     Removes all variants from the `somatic_vcf_file` that are not
-    also in the `somatic_vcf_snv_file` or `somatic_vcf_indel_file`.
+    also in the `somatic_vcf_snv_file`, the `somatic_vcf_indel_file`,
+    or the alteration table.
 
     Also updates the FILTER field in the `somatic_vcf_file` to match
     the FILTER field of the corresponding variant in the
     `somatic_vcf_snv_file` or `somatic_vcf_indel_file`.
+    For variants in the alteration table, the original FILTER field is kept.
     """
     log.info("Pre-filtering somatic VCF file")
 
@@ -48,20 +77,22 @@ def pre_filter_somatic_vcf(
                     extract_filter_from_vcf_line(line)
                 )
 
-    log.info(f"Found {len(valid_variants_with_filters)} valid variants")
+    log.info(f"Found {len(valid_variants_with_filters)} valid variants in the SNV and INDEL files")
 
     output_vcf_path = f"{working_dir}/filtered_somatic.vcf.gz"
     with (
-        open_maybe_gzipped(somatic_vcf_file, "rt") as f,
+        open_maybe_gzipped(somatic_vcf_file, "rt") as r,
         open_maybe_gzipped(output_vcf_path, "wt") as w,
     ):
-        for line in f:
+        for line in r:
             if line.startswith("#"):
                 w.write(line)
             else:
                 key = build_variant_key_from_vcf_line(line)
                 if key in valid_variants_with_filters:
-                    w.write(replace_filter_in_vcf_line(line, valid_variants_with_filters[key]))
+                    w.write(replace_filter_in_line(line, valid_variants_with_filters[key]))
+                elif is_line_in_alteration_table(line, short_variant_table_rows):
+                    w.write(line)
 
     log.info(f"Successfully pre-filtered somatic VCF file to {output_vcf_path}")
     return output_vcf_path

ingestion/nextgen/util/process_cnv.py

@@ -1,20 +1,21 @@
 import pandas as pd
 from logging import Logger
 
-from ingestion.nextgen.util.alteration_table import extract_variant_table
+from ingestion.nextgen.util.alteration_table import AlterationTableRow, CopyNumberVariantGene
 from ingestion.nextgen.util.interpretation import map_interpretation
 
 
 def process_cnv(
-    xml_in_file: str, cnv_in_file: str, root_path: str, prefix: str, log: Logger
+    cnv_in_file: str,
+    copy_number_variant_table_rows: list[AlterationTableRow[CopyNumberVariantGene]],
+    output_dir: str,
+    case_id: str,
+    log: Logger,
 ) -> str | None:
-    copy_number_path_name = f"{root_path}/{prefix}.copynumber.csv"
-    sample_id = prefix
+    copy_number_path_name = f"{output_dir}/{case_id}.copynumber.csv"
+    sample_id = case_id
 
-    copy_number_variant_rows = []
-    copy_number_variant_table = extract_variant_table(
-        xml_in_file=xml_in_file, variant_type="copy number", log=log
-    )
+    copy_number_variant_rows: list[str] = []
 
     with open(cnv_in_file, "r") as f:
         cnv_rows = f.readlines()
@@ -45,20 +46,15 @@ def process_cnv(
         attributes = {}
 
         # Scrape interpretation
-        interpretation = None
-        if not copy_number_variant_table.empty:
-            for index, row in copy_number_variant_table.iterrows():
-                ref_gene = row["gene"].split(" ")[0]
-                ref_coord = row["gene"].split(" ")[1]
-
-                if (
-                    ref_gene == gene_id_only
-                    and ref_coord == f"({chromosome}:{start_position}_{end_position})"
-                ):
-                    interpretation = map_interpretation(row["info"], log)
-
-        if not interpretation:
-            interpretation = "unknown"
+        interpretation = "unknown"
+        for row in copy_number_variant_table_rows:
+            if (
+                row["gene"]["gene"] == gene_id_only
+                and row["gene"]["chr"] == chromosome
+                and row["gene"]["start"] <= int(start_position)
+                and row["gene"]["end"] >= int(end_position)
+            ):
+                interpretation = map_interpretation(row["info"], log)
 
         copy_number_variant_rows.append(
             f"{sample_id},{gene_id_only},{copy_number},{status},{attributes},{chromosome},{start_position},{end_position},{interpretation}\n"

ingestion/nextgen/util/process_manifest.py

@@ -173,45 +173,46 @@ def extract_test_data(patient_info_lines: list, interpretation_lines: list):
 def process_manifest(
     xml_in_file: str,
     source_file_id: str,
-    prefix: str,
+    case_id: str,
     include_copy_number: bool,
     include_structural: bool,
     somatic_translocations: list[str],
-    log: Logger,
+    hyperdiploidy_chromosomes: list[str] | None,
 ):
     test_text = extract_xml_text(xml_in_file)
     interpretation_text = extract_interpretation_text(xml_in_file)
     manifest = extract_test_data(test_text, interpretation_text)
     manifest.update(extract_patient_data(test_text))
 
-    hyperdiploidy_chromosomes = manifest_helpers.extract_hyperdiploidy_chromosomes(xml_in_file, log)
+    file_prefix = f".lifeomic/nextgen/{case_id}/{case_id}"
+
     if hyperdiploidy_chromosomes:
         manifest["hyperdiploidyTrisomies"] = hyperdiploidy_chromosomes
     if somatic_translocations:
         manifest["somaticTranslocations"] = somatic_translocations
 
-    manifest["reportFile"] = f".lifeomic/nextgen/{prefix}/{prefix}.pdf"
+    manifest["reportFile"] = f"{file_prefix}.pdf"
     manifest["sourceFileId"] = source_file_id
     manifest["resources"] = []
 
     manifest["files"] = [
         {
-            "fileName": f".lifeomic/nextgen/{prefix}/{prefix}.modified.somatic.nrm.filtered.vcf.gz",
+            "fileName": f"{file_prefix}.modified.somatic.nrm.filtered.vcf.gz",
             "sequenceType": "somatic",
             "type": "shortVariant",
         },
         {
-            "fileName": f".lifeomic/nextgen/{prefix}/{prefix}.modified.germline.nrm.filtered.vcf.gz",
+            "fileName": f"{file_prefix}.modified.germline.nrm.filtered.vcf.gz",
             "sequenceType": "germline",
             "type": "shortVariant",
         },
         {
-            "fileName": f".lifeomic/nextgen/{prefix}/{prefix}.somatic.updated.bam",
+            "fileName": f"{file_prefix}.somatic.updated.bam",
             "sequenceType": "somatic",
             "type": "read",
         },
         {
-            "fileName": f".lifeomic/nextgen/{prefix}/{prefix}.germline.updated.bam",
+            "fileName": f"{file_prefix}.germline.updated.bam",
             "sequenceType": "germline",
             "type": "read",
         },
@@ -219,7 +220,7 @@ def process_manifest(
     if include_structural:
         manifest["files"].append(
             {
-                "fileName": f".lifeomic/nextgen/{prefix}/{prefix}.structural.csv",
+                "fileName": f"{file_prefix}.structural.csv",
                 "sequenceType": "somatic",
                 "type": "structuralVariant",
             },
@@ -227,7 +228,7 @@ def process_manifest(
     if include_copy_number:
         manifest["files"].append(
             {
-                "fileName": f".lifeomic/nextgen/{prefix}/{prefix}.copynumber.csv",
+                "fileName": f"{file_prefix}.copynumber.csv",
                 "sequenceType": "somatic",
                 "type": "copyNumberVariant",
             }

ingestion/nextgen/util/process_structural.py

@@ -3,12 +3,9 @@ import re
 from typing import TypedDict
 
 from ingestion.shared_util.coords_to_genes import coords_to_genes
-from ingestion.nextgen.util.alteration_table import extract_variant_table
+from ingestion.nextgen.util.alteration_table import AlterationTableRow, StructuralVariantGene
 from ingestion.nextgen.util.interpretation import map_interpretation
-from ingestion.nextgen.util.nextgen_specific_genes import (
-    maybe_get_matching_gene_for_location,
-    nextgen_specific_genes,
-)
+from ingestion.nextgen.util.nextgen_specific_genes import maybe_get_nextgen_specific_gene
 from ingestion.shared_util.open_maybe_gzipped import open_maybe_gzipped
 
 
@@ -46,38 +43,33 @@ def is_del_dup_or_ins(variant: list[str]) -> bool:
     return any([x in variant[2] for x in ["MantaDEL", "MantaDUP", "MantaINS"]])
 
 
-def get_gene_from_coords(
-    chromosome: str, start_position: str, end_position: str, log: Logger
-) -> str:
+def get_center_position(start_position: str, end_position: str) -> int:
     """
-    A number of genes of interest with specific start and end positions have been provided.
-    If a variant falls within the start and end positions of one of those genes of interest, that gene will be used.
-    Otherwise, we fall back to the standard gene lookup.
+    Calculate the center position of a variant based on its start and end positions, useful for finding genes.
     """
-    center_position = int((int(start_position) + int(end_position)) / 2)
-
-    gene = maybe_get_matching_gene_for_location(chromosome, center_position)
-    if gene:
-        return gene
-
-    return coords_to_genes("GRCh38", chromosome, center_position, log)
+    return int((int(start_position) + int(end_position)) / 2)
 
 
 def process_structural(
-    sv_in_file: str, xml_in_file, root_path: str, prefix: str, log: Logger
+    structural_variant_in_file: str,
+    structural_variant_table_rows: list[AlterationTableRow[StructuralVariantGene]],
+    output_dir: str,
+    case_id: str,
+    log: Logger,
 ) -> tuple[str | None, list[str]]:
-    structural_variant_table = extract_variant_table(
-        xml_in_file=xml_in_file, variant_type="structural", log=log
-    )
+    structural_variant_path_name = f"{output_dir}/{case_id}.structural.csv"
+    sample_id = case_id
 
-    structural_variant_path_name = f"{root_path}/{prefix}.structural.csv"
-    sample_id = prefix
-
-    with open_maybe_gzipped(sv_in_file, "rt") as f:
+    with open_maybe_gzipped(structural_variant_in_file, "rt") as f:
         variants = [line for line in f.readlines() if not line.startswith("#")]
 
     structural_variants: list[StructuralVariant] = []
+    formatted_translocations: set[str] = set()
+
     for variant in variants:
+        gene1: str | None = None
+        gene2: str | None = None
+
         working_variant = variant.strip().split("\t")
 
         chromosome1 = f"chr{working_variant[0]}"
@@ -95,7 +87,7 @@ def process_structural(
                 effect = "insertion"
 
             # Get genes from coordinates using center point of start and end positions
-            gene1 = get_gene_from_coords(chromosome1, start_position1, end_position1, log)
+            gene1 = None
             gene2 = "N/A"
 
         else:
@@ -107,32 +99,60 @@ def process_structural(
             end_position2 = alt[1]
             effect = "translocation"
 
-            # Get genes from coordinates using center point of start and end positions
-            gene1 = get_gene_from_coords(chromosome1, start_position1, end_position1, log)
-            gene2 = get_gene_from_coords(chromosome2, start_position2, end_position2, log)
+            gene1 = maybe_get_nextgen_specific_gene(
+                chromosome1, get_center_position(start_position1, end_position1)
+            )
+            gene2 = maybe_get_nextgen_specific_gene(
+                chromosome2, get_center_position(start_position2, end_position2)
+            )
+
+            # Maybe add this variant to the formatted translocations list
+            if (gene1 == "MYC" or gene2 == "MYC") and gene1 != gene2:
+                formatted_translocations.add("t(MYC)")
+            elif gene1 and gene2:
+                # Remove the "chr" prefix and convert to int
+                chr1, chr2 = int(chromosome1[3:]), int(chromosome2[3:])
+                # Don't add translocations between the same chromosome
+                if chr1 == chr2:
+                    continue
+                # Ensure chromosomes are in ascending order
+                if chr1 > chr2:
+                    chr1, chr2 = chr2, chr1
+                formatted_translocations.add(f"t({chr1};{chr2})")
 
         # Scrape interpretation
         interpretation = "unknown"
-        if not structural_variant_table.empty:
-            for _, row in structural_variant_table.iterrows():
-                pattern = r"^.*\(.*(chr\d+:\d+).*;.*(chr\d+:\d+).*\).*$"
-                match = re.match(pattern, row["gene"])
-                if not match:
-                    log.warn(f"Failed to parse gene field for structural variant")
-                    continue
-                ref_coords = set(match.groups())
-                variant_coords = set(
-                    [f"{chromosome1}:{start_position1}", f"{chromosome2}:{start_position2}"]
-                )
+        for row in structural_variant_table_rows:
+            is_match = (
+                row["gene"]["chr1"] == chromosome1
+                and row["gene"]["chr2"] == chromosome2
+                and row["gene"]["pos1"] == int(start_position1)
+                and row["gene"]["pos2"] == int(start_position2)
+            )
+            if not is_match:
+                continue
 
-                if ref_coords == variant_coords:
-                    interpretation = map_interpretation(row["info"], log)
+            interpretation = map_interpretation(row["info"], log)
+            # Use the gene names from the alteration table but only if they are not already set
+            gene1 = gene1 if gene1 else row["gene"]["gene1"]
+            gene2 = gene2 if gene2 else row["gene"]["gene2"]
 
         # Hard-code
         sequence_type = "Somatic"
         in_frame = "Unknown"
         attributes: dict = {}
 
+        # If genes have not been populated from the nextgen specific genes or alteration
+        # table fall back to using the default gene finding method
+        if not gene1:
+            gene1 = coords_to_genes(
+                "GRCh38", chromosome1, get_center_position(start_position1, end_position1), log
+            )
+        if not gene2:
+            gene2 = coords_to_genes(
+                "GRCh38", chromosome2, get_center_position(start_position2, end_position2), log
+            )
+
         structural_variants.append(
             {
                 "sample_id": sample_id,
@@ -163,7 +183,7 @@ def process_structural(
             deduped_structural_variants.append(sv)
 
     if not deduped_structural_variants:
-        log.info(f"Ignoring empty structural variant file {sv_in_file}")
+        log.info(f"Ignoring empty structural variant file {structural_variant_in_file}")
         return (None, [])
 
     log.info(f"Saving file to {structural_variant_path_name}")
@@ -174,28 +194,6 @@ def process_structural(
         for sv in deduped_structural_variants:
             f.write(structural_variant_to_csv_row(sv))
 
-    log.info("Finding structural variant translocations for genes of interest")
-    translocations = [sv for sv in deduped_structural_variants if sv["effect"] == "translocation"]
-    formatted_translocations: set[str] = set()
-    for translocation in translocations:
-        gene1, gene2 = translocation["gene1"], translocation["gene2"]
-        # MYC is a special case
-        if gene1 == "MYC" or gene2 == "MYC":
-            if gene1 == gene2:
-                continue
-            formatted_translocations.add("t(MYC)")
-            continue
-        if gene1 in nextgen_specific_genes and gene2 in nextgen_specific_genes:
-            chr1, chr2 = int(translocation["position1"][0][3:]), int(
-                translocation["position2"][0][3:]
-            )
-            if chr1 == chr2:
-                continue
-            # Ensure chromosomes are in ascending order
-            if chr1 > chr2:
-                chr1, chr2 = chr2, chr1
-            formatted_translocations.add(f"t({chr1};{chr2})")
-
     log.info(f"Found {len(formatted_translocations)} translocations for genes of interest")
 
     return structural_variant_path_name, list(formatted_translocations)

ingestion/nextgen/util/process_vcf.py

@@ -4,7 +4,7 @@ import shutil
 from logging import Logger
 from typing import Literal
 
-from ingestion.nextgen.util.alteration_table import extract_variant_table
+from ingestion.nextgen.util.alteration_table import AlterationTableRow, ShortVariantGene
 
 SequenceType = Literal["somatic", "germline"]
 
@@ -76,14 +76,10 @@ def transform_vcf(
     headers: list,
     variants: list,
     sequence_type: SequenceType,
-    xml_in_file: str,
+    short_variant_table_rows: list[AlterationTableRow[ShortVariantGene]],
     case_id: str,
     log: Logger,
 ) -> str:
-    short_variant_table = extract_variant_table(
-        xml_in_file=xml_in_file, variant_type="short", log=log
-    )
-
     log.info(f"Performing file transformations on {vcf_in_file}")
     approved_chr_list = ["chr" + str(i) for i in range(1, 23)] + ["chrX", "chrY", "chrM"]
     vcf_out: list[str] = []
@@ -135,13 +131,12 @@ def transform_vcf(
             working.calculate_af()
             working.prune_var()
 
-        if sequence_type == "somatic" and not short_variant_table.empty:
+        if sequence_type == "somatic":
             split_var[7] = add_vendsig_to_info(
                 working.pruned_info,
-                short_variant_table,
+                short_variant_table_rows,
                 split_var[0],
                 int(split_var[1]),
-                log,
             )
         else:
             split_var[7] = f"{working.pruned_info};VENDSIG=Unknown"
@@ -160,14 +155,14 @@ def export_vcf(vcf_out: str, vcf_path: str, log: Logger):
 
 def process_vcf(
     vcf_in_file: str,
-    root_path: str,
+    output_dir: str,
     case_id: str,
     sequence_type: SequenceType,
-    xml_in_file: str,
+    short_variant_table_rows: list[AlterationTableRow[ShortVariantGene]],
     log: Logger,
 ):
     line_count = 0
-    vcf_path = f"{root_path}/{case_id}.modified.{sequence_type}.vcf.gz"
+    vcf_path = f"{output_dir}/{case_id}.modified.{sequence_type}.vcf.gz"
 
     headers = []
     variants = []
@@ -196,7 +191,13 @@ def process_vcf(
 
             else:
                 vcf_out = transform_vcf(
-                    vcf_in_file, headers, variants, sequence_type, xml_in_file, case_id, log
+                    vcf_in_file,
+                    headers,
+                    variants,
+                    sequence_type,
+                    short_variant_table_rows,
+                    case_id,
+                    log,
                 )
                 export_vcf(vcf_out, vcf_path, log)
 
@@ -206,7 +207,7 @@ def process_vcf(
 def map_vendsig(vendsig: str) -> str:
     if vendsig in ["Pathogenic"]:
         return "VENDSIG=Pathogenic"
-    elif vendsig in ["Likely Pathogenic"]:
+    elif vendsig in ["Likely Pathogenic", "LikelyPathogenic"]:
         return "VENDSIG=Likely pathogenic"
     elif vendsig in ["VUS"]:
         return "VENDSIG=Uncertain significance"
@@ -214,26 +215,15 @@ def map_vendsig(vendsig: str) -> str:
         raise RuntimeError(f"Unable to map vendor significance: {vendsig}")
 
 
-def extract_chrom_pos_from_gene_string(chr_pos: str, log: Logger) -> tuple[str, int]:
-    """
-    Parses chromosome and position from a gene string from the alteration table.
-
-    Raises if no match found.
-    """
-
-    pattern = r"^.*\((chr\d+|chrX|chrY):(\d+).*\).*$"
-    match = re.match(pattern, chr_pos)
-    if not match:
-        raise RuntimeError(f"Failed to extract chrom and pos from gene string")
-    chrom = match.group(1)
-    pos = int(match.group(2))
-    return (chrom, pos)
-
-
-def add_vendsig_to_info(info: str, short_var_table, chrom: str, pos: int, log: Logger) -> str:
+def add_vendsig_to_info(
+    info: str,
+    short_variant_table_rows: list[AlterationTableRow[ShortVariantGene]],
+    chrom: str,
+    pos: int,
+) -> str:
     mapped_vendsig = None
-    for index, row in short_var_table.iterrows():
-        ref_chrom, ref_pos = extract_chrom_pos_from_gene_string(row["gene"], log)
+    for row in short_variant_table_rows:
+        ref_chrom, ref_pos = row["gene"]["chr"], row["gene"]["pos"]
 
         if ref_chrom == chrom:
             if ref_pos == pos or ref_pos + 1 == pos or ref_pos - 1 == pos:

{phc_ingestion-0.8.31.dist-info → phc_ingestion-0.8.33.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: phc-ingestion
-Version: 0.8.31
+Version: 0.8.33
 Summary: Functions for LifeOmic PHC genomic ingestions
 License: MIT
 Author-email: LifeOmic Development <development@lifeomic.com>
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: jsonschema<5.0.0,>=4.16.0
 Requires-Dist: lifeomic-logging<0.4.0,>=0.3.2
 Requires-Dist: natsort==7.1.1

{phc_ingestion-0.8.31.dist-info → phc_ingestion-0.8.33.dist-info}/RECORD

@@ -29,16 +29,16 @@ ingestion/generic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 ingestion/generic/process.py,sha256=WJHV_-SKhrDZ3JS3fm9DVMoW3Zs2t50GiraSV3vlLHE,1548
 ingestion/generic/utils.py,sha256=1MEIru7uq38IjUdL8lcHqDH0oTki9uWrz1f2e-pmRoU,2814
 ingestion/nextgen/__init__.py,sha256=7LQ-h_Bvc5P1QcHMdzsqi1Qm4fTJn04-ozar2ty9wSc,59
-ingestion/nextgen/process.py,sha256=F0Ms8rTr_4boWPpE13D39C3ljFtyIVtw9XIIjCVI6f8,3849
-ingestion/nextgen/util/alteration_table.py,sha256=h3nqImVRGgMV62P5_8wZBbaD06lr7kJA9JOBqtW3fco,4263
+ingestion/nextgen/process.py,sha256=kDCnU685v7aqJ3i4HpFdb7HqgHRSBKqtYPpuyN7qWmM,3976
+ingestion/nextgen/util/alteration_table.py,sha256=OqstLK6cgoNvRWy8bW6_iABaAn5ggCi1xBM8GOU6wYQ,6060
 ingestion/nextgen/util/interpretation.py,sha256=ozuzb0vozff34zfP6AdOiUmI8Q77hI02jve_nCPZHfE,297
-ingestion/nextgen/util/manifest_helpers.py,sha256=PpSay-pe62jk735nom1tVD9nDE8-CxmzzCrgpBhgtjY,1571
-ingestion/nextgen/util/nextgen_specific_genes.py,sha256=II_E2AgAqv35u_ga25geRn6UHuZy_Uk9itfyu_HybFY,1211
-ingestion/nextgen/util/pre_filter_somatic_vcf.py,sha256=K_gH4EnUXrKB22u_f8FqQVGrOS5LxXNsNO3VBn381eY,2301
-ingestion/nextgen/util/process_cnv.py,sha256=m-AhsXFlYw4LTzgJJaj5vXYbK5n3H7cImzBxD2To6M0,2598
-ingestion/nextgen/util/process_manifest.py,sha256=FOa-m78layb5TFTktaDHkHT9hAGUeH9ZPGeqgBncz64,8585
-ingestion/nextgen/util/process_structural.py,sha256=fUhoVGY5XHXPqLjC9bKD8JpXNDF9VbUQGl3Y3d7vG6E,7703
-ingestion/nextgen/util/process_vcf.py,sha256=TvyV5wyXXSpFy2lc6h3ljqVQ5VeDPGxihHl2_K6LalQ,8432
+ingestion/nextgen/util/manifest_helpers.py,sha256=2xrpEtHbCb1Kea1wJeObkDfTiBklmffQt_o2hMgOSOE,1208
+ingestion/nextgen/util/nextgen_specific_genes.py,sha256=hgam7HVE324FwOf7G4Wk4cUArch9vHIjBZRUUyF3ukg,1206
+ingestion/nextgen/util/pre_filter_somatic_vcf.py,sha256=mIaUihmGLbS38D4Gy_Qtf1lFAfW0A-LgAgQmsrEiI-M,3529
+ingestion/nextgen/util/process_cnv.py,sha256=MIirc8e0k6lsaTZkRM3U3L3IvbrcHmKQ4xlIu585514,2430
+ingestion/nextgen/util/process_manifest.py,sha256=EGYaTcub4M08mFTAh4CNHPRkP8_a5r4jMJaExm9Nkko,8423
+ingestion/nextgen/util/process_structural.py,sha256=FKjkK7BkIlocnLs8rFCjrMC39FCQnD0nQCeWvi7cRoA,7539
+ingestion/nextgen/util/process_vcf.py,sha256=SN0C13F45R_N5UaMaVSUDSCtIMmpHfaMTo7_5PkFkrM,8085
 ingestion/nextgen/util/types.py,sha256=SSzt5gv-kss1PR45eQUelypWrGI-dAfQMO3GSD-T-Wg,22
 ingestion/resources/GRCh37_map.csv.gz,sha256=JOEkjtbYrJpIdyoZdCvfJhvvz2dNfkSve7lXSXkCCD8,408290
 ingestion/resources/GRCh38_map.csv.gz,sha256=qriYO2_buCCb4T6WcuZ-pCwPxMsm0TL2OxAHvJ1cEfA,612373
@@ -54,6 +54,6 @@ ingestion/vcf_standardization/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 ingestion/vcf_standardization/util/af_helpers.py,sha256=dpTzoeIQVeBRt0ETF3a9rp5ojZqznHg4x_hCZ8OPcOg,1061
 ingestion/vcf_standardization/util/dp_helpers.py,sha256=Nq8oLOLObu4_pv16qwwgpALRlUoJVCULrd9cFOD-eoI,823
 ingestion/vcf_standardization/util/read_write.py,sha256=IQotJ27To1MoQcRstc5AbHZtUuJz5cqkkZiHsDNaBvI,2471
-phc_ingestion-0.8.31.dist-info/WHEEL,sha256=B19PGBCYhWaz2p_UjAoRVh767nYQfk14Sn4TpIZ-nfU,87
-phc_ingestion-0.8.31.dist-info/METADATA,sha256=2b0ZTRMP3dMucsx_0wMtLBripA-ak7EgMZsGCUbhT24,552
-phc_ingestion-0.8.31.dist-info/RECORD,,
+phc_ingestion-0.8.33.dist-info/WHEEL,sha256=B19PGBCYhWaz2p_UjAoRVh767nYQfk14Sn4TpIZ-nfU,87
+phc_ingestion-0.8.33.dist-info/METADATA,sha256=CM6kTtndCIkqq55vXl_x2vehEZ_mL29fK7TblsCsz9E,552
+phc_ingestion-0.8.33.dist-info/RECORD,,