phc-ingestion 0.8.36__tar.gz → 0.8.38__tar.gz

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: phc-ingestion
-Version: 0.8.36
+Version: 0.8.38
 Summary: Functions for LifeOmic PHC genomic ingestions
 License: MIT
 Author-email: LifeOmic Development <development@lifeomic.com>

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/ingestion/caris/util/json.py

@@ -84,11 +84,17 @@ def process_caris_json(infile: str, outpath: str, file_name: str, source_file_id
                 ]
 
     # Get patient
-    metadata = extract_metadata(data, file_name, files, source_file_id, log)
+    metadata, is_test_cancelled_permit_vcf_skip = extract_metadata(
+        data, file_name, files, source_file_id, log
+    )
     structural_results = extract_structural(file_name, data, log)
     cnv_results = extract_cnv(file_name, data, log)
     rgel_results = convert_tsv_to_rgel(file_name, files, log)
-    vcf_results = extract_sv(file_name, bool(somatic_filename), bool(germline_filename))
+
+    include_empty = metadata["ihcTests"] and is_test_cancelled_permit_vcf_skip
+    vcf_results = extract_sv(
+        file_name, bool(somatic_filename), bool(germline_filename), include_empty
+    )
 
     # We might not have any of these files but we need an empty json object here.
     file_genome_references = {}
@@ -117,5 +123,7 @@ def process_caris_json(infile: str, outpath: str, file_name: str, source_file_id
         result["somatic_vcf"] = f"{outpath}/{somatic_filename}"
     if germline_filename is not None:
         result["germline_vcf"] = f"{outpath}/{germline_filename}"
+    if not germline_filename and not somatic_filename and include_empty:
+        result["somatic_vcf"] = f"{outpath}/{file_name}.modified.somatic.vcf.gz"
 
     return (result, germline_case_id, file_genome_references, data)

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/ingestion/caris/util/metadata.py

@@ -80,6 +80,7 @@ def is_valid_test_entry(test: dict):
 
 # Build up the manifest iteratively because almost everything is optional
 def extract_metadata(data, prefix, files, source_file_id, log: Logger) -> dict:
+    is_test_cancelled_permit_vcf_skip = False
     metadata = {}
 
     test_details = data["testDetails"]
@@ -158,6 +159,13 @@ def extract_metadata(data, prefix, files, source_file_id, log: Logger) -> dict:
     # if not sufficient quantity we won't have test results
     if test_details["reportType"] != "QNS":
         for test in tests:
+            if "test_cancellation_reason" in test:
+                if test["test_cancellation_reason"] == "Quantitation quantity not sufficient":
+                    # capture cancellation reason before bailing
+                    # this is so we can generate an empty vcf so present biomarkers are
+                    # still ingested: https://lifeomic.atlassian.net/browse/PHC-5748
+                    is_test_cancelled_permit_vcf_skip = True
+
             if not is_valid_test_entry(test):
                 continue
             # Sometimes, if there is only a single test result,
@@ -244,4 +252,4 @@ def extract_metadata(data, prefix, files, source_file_id, log: Logger) -> dict:
                 )
 
     active_metadata = {k: v for k, v in metadata.items() if v is not None}
-    return active_metadata
+    return (active_metadata, is_test_cancelled_permit_vcf_skip)

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/ingestion/caris/util/vcf.py

@@ -1,11 +1,5 @@
 import datetime
 import gzip
-import io
-import os
-import re
-import subprocess
-import sys
-import zipfile
 
 from logging import Logger
 
@@ -13,8 +7,38 @@ from ingestion.caris.util.tests import safely_extract_tests_from_json_data
 from ingestion.vcf_standardization.standardize import standardize_vcf
 
 
+def create_empty_vcf_zip(prefix):
+    vcf_gzip_path = f"{prefix}.modified.somatic.vcf.gz"
+    content = (
+        """##fileformat=VCFv4.1
+##filedate="""
+        + datetime.datetime.now().isoformat()
+        + """
+##FILTER=<ID=PASS,Description="All filters passed">
+##FILTER=<ID=R8,Description="IndelRepeatLength is greater than 8">
+##FILTER=<ID=R8.1,Description="IndelRepeatLength of a monomer is greater than 8">
+##FILTER=<ID=R8.2,Description="IndelRepeatLength of a dimer is greater than 8">
+##FILTER=<ID=sb,Description="Variant strand bias high">
+##FILTER=<ID=sb.s,Description="Variant strand bias significantly high (only for SNV)">
+##FILTER=<ID=rs,Description="Variant with rs (dbSNP) number in a non-core gene">
+##FILTER=<ID=FP,Description="Possibly false positives due to high similarity to off-target regions">
+##FILTER=<ID=NC,Description="Noncoding INDELs on non-core genes">
+##FILTER=<ID=lowDP,Description="low depth variant">
+##FILTER=<ID=Benign,Description="Benign variant">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=AF,Number=1,Type=String,Description="Variant Allele Frequency">
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	"""
+        + prefix
+        + """
+"""
+    )
+
+    with gzip.open(vcf_gzip_path, "wb") as f:
+        f.write(content.encode("utf-8"))
+
+
 # This is done in next step, we are just adding to yaml
-def extract_sv(prefix, include_somatic: bool, include_germline: bool):
+def extract_sv(prefix, include_somatic: bool, include_germline: bool, include_empty: bool):
     vcfs = []
 
     # Hard-code genome reference for Caris VCFs
@@ -40,6 +64,17 @@ def extract_sv(prefix, include_somatic: bool, include_germline: bool):
             }
         )
 
+    if not vcfs and include_empty:
+        create_empty_vcf_zip(prefix)
+        vcfs.append(
+            {
+                "fileName": f".lifeomic/caris/{prefix}/{prefix}.modified.somatic.nrm.filtered.vcf.gz",
+                "sequenceType": "somatic",
+                "type": "shortVariant",
+                "reference": genome_reference,
+            }
+        )
+
     return vcfs
 
 

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/ingestion/foundation/util/vcf_etl.py

@@ -1,6 +1,5 @@
 import gzip
 import xmltodict
-from natsort import natsorted
 from logging import Logger
 import re
 import os

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/ingestion/nextgen/process.py

@@ -68,6 +68,7 @@ def process(
             bool(structural_path_name),
             translocations,
             hyperdiploidy_chromosomes,
+            log,
         )
         pre_filtered_somatic_vcf_path = pre_filter_somatic_vcf(
             vendor_files["somaticVcfFile"],

phc-ingestion-0.8.38/ingestion/nextgen/util/manifest_helpers.py

@@ -0,0 +1,88 @@
+from logging import Logger
+import re
+
+
+def parse_pattern(pattern: str, line: str, name: str) -> str:
+    regex = re.compile(pattern)
+    match = regex.match(line)
+
+    if not match:
+        raise ValueError(f"Could not parse {name} from line")
+
+    return match.group(1).strip()
+
+
+def parse_indication(line: str) -> str:
+    return parse_pattern(r"^.*Reason for Referral:(.*?)(Patient|<).*$", line, "indication")
+
+
+def parse_ordering_md(line: str) -> str:
+    return parse_pattern(r"^.*Physician Name:(.*?)(Reason|<).*$", line, "ordering MD")
+
+
+def parse_patient_name(line: str) -> str:
+    return parse_pattern(r"^.*Patient Name: (.*?)(Accession|<).*$", line, "patient name")
+
+
+def parse_sample_number(line: str) -> str:
+    return parse_pattern(r"^.*Specimen #: (\d*) .*$", line, "sample number")
+
+
+def parse_body_site(line: str) -> str:
+    return parse_pattern(r"^.*Specimen:(.*?)(Age|Birthdate|<).*$", line, "body site")
+
+
+def parse_report_id(line: str) -> str:
+    return parse_pattern(r"^.*Accession #: (.*?) .*$", line, "report ID")
+
+
+def parse_report_date_single_line(line: str) -> str:
+    return parse_pattern(
+        r"^.*Diagnostic Genomics Laboratory.*(\d{2}\/\d{2}\/\d{4}).*$", line, "report date"
+    )
+
+
+def parse_report_date_multiline(patient_info_lines: list[str]) -> str:
+    in_range_trigger = False
+
+    for line in patient_info_lines:
+        if "Laboratory" in line:
+            in_range_trigger = True
+            continue
+        if in_range_trigger:
+            formatted_line = re.sub(r"<\/?T.\/?>", "", line).strip()
+            if not formatted_line:
+                continue
+            return parse_pattern(
+                r"^.*(\d{2}\/\d{2}\/\d{4}).*$", formatted_line, "report date from multiline"
+            )
+
+    raise ValueError("Could not parse report date from lines")
+
+
+def parse_report_date(patient_info_lines: list[str], log: Logger) -> str:
+    """
+    Typically, the report date is in a form like:
+    ```
+    Diagnostic Genomics Laboratory 01/01/2021
+    ```
+
+    However, sometimes the date is split across multiple lines, like:
+    ```
+    Diagnostic Genomics Laboratory
+    ...random empty lines or lines with only tags...
+    01/01/2021
+    ```
+    This function attempts to first parse the date from a single line, and if that fails,
+    it will attempt to parse it from multiple lines.
+    """
+    for line in patient_info_lines:
+        if "Laboratory" in line:
+            try:
+                report_date = parse_report_date_single_line(line)
+                return report_date
+            except ValueError:
+                log.warning("Could not parse report date from single line")
+                break
+
+    return parse_report_date_multiline(patient_info_lines)

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/ingestion/nextgen/util/process_manifest.py

@@ -17,7 +17,7 @@ def search_and_grab(array: list, search_item: str, grab_index: int):
     return array[array.index([i for i in array if re.search(search_item, i)][0]) + grab_index]
 
 
-def extract_xml_text(xml_in_file: str):
+def extract_xml_text(xml_in_file: str) -> list[str]:
     with open(xml_in_file, "r") as f:
         xml_lines = f.readlines()
 
@@ -35,7 +35,7 @@ def extract_xml_text(xml_in_file: str):
     return patient_info_lines
 
 
-def extract_interpretation_text(xml_in_file: str):
+def extract_interpretation_text(xml_in_file: str) -> list[str]:
     with open(xml_in_file, "r") as f:
         xml_lines = f.readlines()
 
@@ -111,7 +111,7 @@ def extract_patient_data(patient_info_lines: list[str]):
     return patient_data
 
 
-def extract_test_data(patient_info_lines: list, interpretation_lines: list):
+def extract_test_data(patient_info_lines: list[str], interpretation_lines: list[str], log: Logger):
     # Initialize manifest and hard-code some values
     manifest: dict[str, Any] = {}
     manifest["testType"] = "Plasma Cell Myeloma Panel"
@@ -128,12 +128,11 @@ def extract_test_data(patient_info_lines: list, interpretation_lines: list):
     manifest["medFacilID"] = ""
     manifest["medFacilName"] = "IU Health"
 
-    for line in patient_info_lines:
-        if "reportDate" not in manifest and "Laboratory" in line:
-            report_date = manifest_helpers.parse_report_date(line)
-            manifest["reportDate"] = transform_date(report_date)
-            manifest["indexedDate"] = manifest["reportDate"]
+    report_date = manifest_helpers.parse_report_date(patient_info_lines, log)
+    manifest["reportDate"] = transform_date(report_date)
+    manifest["indexedDate"] = manifest["reportDate"]
 
+    for line in patient_info_lines:
         if "collDate" not in manifest and "Collected" in line:
             collArray = line.split(" ")
             coll_date = search_and_grab(collArray, "Collected", 1)
@@ -178,10 +177,11 @@ def process_manifest(
     include_structural: bool,
     somatic_translocations: list[str],
     hyperdiploidy_chromosomes: list[str] | None,
+    log: Logger,
 ):
     test_text = extract_xml_text(xml_in_file)
     interpretation_text = extract_interpretation_text(xml_in_file)
-    manifest = extract_test_data(test_text, interpretation_text)
+    manifest = extract_test_data(test_text, interpretation_text, log)
     manifest.update(extract_patient_data(test_text))
 
     file_prefix = f".lifeomic/nextgen/{case_id}/{case_id}"

{phc-ingestion-0.8.36 → phc-ingestion-0.8.38}/pyproject.toml

@@ -1,19 +1,20 @@
 [project]
 name = "phc-ingestion"
-version = "0.8.36"
+version = "0.8.38"
 description = "Functions for LifeOmic PHC genomic ingestions"
 authors = [
     { name = "LifeOmic Development", email = "development@lifeomic.com" },
 ]
 dependencies = [
     "lifeomic-logging>=0.3.2,<0.4.0",
-    "xmltodict==0.13.0",
+    "xmltodict>=0.14.2",
     "natsort==7.1.1",
     "ruamel.yaml==0.17.21",
-    "pandas>=1.5.0,<1.6.0",
+    "pandas>=2.2.3",
     "jsonschema>=4.16.0,<5.0.0",
     "schema>=0.7.5",
     "packaging>=23.1",
+    "numpy>=2.1.2",
 ]
 requires-python = ">=3.11"
 readme = "PYPI.md"

phc-ingestion-0.8.36/ingestion/nextgen/util/manifest_helpers.py

@@ -1,42 +0,0 @@
-from logging import Logger
-import re
-
-
-def parse_pattern(pattern: str, line: str, name: str) -> str:
-    regex = re.compile(pattern)
-    match = regex.match(line)
-
-    if not match:
-        raise ValueError(f"Could not parse {name} from line")
-
-    return match.group(1).strip()
-
-
-def parse_indication(line: str) -> str:
-    return parse_pattern(r"^.*Reason for Referral:(.*?)(Patient|<).*$", line, "indication")
-
-
-def parse_ordering_md(line: str) -> str:
-    return parse_pattern(r"^.*Physician Name:(.*?)(Reason|<).*$", line, "ordering MD")
-
-
-def parse_patient_name(line: str) -> str:
-    return parse_pattern(r"^.*Patient Name: (.*?)(Accession|<).*$", line, "patient name")
-
-
-def parse_sample_number(line: str) -> str:
-    return parse_pattern(r"^.*Specimen #: (\d*) .*$", line, "sample number")
-
-
-def parse_body_site(line: str) -> str:
-    return parse_pattern(r"^.*Specimen:(.*?)(Age|Birthdate|<).*$", line, "body site")
-
-
-def parse_report_id(line: str) -> str:
-    return parse_pattern(r"^.*Accession #: (.*?) .*$", line, "report ID")
-
-
-def parse_report_date(line: str) -> str:
-    return parse_pattern(
-        r"^.*Diagnostic Genomics Laboratory.*(\d{2}\/\d{2}\/\d{4}).*$", line, "report date"
-    )