phc-ingestion 0.9.3__tar.gz → 0.10.0__tar.gz

{phc-ingestion-0.9.3 → phc-ingestion-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: phc-ingestion
-Version: 0.9.3
+Version: 0.10.0
 Summary: Functions for LifeOmic PHC genomic ingestions
 License: MIT
 Author-email: LifeOmic Development <development@lifeomic.com>

{phc-ingestion-0.9.3 → phc-ingestion-0.10.0}/ingestion/caris/util/json.py

@@ -4,13 +4,13 @@ import json
 import os
 import shutil
 
-from ingestion.caris.util.tar import unpack
+from ingestion.shared_util.tar import unpack
 from ingestion.caris.util.metadata import extract_metadata
 from ingestion.caris.util.structural import extract_structural
 from ingestion.caris.util.cnv import extract_cnv
 from ingestion.caris.util.tsv import convert_tsv_to_rgel
 from ingestion.caris.util.vcf import extract_sv
-from ingestion.caris.util.ga4gh import create_yaml
+from ingestion.shared_util.ga4gh import create_yaml
 from logging import Logger
 
 CARIS_CASE_ID_LENGTH = len("TN22-000000")

phc-ingestion-0.10.0/ingestion/nebula/process.py

@@ -0,0 +1,25 @@
+import os
+
+from ingestion.nebula.util.json import process_nebula_json
+from ingestion.nebula.util.vcf import process_nebula_vcf
+from lifeomic_logging import scoped_logger
+
+
+def process_nebula(infile, outpath, file_name, source_file_id):
+
+    with scoped_logger(__name__) as log:
+        os.makedirs(f"{outpath}", exist_ok=True)
+        result, file_genome_references, json_data = process_nebula_json(
+            infile, outpath, file_name, source_file_id, log
+        )
+        somatic_vcf_line_count = process_nebula_vcf(
+            result["somatic_vcf"], json_data, outpath, file_name, log
+        )
+        case_metadata = {
+            "somatic_vcf_line_count": somatic_vcf_line_count,
+        }
+
+        if file_genome_references != {}:
+            case_metadata.update(file_genome_references)
+
+        return case_metadata

phc-ingestion-0.10.0/ingestion/nebula/util/json.py

@@ -0,0 +1,72 @@
+import glob
+import gzip
+import os
+import shutil
+
+from ingestion.shared_util.tar import unpack
+from ingestion.nebula.util.vcf import extract_sv
+from ingestion.shared_util.ga4gh import create_yaml
+from logging import Logger
+
+
+def handle_tsv(file: str, file_list: list[str]) -> dict[str, str]:
+    multiple_tsv = len([file for file in file_list if file.endswith("tsv")]) > 1
+
+    if not multiple_tsv or "Transformed" in file:
+        return {
+            "tsv": file,
+        }
+    return {}
+
+
+def process_nebula_json(
+    infile: str, outpath: str, file_name: str, source_file_id: str, log: Logger
+):
+    # Unpack tarball and go into the new directory
+    unpack(infile, outpath)
+    os.chdir(outpath)
+
+    file_list = glob.glob("*")
+    files: dict[str, str] = {}
+
+    for file in file_list:
+        extension = ".".join(file.split(".")[1:])
+        if file.endswith("vcf"):
+            files["somatic.vcf"] = file
+        else:
+            # There should only be the vcf file
+            files[extension] = file
+
+    log.info(f"Files in tarball input: {file_list}")
+
+    somatic_filename = None
+    data = {}
+    metadata = {}
+
+    # Sometimes they don't come in gzipped
+    for key in files.keys():
+        if "somatic.vcf" in key:
+            somatic_filename = files["somatic.vcf"].replace(".vcf", ".somatic.vcf") + ".gz"
+            with open(files["somatic.vcf"], "rb") as f_in:
+                with gzip.open(somatic_filename, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+
+    vcf_results = extract_sv(file_name, bool(somatic_filename), False, False)
+
+    # We might not have any of these files but we need an empty json object here.
+    file_genome_references = {}
+    if vcf_results:
+        metadata["files"] = metadata["files"] + vcf_results
+        for vcf in vcf_results:
+            seq_type = vcf.get("sequenceType")
+            file_genome_references[f"{seq_type}_genome_reference"] = vcf["reference"]
+
+    create_yaml(metadata, file_name)
+
+    # Return VCF files for immediate processing, and JSON data for adding vendsig
+    result = {}
+
+    if somatic_filename is not None:
+        result["somatic_vcf"] = f"{outpath}/{somatic_filename}"
+
+    return (result, file_genome_references, data)

phc-ingestion-0.10.0/ingestion/nebula/util/vcf.py

@@ -0,0 +1,100 @@
+import datetime
+import gzip
+
+from logging import Logger
+
+from ingestion.vcf_standardization.standardize import standardize_vcf
+
+
+def create_empty_vcf_zip(prefix):
+    vcf_gzip_path = f"{prefix}.modified.somatic.vcf.gz"
+    content = (
+        """##fileformat=VCFv4.1
+##filedate="""
+        + datetime.datetime.now().isoformat()
+        + """
+##FILTER=<ID=PASS,Description="All filters passed">
+##FILTER=<ID=R8,Description="IndelRepeatLength is greater than 8">
+##FILTER=<ID=R8.1,Description="IndelRepeatLength of a monomer is greater than 8">
+##FILTER=<ID=R8.2,Description="IndelRepeatLength of a dimer is greater than 8">
+##FILTER=<ID=sb,Description="Variant strand bias high">
+##FILTER=<ID=sb.s,Description="Variant strand bias significantly high (only for SNV)">
+##FILTER=<ID=rs,Description="Variant with rs (dbSNP) number in a non-core gene">
+##FILTER=<ID=FP,Description="Possibly false positives due to high similarity to off-target regions">
+##FILTER=<ID=NC,Description="Noncoding INDELs on non-core genes">
+##FILTER=<ID=lowDP,Description="low depth variant">
+##FILTER=<ID=Benign,Description="Benign variant">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=AF,Number=1,Type=String,Description="Variant Allele Frequency">
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	"""
+        + prefix
+        + """
+"""
+    )
+
+    with gzip.open(vcf_gzip_path, "wb") as f:
+        f.write(content.encode("utf-8"))
+
+
+# This is done in next step, we are just adding to yaml
+def extract_sv(prefix, include_somatic: bool):
+    vcfs = []
+
+    # Hard-code genome reference for Nebula VCFs
+    genome_reference = "GRCh38"
+
+    if include_somatic:
+        vcfs.append(
+            {
+                "fileName": f".lifeomic/nebula/{prefix}/{prefix}.modified.somatic.nrm.filtered.vcf.gz",
+                "sequenceType": "somatic",
+                "type": "shortVariant",  # todo: Is this always going to be shortVariant?
+                "reference": genome_reference,
+            }
+        )
+
+    return vcfs
+
+
+def get_vendsig_dict(json_data, log: Logger):
+    # Return a dicitionary of {'chr:star_pos:ref:alt' : 'vendsig'}
+    vendsig_dict = {"vendor": "nebula"}
+
+    return vendsig_dict
+
+
+def map_vendsig(ci: str) -> str:
+    if ci in ["Pathogenic Variant", "Pathogenic"]:
+        return "Pathogenic"
+    elif ci in ["Likely Pathogenic Variant", "Likely Pathogenic"]:
+        return "Likely pathogenic"
+    elif ci in ["Benign Variant", "Benign"]:
+        return "Benign"
+    elif ci in ["Likely Benign Variant", "Likely Benign"]:
+        return "Likely benign"
+    elif ci in ["Variant of Uncertain Significance", "VUS"]:
+        return "Uncertain significance"
+    else:
+        return "Unknown"
+
+
+def process_nebula_vcf(infile, json_data, outpath, file_name, log: Logger):
+    line_count = 0
+    vendsig_dict = {"vendor": "nebula"}
+
+    outfile = f"{file_name}.modified.somatic.vcf"
+    sample_name = file_name
+    # Read in a dictionary of variants with VENDSIG from the JSON file for somatic only
+    vendsig_dict = get_vendsig_dict(json_data, log)
+
+    line_count = standardize_vcf(
+        infile=infile,
+        outfile=outfile,
+        out_path=outpath,
+        case_id=sample_name,
+        log=log,
+        vendsig_dict=vendsig_dict,
+        compression=True,
+    )
+
+    return line_count

{phc-ingestion-0.9.3 → phc-ingestion-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "phc-ingestion"
-version = "0.9.3"
+version = "0.10.0"
 description = "Functions for LifeOmic PHC genomic ingestions"
 authors = [
     { name = "LifeOmic Development", email = "development@lifeomic.com" },

phc-ingestion-0.9.3/ingestion/nebula/process.py

@@ -1,46 +0,0 @@
-import os
-from pathlib import Path
-from ingestion.vcf_standardization.standardize import standardize_vcf
-from lifeomic_logging import scoped_logger
-
-
-def process(vcf_file: str, source_file_id: str, out_path: str, case_id: str) -> dict:
-    with scoped_logger(__name__) as log:
-        # TODO: do we need to take in + process the manifest file here?
-
-        # Process VCF
-        base_vcf_file = os.path.basename(vcf_file)
-        vcf_out = base_vcf_file.replace(".vcf", ".modified.vcf")
-        vcf_final = base_vcf_file.replace(".vcf", ".modified.nrm.filtered.vcf")
-        if not vcf_final.endswith(".gz"):
-            vcf_final = vcf_final + ".gz"
-
-        # Assuming Nebula VCFs are germline
-        sample_name = f"germline_{case_id}"
-        vcf_line_count = standardize_vcf(
-            vcf_file, vcf_out, out_path, sample_name, log, compression=True
-        )
-
-        # Create a basic manifest for the Nebula VCF
-        manifest = {
-            "testType": "Nebula",
-            "sourceFileId": source_file_id,
-            "reference": "GRCh38",  # Assuming GRCh38, adjust as needed
-            "resources": [{"fileName": f".lifeomic/nebula/{case_id}/{base_vcf_file}"}],
-            "files": [
-                {
-                    "fileName": f".lifeomic/nebula/{case_id}/{vcf_final}",
-                    "sequenceType": "germline",
-                    "type": "shortVariant",
-                }
-            ],
-        }
-
-        case_metadata = {
-            "test_type": "Nebula",
-            "vcf_line_count": vcf_line_count,
-            "case_id": case_id,
-            "germline_genome_reference": manifest["reference"],
-        }
-
-        return case_metadata, manifest