phc-ingestion 0.9.3__py3-none-any.whl → 0.10.1__py3-none-any.whl

ingestion/caris/util/json.py

@@ -4,13 +4,13 @@ import json
 import os
 import shutil
 
-from ingestion.caris.util.tar import unpack
+from ingestion.shared_util.tar import unpack
 from ingestion.caris.util.metadata import extract_metadata
 from ingestion.caris.util.structural import extract_structural
 from ingestion.caris.util.cnv import extract_cnv
 from ingestion.caris.util.tsv import convert_tsv_to_rgel
 from ingestion.caris.util.vcf import extract_sv
-from ingestion.caris.util.ga4gh import create_yaml
+from ingestion.shared_util.ga4gh import create_yaml
 from logging import Logger
 
 CARIS_CASE_ID_LENGTH = len("TN22-000000")

ingestion/nebula/process.py

@@ -1,46 +1,26 @@
 import os
-from pathlib import Path
-from ingestion.vcf_standardization.standardize import standardize_vcf
-from lifeomic_logging import scoped_logger
 
+from ingestion.nebula.util.json import process_nebula_json
+from ingestion.nebula.util.vcf import process_nebula_vcf
+from lifeomic_logging import scoped_logger
 
-def process(vcf_file: str, source_file_id: str, out_path: str, case_id: str) -> dict:
-    with scoped_logger(__name__) as log:
-        # TODO: do we need to take in + process the manifest file here?
 
-        # Process VCF
-        base_vcf_file = os.path.basename(vcf_file)
-        vcf_out = base_vcf_file.replace(".vcf", ".modified.vcf")
-        vcf_final = base_vcf_file.replace(".vcf", ".modified.nrm.filtered.vcf")
-        if not vcf_final.endswith(".gz"):
-            vcf_final = vcf_final + ".gz"
+def process(infile, outpath, file_name, source_file_id):
 
-        # Assuming Nebula VCFs are germline
-        sample_name = f"germline_{case_id}"
-        vcf_line_count = standardize_vcf(
-            vcf_file, vcf_out, out_path, sample_name, log, compression=True
+    with scoped_logger(__name__) as log:
+        log.info(f"Beginning Nebula ingestion for file: {file_name}")
+        os.makedirs(f"{outpath}", exist_ok=True)
+        result, file_genome_references, json_data = process_nebula_json(
+            infile, outpath, file_name, source_file_id, log
+        )
+        somatic_vcf_line_count = process_nebula_vcf(
+            result["somatic_vcf"], json_data, outpath, file_name, log
         )
-
-        # Create a basic manifest for the Nebula VCF
-        manifest = {
-            "testType": "Nebula",
-            "sourceFileId": source_file_id,
-            "reference": "GRCh38",  # Assuming GRCh38, adjust as needed
-            "resources": [{"fileName": f".lifeomic/nebula/{case_id}/{base_vcf_file}"}],
-            "files": [
-                {
-                    "fileName": f".lifeomic/nebula/{case_id}/{vcf_final}",
-                    "sequenceType": "germline",
-                    "type": "shortVariant",
-                }
-            ],
-        }
-
         case_metadata = {
-            "test_type": "Nebula",
-            "vcf_line_count": vcf_line_count,
-            "case_id": case_id,
-            "germline_genome_reference": manifest["reference"],
+            "somatic_vcf_line_count": somatic_vcf_line_count,
         }
 
-        return case_metadata, manifest
+        if file_genome_references != {}:
+            case_metadata.update(file_genome_references)
+
+        return case_metadata

ingestion/nebula/util/json.py

@@ -0,0 +1,72 @@
+import glob
+import gzip
+import os
+import shutil
+
+from ingestion.shared_util.tar import unpack
+from ingestion.nebula.util.vcf import extract_sv
+from ingestion.shared_util.ga4gh import create_yaml
+from logging import Logger
+
+
+def handle_tsv(file: str, file_list: list[str]) -> dict[str, str]:
+    multiple_tsv = len([file for file in file_list if file.endswith("tsv")]) > 1
+
+    if not multiple_tsv or "Transformed" in file:
+        return {
+            "tsv": file,
+        }
+    return {}
+
+
+def process_nebula_json(
+    infile: str, outpath: str, file_name: str, source_file_id: str, log: Logger
+):
+    # Unpack tarball and go into the new directory
+    unpack(infile, outpath)
+    os.chdir(outpath)
+
+    file_list = glob.glob("*")
+    files: dict[str, str] = {}
+
+    for file in file_list:
+        extension = ".".join(file.split(".")[1:])
+        if file.endswith("vcf"):
+            files["somatic.vcf"] = file
+        else:
+            # There should only be the vcf file
+            files[extension] = file
+
+    log.info(f"Files in tarball input: {file_list}")
+
+    somatic_filename = None
+    data = {}
+    metadata = {}
+
+    # Sometimes they don't come in gzipped
+    for key in files.keys():
+        if "somatic.vcf" in key:
+            somatic_filename = files["somatic.vcf"].replace(".vcf", ".somatic.vcf") + ".gz"
+            with open(files["somatic.vcf"], "rb") as f_in:
+                with gzip.open(somatic_filename, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+
+    vcf_results = extract_sv(file_name, bool(somatic_filename), False, False)
+
+    # We might not have any of these files but we need an empty json object here.
+    file_genome_references = {}
+    if vcf_results:
+        metadata["files"] = metadata["files"] + vcf_results
+        for vcf in vcf_results:
+            seq_type = vcf.get("sequenceType")
+            file_genome_references[f"{seq_type}_genome_reference"] = vcf["reference"]
+
+    create_yaml(metadata, file_name)
+
+    # Return VCF files for immediate processing, and JSON data for adding vendsig
+    result = {}
+
+    if somatic_filename is not None:
+        result["somatic_vcf"] = f"{outpath}/{somatic_filename}"
+
+    return (result, file_genome_references, data)

ingestion/nebula/util/vcf.py

@@ -0,0 +1,100 @@
+import datetime
+import gzip
+
+from logging import Logger
+
+from ingestion.vcf_standardization.standardize import standardize_vcf
+
+
+def create_empty_vcf_zip(prefix):
+    vcf_gzip_path = f"{prefix}.modified.somatic.vcf.gz"
+    content = (
+        """##fileformat=VCFv4.1
+##filedate="""
+        + datetime.datetime.now().isoformat()
+        + """
+##FILTER=<ID=PASS,Description="All filters passed">
+##FILTER=<ID=R8,Description="IndelRepeatLength is greater than 8">
+##FILTER=<ID=R8.1,Description="IndelRepeatLength of a monomer is greater than 8">
+##FILTER=<ID=R8.2,Description="IndelRepeatLength of a dimer is greater than 8">
+##FILTER=<ID=sb,Description="Variant strand bias high">
+##FILTER=<ID=sb.s,Description="Variant strand bias significantly high (only for SNV)">
+##FILTER=<ID=rs,Description="Variant with rs (dbSNP) number in a non-core gene">
+##FILTER=<ID=FP,Description="Possibly false positives due to high similarity to off-target regions">
+##FILTER=<ID=NC,Description="Noncoding INDELs on non-core genes">
+##FILTER=<ID=lowDP,Description="low depth variant">
+##FILTER=<ID=Benign,Description="Benign variant">
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=AF,Number=1,Type=String,Description="Variant Allele Frequency">
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	"""
+        + prefix
+        + """
+"""
+    )
+
+    with gzip.open(vcf_gzip_path, "wb") as f:
+        f.write(content.encode("utf-8"))
+
+
+# This is done in next step, we are just adding to yaml
+def extract_sv(prefix, include_somatic: bool):
+    vcfs = []
+
+    # Hard-code genome reference for Nebula VCFs
+    genome_reference = "GRCh38"
+
+    if include_somatic:
+        vcfs.append(
+            {
+                "fileName": f".lifeomic/nebula/{prefix}/{prefix}.modified.somatic.nrm.filtered.vcf.gz",
+                "sequenceType": "somatic",
+                "type": "shortVariant",  # todo: Is this always going to be shortVariant?
+                "reference": genome_reference,
+            }
+        )
+
+    return vcfs
+
+
+def get_vendsig_dict(json_data, log: Logger):
+    # Return a dicitionary of {'chr:star_pos:ref:alt' : 'vendsig'}
+    vendsig_dict = {"vendor": "nebula"}
+
+    return vendsig_dict
+
+
+def map_vendsig(ci: str) -> str:
+    if ci in ["Pathogenic Variant", "Pathogenic"]:
+        return "Pathogenic"
+    elif ci in ["Likely Pathogenic Variant", "Likely Pathogenic"]:
+        return "Likely pathogenic"
+    elif ci in ["Benign Variant", "Benign"]:
+        return "Benign"
+    elif ci in ["Likely Benign Variant", "Likely Benign"]:
+        return "Likely benign"
+    elif ci in ["Variant of Uncertain Significance", "VUS"]:
+        return "Uncertain significance"
+    else:
+        return "Unknown"
+
+
+def process_nebula_vcf(infile, json_data, outpath, file_name, log: Logger):
+    line_count = 0
+    vendsig_dict = {"vendor": "nebula"}
+
+    outfile = f"{file_name}.modified.somatic.vcf"
+    sample_name = file_name
+    # Read in a dictionary of variants with VENDSIG from the JSON file for somatic only
+    vendsig_dict = get_vendsig_dict(json_data, log)
+
+    line_count = standardize_vcf(
+        infile=infile,
+        outfile=outfile,
+        out_path=outpath,
+        case_id=sample_name,
+        log=log,
+        vendsig_dict=vendsig_dict,
+        compression=True,
+    )
+
+    return line_count

{phc_ingestion-0.9.3.dist-info → phc_ingestion-0.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: phc-ingestion
-Version: 0.9.3
+Version: 0.10.1
 Summary: Functions for LifeOmic PHC genomic ingestions
 License: MIT
 Author-email: LifeOmic Development <development@lifeomic.com>

{phc_ingestion-0.9.3.dist-info → phc_ingestion-0.10.1.dist-info}/RECORD

@@ -4,15 +4,13 @@ ingestion/caris/process.py,sha256=7GaSTg9LjqSn7iTNbY3FJIPFvVjkd6wgXiVC00pn-7A,14
 ingestion/caris/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestion/caris/util/cnv.py,sha256=Nrc0aoG2k4tmrqHb69hAuXr0adDZIVVJRRjf9_sO91E,4441
 ingestion/caris/util/detect_genome_ref.py,sha256=MpiPa71QmlO3MWvjxPzNdbEHyOhorOcpQWWlwE5BO4c,1640
-ingestion/caris/util/ga4gh.py,sha256=-jNQj79zspxG67MxHzOfwAhLbb9je55M1h4-i5ri-tU,507
 ingestion/caris/util/hla.py,sha256=X_t6ngBRvmdG3m4I2_KnPFeWn3BaH-3IWHtOvDbS32A,770
 ingestion/caris/util/ihc.py,sha256=vegxudxHj7tLihrXGbEx_ptwkSsu3YCCB1nZVwoiYXg,12312
 ingestion/caris/util/interpretation.py,sha256=CghNurqeVA5VTBBorU8-ZTN-PVNPnR8wrmTwKCH3568,555
-ingestion/caris/util/json.py,sha256=HBU3Tf-XSi9fGHANYUtD8maXNYqmmnpncGh0KCDaPEU,5018
+ingestion/caris/util/json.py,sha256=aifO1hnZwNSS-ZtY20otyGbfIoc23w9HMWJ5D56lhFo,5020
 ingestion/caris/util/metadata.py,sha256=a6NToMtGtIRrlMd3CQwq4IRjGGmIiBA9JFwsATjNEoQ,10287
 ingestion/caris/util/specimen_details.py,sha256=wDTIelrGelAuSljmE6dAoJRjb9kwrlhmB2f1_vcCiUc,2963
 ingestion/caris/util/structural.py,sha256=EUcMIea_WnafoVmFLIyEqlJ_HtYIj_g6qkekXa7QNQs,4628
-ingestion/caris/util/tar.py,sha256=BGR_2vBbxyMgF-GzJ3SrihsPdOzII4SFVz9tvKV5vo0,482
 ingestion/caris/util/tests.py,sha256=mcG3A8TW81_sn2Bfoa-Gd6Q1sR3_R4FX2BNskD4DkJk,372
 ingestion/caris/util/tmb.py,sha256=DVi1wPSjVr_32ZCc6Yb51tGqUlcxUx40yCvuqvNuDx4,1027
 ingestion/caris/util/tsv.py,sha256=xeIfDUtqG_5ewkbaPLakqm4kQlu6ClgkAf4tefqKlJA,1595
@@ -29,7 +27,9 @@ ingestion/generic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 ingestion/generic/process.py,sha256=ZaVnZ_gx9faDUsuresI1A0oCegTa-dPQT7DBFMeZGyY,1777
 ingestion/generic/utils.py,sha256=1MEIru7uq38IjUdL8lcHqDH0oTki9uWrz1f2e-pmRoU,2814
 ingestion/nebula/__init__.py,sha256=VauK-rup_N8ZXVohx3HYqHX_PE_WoPyMUhdv2R7al4o,45
-ingestion/nebula/process.py,sha256=Ss2hReq9dAYFcBC4r89RYKcdE6cdUNXvgH2VMBijJPM,1687
+ingestion/nebula/process.py,sha256=r4zKxVRPzl0pI-OInGIy8V_Z2K4UW_iIf9ggUvGZZlk,893
+ingestion/nebula/util/json.py,sha256=ChThuJjF7o4xMQ4gkX7s2w9wgZ5_k528XxiSI3-gOCU,2230
+ingestion/nebula/util/vcf.py,sha256=83JWG6ndPxHYZNXVj6-Gon9AY9JnGeR1JMivUQoSHjM,3311
 ingestion/nextgen/__init__.py,sha256=7LQ-h_Bvc5P1QcHMdzsqi1Qm4fTJn04-ozar2ty9wSc,59
 ingestion/nextgen/process.py,sha256=5Z0RfclwTAYZruGDiLPutjPCYFh1DJpoWY9dnttghT4,3993
 ingestion/nextgen/util/alteration_table.py,sha256=JTWBL1Fqj_pGsH5vwuVEnCUJle2wOBk6VYImHYCF9vg,6129
@@ -46,8 +46,10 @@ ingestion/resources/GRCh37_map.csv.gz,sha256=JOEkjtbYrJpIdyoZdCvfJhvvz2dNfkSve7l
 ingestion/resources/GRCh38_map.csv.gz,sha256=qriYO2_buCCb4T6WcuZ-pCwPxMsm0TL2OxAHvJ1cEfA,612373
 ingestion/shared_util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestion/shared_util/coords_to_genes.py,sha256=vz9EfgFm3BS6pEPnslbEka8cJKlQZtHJdH2WRCCUMdE,1669
+ingestion/shared_util/ga4gh.py,sha256=-jNQj79zspxG67MxHzOfwAhLbb9je55M1h4-i5ri-tU,507
 ingestion/shared_util/gene_to_coords.py,sha256=M-q5ateLSQ4fCF0uMk5TX2uBLRrcZzXqXEf05TPaLsU,876
 ingestion/shared_util/open_maybe_gzipped.py,sha256=FrOPJ4OgfpQGyT3f1Su1rFeuuYYu6QJ-nVIBIosbfhw,232
+ingestion/shared_util/tar.py,sha256=BGR_2vBbxyMgF-GzJ3SrihsPdOzII4SFVz9tvKV5vo0,482
 ingestion/shared_util/types.py,sha256=u9AD2OrTQWMBtK_7VXHsD8Rv6HFs-7ZUItNl4KXdL7k,68
 ingestion/vcf_standardization/Variant.py,sha256=aoSqT4XAECxCF0JZgv1YRmxuw20WGeWkwFTw0x6FmWc,5475
 ingestion/vcf_standardization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -56,6 +58,6 @@ ingestion/vcf_standardization/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 ingestion/vcf_standardization/util/af_helpers.py,sha256=dpTzoeIQVeBRt0ETF3a9rp5ojZqznHg4x_hCZ8OPcOg,1061
 ingestion/vcf_standardization/util/dp_helpers.py,sha256=Nq8oLOLObu4_pv16qwwgpALRlUoJVCULrd9cFOD-eoI,823
 ingestion/vcf_standardization/util/read_write.py,sha256=x3Pf6Dq8tmolblbCS5CrNmrcHS3FGfqBSFpFgvFGC4g,2526
-phc_ingestion-0.9.3.dist-info/WHEEL,sha256=B19PGBCYhWaz2p_UjAoRVh767nYQfk14Sn4TpIZ-nfU,87
-phc_ingestion-0.9.3.dist-info/METADATA,sha256=BeKjYNu_HC4PvcMS57E38mys7Q2gvpcrDTvaI0vvdPs,572
-phc_ingestion-0.9.3.dist-info/RECORD,,
+phc_ingestion-0.10.1.dist-info/WHEEL,sha256=B19PGBCYhWaz2p_UjAoRVh767nYQfk14Sn4TpIZ-nfU,87
+phc_ingestion-0.10.1.dist-info/METADATA,sha256=TKRNABWds66BCmH2r2QoLxqcqNEZzUsom92zWJdx2aw,573
+phc_ingestion-0.10.1.dist-info/RECORD,,