phc-ingestion 0.10.1__py3-none-any.whl → 0.10.2__py3-none-any.whl

ingestion/nebula/process.py

@@ -1,26 +1,51 @@
 import os
 
-from ingestion.nebula.util.json import process_nebula_json
-from ingestion.nebula.util.vcf import process_nebula_vcf
+from ingestion.vcf_standardization.standardize import standardize_vcf
 from lifeomic_logging import scoped_logger
 
 
-def process(infile, outpath, file_name, source_file_id):
+def process(vcf_file, out_path, file_name, source_file_id):
 
     with scoped_logger(__name__) as log:
-        log.info(f"Beginning Nebula ingestion for file: {file_name}")
-        os.makedirs(f"{outpath}", exist_ok=True)
-        result, file_genome_references, json_data = process_nebula_json(
-            infile, outpath, file_name, source_file_id, log
+        log.info(
+            f"Beginning Nebula ingestion for vcf_file: {vcf_file}, file_name: {file_name}, out_path: {out_path}, source_file_id: {source_file_id}"
         )
-        somatic_vcf_line_count = process_nebula_vcf(
-            result["somatic_vcf"], json_data, outpath, file_name, log
+
+        manifest = {}
+        case_id = file_name
+        base_vcf_file = os.path.basename(vcf_file)
+        vcf_out = base_vcf_file.replace(".vcf", ".modified.vcf")
+        vcf_final = base_vcf_file.replace(".vcf", ".modified.nrm.filtered.vcf")
+        if not vcf_final.endswith(".gz"):
+            vcf_final = vcf_final + ".gz"
+
+        # All Nebula VCF ingestions are germline, so ensure the
+        # sample name is prefixed with "germline_". This matches
+        # the downstream logic in genomic-manifest
+        sample_name = f"germline_{case_id}"
+        vcf_line_count = standardize_vcf(
+            vcf_file, vcf_out, out_path, sample_name, log, compression=True
         )
+
+        # Add to manifest
+        manifest["sourceFileId"] = source_file_id
+        manifest["resources"] = [{"fileName": f".lifeomic/vcf-ingest/{case_id}/{base_vcf_file}"}]
+        manifest["files"] = [
+            {
+                "fileName": f".lifeomic/vcf-ingest/{case_id}/{vcf_final}",
+                "sequenceType": "germline",
+                "type": "shortVariant",
+            }
+        ]
+
+        # Hard-code genome reference for Nebula VCFs
+        genome_reference = "GRCh38"
+
         case_metadata = {
-            "somatic_vcf_line_count": somatic_vcf_line_count,
+            "test_type": manifest["testType"],
+            "vcf_line_count": vcf_line_count,
+            "case_id": manifest["reportID"],
+            "germline_genome_reference": genome_reference,
         }
 
-        if file_genome_references != {}:
-            case_metadata.update(file_genome_references)
-
-        return case_metadata
+        return case_metadata, manifest

{phc_ingestion-0.10.1.dist-info → phc_ingestion-0.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: phc-ingestion
-Version: 0.10.1
+Version: 0.10.2
 Summary: Functions for LifeOmic PHC genomic ingestions
 License: MIT
 Author-email: LifeOmic Development <development@lifeomic.com>

{phc_ingestion-0.10.1.dist-info → phc_ingestion-0.10.2.dist-info}/RECORD

@@ -27,9 +27,7 @@ ingestion/generic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 ingestion/generic/process.py,sha256=ZaVnZ_gx9faDUsuresI1A0oCegTa-dPQT7DBFMeZGyY,1777
 ingestion/generic/utils.py,sha256=1MEIru7uq38IjUdL8lcHqDH0oTki9uWrz1f2e-pmRoU,2814
 ingestion/nebula/__init__.py,sha256=VauK-rup_N8ZXVohx3HYqHX_PE_WoPyMUhdv2R7al4o,45
-ingestion/nebula/process.py,sha256=r4zKxVRPzl0pI-OInGIy8V_Z2K4UW_iIf9ggUvGZZlk,893
-ingestion/nebula/util/json.py,sha256=ChThuJjF7o4xMQ4gkX7s2w9wgZ5_k528XxiSI3-gOCU,2230
-ingestion/nebula/util/vcf.py,sha256=83JWG6ndPxHYZNXVj6-Gon9AY9JnGeR1JMivUQoSHjM,3311
+ingestion/nebula/process.py,sha256=kIFhndcXFUICKMg4PfFhEsBxb_0oPkGPnpVNHhHFJ7k,1853
 ingestion/nextgen/__init__.py,sha256=7LQ-h_Bvc5P1QcHMdzsqi1Qm4fTJn04-ozar2ty9wSc,59
 ingestion/nextgen/process.py,sha256=5Z0RfclwTAYZruGDiLPutjPCYFh1DJpoWY9dnttghT4,3993
 ingestion/nextgen/util/alteration_table.py,sha256=JTWBL1Fqj_pGsH5vwuVEnCUJle2wOBk6VYImHYCF9vg,6129
@@ -58,6 +56,6 @@ ingestion/vcf_standardization/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 ingestion/vcf_standardization/util/af_helpers.py,sha256=dpTzoeIQVeBRt0ETF3a9rp5ojZqznHg4x_hCZ8OPcOg,1061
 ingestion/vcf_standardization/util/dp_helpers.py,sha256=Nq8oLOLObu4_pv16qwwgpALRlUoJVCULrd9cFOD-eoI,823
 ingestion/vcf_standardization/util/read_write.py,sha256=x3Pf6Dq8tmolblbCS5CrNmrcHS3FGfqBSFpFgvFGC4g,2526
-phc_ingestion-0.10.1.dist-info/WHEEL,sha256=B19PGBCYhWaz2p_UjAoRVh767nYQfk14Sn4TpIZ-nfU,87
-phc_ingestion-0.10.1.dist-info/METADATA,sha256=TKRNABWds66BCmH2r2QoLxqcqNEZzUsom92zWJdx2aw,573
-phc_ingestion-0.10.1.dist-info/RECORD,,
+phc_ingestion-0.10.2.dist-info/WHEEL,sha256=B19PGBCYhWaz2p_UjAoRVh767nYQfk14Sn4TpIZ-nfU,87
+phc_ingestion-0.10.2.dist-info/METADATA,sha256=HuebNLig9ccPPhZrF9It2bxP7p4ZWf0LGiz_rt__1Zk,573
+phc_ingestion-0.10.2.dist-info/RECORD,,

ingestion/nebula/util/json.py

@@ -1,72 +0,0 @@
-import glob
-import gzip
-import os
-import shutil
-
-from ingestion.shared_util.tar import unpack
-from ingestion.nebula.util.vcf import extract_sv
-from ingestion.shared_util.ga4gh import create_yaml
-from logging import Logger
-
-
-def handle_tsv(file: str, file_list: list[str]) -> dict[str, str]:
-    multiple_tsv = len([file for file in file_list if file.endswith("tsv")]) > 1
-
-    if not multiple_tsv or "Transformed" in file:
-        return {
-            "tsv": file,
-        }
-    return {}
-
-
-def process_nebula_json(
-    infile: str, outpath: str, file_name: str, source_file_id: str, log: Logger
-):
-    # Unpack tarball and go into the new directory
-    unpack(infile, outpath)
-    os.chdir(outpath)
-
-    file_list = glob.glob("*")
-    files: dict[str, str] = {}
-
-    for file in file_list:
-        extension = ".".join(file.split(".")[1:])
-        if file.endswith("vcf"):
-            files["somatic.vcf"] = file
-        else:
-            # There should only be the vcf file
-            files[extension] = file
-
-    log.info(f"Files in tarball input: {file_list}")
-
-    somatic_filename = None
-    data = {}
-    metadata = {}
-
-    # Sometimes they don't come in gzipped
-    for key in files.keys():
-        if "somatic.vcf" in key:
-            somatic_filename = files["somatic.vcf"].replace(".vcf", ".somatic.vcf") + ".gz"
-            with open(files["somatic.vcf"], "rb") as f_in:
-                with gzip.open(somatic_filename, "wb") as f_out:
-                    shutil.copyfileobj(f_in, f_out)
-
-    vcf_results = extract_sv(file_name, bool(somatic_filename), False, False)
-
-    # We might not have any of these files but we need an empty json object here.
-    file_genome_references = {}
-    if vcf_results:
-        metadata["files"] = metadata["files"] + vcf_results
-        for vcf in vcf_results:
-            seq_type = vcf.get("sequenceType")
-            file_genome_references[f"{seq_type}_genome_reference"] = vcf["reference"]
-
-    create_yaml(metadata, file_name)
-
-    # Return VCF files for immediate processing, and JSON data for adding vendsig
-    result = {}
-
-    if somatic_filename is not None:
-        result["somatic_vcf"] = f"{outpath}/{somatic_filename}"
-
-    return (result, file_genome_references, data)

ingestion/nebula/util/vcf.py

@@ -1,100 +0,0 @@
-import datetime
-import gzip
-
-from logging import Logger
-
-from ingestion.vcf_standardization.standardize import standardize_vcf
-
-
-def create_empty_vcf_zip(prefix):
-    vcf_gzip_path = f"{prefix}.modified.somatic.vcf.gz"
-    content = (
-        """##fileformat=VCFv4.1
-##filedate="""
-        + datetime.datetime.now().isoformat()
-        + """
-##FILTER=<ID=PASS,Description="All filters passed">
-##FILTER=<ID=R8,Description="IndelRepeatLength is greater than 8">
-##FILTER=<ID=R8.1,Description="IndelRepeatLength of a monomer is greater than 8">
-##FILTER=<ID=R8.2,Description="IndelRepeatLength of a dimer is greater than 8">
-##FILTER=<ID=sb,Description="Variant strand bias high">
-##FILTER=<ID=sb.s,Description="Variant strand bias significantly high (only for SNV)">
-##FILTER=<ID=rs,Description="Variant with rs (dbSNP) number in a non-core gene">
-##FILTER=<ID=FP,Description="Possibly false positives due to high similarity to off-target regions">
-##FILTER=<ID=NC,Description="Noncoding INDELs on non-core genes">
-##FILTER=<ID=lowDP,Description="low depth variant">
-##FILTER=<ID=Benign,Description="Benign variant">
-##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
-##FORMAT=<ID=AF,Number=1,Type=String,Description="Variant Allele Frequency">
-#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	"""
-        + prefix
-        + """
-"""
-    )
-
-    with gzip.open(vcf_gzip_path, "wb") as f:
-        f.write(content.encode("utf-8"))
-
-
-# This is done in next step, we are just adding to yaml
-def extract_sv(prefix, include_somatic: bool):
-    vcfs = []
-
-    # Hard-code genome reference for Nebula VCFs
-    genome_reference = "GRCh38"
-
-    if include_somatic:
-        vcfs.append(
-            {
-                "fileName": f".lifeomic/nebula/{prefix}/{prefix}.modified.somatic.nrm.filtered.vcf.gz",
-                "sequenceType": "somatic",
-                "type": "shortVariant",  # todo: Is this always going to be shortVariant?
-                "reference": genome_reference,
-            }
-        )
-
-    return vcfs
-
-
-def get_vendsig_dict(json_data, log: Logger):
-    # Return a dicitionary of {'chr:star_pos:ref:alt' : 'vendsig'}
-    vendsig_dict = {"vendor": "nebula"}
-
-    return vendsig_dict
-
-
-def map_vendsig(ci: str) -> str:
-    if ci in ["Pathogenic Variant", "Pathogenic"]:
-        return "Pathogenic"
-    elif ci in ["Likely Pathogenic Variant", "Likely Pathogenic"]:
-        return "Likely pathogenic"
-    elif ci in ["Benign Variant", "Benign"]:
-        return "Benign"
-    elif ci in ["Likely Benign Variant", "Likely Benign"]:
-        return "Likely benign"
-    elif ci in ["Variant of Uncertain Significance", "VUS"]:
-        return "Uncertain significance"
-    else:
-        return "Unknown"
-
-
-def process_nebula_vcf(infile, json_data, outpath, file_name, log: Logger):
-    line_count = 0
-    vendsig_dict = {"vendor": "nebula"}
-
-    outfile = f"{file_name}.modified.somatic.vcf"
-    sample_name = file_name
-    # Read in a dictionary of variants with VENDSIG from the JSON file for somatic only
-    vendsig_dict = get_vendsig_dict(json_data, log)
-
-    line_count = standardize_vcf(
-        infile=infile,
-        outfile=outfile,
-        out_path=outpath,
-        case_id=sample_name,
-        log=log,
-        vendsig_dict=vendsig_dict,
-        compression=True,
-    )
-
-    return line_count