nci-cidc-schemas 0.28.7__py2.py3-none-any.whl → 0.28.13__py2.py3-none-any.whl

cidc_schemas/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """NCI"""
 __email__ = "nci-cidc-tools-admin@mail.nih.gov"
-__version__ = "0.28.7"
+__version__ = "0.28.13"

cidc_schemas/ngs_pipeline_api/__init__.py

@@ -3,7 +3,6 @@
 import os
 from json import load
 
-
 # __author__ = """NCI"""
 # __email__ = "nci-cidc-tools-admin@mail.nih.gov"
 # __version__ = "0.1.25"

cidc_schemas/ngs_pipeline_api/wes/wes_output_API.py

@@ -485,7 +485,7 @@ def main():
         default=False,
         action="store_true",
     )
-    (options, args) = optparser.parse_args(sys.argv)
+    options, args = optparser.parse_args(sys.argv)
 
     run_id_files = [
         r for r in map(lambda x: evalWildcards(x, "{run}", "{run id}"), run_files)

cidc_schemas/prism/constants.py

@@ -2,7 +2,6 @@
 
 from typing import Dict
 
-
 PROTOCOL_ID_FIELD_NAME = "protocol_identifier"
 
 SUPPORTED_ASSAYS = [
@@ -30,9 +29,11 @@ SUPPORTED_ASSAYS = [
     "visium",
     "nulisa",
     "maldi_glycan",
+    "olink_ht",
+    "tcrseq_rna",
 ]
 
-SUPPORTED_SHIPPING_MANIFESTS = [
+SUPPORTED_MANIFESTS = [
     "pbmc",
     "plasma",
     "tissue_slide",
@@ -43,12 +44,6 @@ SUPPORTED_SHIPPING_MANIFESTS = [
     "h_and_e",
     "microbiome_dna",
 ]
-# weird non shipping manifest
-SUPPORTED_WEIRD_MANIFESTS = [
-    "tumor_normal_pairing",
-]
-SUPPORTED_MANIFESTS = SUPPORTED_SHIPPING_MANIFESTS + SUPPORTED_WEIRD_MANIFESTS
-
 SUPPORTED_ANALYSES = [
     "atacseq_analysis",
     "cytof_analysis",
@@ -147,6 +142,8 @@ ASSAY_TO_FILEPATH: Dict[str, str] = {
             "visium_analysis",
             "nulisa",
             "maldi_glycan",
+            "olink_ht",
+            "tcrseq_rna",
         ]
     },
 }

cidc_schemas/prism/pipelines.py

@@ -9,7 +9,7 @@ from typing import Dict, List, NamedTuple, Union
 import jinja2
 import pandas as pd
 
-from .constants import PROTOCOL_ID_FIELD_NAME, SUPPORTED_SHIPPING_MANIFESTS
+from .constants import PROTOCOL_ID_FIELD_NAME, SUPPORTED_MANIFESTS
 from ..template import Template
 from ..util import get_logger, load_pipeline_config_template, participant_id_from_cimac
 
@@ -108,10 +108,6 @@ class _Wes_pipeline_config:
             for both - normal and tumor sample - and we render tumor/normal pipeline config for those
             and tumor_only config otherwise
 
-        As a result each assay or tumor_normal_pairing upload that "completes" some
-        analysis run/pair (making all 3 pieces available - both samples data and a pairing)
-        will be rendered only once - right after that upload.
-
         Patch is expected to be already merged into full_ct.
         """
         if upload_type not in ["assay", "pairing"]:
@@ -552,15 +548,8 @@ class _Wes_pipeline_config:
     ) -> Dict[str, Union[bytes, str]]:
         """
         Generates a mapping from filename to the files to attach.
-        Attachments differ depending on whether the upload is wes_[bam/fastq] vs tumor_normal_pairing
         For wes_[bam/faq]:
             generates a pairing.csv file attempting to pair WES samples
-        For tumor_normal_pairing:
-            generates a pairing.csv file attempting to pair WES samples
-            generates WES Monitor .yaml configs for each batch of 20 affected samples
-            generates ingestion .xlsx templates for each affected sample
-                for paired samples, for wes_analysis
-                for unpaired samples, for wes_tumor_only_analysis
 
         For the pairing.csv
             - tumor samples already in a paired analysis are ignored
@@ -652,16 +641,6 @@ class _Wes_pipeline_config:
             + "_pairing.csv": self._generate_pairing_csv(trial_id, tumor_pair_list)
         }
 
-        # for tumor_normal_pairing, also generate configs and ingestion sheets
-        if "pairing" in self.upload_type:
-            res.update(
-                self._generate_configs_and_ingestion_sheets(
-                    trial_id=trial_id,
-                    patch=patch,
-                    data_bucket=data_bucket,
-                )
-            )
-
         return res
 
 
@@ -821,7 +800,6 @@ _ANALYSIS_CONF_GENERATORS = {
     "wes_bam": _Wes_pipeline_config("assay"),
     "tcr_adaptive": _tcr_pipeline_config,
     "tcr_fastq": _tcr_pipeline_config,
-    "tumor_normal_pairing": _Wes_pipeline_config("pairing"),
     "rna_fastq": _rna_level1_pipeline_config,
     "rna_bam": _rna_level1_pipeline_config,
 }
@@ -843,7 +821,7 @@ def generate_analysis_configs_from_upload_patch(
     """
     ret = {}
 
-    if template_type in SUPPORTED_SHIPPING_MANIFESTS:
+    if template_type in SUPPORTED_MANIFESTS:
         ret.update(_shipping_manifest_new_participants(ct, patch, data_bucket))
 
     if template_type in _ANALYSIS_CONF_GENERATORS:

cidc_schemas/schemas/artifacts/artifact_parquet.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "metaschema/strict_meta_schema.json#",
+  "$id": "parquet_artifact",
+  "title": "Parquet Artifact",
+  "type": "object",
+  "description": "Information about a parquet file.",
+  "additionalProperties": false,
+  "properties": {
+    "data_format": {
+      "description": "Data format.",
+      "const": "parquet"
+    },
+    "upload_placeholder": {
+      "$ref": "artifacts/artifact_core.json#properties/upload_placeholder"
+    },
+    "artifact_creator": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_creator"
+    },
+    "uploader": { "$ref": "artifacts/artifact_core.json#properties/uploader" },
+    "uuid": { "$ref": "artifacts/artifact_core.json#properties/uuid" },
+    "file_name": {
+      "$ref": "artifacts/artifact_core.json#properties/file_name"
+    },
+    "object_url": {
+      "$ref": "artifacts/artifact_core.json#properties/object_url"
+    },
+    "uploaded_timestamp": {
+      "$ref": "artifacts/artifact_core.json#properties/uploaded_timestamp"
+    },
+    "file_size_bytes": {
+      "$ref": "artifacts/artifact_core.json#properties/file_size_bytes"
+    },
+    "md5_hash": { "$ref": "artifacts/artifact_core.json#properties/md5_hash" },
+    "crc32c_hash": {
+      "$ref": "artifacts/artifact_core.json#properties/crc32c_hash"
+    },
+    "visible": { "$ref": "artifacts/artifact_core.json#properties/visible" },
+    "artifact_category": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_category"
+    },
+    "facet_group": {
+      "$ref": "artifacts/artifact_core.json#properties/facet_group"
+    }
+  },
+  "allOf": [
+    {
+      "$ref": "artifacts/artifact_core.json"
+    }
+  ],
+  "mergeStrategy": "objectMerge"
+}

cidc_schemas/schemas/assays/atacseq_assay.json

@@ -55,6 +55,10 @@
 		"batch_id": {
             "description": "ATACseq batch identification number. Unique to every batch.",
             "type": "string"
+        },
+		"optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
         },
 		"error": {"$ref": "artifacts/artifact_yaml.json"},
 		"assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},

cidc_schemas/schemas/assays/components/available_assays.json

@@ -77,7 +77,7 @@
       },
       "mergeStrategy": "arrayMergeById",
       "mergeOptions": {
-        "idRef": "assay_run_id"
+        "idRef": "batch_id"
       }
     },
     "olink": {
@@ -189,7 +189,28 @@
       "mergeOptions": {
         "idRef": "batch_id"
       }
+    },
+    "olink_ht": {
+      "type": "array",
+      "description": "Olink HT",
+      "items": {
+        "$ref": "assays/olink_ht_assay.json"
+      },
+      "mergeStrategy": "arrayMergeById",
+      "mergeOptions": {
+        "idRef": "batch_id"
+      }
+    },
+    "tcrseq_rna": {
+      "type": "array",
+      "description": "TCR RNA",
+      "items": {
+        "$ref": "assays/tcrseq_rna_assay.json"
+      },
+      "mergeStrategy": "arrayMergeById",
+      "mergeOptions": {
+        "idRef": "batch_id"
+      }
     }
-
   }
 }

cidc_schemas/schemas/assays/components/ngs/atacseq/atacseq_analysis.json

@@ -55,6 +55,10 @@
         "batch_id":{
             "$ref": "assays/atacseq_assay.json#properties/batch_id"
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "records": {
             "type": "array",
             "mergeStrategy": "arrayMergeById",

cidc_schemas/schemas/assays/components/ngs/rna/rna_analysis.json

@@ -18,6 +18,15 @@
     },
     "excluded_samples": {
       "$ref": "assays/components/excluded_samples.json"
+    },
+    "batch_id": {
+      "description": "Batch identification number. Unique to every upload.",
+      "type": "string"
+    },
+    "optional_id": {
+      "description": "Optional user provided id. Currently, comes from CIDC submission ID",
+      "type": "string"
     }
-  }
+  },
+  "required": ["batch_id"]
 }

cidc_schemas/schemas/assays/components/ngs_assay_core.json

@@ -16,7 +16,8 @@
         "Illumina - NextSeq 550",
         "Illumina - HiSeq 4000",
         "Illumina - NovaSeq 6000",
-        "MiSeq"
+        "MiSeq",
+        "Oxford Nanosphere"
       ]
     },
     "library_kit": {

cidc_schemas/schemas/assays/ctdna_assay.json

@@ -89,6 +89,10 @@
 		"batch_id": {
             "description": "ctDNA batch identification number. Unique to every batch.",
             "type": "string"
+        },
+		"optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
         },
 		"summary_plots": {"$ref": "artifacts/artifact_pdf.json"},
 		"records": {

cidc_schemas/schemas/assays/cytof_assay.json

@@ -21,6 +21,10 @@
         "barcodes": {"$ref": "assays/cytof_assay_core.json#properties/barcodes"},
         "controls": {"$ref": "assays/cytof_assay_core.json#properties/controls"},
         "batch_id": {"$ref": "assays/cytof_assay_core.json#properties/batch_id"},
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "excluded_samples": {"$ref": "assays/cytof_assay_core.json#properties/excluded_samples"},
         "injector": {
             "description": "Name of Injector component used as part of the CyTOF software",

cidc_schemas/schemas/assays/cytof_assay_core.json

@@ -31,6 +31,10 @@
             "description": "CyTOF batch identification number. Unique to every batch.",
             "type": "string"
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "controls": {
             "type": "array",
             "mergeStrategy": "arrayMergeById",

cidc_schemas/schemas/assays/elisa_assay.json

@@ -52,9 +52,13 @@
 
   "properties": {
     "assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},
-    "assay_run_id": {
-      "type": "string",
-      "description": "User defined unique identifier for this assay run."
+    "batch_id": {
+      "description": "Batch identification number. Unique to every upload.",
+      "type": "string"
+    },
+    "optional_id": {
+      "description": "User provided optional id. Currently comes from CIDC Submission ID",
+      "type": "string"
     },
     "assay_xlsx": {
       "$ref": "artifacts/artifact_elisa_xlsx.json"
@@ -72,5 +76,5 @@
   },
   "mergeStrategy": "objectMerge",
   "additionalProperties": false,
-  "required": ["assay_run_id", "assay_xlsx", "antigens"]
+  "required": ["batch_id", "assay_xlsx", "antigens"]
 }

cidc_schemas/schemas/assays/hande_assay.json

@@ -102,6 +102,14 @@
   
   "properties": {
       "assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},
+      "batch_id": {
+        "description": "Batch identification number. Unique to every upload.",
+        "type": "string"
+      },
+      "optional_id": {
+        "description": "User provided optional id. Currently comes from CIDC Submission ID",
+        "type": "string"
+      },
       "records": {
             "type": "array",
             "description": "A single data record from H & E assay.",
@@ -116,6 +124,7 @@
   },
   "required": [
     "records",
-    "assay_creator"
+    "assay_creator",
+    "batch_id"
   ]
 }

cidc_schemas/schemas/assays/ihc_assay.json

@@ -122,7 +122,14 @@
 
     "properties": {
         "assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},
-        
+        "batch_id": {
+            "description": "Batch identification number. Unique to every upload.",
+            "type": "string"
+        },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "slide_scanner_model": {
             "description": "Model version of the slide scanner instrument.",
             "type": "string",
@@ -165,6 +172,7 @@
         }
     },
     "required": [
+        "batch_id",
         "slide_scanner_model",
         "staining_platform",
         "autostainer_model",

cidc_schemas/schemas/assays/maldi_glycan_assay.json

@@ -35,6 +35,10 @@
             "description": "Batch identification number. Unique to every upload.",
             "type": "string"
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "molecular_assignments_file": {
             "$ref": "artifacts/artifact_tsv.json"
         },

cidc_schemas/schemas/assays/mibi_assay.json

@@ -35,6 +35,10 @@
             "type": "string",
             "description": "Batch identification number. Unique to every batch."
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "metadata_tsv": {
             "description": "Deidentified metadata as a TSV file",
             "$ref": "artifacts/artifact_tsv.json"

cidc_schemas/schemas/assays/microbiome_assay.json

@@ -50,6 +50,10 @@
       "description": "Microbiome batch identification number.",
       "type": "string"
     },
+    "optional_id": {
+      "description": "User provided optional id. Currently comes from CIDC Submission ID",
+      "type": "string"
+    },
     "sequencer_platform": { "$ref": "assays/components/ngs_assay_core.json#properties/sequencer_platform" },
     "forward_fastq": {"$ref": "artifacts/artifact_fastq_gz.json"},
     "forward_index": {"$ref": "artifacts/artifact_fastq_gz.json"},
@@ -68,4 +72,4 @@
       }
     }
   }
-}
+}

cidc_schemas/schemas/assays/mif_assay.json

@@ -4,7 +4,7 @@
   "title": "mIF Assay",
   "type": "object",
 
-	"$comment": "description used in docs/index.html and docs/assays.mif.html",
+  "$comment": "description used in docs/index.html and docs/assays.mif.html",
   "description": "Multiplex immunofluorescence.",
   
   "additionalProperties": false,
@@ -28,7 +28,14 @@
     "imaging_status": {"$ref": "assays/components/imaging_data.json#properties/imaging_status"},
 
     "qc_report": {"$ref": "artifacts/artifact_zip.json"},
-
+    "batch_id": {
+      "description": "Batch identification number. Unique to every upload.",
+      "type": "string"
+    },
+    "optional_id": {
+      "description": "User provided optional id. Currently comes from CIDC Submission ID",
+      "type": "string"
+    },
     "panel": {
       "type": "string",
       "enum": [
@@ -65,6 +72,6 @@
       }
     }
   },
-  "required": ["records", "panel", "antibodies"],
+  "required": ["records", "panel", "antibodies", "batch_id"],
   "mergeStrategy": "objectMerge"
 }

cidc_schemas/schemas/assays/mihc_assay.json

@@ -46,7 +46,10 @@
             "description": "Batch identification number. Unique to every upload.",
             "type": "string"
         },
-
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "samples_file": {"$ref": "artifacts/artifact_csv.json"},
 
         "panel": {"type": "string"},

cidc_schemas/schemas/assays/misc_data.json

@@ -43,6 +43,14 @@
         "assay_creator": {
             "$ref": "assays/components/assay_core.json#properties/assay_creator"
         },
+        "batch_id": {
+            "description": "Batch identification number. Unique to every upload.",
+            "type": "string"
+        },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "files": {
             "type": "array",
             "description": "A single file containing miscellaneous data.",
@@ -59,4 +67,4 @@
         "files",
         "assay_creator"
     ]
-}
+}

cidc_schemas/schemas/assays/nanostring_assay.json

@@ -61,6 +61,10 @@
       "type": "string",
       "description": "A unique ID that separates the different batches of Nanostring."
     },
+    "optional_id": {
+      "description": "User provided optional id. Currently comes from CIDC Submission ID",
+      "type": "string"
+    },
     "data": {
       "type": "object",
       "additionalProperties": false,
@@ -82,5 +86,5 @@
       "items": { "$ref": "assays/nanostring_assay.json#definitions/run" }
     }
   },
-  "required": ["data", "runs"]
+  "required": ["data", "runs", "batch_id"]
 }

cidc_schemas/schemas/assays/nulisa_assay.json

@@ -35,6 +35,10 @@
             "description": "Batch identification number. Unique to every upload.",
             "type": "string"
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "npq_file": {
             "$ref": "artifacts/artifact_csv.json"
         },

cidc_schemas/schemas/assays/olink_assay.json

@@ -119,6 +119,10 @@
           "type": "string",
           "description": "A unique identifier for this Olink upload"
         },
+        "optional_id": {
+          "description": "User provided optional id. Currently comes from CIDC Submission ID",
+          "type": "string"
+        },
         "assay_creator": {
           "$ref": "assays/components/assay_core.json#properties/assay_creator"
         },

cidc_schemas/schemas/assays/olink_ht_assay.json

@@ -0,0 +1,100 @@
+{
+    "$schema": "metaschema/strict_meta_schema.json#",
+    "$id": "olink_ht_assay",
+    "title": "Olink HT Assay",
+    "type": "object",
+    "$comment": "description used in docs/index.html and docs/assays.olink_ht.html",
+    "description": "Olink HT assay",
+    "additionalProperties": false,
+    "definitions": {
+        "record": {
+            "type": "object",
+            "mergeStrategy": "objectMerge",
+            "description": "A single data record from an Olink HT assay.",
+            "additionalProperties": false,
+            "required": [
+                "cimac_id",
+                "npx_run_file",
+                "instrument",
+                "npx_manager_version"
+            ],
+            "properties": {
+                "cimac_id": {
+                    "description": "Id of an sample within this clinical trial, that this assay record is based upon.",
+                    "$ref": "sample.json#properties/cimac_id"
+                },
+                "npx_run_file": {
+                    "$ref": "artifacts/artifact_parquet.json",
+                    "$comment": "Parquet file of the overall run the sample was in."
+                },
+                "instrument": {
+                    "description": "Instrument used for the Olink HT assay.",
+                    "type": "string"
+                },
+                "npx_manager_version": {
+                    "description": "Version of NPX manager used for the analysis.",
+                    "type": "string"
+                },
+                "comments": {
+                    "description": "Comments on the Olink HT assay sample.",
+                    "type": "string"
+                }
+            }
+        }
+    },
+    "properties": {
+        "assay_creator": {
+            "$ref": "assays/components/assay_core.json#properties/assay_creator"
+        },
+        "batch_id": {
+            "description": "Batch identification number. Unique to every upload.",
+            "type": "string"
+        },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
+        "panel": {
+            "description": "Panel used for the assay.",
+            "type": "string"
+        },
+        "assay_panel_lot": {
+            "description": "Lot number of the assay panel.",
+            "type": "string"
+        },
+        "study_level_combined_file": {
+            "$comment": "Parquet file of the study level run.",
+            "$ref": "artifacts/artifact_parquet.json"
+        },
+        "study_level_combined_npx_manager_version": {
+            "description": "Version of NPX manager used for the analysis.",
+            "type": "string"
+        },
+        "batch_level_combined_file": {
+            "$comment": "Parquet file of the batch level run.",
+            "$ref": "artifacts/artifact_parquet.json"
+        },
+        "batch_level_combined_npx_manager_version": {
+            "description": "Version of NPX manager used for the analysis.",
+            "type": "string"
+        },
+        "records": {
+            "type": "array",
+            "description": "A single data record from an Olink HT assay.",
+            "items": {
+                "$ref": "assays/olink_ht_assay.json#definitions/record"
+            },
+            "mergeStrategy": "append"
+        }
+    },
+    "required": [
+        "assay_creator",
+        "batch_id",
+        "panel",
+        "assay_panel_lot",
+        "study_level_combined_file",
+        "study_level_combined_npx_manager_version",
+        "batch_level_combined_file",
+        "batch_level_combined_npx_manager_version"
+    ]
+}