nci-cidc-schemas 0.28.3__py2.py3-none-any.whl → 0.28.13__py2.py3-none-any.whl

cidc_schemas/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """NCI"""
 __email__ = "nci-cidc-tools-admin@mail.nih.gov"
-__version__ = "0.28.3"
+__version__ = "0.28.13"

cidc_schemas/ngs_pipeline_api/__init__.py

@@ -3,7 +3,6 @@
 import os
 from json import load
 
-
 # __author__ = """NCI"""
 # __email__ = "nci-cidc-tools-admin@mail.nih.gov"
 # __version__ = "0.1.25"

cidc_schemas/ngs_pipeline_api/wes/wes_output_API.py

@@ -485,7 +485,7 @@ def main():
         default=False,
         action="store_true",
     )
-    (options, args) = optparser.parse_args(sys.argv)
+    options, args = optparser.parse_args(sys.argv)
 
     run_id_files = [
         r for r in map(lambda x: evalWildcards(x, "{run}", "{run id}"), run_files)

cidc_schemas/prism/constants.py

@@ -2,7 +2,6 @@
 
 from typing import Dict
 
-
 PROTOCOL_ID_FIELD_NAME = "protocol_identifier"
 
 SUPPORTED_ASSAYS = [
@@ -29,9 +28,12 @@ SUPPORTED_ASSAYS = [
     "scrnaseq",
     "visium",
     "nulisa",
+    "maldi_glycan",
+    "olink_ht",
+    "tcrseq_rna",
 ]
 
-SUPPORTED_SHIPPING_MANIFESTS = [
+SUPPORTED_MANIFESTS = [
     "pbmc",
     "plasma",
     "tissue_slide",
@@ -42,12 +44,6 @@ SUPPORTED_SHIPPING_MANIFESTS = [
     "h_and_e",
     "microbiome_dna",
 ]
-# weird non shipping manifest
-SUPPORTED_WEIRD_MANIFESTS = [
-    "tumor_normal_pairing",
-]
-SUPPORTED_MANIFESTS = SUPPORTED_SHIPPING_MANIFESTS + SUPPORTED_WEIRD_MANIFESTS
-
 SUPPORTED_ANALYSES = [
     "atacseq_analysis",
     "cytof_analysis",
@@ -145,6 +141,9 @@ ASSAY_TO_FILEPATH: Dict[str, str] = {
             "visium",
             "visium_analysis",
             "nulisa",
+            "maldi_glycan",
+            "olink_ht",
+            "tcrseq_rna",
         ]
     },
 }

cidc_schemas/prism/pipelines.py

@@ -9,7 +9,7 @@ from typing import Dict, List, NamedTuple, Union
 import jinja2
 import pandas as pd
 
-from .constants import PROTOCOL_ID_FIELD_NAME, SUPPORTED_SHIPPING_MANIFESTS
+from .constants import PROTOCOL_ID_FIELD_NAME, SUPPORTED_MANIFESTS
 from ..template import Template
 from ..util import get_logger, load_pipeline_config_template, participant_id_from_cimac
 
@@ -108,10 +108,6 @@ class _Wes_pipeline_config:
             for both - normal and tumor sample - and we render tumor/normal pipeline config for those
             and tumor_only config otherwise
 
-        As a result each assay or tumor_normal_pairing upload that "completes" some
-        analysis run/pair (making all 3 pieces available - both samples data and a pairing)
-        will be rendered only once - right after that upload.
-
         Patch is expected to be already merged into full_ct.
         """
         if upload_type not in ["assay", "pairing"]:
@@ -552,15 +548,8 @@ class _Wes_pipeline_config:
     ) -> Dict[str, Union[bytes, str]]:
         """
         Generates a mapping from filename to the files to attach.
-        Attachments differ depending on whether the upload is wes_[bam/fastq] vs tumor_normal_pairing
         For wes_[bam/faq]:
             generates a pairing.csv file attempting to pair WES samples
-        For tumor_normal_pairing:
-            generates a pairing.csv file attempting to pair WES samples
-            generates WES Monitor .yaml configs for each batch of 20 affected samples
-            generates ingestion .xlsx templates for each affected sample
-                for paired samples, for wes_analysis
-                for unpaired samples, for wes_tumor_only_analysis
 
         For the pairing.csv
             - tumor samples already in a paired analysis are ignored
@@ -652,16 +641,6 @@ class _Wes_pipeline_config:
             + "_pairing.csv": self._generate_pairing_csv(trial_id, tumor_pair_list)
         }
 
-        # for tumor_normal_pairing, also generate configs and ingestion sheets
-        if "pairing" in self.upload_type:
-            res.update(
-                self._generate_configs_and_ingestion_sheets(
-                    trial_id=trial_id,
-                    patch=patch,
-                    data_bucket=data_bucket,
-                )
-            )
-
         return res
 
 
@@ -821,7 +800,6 @@ _ANALYSIS_CONF_GENERATORS = {
     "wes_bam": _Wes_pipeline_config("assay"),
     "tcr_adaptive": _tcr_pipeline_config,
     "tcr_fastq": _tcr_pipeline_config,
-    "tumor_normal_pairing": _Wes_pipeline_config("pairing"),
     "rna_fastq": _rna_level1_pipeline_config,
     "rna_bam": _rna_level1_pipeline_config,
 }
@@ -843,7 +821,7 @@ def generate_analysis_configs_from_upload_patch(
     """
     ret = {}
 
-    if template_type in SUPPORTED_SHIPPING_MANIFESTS:
+    if template_type in SUPPORTED_MANIFESTS:
         ret.update(_shipping_manifest_new_participants(ct, patch, data_bucket))
 
     if template_type in _ANALYSIS_CONF_GENERATORS:

cidc_schemas/schemas/artifacts/artifact_ibd.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "metaschema/strict_meta_schema.json#",
+  "$id": "ibd_artifact",
+  "title": "IBD Artifact",
+  "type": "object",
+  "description": "Information about a IBD file.",
+  "additionalProperties": false,
+  "properties": {
+    "data_format": {
+      "description": "Data format.",
+      "const": "IBD"
+    },
+    "upload_placeholder": {
+      "$ref": "artifacts/artifact_core.json#properties/upload_placeholder"
+    },
+    "artifact_creator": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_creator"
+    },
+    "uploader": { "$ref": "artifacts/artifact_core.json#properties/uploader" },
+    "uuid": { "$ref": "artifacts/artifact_core.json#properties/uuid" },
+    "file_name": {
+      "$ref": "artifacts/artifact_core.json#properties/file_name"
+    },
+    "object_url": {
+      "$ref": "artifacts/artifact_core.json#properties/object_url"
+    },
+    "uploaded_timestamp": {
+      "$ref": "artifacts/artifact_core.json#properties/uploaded_timestamp"
+    },
+    "file_size_bytes": {
+      "$ref": "artifacts/artifact_core.json#properties/file_size_bytes"
+    },
+    "md5_hash": { "$ref": "artifacts/artifact_core.json#properties/md5_hash" },
+    "crc32c_hash": {
+      "$ref": "artifacts/artifact_core.json#properties/crc32c_hash"
+    },
+    "visible": { "$ref": "artifacts/artifact_core.json#properties/visible" },
+    "artifact_category": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_category"
+    },
+    "facet_group": {
+      "$ref": "artifacts/artifact_core.json#properties/facet_group"
+    }
+  },
+  "allOf": [
+    {
+      "$ref": "artifacts/artifact_core.json"
+    }
+  ],
+  "mergeStrategy": "objectMerge"
+}

cidc_schemas/schemas/artifacts/artifact_imzml.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "metaschema/strict_meta_schema.json#",
+  "$id": "imzml_artifact",
+  "title": "IMZML Artifact",
+  "type": "object",
+  "description": "Information about an IMZML file.",
+  "additionalProperties": false,
+  "properties": {
+    "data_format": {
+      "description": "Data format.",
+      "const": "IMZML"
+    },
+    "upload_placeholder": {
+      "$ref": "artifacts/artifact_core.json#properties/upload_placeholder"
+    },
+    "artifact_creator": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_creator"
+    },
+    "uploader": { "$ref": "artifacts/artifact_core.json#properties/uploader" },
+    "uuid": { "$ref": "artifacts/artifact_core.json#properties/uuid" },
+    "file_name": {
+      "$ref": "artifacts/artifact_core.json#properties/file_name"
+    },
+    "object_url": {
+      "$ref": "artifacts/artifact_core.json#properties/object_url"
+    },
+    "uploaded_timestamp": {
+      "$ref": "artifacts/artifact_core.json#properties/uploaded_timestamp"
+    },
+    "file_size_bytes": {
+      "$ref": "artifacts/artifact_core.json#properties/file_size_bytes"
+    },
+    "md5_hash": { "$ref": "artifacts/artifact_core.json#properties/md5_hash" },
+    "crc32c_hash": {
+      "$ref": "artifacts/artifact_core.json#properties/crc32c_hash"
+    },
+    "visible": { "$ref": "artifacts/artifact_core.json#properties/visible" },
+    "artifact_category": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_category"
+    },
+    "facet_group": {
+      "$ref": "artifacts/artifact_core.json#properties/facet_group"
+    }
+  },
+  "allOf": [
+    {
+      "$ref": "artifacts/artifact_core.json"
+    }
+  ],
+  "mergeStrategy": "objectMerge"
+}

cidc_schemas/schemas/artifacts/artifact_parquet.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "metaschema/strict_meta_schema.json#",
+  "$id": "parquet_artifact",
+  "title": "Parquet Artifact",
+  "type": "object",
+  "description": "Information about a parquet file.",
+  "additionalProperties": false,
+  "properties": {
+    "data_format": {
+      "description": "Data format.",
+      "const": "parquet"
+    },
+    "upload_placeholder": {
+      "$ref": "artifacts/artifact_core.json#properties/upload_placeholder"
+    },
+    "artifact_creator": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_creator"
+    },
+    "uploader": { "$ref": "artifacts/artifact_core.json#properties/uploader" },
+    "uuid": { "$ref": "artifacts/artifact_core.json#properties/uuid" },
+    "file_name": {
+      "$ref": "artifacts/artifact_core.json#properties/file_name"
+    },
+    "object_url": {
+      "$ref": "artifacts/artifact_core.json#properties/object_url"
+    },
+    "uploaded_timestamp": {
+      "$ref": "artifacts/artifact_core.json#properties/uploaded_timestamp"
+    },
+    "file_size_bytes": {
+      "$ref": "artifacts/artifact_core.json#properties/file_size_bytes"
+    },
+    "md5_hash": { "$ref": "artifacts/artifact_core.json#properties/md5_hash" },
+    "crc32c_hash": {
+      "$ref": "artifacts/artifact_core.json#properties/crc32c_hash"
+    },
+    "visible": { "$ref": "artifacts/artifact_core.json#properties/visible" },
+    "artifact_category": {
+      "$ref": "artifacts/artifact_core.json#properties/artifact_category"
+    },
+    "facet_group": {
+      "$ref": "artifacts/artifact_core.json#properties/facet_group"
+    }
+  },
+  "allOf": [
+    {
+      "$ref": "artifacts/artifact_core.json"
+    }
+  ],
+  "mergeStrategy": "objectMerge"
+}

cidc_schemas/schemas/assays/atacseq_assay.json

@@ -55,6 +55,10 @@
 		"batch_id": {
             "description": "ATACseq batch identification number. Unique to every batch.",
             "type": "string"
+        },
+		"optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
         },
 		"error": {"$ref": "artifacts/artifact_yaml.json"},
 		"assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},

cidc_schemas/schemas/assays/components/available_assays.json

@@ -77,7 +77,7 @@
       },
       "mergeStrategy": "arrayMergeById",
       "mergeOptions": {
-        "idRef": "assay_run_id"
+        "idRef": "batch_id"
       }
     },
     "olink": {
@@ -178,6 +178,39 @@
       "mergeOptions": {
         "idRef": "batch_id"
       }
+    },
+    "maldi_glycan": {
+      "type": "array",
+      "description": "MALDI Glycan",
+      "items": {
+        "$ref": "assays/maldi_glycan_assay.json"
+      },
+      "mergeStrategy": "arrayMergeById",
+      "mergeOptions": {
+        "idRef": "batch_id"
+      }
+    },
+    "olink_ht": {
+      "type": "array",
+      "description": "Olink HT",
+      "items": {
+        "$ref": "assays/olink_ht_assay.json"
+      },
+      "mergeStrategy": "arrayMergeById",
+      "mergeOptions": {
+        "idRef": "batch_id"
+      }
+    },
+    "tcrseq_rna": {
+      "type": "array",
+      "description": "TCR RNA",
+      "items": {
+        "$ref": "assays/tcrseq_rna_assay.json"
+      },
+      "mergeStrategy": "arrayMergeById",
+      "mergeOptions": {
+        "idRef": "batch_id"
+      }
     }
   }
 }

cidc_schemas/schemas/assays/components/maldi_glycan_input.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "metaschema/strict_meta_schema.json#",
+  "$id": "maldi_glycan_input",
+  "title": "MALDI Glycan Input Files and Data",
+  "type": "object",
+  "description": "MALDI Glycan assay input files and data",
+  "inheritableBase": true,
+  "properties": {
+    "ibd_file": {
+      "$ref": "artifacts/artifact_ibd.json"
+    },
+    "imzml_file": {
+      "$ref": "artifacts/artifact_imzml.json"
+    },
+    "channels_csv": {
+      "$ref": "artifacts/artifact_csv.json"
+    },
+    "tiff_zip": {
+      "$ref": "artifacts/artifact_zip.json"
+    }
+  },
+  "required": [
+    "ibd_file",
+    "imzml_file",
+    "channels_csv",
+    "tiff_zip"
+  ]
+}

cidc_schemas/schemas/assays/components/ngs/atacseq/atacseq_analysis.json

@@ -55,6 +55,10 @@
         "batch_id":{
             "$ref": "assays/atacseq_assay.json#properties/batch_id"
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "records": {
             "type": "array",
             "mergeStrategy": "arrayMergeById",

cidc_schemas/schemas/assays/components/ngs/rna/rna_analysis.json

@@ -18,6 +18,15 @@
     },
     "excluded_samples": {
       "$ref": "assays/components/excluded_samples.json"
+    },
+    "batch_id": {
+      "description": "Batch identification number. Unique to every upload.",
+      "type": "string"
+    },
+    "optional_id": {
+      "description": "Optional user provided id. Currently, comes from CIDC submission ID",
+      "type": "string"
     }
-  }
+  },
+  "required": ["batch_id"]
 }

cidc_schemas/schemas/assays/components/ngs_assay_core.json

@@ -16,7 +16,8 @@
         "Illumina - NextSeq 550",
         "Illumina - HiSeq 4000",
         "Illumina - NovaSeq 6000",
-        "MiSeq"
+        "MiSeq",
+        "Oxford Nanosphere"
       ]
     },
     "library_kit": {

cidc_schemas/schemas/assays/ctdna_assay.json

@@ -89,6 +89,10 @@
 		"batch_id": {
             "description": "ctDNA batch identification number. Unique to every batch.",
             "type": "string"
+        },
+		"optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
         },
 		"summary_plots": {"$ref": "artifacts/artifact_pdf.json"},
 		"records": {

cidc_schemas/schemas/assays/cytof_assay.json

@@ -21,6 +21,10 @@
         "barcodes": {"$ref": "assays/cytof_assay_core.json#properties/barcodes"},
         "controls": {"$ref": "assays/cytof_assay_core.json#properties/controls"},
         "batch_id": {"$ref": "assays/cytof_assay_core.json#properties/batch_id"},
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "excluded_samples": {"$ref": "assays/cytof_assay_core.json#properties/excluded_samples"},
         "injector": {
             "description": "Name of Injector component used as part of the CyTOF software",

cidc_schemas/schemas/assays/cytof_assay_core.json

@@ -31,6 +31,10 @@
             "description": "CyTOF batch identification number. Unique to every batch.",
             "type": "string"
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "controls": {
             "type": "array",
             "mergeStrategy": "arrayMergeById",

cidc_schemas/schemas/assays/elisa_assay.json

@@ -52,9 +52,13 @@
 
   "properties": {
     "assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},
-    "assay_run_id": {
-      "type": "string",
-      "description": "User defined unique identifier for this assay run."
+    "batch_id": {
+      "description": "Batch identification number. Unique to every upload.",
+      "type": "string"
+    },
+    "optional_id": {
+      "description": "User provided optional id. Currently comes from CIDC Submission ID",
+      "type": "string"
     },
     "assay_xlsx": {
       "$ref": "artifacts/artifact_elisa_xlsx.json"
@@ -72,5 +76,5 @@
   },
   "mergeStrategy": "objectMerge",
   "additionalProperties": false,
-  "required": ["assay_run_id", "assay_xlsx", "antigens"]
+  "required": ["batch_id", "assay_xlsx", "antigens"]
 }

cidc_schemas/schemas/assays/hande_assay.json

@@ -27,6 +27,10 @@
             {
               "$comment": "Path to the H & E image in JPG format.",
               "$ref": "artifacts/artifact_jpg.json"
+            },
+            {
+              "$comment": "Path to the H & E image in OME-TIFF format.",
+              "$ref": "artifacts/artifact_jpg.json"
             }
           ]
         },
@@ -41,6 +45,10 @@
             {
               "$comment": "Path to the H & E annotated image in JPG format.",
               "$ref": "artifacts/artifact_jpg.json"
+            },
+            {
+              "$comment": "Path to the H & E annotated image in OME-TIFF format.",
+              "$ref": "artifacts/artifact_jpg.json"
             }
           ]
         }
@@ -94,6 +102,14 @@
   
   "properties": {
       "assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},
+      "batch_id": {
+        "description": "Batch identification number. Unique to every upload.",
+        "type": "string"
+      },
+      "optional_id": {
+        "description": "User provided optional id. Currently comes from CIDC Submission ID",
+        "type": "string"
+      },
       "records": {
             "type": "array",
             "description": "A single data record from H & E assay.",
@@ -108,6 +124,7 @@
   },
   "required": [
     "records",
-    "assay_creator"
+    "assay_creator",
+    "batch_id"
   ]
 }

cidc_schemas/schemas/assays/ihc_assay.json

@@ -122,7 +122,14 @@
 
     "properties": {
         "assay_creator": {"$ref": "assays/components/assay_core.json#properties/assay_creator"},
-        
+        "batch_id": {
+            "description": "Batch identification number. Unique to every upload.",
+            "type": "string"
+        },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "slide_scanner_model": {
             "description": "Model version of the slide scanner instrument.",
             "type": "string",
@@ -165,6 +172,7 @@
         }
     },
     "required": [
+        "batch_id",
         "slide_scanner_model",
         "staining_platform",
         "autostainer_model",

cidc_schemas/schemas/assays/maldi_glycan_assay.json

@@ -0,0 +1,72 @@
+{
+    "$schema": "metaschema/strict_meta_schema.json#",
+    "$id": "maldi_glycan_assay",
+    "title": "MALDI Glycan Assay",
+    "type": "object",
+    "$comment": "description used in docs/index.html and docs/assays.maldi_glycan.html",
+    "description": "MALDI Glycan assay",
+    "additionalProperties": false,
+    "definitions": {
+        "record": {
+            "type": "object",
+            "mergeStrategy": "objectMerge",
+            "description": "A single data record from a MALDI Glycan assay.",
+            "additionalProperties": false,
+            "required": [
+                "cimac_id",
+                "details"
+            ],
+            "properties": {
+                "cimac_id": {
+                    "description": "Id of an sample within this clinical trial, that this assay record is based upon.",
+                    "$ref": "sample.json#properties/cimac_id"
+                },
+                "details": {
+                    "$ref": "assays/components/maldi_glycan_input.json"
+                }
+            }
+        }
+    },
+    "properties": {
+        "assay_creator": {
+            "$ref": "assays/components/assay_core.json#properties/assay_creator"
+        },
+        "batch_id": {
+            "description": "Batch identification number. Unique to every upload.",
+            "type": "string"
+        },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
+        "molecular_assignments_file": {
+            "$ref": "artifacts/artifact_tsv.json"
+        },
+        "metadata_tsv": {"$ref": "artifacts/artifact_tsv.json"},
+
+        "antibodies": {
+            "type": "array",
+            "items": {
+                "$ref": "assays/components/mibi_antibody.json"
+            },
+            "mergeStrategy": "arrayMergeById",
+            "mergeOptions": {
+                "idRef": "antibody"
+            }
+        },
+        "records": {
+            "type": "array",
+            "description": "A single data record from a MALDI Glycan assay.",
+            "items": {
+                "$ref": "assays/maldi_glycan_assay.json#definitions/record"
+            },
+            "mergeStrategy": "append"
+        }
+    },
+    "required": [
+        "assay_creator",
+        "batch_id",
+        "molecular_assignments_file",
+        "records"
+    ]
+}

cidc_schemas/schemas/assays/mibi_assay.json

@@ -35,6 +35,10 @@
             "type": "string",
             "description": "Batch identification number. Unique to every batch."
         },
+        "optional_id": {
+            "description": "User provided optional id. Currently comes from CIDC Submission ID",
+            "type": "string"
+        },
         "metadata_tsv": {
             "description": "Deidentified metadata as a TSV file",
             "$ref": "artifacts/artifact_tsv.json"

cidc_schemas/schemas/assays/microbiome_assay.json

@@ -50,6 +50,10 @@
       "description": "Microbiome batch identification number.",
       "type": "string"
     },
+    "optional_id": {
+      "description": "User provided optional id. Currently comes from CIDC Submission ID",
+      "type": "string"
+    },
     "sequencer_platform": { "$ref": "assays/components/ngs_assay_core.json#properties/sequencer_platform" },
     "forward_fastq": {"$ref": "artifacts/artifact_fastq_gz.json"},
     "forward_index": {"$ref": "artifacts/artifact_fastq_gz.json"},
@@ -68,4 +72,4 @@
       }
     }
   }
-}
+}