nci-cidc-schemas 0.26.38__py2.py3-none-any.whl → 0.27.1__py2.py3-none-any.whl

cidc_schemas/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """NCI"""
 __email__ = "nci-cidc-tools-admin@mail.nih.gov"
-__version__ = "0.26.38"
+__version__ = "0.27.01"

cidc_schemas/json_validation.py

@@ -514,6 +514,49 @@ def convert(fmt: str, value: str) -> str:
         raise jsonschema.ValidationError(e)
 
 
+def strip_metadata_for_validation(full_metadata: dict) -> dict:
+    """
+    Returns a stripped down version of the full_metadata to only include necessary manifest data prior to validation.
+    The 'shipments' section is removed entirely, and the 'participants' section is stripped down to only include
+    cimac_participant_id and the cimac_id for each of the 'samples' in each participant.
+
+    The intention of this function is to be used prior to calling <validator>.iter_error_messages(<metadata>),
+    which will run a full validation against the current schema. The stripping done here allows for existing
+    manifest data (shipments, participants, samples) in the system to not be re-validated against the latest schema,
+    as many breaking schema changes (for older data) have been introduced via CSMS Integration (CIDC-1847). Any
+    new manifest data coming in should already be validated against the latest schema, so no need to re-validate here.
+    The cimac_participant_ids and cimac_ids are kept, as those are used beyond just simple schema field validation -
+    for cross-validating with properties in other areas of the full_metadata object.
+
+    This ensures that relevant fields are validated, while other existing shipment and participant/sample data
+    (which were previously valid and accepted into the system, but may no longer conform to the new schema)
+    can be ignored during the iter_error_messages part of the validation.
+
+    :param dict full_metadata: full metadata object that will be copied to create stripped down version for validation.
+    """
+    # Copy so we don't modify the original full_metadata
+    stripped_metadata = copy.deepcopy(full_metadata)
+    # Remove the 'shipments' section entirely
+    stripped_metadata.pop("shipments", None)
+    if "participants" not in stripped_metadata:
+        return stripped_metadata
+    participants = stripped_metadata["participants"]
+    stripped_participants = []
+    # Strip 'participants' and 'samples' data to only include necessary fields
+    for participant in participants:
+        stripped_samples = []
+        for sample in participant["samples"]:
+            stripped_samples.append({"cimac_id": sample["cimac_id"]})
+        stripped_participants.append(
+            {
+                "samples": stripped_samples,
+                "cimac_participant_id": participant["cimac_participant_id"],
+            }
+        )
+    stripped_metadata["participants"] = stripped_participants
+    return stripped_metadata
+
+
 REQUIRED_PROPERTY_MSG = " is a required property"
 
 

cidc_schemas/prism/merger.py

@@ -7,7 +7,11 @@ import jsonschema
 from jsonmerge import Merger, strategies
 from deepdiff import DeepSearch
 
-from ..json_validation import load_and_validate_schema, _Validator
+from ..json_validation import (
+    load_and_validate_schema,
+    strip_metadata_for_validation,
+    _Validator,
+)
 from ..util import get_path, get_source
 from .extra_metadata import EXTRA_METADATA_PARSERS
 from .constants import PROTOCOL_ID_FIELD_NAME
@@ -228,15 +232,16 @@ class ThrowOnOverwrite(strategies.Strategy):
     Similar to the jsonmerge's built in 'discard' strategy,
     but throws an error if the value already exists. This is
     used to prevent updates in `merge_clinical_trial_metadata`.
+    There is an exception for allowing updates to properties
+    defined in ALLOW_MERGE_PROPERTIES.
     """
 
     def merge(self, walk, base, head, schema, meta, **kwargs):
-        if base.is_undef():
-            return head
-        if base.val != head.val:
+        if not base.is_undef() and base.val != head.val:
             prop_name = base.ref.rsplit("/", 1)[-1]
-            raise MergeCollisionException(prop_name, base.val, head.val)
-        return base
+            if prop_name not in ALLOW_MERGE_PROPERTIES:
+                raise MergeCollisionException(prop_name, base.val, head.val)
+        return head
 
     def get_schema(self, walk, schema, **kwargs):
         return schema
@@ -280,6 +285,7 @@ PRISM_MERGE_STRATEGIES = {
     # Alias the builtin jsonmerge overwrite strategy
     "overwriteAny": strategies.Overwrite(),
 }
+ALLOW_MERGE_PROPERTIES = ["cohort_name", "collection_event_name"]
 
 
 def merge_clinical_trial_metadata(patch: dict, target: dict) -> Tuple[dict, List[str]]:
@@ -316,5 +322,8 @@ def merge_clinical_trial_metadata(patch: dict, target: dict) -> Tuple[dict, List
     # merge the two documents
     merger = Merger(validator.schema, strategies=PRISM_MERGE_STRATEGIES)
     merged = merger.merge(target, patch)
-
-    return merged, list(validator.iter_error_messages(merged))
+    # Prior to running validator.iter_error_messages on the merged metadata, strip out unnecessary manifest data for the
+    # validation so that existing manifest data that no longer conform to the post-CSMS-integration schema can be kept.
+    # See more details in the strip_metadata_for_validation function docs.
+    merged_to_validate = strip_metadata_for_validation(merged)
+    return merged, list(validator.iter_error_messages(merged_to_validate))

cidc_schemas/schemas/clinical_trial.json

@@ -83,17 +83,15 @@
       "description": "A list of assays the CIDC expects to receive for this trial."
     },
     "allowed_collection_event_names": {
-      "description": "Allowed values for sample.json#properties/collection_event_name for this trial.",
+      "description": "Allowed values for sample.json#properties/collection_event_name for this trial. Deprecated with CSMS Integration.",
       "type": "array",
-      "$comment": "This works together with sample.json#properties/collection_event_name/in_doc_ref_pattern.",
       "items": {
         "type": "string"
       }
     },
     "allowed_cohort_names": {
-      "description": "Allowed values for sample.json#properties/cohort_name for this trial.",
+      "description": "Allowed values for sample.json#properties/cohort_name for this trial. Deprecated with CSMS Integration.",
       "type": "array",
-      "$comment": "This works together with sample.json#properties/cohort_name/in_doc_ref_pattern.",
       "items": {
         "type": "string"
       }
@@ -130,8 +128,6 @@
   },
   "required": [
     "protocol_identifier",
-    "participants",
-    "allowed_cohort_names",
-    "allowed_collection_event_names"
+    "participants"
   ]
 }

cidc_schemas/schemas/participant.json

@@ -25,9 +25,7 @@
     "cohort_name": {
       "type": "string",
       "description": "Categorical description of cohorts, arms, and treatment groups.",
-      "example": "Arm_Z",
-      "in_doc_ref_pattern": "/allowed_cohort_names/*",
-      "$comment": "Values in this field are validated dynamically by `in_doc_ref_pattern`."
+      "example": "Arm_Z"
     },
     "essential_patient_entry_number": {
       "description": "Deprecated",

cidc_schemas/schemas/sample.json

@@ -67,9 +67,7 @@
         "collection_event_name": {
             "title": "Collection Event Name",
             "description": "Categorical description of timepoint at which the sample was taken.",
-            "type": "string",
-            "in_doc_ref_pattern": "/allowed_collection_event_names/*",
-            "$comment": "Values in this field are validated dynamically by `in_doc_ref_pattern`."
+            "type": "string"
         },
         "sample_location": {
             "description": "Sample location within the shipping container. Example: A1.",
@@ -79,25 +77,92 @@
             "description": "Type of sample sent.",
             "type": "string",
             "enum": [
-                "Tumor Tissue",
-                "Normal Tissue",
+                "BAL Cells",
+                "BAL Cell Supernatant",
+                "BAL Fluid",
+                "BMMC",
+                "BMMC Supernatant",
+                "Bone Marrow Aspirate",
+                "Bone Marrow Core",
+                "Bone Marrow Film",
+                "Buccal Cells",
+                "Buffy Coat",
+                "CAR-T Cells",
+                "cfDNA",
+                "CSF",
+                "CSF Cells",
+                "CSF Cell Supernatant",
+                "CTC Cells",
+                "CTC Cell Supernatant",
+                "ctDNA",
+                "Cytospin Film",
+                "DNA",
+                "FFPE Block",
+                "FFPE Block Punch",
+                "FFPE Section",
+                "FFPE Tissue Core",
+                "FFPE Tissue Curl",
+                "FFPE Tissue Scroll",
+                "Fine Needle Aspirate",
+                "Fixed Tissue Slide",
+                "Formalin Fixed Tissue",
+                "Fresh Tissue",
+                "Fresh Tissue Core",
+                "Frozen Tissue",
+                "Frozen Tissue Block",
+                "Frozen Tissue Curl",
+                "Frozen Tissue Core",
+                "Frozen Tissue Section",
+                "Germline DNA",
+                "Germline Nucleic Acid",
+                "Germline RNA",
+                "H&E Fixed Tissue Slide",
+                "Leukapheresis Cells",
+                "Lymph Node Tissue",
+                "Nucleic Acid",
+                "OCT Frozen Tissue",
+                "OCT Frozen Tissue Block",
+                "OCT Frozen Tissue Core",
+                "OCT Frozen Tissue Curl",
+                "OCT Frozen Tissue Section",
+                "PBMC",
+                "PBMC Supernatant",
+                "PBSC",
+                "PBSC Supernatant",
+                "Peptides",
+                "Pericardial Fluid",
+                "Peritoneal Cells",
+                "Peritoneal Cell Supernatant",
+                "Peritoneal Fluid",
+                "Peritoneal Lavage Fluid",
+                "Plasma",
+                "Pleural Cells",
+                "Pleural Cell Supernatant",
+                "Pleural Fluid",
+                "Protein Lysate",
+                "RNA",
+                "Saliva",
+                "Serum",
                 "Skin Tissue",
-                "Blood",
-                "Bone Marrow",
-                "Cerebrospinal Fluid",
-                "Lymph Node",
                 "Stool",
-                "Cell Product",
-                "White Blood Cell Apheresis",
+                "Synovial Fluid",
+                "Tissue Core",
                 "Urine",
-                "Not Reported",
+                "WBC",
+                "Whole Blood",
+                "Whole Blood Film",
                 "Other"
             ]
         },
+        "description_of_sample": {
+            "description": "The description of the sample.",
+            "type": "string",
+            "enum": ["Tumor", "Normal"]
+        },
         "type_of_tumor_sample": {
             "description": "The type of tumor sample obtained (primary or metastatic).",
             "type": "string",
-            "enum": ["Metastatic Tumor", "Primary Tumor", "Not Reported", "Other"]
+            "enum": ["Metastatic Tumor", "Primary Tumor", "Not Reported"]
         },
         "sample_collection_procedure": {
             "description": "Indicates the specimen source of the sample shipped. Example: Na Heparin blood draw aliquots (2 of three), FFPE block #52",
@@ -117,19 +182,27 @@
                 "Other"
             ]
         },
+        "sample_collection_procedure_other": {
+            "description": "Details of sample collection procedure when 'Other' is set.",
+            "type": "string"
+        },
         "core_number": {
             "description": "The biopsy core number from which the sample was used.",
-            "type": "number"
+            "type": "string"
         },
         "fixation_stabilization_type": {
             "description": "Type of specimen fixation or stabilization that was employed by the site directly after collection.",
             "type": "string",
             "enum": [
-                "Archival FFPE",
-                "Fresh Specimen",
-                "Frozen Specimen",
-                "Formalin-Fixed Paraffin-Embedded",
-                "Optimum cutting temperature medium",
+                "Ficoll",
+                "Formulin Fixation",
+                "Formulin-Fixed Paraffin-Embedded (FFPE)",
+                "Frozen",
+                "70% Ethanol",
+                "H&E",
+                "Liquid Nitrogen (Frozen)",
+                "OCT (Frozen)",
+                "Proteomic Stabilization",
                 "Thaw-Lyse",
                 "Not Reported",
                 "Other"
@@ -139,15 +212,43 @@
             "description": "The format in which the sample was sent.",
             "type": "string",
             "enum": [
-                "Sodium heparin",
-                "Blood specimen container with EDTA",
-                "Potassium EDTA",
-                "Streck Blood Collection Tube",
-                "Stool collection container with DNA stabilizer",
-                "Not Reported",
+                "ACD-A Tube",
+                "ACD-B Tube",
+                "Bag",
+                "Box",
+                "CellSave Tube",
+                "Conical Tube",
+                "Container",
+                "CPT Citrate Tube",
+                "CPT Heparin Tube",
+                "Cryovial",
+                "EDTA Tube",
+                "Fecal Collection Container with NA Stabilizer",
+                "FFPE Tissue Cassette",
+                "Formalin Jar",
+                "Lithium Heparin Tube",
+                "OMNIgene",
+                "OMNImet",
+                "PAXgene DNA Tube",
+                "PAXgene RNA Tube",
+                "Plain Red Top Tube",
+                "PPT Tube",
+                "Saliva Tube",
+                "Screw Top Jar",
+                "Slide",
+                "Slide Cassette",
+                "Smart Tube",
+                "Sodium Citrate Tube",
+                "Sodium Heparin Tube",
+                "SST Tube",
+                "Streck Tube",
                 "Other"
             ]
         },
+        "type_of_primary_container_other": {
+            "description": "The input for type of primary container if value is other.",
+            "type": "string"
+        },
         "sample_volume": {
             "description": "Volume of the parent sample (e.g. Heparin tube volume)",
             "type": "number"
@@ -161,20 +262,50 @@
             "description": "The type of processing that was performed on the collected specimen by the Biobank for storage.",
             "type": "string",
             "enum": [
-                "Whole Blood",
+                "BAL Cells",
+                "BAL Cell Supernatant",
+                "BMMC",
+                "BMMC Supernatant",
+                "Bone Marrow Film",
+                "Buccal Cells",
+                "Buffy Coat",
+                "cfDNA",
+                "CSF Cell Supernatant",
+                "CSF Cells",
+                "CTC Cell Supernatant",
+                "CTC Cells",
+                "ctDNA",
+                "DNA",
+                "FFPE Block",
+                "FFPE Block Punch",
+                "FFPE Section",
+                "FFPE Tissue Core",
+                "FFPE Tissue Curl",
+                "FFPE Tissue Scroll",
+                "Formalin Fixed Tissue",
+                "Germline DNA",
+                "Germline RNA",
+                "Leukapheresis Cells",
+                "Other",
+                "PBMC",
+                "PBMC Supernatant",
+                "PBSC",
+                "PBSC Supernatant",
+                "Peritoneal Cell Supernatant",
+                "Peritoneal Cells",
                 "Plasma",
+                "Pleural Cell Supernatant",
+                "Pleural Cells",
+                "RNA",
                 "Serum",
-                "PBMC",
-                "Buffy Coat",
-                "Bone Marrow Mononuclear Cell",
-                "Supernatant",
-                "Cell Pellet",
-                "H&E-Stained Fixed Tissue Slide Specimen",
-                "Fixed Slide",
-                "Tissue Scroll",
-                "FFPE Punch",
-                "Not Reported",
-                "Other"
+                "Whole Blood Film",
+                "Protein Lysate",
+                "Peptides",
+                "Germline Nucleic Acid",
+                "Nucleic Acid",
+                "H&E Fixed Tissue Slide",
+                "Fixed Tissue Slide",
+                "WBC"
             ]
         },
         "processed_sample_volume": {
@@ -210,13 +341,16 @@
             "description": "The type of derivative or analyte extracted from the specimen to be shipped for testing.",
             "type": "string",
             "enum": [
+                "cfDNA",
+                "ctDNA",
+                "Germline DNA",
+                "Germline RNA",
+                "Other",
+                "Protein Lysate",
+                "Peptides",
                 "Tumor DNA",
                 "Tumor RNA",
-                "Germline DNA",
-                "Circulating Tumor-Derived DNA",
-                "Stool DNA",
-                "Not Reported",
-                "Other"
+                "Stool DNA"
             ]
         },
         "sample_derivative_volume": {
@@ -340,12 +474,12 @@
         "material_storage_condition": {
             "description": "Storage condition of the material once it was received.",
             "type": "string",
-            "enum": ["RT", "4oC", "(-20)oC", "(-80)oC", "LN", "Not Reported", "Other"]
+            "enum": ["Ambient", "RT", "4 Celsius Degree", "-20 Degrees Celsius Or Minus 20 Degrees Celsius", "-80 Degrees Celsius Or Minus 80 Degrees Celsius", "-150 Degrees Celsius Or Minus 150 Degrees Celsius", "LN", "Dry Ice", "Not Reported", "Other"]
         },
         "quality_of_sample": {
             "description": "Final status of sample after QC and pathology review.",
             "type": "string",
-            "enum": ["Pass", "Pass at Risk", "Fail", "Not Reported", "Other"]
+            "enum": ["Pass", "Pass at Risk", "Fail"]
         },
         "sample_replacement": {
             "description": "Indication if sample replacement is/was requested.",

cidc_schemas/schemas/shipping_core.json

@@ -60,16 +60,12 @@
     "courier": {
       "description": "Courier utilized for shipment.",
       "type": "string",
-      "enum": ["FEDEX", "USPS", "UPS", "Inter-Site Delivery"]
+      "enum": ["FedEx", "USPS", "UPS", "Inter-Site Delivery", "DHL"]
     },
     "tracking_number": {
       "description": "Air bill number assigned to shipment. Example: 4567788343.",
       "type": "string"
     },
-    "account_number": {
-      "description": "Courier account number to pay for shipping if available. Example: 45465732.",
-      "type": "string"
-    },
     "shipping_condition": {
       "description": "Type of shipment made.",
       "type": "string",
@@ -82,6 +78,10 @@
         "Other"
       ]
     },
+    "shipping_condition_other": {
+      "description": "Details of shipping condition when shipping_condition value 'Other' is set.",
+      "type": "string"
+    },
     "date_shipped": {
       "description": "Date of shipment.",
       "type": "string",
@@ -96,43 +96,83 @@
         "description": "Indication that specimens were received in good condition.",
         "type": "string",
         "enum": [
-            "Specimen shipment received in good condition",
-            "Specimen shipment received in poor condition",
-            "Not Reported",
-            "Other"
+            "Normal",
+            "Damaged",
+            "Leakage",
+            "Opened"
         ]
     },
-    "ship_from": {
-      "description": "Contact information for shipment.",
+    "ship_from_sender_person_name": {
+      "description": "Sender person name.",
+      "type": "string"
+    },
+    "ship_from_sender_email": {
+      "description": "Sender email.",
+      "type": "string"
+    },
+    "ship_from_sender_phone": {
+      "description": "Ship from sender phone.",
+      "type": "string"
+    },
+    "ship_from_name": {
+      "description": "Ship from name.",
+      "type": "string"
+    },
+    "ship_from_street1": {
+      "description": "Ship from street1.",
+      "type": "string"
+    },
+    "ship_from_street2": {
+      "description": "Ship from street2.",
+      "type": "string"
+    },
+    "ship_from_city": {
+      "description": "Ship from city.",
+      "type": "string"
+    },
+    "ship_from_state": {
+      "description": "Ship from state.",
+      "type": "string"
+    },
+    "ship_from_zip": {
+      "description": "ship from zip.",
+      "type": "string"
+    },
+    "ship_from_country": {
+      "description": "Ship from country.",
+      "type": "string"
+    },
+    "ship_to_name": {
+      "description": "Ship to name.",
       "type": "string"
     },
-    "ship_to": {
-      "description": "Physical shipping address of the destination.",
+    "ship_to_street1": {
+      "description": "Ship to street1.",
+      "type": "string"
+    },
+    "ship_to_street2": {
+      "description": "ship to street2.",
+      "type": "string"
+    },
+    "ship_to_city": {
+      "description": "Ship to city.",
+      "type": "string"
+    },
+    "ship_to_state": {
+      "description": "Ship to state.",
+      "type": "string"
+    },
+    "ship_to_zip": {
+      "description": "Ship to zip.",
+      "type": "string"
+    },
+    "ship_to_country": {
+      "description": "Ship to country.",
       "type": "string"
     },
     "receiving_party": {
       "description": "Site where sample was shipped to be assayed.",
-      "type": "string",
-      "enum": [
-        "MDA_Wistuba",
-        "MDA_Bernatchez",
-        "MDA_Al-Atrash",
-        "MSSM_Gnjatic",
-        "MSSM_Rahman",
-        "MSSM_Kim-Schulze",
-        "MSSM_Bongers",
-        "MSSM_MTC",
-        "DFCI_Wu",
-        "DFCI_Hodi",
-        "DFCI_Severgnini",
-        "DFCI_Livak",
-        "Broad_Cibulskis",
-        "Stanf_Maecker",
-        "Stanf_Bendall",
-        "NCH",
-        "Adaptive",
-        "FNLCR_MoCha"
-      ]
+      "type": "string"
     }
   }
 }