nci-cidc-schemas 0.27.15__tar.gz → 0.27.16__tar.gz

{nci_cidc_schemas-0.27.15/nci_cidc_schemas.egg-info → nci_cidc_schemas-0.27.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_schemas
-Version: 0.27.15
+Version: 0.27.16
 Summary: The CIDC data model and tools for working with it.
 Home-page: https://github.com/NCI-CIDC/cidc-schemas
 Author: NCI

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """NCI"""
 __email__ = "nci-cidc-tools-admin@mail.nih.gov"
-__version__ = "0.27.15"
+__version__ = "0.27.16"

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/json_validation.py

@@ -7,7 +7,6 @@ import copy
 import functools
 import json
 import collections.abc
-import logging
 from contextlib import contextmanager
 from typing import Optional, Callable, Union
 
@@ -18,9 +17,9 @@ from jsonschema.exceptions import ValidationError, RefResolutionError
 from jsonpointer import resolve_pointer
 
 from .constants import SCHEMA_DIR, METASCHEMA_PATH
-from .util import JSON
+from .util import get_logger, JSON
 
-logger = logging.getLogger("cidc_schemas.json_validation")
+logger = get_logger("cidc_schemas.json_validation")
 
 
 class InDocRefNotFoundError(ValidationError):

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/prism/core.py

@@ -1,6 +1,5 @@
 """Build metadata dictionaries from Excel files."""
 
-import logging
 import base64
 import hmac
 from typing import List, Tuple, Union
@@ -16,12 +15,13 @@ from cidc_schemas.template_reader import XlTemplateReader
 from cidc_schemas.template_writer import RowType
 from cidc_schemas.constants import SCHEMA_DIR
 from .merger import PRISM_MERGE_STRATEGIES, MergeCollisionException
+from ..util import get_logger
 from jsonmerge import Merger
 from jsonpointer import EndOfList, JsonPointer, JsonPointerException, resolve_pointer
 
 from .constants import SUPPORTED_TEMPLATES
 
-logger = logging.getLogger(__file__)
+logger = get_logger(__file__)
 # logger.setLevel(logging.DEBUG)
 
 
@@ -453,7 +453,7 @@ def prismify(
             # for row in data:
             for row in data:
 
-                logging.debug(f"  next data row {row!r}")
+                logger.debug(f"  next data row {row!r}")
 
                 # creating data obj
                 data_obj = {}

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/prism/extra_metadata.py

@@ -1,6 +1,5 @@
 """Parsers for extracting extra metadata from files containing molecular data."""
 
-import logging
 import re
 from codecs import BOM_UTF8
 from typing import BinaryIO
@@ -9,8 +8,9 @@ import openpyxl
 import pandas as pd
 
 from ..json_validation import load_and_validate_schema
+from ..util import get_logger
 
-logger = logging.getLogger(__file__)
+logger = get_logger(__file__)
 
 # Build a regex from the CIMAC ID pattern in the schema
 cimac_id_regex = re.compile(

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/prism/merger.py

@@ -1,6 +1,5 @@
 """Merge CIDC schemas metadata dictionaries."""
 
-import logging
 from typing import BinaryIO, List, NamedTuple, Optional, Tuple
 
 import jsonschema
@@ -12,11 +11,11 @@ from ..json_validation import (
     strip_metadata_for_validation,
     _Validator,
 )
-from ..util import get_path, get_source
+from ..util import get_logger, get_path, get_source
 from .extra_metadata import EXTRA_METADATA_PARSERS
 from .constants import PROTOCOL_ID_FIELD_NAME
 
-logger = logging.getLogger(__file__)
+logger = get_logger(__file__)
 
 
 def merge_artifact(

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/prism/pipelines.py

@@ -3,7 +3,6 @@
 from collections import defaultdict
 from datetime import datetime
 from io import BytesIO
-import logging
 from tempfile import NamedTemporaryFile
 from typing import Dict, List, NamedTuple, Union
 
@@ -12,9 +11,9 @@ import pandas as pd
 
 from .constants import PROTOCOL_ID_FIELD_NAME, SUPPORTED_SHIPPING_MANIFESTS
 from ..template import Template
-from ..util import load_pipeline_config_template, participant_id_from_cimac
+from ..util import get_logger, load_pipeline_config_template, participant_id_from_cimac
 
-logger = logging.getLogger(__file__)
+logger = get_logger(__file__)
 
 
 # Note, bucket names must be all lowercase, dash, and underscore

nci_cidc_schemas-0.27.16/cidc_schemas/schemas/assays/components/mihc_input.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "metaschema/strict_meta_schema.json#",
+  "$id": "mihc_input",
+  "title": "MIHC Input Files",
+  "type": "object",
+  "description": "MIHC assay input files.",
+  "inheritableBase": true,
+  "properties": {
+    "tilemap_file": {
+      "$ref": "artifacts/artifact_ome_tiff.json"
+    },
+    "tilestats_file": {
+      "$ref": "artifacts/artifact_csv.json"
+    },
+    "h5ad_file": {
+      "$ref": "artifacts/artifact_h5ad.json"
+    },
+    "multitiffs_gzip": {
+      "$ref": "artifacts/artifact_gz.json"
+    },
+    "stain_zip": {
+      "$ref": "artifacts/artifact_zip.json"
+    },
+    "geojsons_zip": {
+      "$ref": "artifacts/artifact_zip.json"
+    }
+  },
+  "required": [
+    "tilemap_file",
+    "tilestats_file",
+    "h5ad_file",
+    "multitiffs_gzip",
+    "stain_zip",
+    "geojsons_zip"
+  ]
+}

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/schemas/assays/mihc_assay.json

@@ -14,35 +14,15 @@
             "additionalProperties": false,
             "required": [
                 "cimac_id",
-                "tilemap_file",
-                "tilestats_file",
-                "h5ad_file",
-                "multitiffs_gzip",
-                "stain_zip",
-                "geojsons_zip"
+                "files"
             ],
             "properties": {
                 "cimac_id": {
                     "description": "Id of an sample within this clinical trial, that this assay record is based upon.",
                     "$ref": "sample.json#properties/cimac_id"
                 },
-                "tilemap_file": {
-                    "$ref": "artifacts/artifact_ome_tiff.json"
-                },
-                "tilestats_file": {
-                    "$ref": "artifacts/artifact_csv.json"
-                },
-                "h5ad_file": {
-                    "$ref": "artifacts/artifact_h5ad.json"
-                },
-                "multitiffs_gzip": {
-                    "$ref": "artifacts/artifact_gz.json"
-                },
-                "stain_zip": {
-                    "$ref": "artifacts/artifact_zip.json"
-                },
-                "geojsons_zip": {
-                    "$ref": "artifacts/artifact_zip.json"
+                "files": {
+                    "$ref": "assays/components/mihc_input.json"
                 }
             }
         }
@@ -82,11 +62,6 @@
             }
         },
 
-
-        "excluded_samples": {
-            "$ref": "assays/components/excluded_samples.json"
-        },
-
         "records": {
             "type": "array",
             "description": "A single data record from a mIHC assay.",

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/schemas/templates/assays/mihc_template.json

@@ -98,24 +98,21 @@
                         {
                             "merge_pointer": "0/cimac_id",
                             "type_ref": "sample.json#properties/cimac_id"
-                        }
-                    },
-                    "Run info":
-                    {
+                        },
                         "Tilemap file": {
-                            "merge_pointer": "0/tilemap_file",
+                            "merge_pointer": "0/files/tilemap_file",
                             "is_artifact": 1,
                             "gcs_uri_format": "{protocol identifier}/mihc/{cimac id}/tilemap.ome.tiff",
                             "type_ref": "assays/components/local_file.json#properties/file_path"
                         },
                         "Tilestats file": {
-                            "merge_pointer": "0/tilestats_file",
+                            "merge_pointer": "0/files/tilestats_file",
                             "is_artifact": 1,
                             "gcs_uri_format": "{protocol identifier}/mihc/{cimac id}/tilestats.csv",
                             "type_ref": "assays/components/local_file.json#properties/file_path"
                         },
                         "H5ad file": {
-                            "merge_pointer": "0/h5ad_file",
+                            "merge_pointer": "0/files/h5ad_file",
                             "is_artifact": 1,
                             "gcs_uri_format": "{protocol identifier}/mihc/{cimac id}/h5ad.h5ad",
                             "type_ref": "assays/components/local_file.json#properties/file_path"
@@ -127,13 +124,13 @@
                             "type_ref": "assays/components/local_file.json#properties/file_path"
                         },
                         "Stain Zip": {
-                            "merge_pointer": "0/stain_zip",
+                            "merge_pointer": "0/files/stain_zip",
                             "is_artifact": 1,
                             "gcs_uri_format": "{protocol identifier}/mihc/{cimac id}/stains.zip",
                             "type_ref": "assays/components/local_file.json#properties/file_path"
                         },
                         "Geojsons Zip": {
-                            "merge_pointer": "0/geojsons_zip",
+                            "merge_pointer": "0/files/geojsons_zip",
                             "is_artifact": 1,
                             "gcs_uri_format": "{protocol identifier}/mihc/{cimac id}/geo_jsons.zip",
                             "type_ref": "assays/components/local_file.json#properties/file_path"
@@ -222,21 +219,6 @@
                         }
                     }
                 }
-            },
-            "Excluded Samples": {
-                "prism_data_object_pointer": "/excluded_samples/-",
-                "data_columns": {
-                  "Samples Excluded From Analysis": {
-                    "cimac id": {
-                      "type_ref": "sample.json#properties/cimac_id",
-                      "merge_pointer": "0/cimac_id"
-                    },
-                    "reason": {
-                      "type_ref": "assays/components/excluded_samples.json#items/properties/reason_excluded",
-                      "merge_pointer": "0/reason_excluded"
-                    }
-                  }
-                }
             }
         }
     }

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/template.py

@@ -4,7 +4,6 @@
 
 import os
 import os.path
-import logging
 import uuid
 import json
 import jsonschema
@@ -24,11 +23,11 @@ from collections import defaultdict
 
 from .constants import ANALYSIS_TEMPLATE_DIR, SCHEMA_DIR, TEMPLATE_DIR
 from .json_validation import _load_dont_validate_schema
-from .util import get_file_ext
+from .util import get_file_ext, get_logger
 
 from cidc_ngs_pipeline_api import OUTPUT_APIS
 
-logger = logging.getLogger("cidc_schemas.template")
+logger = get_logger("cidc_schemas.template")
 
 
 POSSIBLE_FILE_EXTS = [

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/template_reader.py

@@ -2,7 +2,6 @@
 
 """Defines the `XlTemplateReader` class for reading/validating templates from Excel templates."""
 
-import logging
 from itertools import dropwhile, zip_longest
 from typing import Dict, List, Tuple, Union, BinaryIO, NamedTuple, Optional
 from warnings import filterwarnings
@@ -18,9 +17,10 @@ import openpyxl
 
 from .template import Template
 from .template_writer import RowType, row_type_from_string
+from .util import get_logger
 from .json_validation import validate_instance
 
-logger = logging.getLogger("cidc_schemas.template_reader")
+logger = get_logger("cidc_schemas.template_reader")
 
 
 # A template row is any tuple whose first member is a RowType

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/template_writer.py

@@ -2,7 +2,6 @@
 
 """Defines the `XlTemplateWriter` class for writing `Template`s to Excel templates."""
 
-import logging
 from typing import Dict, Optional
 from enum import Enum
 from datetime import time
@@ -11,8 +10,9 @@ import xlsxwriter
 from xlsxwriter.utility import xl_rowcol_to_cell, xl_range
 
 from .template import Template
+from .util import get_logger
 
-logger = logging.getLogger("cidc_schemas.template_writer")
+logger = get_logger("cidc_schemas.template_writer")
 
 
 class RowType(Enum):

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/cidc_schemas/util.py

@@ -1,11 +1,23 @@
+import logging
 import os
 import re
+import sys
+from typing import List, Optional, Union
+
 import jinja2
-from typing import List, Union
+from deepdiff import grep
 
 JSON = Union[dict, float, int, list, str]
 
-from deepdiff import grep
+
+def get_logger(name: Optional[str], log_level=logging.INFO) -> logging.Logger:
+    """Get a configured logger with the given `name`."""
+    logger = logging.getLogger(name)
+    handler = logging.StreamHandler(sys.stdout)
+    logger.addHandler(handler)
+    logger.setLevel(log_level)
+    logger.propagate = False
+    return logger
 
 
 def load_pipeline_config_template(name: str) -> jinja2.Template:

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16/nci_cidc_schemas.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_schemas
-Version: 0.27.15
+Version: 0.27.16
 Summary: The CIDC data model and tools for working with it.
 Home-page: https://github.com/NCI-CIDC/cidc-schemas
 Author: NCI

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/nci_cidc_schemas.egg-info/SOURCES.txt

@@ -105,6 +105,7 @@ cidc_schemas/schemas/assays/components/mapping.json
 cidc_schemas/schemas/assays/components/mibi_antibody.json
 cidc_schemas/schemas/assays/components/mif_antibody.json
 cidc_schemas/schemas/assays/components/mihc_antibody.json
+cidc_schemas/schemas/assays/components/mihc_input.json
 cidc_schemas/schemas/assays/components/multiple_local_files.json
 cidc_schemas/schemas/assays/components/ngs_assay_core.json
 cidc_schemas/schemas/assays/components/ngs_assay_record.json

{nci_cidc_schemas-0.27.15 → nci_cidc_schemas-0.27.16}/tests/prism/cidc_test_data/assay_data.py

@@ -3385,23 +3385,48 @@ def mihc() -> PrismTestData:
                     "records": [
                         {
                             "cimac_id": "CTTTPP111.01",
-                            "tilemap_file": {
-                                "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b2e69"
-                            },
-                            "tilestats_file": {
-                                "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b2221"
-                            },
-                            "h5ad_file": {
-                                "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b1244"
-                            },
-                            "multitiffs_gzip": {
-                                "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b1564"
-                            },
-                            "stain_zip": {
-                                "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b1999"
+                            "files": {
+                                "tilemap_file": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b2e69"
+                                },
+                                "tilestats_file": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b2221"
+                                },
+                                "h5ad_file": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b1244"
+                                },
+                                "multitiffs_gzip": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b1564"
+                                },
+                                "stain_zip": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b1999"
+                                },
+                                "geojsons_zip": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b329"
+                                },
                             },
-                            "geojsons_zip": {
-                                "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b329"
+                        },
+                        {
+                            "cimac_id": "CTTTPP121.01",
+                            "files": {
+                                "tilemap_file": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b2e70"
+                                },
+                                "tilestats_file": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b23280"
+                                },
+                                "h5ad_file": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b122111"
+                                },
+                                "multitiffs_gzip": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b158188"
+                                },
+                                "stain_zip": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b133312"
+                                },
+                                "geojsons_zip": {
+                                    "upload_placeholder": "865f4bae-bca8-42f6-a3b7-cb4db27b4377"
+                                },
                             },
                         },
                     ],
@@ -3410,12 +3435,6 @@ def mihc() -> PrismTestData:
                     "staining": "Bond RX",
                     "imaging_status": "Yes",
                     "batch_id": "1234",
-                    "excluded_samples": [
-                        {
-                            "cimac_id": "CTTTPP131.01",
-                            "reason_excluded": "Staining quality issue",
-                        },
-                    ],
                 }
             ]
         },
@@ -3470,13 +3489,54 @@ def mihc() -> PrismTestData:
             metadata_availability=False,
             allow_empty=False,
         ),
+        LocalFileUploadEntry(
+            local_path="samples2.ome.tiff",
+            gs_key="test_prism_trial_id/mihc/CTTTPP121.01/tilemap.ome.tiff",
+            upload_placeholder="865f4bae-bca8-42f6-a3b7-cb4db27b2e70",
+            metadata_availability=False,
+            allow_empty=False,
+        ),
+        LocalFileUploadEntry(
+            local_path="samples2.csv",
+            gs_key="test_prism_trial_id/mihc/CTTTPP121.01/tilestats.csv",
+            upload_placeholder="865f4bae-bca8-42f6-a3b7-cb4db27b23280",
+            metadata_availability=False,
+            allow_empty=False,
+        ),
+        LocalFileUploadEntry(
+            local_path="run2.h5ad",
+            gs_key="test_prism_trial_id/mihc/CTTTPP121.01/h5ad.h5ad",
+            upload_placeholder="865f4bae-bca8-42f6-a3b7-cb4db27b122111",
+            metadata_availability=False,
+            allow_empty=False,
+        ),
+        LocalFileUploadEntry(
+            local_path="run2.tar.gz",
+            gs_key="test_prism_trial_id/mihc/CTTTPP121.01/multitiffs.tar.gz",
+            upload_placeholder="865f4bae-bca8-42f6-a3b7-cb4db27b158188",
+            metadata_availability=False,
+            allow_empty=False,
+        ),
+        LocalFileUploadEntry(
+            local_path="stains2.zip",
+            gs_key="test_prism_trial_id/mihc/CTTTPP121.01/stains.zip",
+            upload_placeholder="865f4bae-bca8-42f6-a3b7-cb4db27b133312",
+            metadata_availability=False,
+            allow_empty=False,
+        ),
+        LocalFileUploadEntry(
+            local_path="geojsons.zip",
+            gs_key="test_prism_trial_id/mihc/CTTTPP121.01/geo_jsons.zip",
+            upload_placeholder="865f4bae-bca8-42f6-a3b7-cb4db27b4377",
+            metadata_availability=False,
+            allow_empty=False,
+        ),
     ]
 
     cimac_ids = [
         record["cimac_id"]
         for batch in prismify_patch["assays"]["mihc"]
         for record in batch["records"]
-        for record in [*batch["records"], *batch["excluded_samples"]]
     ]
     base_trial = get_test_trial(cimac_ids)