nci-cidc-api-modules 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

cidc_api/config/db.py

@@ -52,7 +52,7 @@ def get_sqlalchemy_database_uri(testing: bool = False) -> str:
                 "host": f'{socket_dir}{environ.get("CLOUD_SQL_INSTANCE_NAME")}'
             }
         else:
-            raise Exception(
+            raise RuntimeError(
                 "Either POSTGRES_URI or CLOUD_SQL_INSTANCE_NAME must be defined to connect "
                 + "to a database."
             )

cidc_api/config/secrets.py

@@ -11,8 +11,8 @@ def get_secrets_manager(is_testing=False):
 
         # If we're testing, we shouldn't need access to secrets in Secret Manager
         return MagicMock()
-    else:
-        return GoogleSecretManager()
+
+    return GoogleSecretManager()
 
 
 class GoogleSecretManager:

cidc_api/config/settings.py

@@ -21,10 +21,9 @@ DEV_USE_GCS = environ.get("DEV_USE_GCS") == "True"
 assert ENV in (
     "dev",
     "dev-int",
-    "featuredev",
     "staging",
     "prod",
-), "ENV environment variable must be set to 'dev', 'dev-int', 'featuredev', 'staging', or 'prod'"
+), "ENV environment variable must be set to 'dev', 'dev-int', 'staging', or 'prod'"
 DEBUG = environ.get("DEBUG") == "True"
 assert ENV == "dev" if DEBUG else True, "DEBUG mode is only allowed when ENV='dev'"
 TESTING = environ.get("TESTING") == "True"

cidc_api/csms/auth.py

@@ -1,9 +1,9 @@
 __all__ = ["get_token", "get_with_authorization", "get_with_paging"]
-import os
 
-os.environ["TZ"] = "UTC"
+import os
 from datetime import datetime, timedelta
 from typing import Any, Dict, Iterator
+
 import requests
 
 from ..config.settings import (
@@ -13,7 +13,9 @@ from ..config.settings import (
     CSMS_TOKEN_URL,
 )
 
+os.environ["TZ"] = "UTC"
 
+TIMEOUT_IN_SECONDS = 20
 _TOKEN, _TOKEN_EXPIRY = None, datetime.now()
 
 
@@ -29,13 +31,14 @@ def get_token():
                     "client_id": CSMS_CLIENT_ID,
                     "client_secret": CSMS_CLIENT_SECRET,
                 },
+                timeout=TIMEOUT_IN_SECONDS,
             ).json(),
             datetime.now(),
         )
 
         # res definition from https://developer.okta.com/docs/reference/api/oidc/#response-example-error-7
         if "errorCode" in res:
-            raise Exception(res["errorCode"] + ": " + res.get("errorSummary"))
+            raise RuntimeError(res["errorCode"] + ": " + res.get("errorSummary"))
 
         _TOKEN = res["access_token"]
         _TOKEN_EXPIRY = time + timedelta(seconds=res["expires_in"])
@@ -54,7 +57,11 @@ def get_with_authorization(url: str, **kwargs) -> requests.Response:
     kwargs["headers"] = headers
     if not url.startswith(CSMS_BASE_URL):
         url = CSMS_BASE_URL + url
-    return requests.get(url, **kwargs)
+    return requests.get(
+        url,
+        **kwargs,
+        timeout=TIMEOUT_IN_SECONDS,
+    )
 
 
 def get_with_paging(
@@ -88,7 +95,7 @@ def get_with_paging(
         else:
             limit = 1
 
-    kwargs.update(dict(limit=limit, offset=offset))
+    kwargs.update({"limit": limit, "offset": offset})
 
     res = get_with_authorization(url, params=kwargs)
     while res.status_code < 300 and len(res.json().get("data", [])) > 0:
@@ -96,5 +103,5 @@ def get_with_paging(
         yield from res.json()["data"]
         kwargs["offset"] += 1  # get the next page
         res = get_with_authorization(url, params=kwargs)
-    else:
-        res.raise_for_status()
+
+    res.raise_for_status()

cidc_api/models/csms_api.py

@@ -7,11 +7,8 @@ __all__ = [
 
 import os
 import re
-
-os.environ["TZ"] = "UTC"
 from collections import defaultdict
 from datetime import date, datetime, time
-from sqlalchemy.orm.session import Session
 from typing import (
     Any,
     Callable,
@@ -25,6 +22,9 @@ from typing import (
     Union,
 )
 
+from sqlalchemy.orm.session import Session
+
+from cidc_schemas.prism.merger import merge_clinical_trial_metadata
 from cidc_schemas.prism.core import (
     _check_encrypt_init,
     _encrypt,
@@ -37,13 +37,16 @@ from .models import with_default_session
 from ..config.logging import get_logger
 from ..config.settings import PRISM_ENCRYPT_KEY
 
+
+os.environ["TZ"] = "UTC"
 logger = get_logger(__name__)
 
-cimac_id_to_cimac_participant_id = lambda cimac_id, _: cimac_id[:7]
-cimac_id_regex_pattern = "^C[A-Z0-9]{3}[A-Z0-9]{3}[A-Z0-9]{2}.[0-9]{2}$"
-cimac_id_regex = re.compile(cimac_id_regex_pattern)
+
+def cimac_id_to_cimac_participant_id(cimac_id, _):
+    return cimac_id[:7]
 
 
+CIMAC_ID_REGEX = re.compile("^C[A-Z0-9]{3}[A-Z0-9]{3}[A-Z0-9]{2}.[0-9]{2}$")
 SAMPLE_SCHEMA: dict = load_and_validate_schema("sample.json")
 PARTICIPANT_SCHEMA: dict = load_and_validate_schema("participant.json")
 SHIPMENT_SCHEMA: dict = load_and_validate_schema("shipping_core.json")
@@ -54,13 +57,14 @@ TARGET_PROPERTIES_MAP: Dict[str, dict] = {
 }
 
 # make sure that the encryption key is set
+# NOTE: Exception is raised in external core module
 try:
     _check_encrypt_init()
-except:
+except Exception:
     set_prism_encrypt_key(PRISM_ENCRYPT_KEY)
 
 
-def _get_all_values(target: str, old: dict, drop: List[str] = []) -> Dict[str, Any]:
+def _get_all_values(target: str, old: dict, drop: List[str] = None) -> Dict[str, Any]:
     """
     Parameters
     ----------
@@ -74,6 +78,9 @@ def _get_all_values(target: str, old: dict, drop: List[str] = []) -> Dict[str, A
         all of the values from `old` that are in `target` excepting anything keys in `drop`
     """
 
+    if drop is None:
+        drop = []
+
     ret = {
         p: old[p]
         for p in TARGET_PROPERTIES_MAP[target].keys()
@@ -83,7 +90,7 @@ def _get_all_values(target: str, old: dict, drop: List[str] = []) -> Dict[str, A
     return ret
 
 
-class NewManifestError(Exception):
+class NewManifestError(RuntimeError):
     pass
 
 
@@ -94,7 +101,7 @@ def _parse_upload_type(sample: dict, upload_type: Set[str]) -> str:
         # safety
         return
 
-    elif sample_manifest_type == "biofluid_cellular":
+    if sample_manifest_type == "biofluid_cellular":
         upload_type.add("pbmc")
     elif sample_manifest_type == "tissue_slides":
         upload_type.add("tissue_slide")
@@ -162,9 +169,9 @@ def _get_and_check(
         ret = obj.get(key, default)
 
     if not check(ret):
-        raise Exception(msg)
-    else:
-        return ret
+        raise RuntimeError(msg)
+
+    return ret
 
 
 def _extract_info_from_manifest(
@@ -181,7 +188,7 @@ def _extract_info_from_manifest(
     str : manifest_id
     List[Dict[str, Any]] : samples
 
-    Exceptions Raised
+    RuntimeErrors Raised
     -----------------
     - "Cannot add a manifest that is not qc_complete"
         if manifest's status is not qc_complete (or null)
@@ -223,7 +230,7 @@ def _extract_details_from_trial(csms_samples: List[Dict[str, Any]]):
     str : assay_priority
     str : assay_type
 
-    Exceptions Raised
+    RuntimeErrors Raised
     -----------------
     - f"No assay_priority defined for manifest_id={manifest_id} for trial {trial_id}"
     - f"No assay_type defined for manifest_id={manifest_id} for trial {trial_id}"
@@ -246,11 +253,11 @@ def _extract_details_from_trial(csms_samples: List[Dict[str, Any]]):
 def _process_csms_sample(csms_sample: dict):
     event_name = csms_sample.get("standardized_collection_event_name")
     if event_name is None:
-        raise Exception(
+        raise RuntimeError(
             f"No standardized_collection_event_name defined for sample {csms_sample.get('cimac_id', '')} on manifest {csms_sample['manifest_id']} for trial {csms_sample['protocol_identifier']}"
         )
-    else:
-        csms_sample["collection_event_name"] = event_name
+
+    csms_sample["collection_event_name"] = event_name
 
     # encrypt participant ids if not already encrypted
     if (
@@ -299,18 +306,18 @@ def _convert_csms_samples(
     trial_id: str,
     manifest_id: str,
     csms_samples: List[Dict[str, Any]],
-    existing_cimac_ids: List[str] = [],
+    existing_cimac_ids: List[str] = None,
 ) -> Iterator[Tuple[str, Dict[str, Any]]]:
     """
     Convert a list of CSMS-style samples into an iterator returning CIMAC IDs and CIDC-style samples
-    Exceptions are raised during the call for each sample; full validation is NOT done first.
+    RuntimeErrors are raised during the call for each sample; full validation is NOT done first.
 
     Returns
     -------
     iterator yielding (str, dict)
         cimac_id, converted CSMS sample
 
-    Exceptions Raised
+    RuntimeErrors Raised
     -----------------
     - f"No standardized_collection_event_name defined for sample {sample['cimac_id']} on manifest {sample['manifest_id']} for trial {sample['protocol_identifier']}"
     - f"No cimac_id defined for samples[{n}] on manifest_id={manifest_id} for trial {trial_id}"
@@ -319,6 +326,10 @@ def _convert_csms_samples(
     - f"Sample with no local participant_id given:\n{sample}"
         if participant_id and trial_participant_id are both undefined
     """
+
+    if existing_cimac_ids is None:
+        existing_cimac_ids = []
+
     for n, sample in enumerate(csms_samples):
         # process the sample
         _process_csms_sample(csms_sample=sample)
@@ -329,12 +340,13 @@ def _convert_csms_samples(
             key="cimac_id",
             msg=f"No cimac_id defined for samples[{n}] on manifest_id={manifest_id} for trial {trial_id}",
         )
-        if not cimac_id_regex.match(cimac_id):
-            raise Exception(
+        if not CIMAC_ID_REGEX.match(cimac_id):
+            raise RuntimeError(
                 f"Malformatted cimac_id={cimac_id} on manifest_id={manifest_id} for trial {trial_id}"
             )
-        elif cimac_id in existing_cimac_ids:
-            raise Exception(
+
+        if cimac_id in existing_cimac_ids:
+            raise RuntimeError(
                 f"Sample with cimac_id={cimac_id} already exists for trial {trial_id}\nNew samples: {sample}"
             )
 
@@ -354,7 +366,7 @@ def insert_manifest_into_blob(
     Given a CSMS-style manifest, add it into the JSON metadata blob
     If `dry_run`, calls `session.rollback` instead of `session.commit`
 
-    Exceptions Raised
+    RuntimeErrors Raised
     -----------------
     - "Cannot add a manifest that is not qc_complete"
         if manifest's status is not qc_complete (or null)
@@ -376,13 +388,11 @@ def insert_manifest_into_blob(
 
     - "prism errors: [{errors from merge_clinical_trial_metadata}]"
     """
-    # schemas import here to keep JSON-blob code together
-    from cidc_schemas.prism.merger import merge_clinical_trial_metadata
 
     trial_id, manifest_id, csms_samples = _extract_info_from_manifest(manifest)
     trial_md = TrialMetadata.select_for_update_by_trial_id(trial_id, session=session)
     if manifest_id in [s["manifest_id"] for s in trial_md.metadata_json["shipments"]]:
-        raise Exception(
+        raise RuntimeError(
             f"Manifest with manifest_id={manifest_id} already exists for trial {trial_id}"
         )
 
@@ -418,9 +428,9 @@ def insert_manifest_into_blob(
 
     # each participant has a list of samples
     for cimac_participant_id, partic_samples in sample_map.items():
-        partic = dict(
-            cimac_participant_id=cimac_participant_id,
-            participant_id=partic_samples[0]["participant_id"],
+        partic = {
+            "cimac_participant_id": cimac_participant_id,
+            "participant_id": partic_samples[0]["participant_id"],
             **_get_all_values(
                 target="participant",
                 old=partic_samples[0],
@@ -432,7 +442,7 @@ def insert_manifest_into_blob(
                     "trial_participant_id",
                 ],
             ),
-        )
+        }
         partic["samples"] = [
             _get_all_values(
                 target="sample",
@@ -444,12 +454,12 @@ def insert_manifest_into_blob(
 
         patch["participants"].append(partic)
 
-    logger.info(f"Patch for {trial_id} manifest {manifest_id}:\n{patch}")
+    logger.info("Patch for %s manifest %s:\n%s", trial_id, manifest_id, patch)
     # merge and validate the data
     # the existence of the correct cohort and collection_event names are checked here
     merged, errs = merge_clinical_trial_metadata(patch, trial_md.metadata_json)
-    if len(errs):
-        raise Exception({"prism errors": [str(e) for e in errs]})
+    if errs:
+        raise RuntimeError({"prism errors": [str(e) for e in errs]})
 
     # save it
     trial_md.update(changes={"metadata_json": merged}, commit=False, session=session)
@@ -478,15 +488,17 @@ class Change:
         trial_id: str,
         manifest_id: str,
         cimac_id: str = None,
-        changes: Dict[str, Tuple[Any, Any]] = [],
+        changes: Dict[str, Tuple[Any, Any]] = None,
     ):
+        if changes is None:
+            changes = []
+
         if entity_type not in ["sample", "shipment", "upload"]:
             raise ValueError(
                 f"entity_type must be in: sample, shipment, upload\nnot: {entity_type}"
             )
-        else:
-            self.entity_type = entity_type
 
+        self.entity_type = entity_type
         self.trial_id = trial_id
         self.manifest_id = manifest_id
         self.cimac_id = cimac_id
@@ -514,28 +526,7 @@ def _calc_difference(
     manifest_id: str,
     cidc: Dict[str, Any],
     csms: Dict[str, Any],
-    ignore=[
-        "barcode",
-        "biobank_id",
-        "cimac_participant_id",
-        "entry_number",
-        "event",
-        "excluded",
-        "json_data",
-        "modified_time",
-        "modified_timestamp",
-        "protocol_identifier",
-        "qc_comments",
-        "reason",
-        "sample_approved",
-        "sample_manifest_type",
-        "samples",
-        "status",
-        "status_log",
-        "study_encoding",
-        "submitter",
-        "trial_id",
-    ],
+    ignore=None,
 ) -> Dict[str, Tuple[Any, Any]]:
     """
     The actual comparison function that handles comparing values
@@ -544,11 +535,38 @@ def _calc_difference(
     Do not perform a comparison for ignored keys
     Add constant critical fields back to anything that changes
     """
+
+    if ignore is None:
+        ignore = [
+            "barcode",
+            "biobank_id",
+            "cimac_participant_id",
+            "entry_number",
+            "event",
+            "excluded",
+            "json_data",
+            "modified_time",
+            "modified_timestamp",
+            "protocol_identifier",
+            "qc_comments",
+            "reason",
+            "sample_approved",
+            "sample_manifest_type",
+            "samples",
+            "status",
+            "status_log",
+            "study_encoding",
+            "submitter",
+            "trial_id",
+        ]
+
     # handle formatting and ignore
     cidc1: Dict[str, Any] = {
-        k: datetime.strftime(v, "%Y-%m-%d %H:%M:%S")
-        if isinstance(v, (date, time, datetime))
-        else v
+        k: (
+            datetime.strftime(v, "%Y-%m-%d %H:%M:%S")
+            if isinstance(v, (date, time, datetime))
+            else v
+        )
         for k, v in cidc.items()
         if k not in ignore
     }
@@ -607,15 +625,15 @@ def _get_csms_sample_map(
 ) -> Dict[str, Dict[str, Any]]:
     """Returns a map of CIMAC IDs to the relevant sample details from CSMS"""
     return {
-        csms_cimac_id: dict(
+        csms_cimac_id: {
             # participant-level critical field
-            cohort_name=csms_sample["cohort_name"],
+            "cohort_name": csms_sample["cohort_name"],
             # name changes
-            trial_id=csms_sample["protocol_identifier"],
-            participant_id=csms_sample["participant_id"],
+            "trial_id": csms_sample["protocol_identifier"],
+            "participant_id": csms_sample["participant_id"],
             # not in CSMS
-            cimac_participant_id=cimac_id_to_cimac_participant_id(csms_cimac_id, {}),
-            sample_manifest_type=csms_sample.get("sample_manifest_type"),
+            "cimac_participant_id": cimac_id_to_cimac_participant_id(csms_cimac_id, {}),
+            "sample_manifest_type": csms_sample.get("sample_manifest_type"),
             # the rest of the values
             **_get_all_values(
                 target="sample",
@@ -628,7 +646,7 @@ def _get_csms_sample_map(
                     "trial_id",
                 ],
             ),
-        )
+        }
         for csms_cimac_id, csms_sample in _convert_csms_samples(
             trial_id, manifest_id, csms_samples
         )
@@ -651,7 +669,7 @@ def _cross_validate_samples(
                 manifest_id,
                 cidc_sample["cimac_id"],
             )
-            raise Exception(
+            raise RuntimeError(
                 f"Missing sample: {formatted} on CSMS {(trial_id, manifest_id)}"
             )
     # make sure that all of the CSMS samples are in CIDC
@@ -682,9 +700,9 @@ def _cross_validate_samples(
                     cidc_sample["cimac_id"],
                 )
                 if cidc_sample is not None
-                else f"<no sample found>"
+                else "<no sample found>"
             )
-            raise Exception(
+            raise RuntimeError(
                 f"Change in critical field for: {formatted} to CSMS {(trial_id, manifest_id, cimac_id)}"
             )
 
@@ -705,7 +723,7 @@ def _initial_manifest_validation(
     UploadJobs : cidc_uploadjob
 
 
-    Exceptions Raised
+    RuntimeErrors Raised
     -----------------
     - "Cannot add a manifest that is not qc_complete"
         if manifest's status is not qc_complete (or null)
@@ -753,7 +771,7 @@ def _initial_manifest_validation(
     cidc_sample_map = _get_cidc_sample_map(cidc_shipment.metadata_patch)
     csms_sample_map = _get_csms_sample_map(trial_id, manifest_id, csms_samples)
 
-    # raises Exceptions if something is amiss
+    # raises RuntimeErrors if something is amiss
     _cross_validate_samples(
         trial_id=trial_id,
         manifest_id=manifest_id,
@@ -790,8 +808,8 @@ def _handle_shipment_differences(
     )
     if change:
         return change
-    else:
-        return None
+
+    return None
 
 
 def _handle_sample_differences(
@@ -854,8 +872,8 @@ def _handle_upload_differences(
     )
     if change:
         return change
-    else:
-        return None
+
+    return None
 
 
 @with_default_session
@@ -877,7 +895,7 @@ def detect_manifest_changes(
     ------
     NewManifestError
         if the manifest_id doesn't correspond to anything in CIDC
-    Exception
+    RuntimeError
         if the connections between any critical fields is changed
         namely trial_id, manifest_id, cimac_id
     """
@@ -886,12 +904,12 @@ def detect_manifest_changes(
         obj=csms_manifest,
         key="excluded",
         default=False,
-        msg=f"not called",
+        msg="not called",
         check=lambda _: True,
     ):
         return []
 
-    # ----- Initial validation, raises Exception if issues -----
+    # ----- Initial validation, raises RuntimeError if issues -----
     ret = []
     (
         trial_id,