nci-cidc-api-modules 1.2.15__tar.gz → 1.2.16__tar.gz

{nci_cidc_api_modules-1.2.15/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.15
+Version: 1.2.16
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -12,7 +12,7 @@ Requires-Dist: cloud-sql-python-connector[pg8000]>=1.18.5
 Requires-Dist: flask>=3.1.2
 Requires-Dist: flask-migrate>=4.1.0
 Requires-Dist: flask-sqlalchemy>=3.1.1
-Requires-Dist: google-auth>=2.42.0
+Requires-Dist: google-auth==2.41.1
 Requires-Dist: google-api-python-client>=2.185.0
 Requires-Dist: google-cloud-bigquery>=3.38.0
 Requires-Dist: google-cloud-pubsub>=2.32.0

{nci_cidc_api_modules-1.2.15 → nci_cidc_api_modules-1.2.16}/cidc_api/models/models.py

@@ -26,6 +26,7 @@ __all__ = [
     "FileValidationErrors",
     "IngestionJobs",
     "JobFileCategories",
+    "CategoryDataElements",
     "ValidationConfigs",
     "TRIAL_APPENDIX_A",
     "TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER",
@@ -131,7 +132,6 @@ from ..config.settings import (
     MAX_PAGINATION_PAGE_SIZE,
     TESTING,
     INACTIVE_USER_DAYS,
-    GOOGLE_CLINICAL_DATA_BUCKET,
 )
 from ..shared import emails
 from ..shared.gcloud_client import (
@@ -145,7 +145,6 @@ from ..shared.gcloud_client import (
     revoke_intake_access,
     revoke_lister_access,
     revoke_bigquery_access,
-    gcs_xlsx_or_csv_file_to_pandas_dataframe,
 )
 
 os.environ["TZ"] = "UTC"
@@ -3450,30 +3449,9 @@ class IngestionJobs(CommonColumns):
 
     @with_default_session
     def transition_status(self, status: str, session: Session):
-        # create required categories after opening job for submission
-        if self.status == "DRAFT" and status == "INITIAL SUBMISSION":
-            for category in self.derive_required_categories_from_appendix_a():
-                JobFileCategories.create(category=category, job_id=self.id, type="required")
         self.status = status
         self.update(session=session)
 
-    def derive_required_categories_from_appendix_a(self) -> List:
-        appendix_a = PreprocessedFiles.get_files_by_category_and_status(TRIAL_APPENDIX_A, "current", job_id=self.id)[0]
-        df = gcs_xlsx_or_csv_file_to_pandas_dataframe(GOOGLE_CLINICAL_DATA_BUCKET, appendix_a.object_url)
-        categories = []
-        headers_ended = False
-        for _index, row in df.iterrows():
-            cell = str(row.iloc[0])
-            if headers_ended:
-                if cell != "nan" and cell not in categories:
-                    categories.append(cell)
-            elif cell.lower() == TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER.lower():
-                headers_ended = True
-        if "data_dictionary" not in categories:
-            # Ensure Data_Dictionary is always a required file category
-            categories.append("data_dictionary")
-        return categories
-
     @classmethod
     @with_default_session
     def atomic_set_job_as_pending(cls, job_id: int, session: Session) -> Boolean:
@@ -3553,6 +3531,7 @@ class JobFileCategories(CommonColumns):
     category = Column(String)
     job_id = Column(Integer)
     type = Column(Enum("required", "optional", name="type"))
+    is_custom = Column(Boolean, default=False, server_default="false")
 
     @staticmethod
     @with_default_session
@@ -3560,12 +3539,14 @@ class JobFileCategories(CommonColumns):
         category: str,
         job_id: int,
         type: str,
+        is_custom: bool = False,
         session: Session = None,
     ):
         new_category = JobFileCategories(
             category=category,
             job_id=job_id,
             type=type,
+            is_custom=is_custom,
         )
         new_category.insert(session=session)
         return new_category
@@ -3577,6 +3558,28 @@ class JobFileCategories(CommonColumns):
         return [c.category for c in categories]
 
 
+class CategoryDataElements(CommonColumns):
+    __tablename__ = "category_data_elements"
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ["category_id"],
+            ["job_file_categories.id"],
+            ondelete="CASCADE",
+        ),
+        Index(
+            "idx_elements_category_id" "category_id",
+            "name",
+            unique=True,
+        ),
+    )
+
+    category_id = Column(Integer, nullable=False)
+    name = Column(String, nullable=False)
+    is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
+    element_type = Column(String, nullable=False)
+    cardinality = Column(String, nullable=True)
+
+
 class FileValidationErrors(CommonColumns):
     __tablename__ = "file_validation_errors"
     __table_args__ = (

{nci_cidc_api_modules-1.2.15 → nci_cidc_api_modules-1.2.16}/cidc_api/shared/gcloud_client.py

@@ -1,6 +1,6 @@
 """Utilities for interacting with the Google Cloud Platform APIs."""
 
-# pylint: disable=logging-fstring-interpolation
+# pylint: disable=logging-fstring-interpolation,too-many-lines
 
 import base64
 import datetime
@@ -37,8 +37,8 @@ from sqlalchemy.orm.session import Session
 from werkzeug.datastructures import FileStorage
 from werkzeug.utils import secure_filename
 
-from cidc_api.config.secrets import get_secrets_manager
 from ..config.logging import get_logger
+from ..config.secrets import get_secrets_manager
 from ..config.settings import (
     DEV_USE_GCS,
     GOOGLE_INTAKE_ROLE,
@@ -62,6 +62,8 @@ from ..config.settings import (
     DEV_CFUNCTIONS_SERVER,
     INACTIVE_USER_DAYS,
 )
+from ..shared.utils import strip_whitespaces
+
 
 os.environ["TZ"] = "UTC"
 logger = get_logger(__name__)
@@ -427,9 +429,9 @@ def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
 
     # TODO: specify sheet in xlsx file and/or accept tsv and xls files
     if blob_name[-3:] == "csv":
-        return pd.read_csv(temp_file)
+        return strip_whitespaces(pd.read_csv(temp_file))
     elif blob_name[-4:] == "xlsx":
-        return pd.read_excel(temp_file)
+        return strip_whitespaces(pd.read_excel(temp_file))
     else:
         raise Exception("Can only read csv or xlsx files")
 

nci_cidc_api_modules-1.2.16/cidc_api/shared/utils.py

@@ -0,0 +1,8 @@
+def strip_whitespaces(df):
+    def stripper(x):
+        if x and isinstance(x, str):
+            return x.strip()
+        else:
+            return x
+
+    return df.map(stripper)

{nci_cidc_api_modules-1.2.15 → nci_cidc_api_modules-1.2.16/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.15
+Version: 1.2.16
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -12,7 +12,7 @@ Requires-Dist: cloud-sql-python-connector[pg8000]>=1.18.5
 Requires-Dist: flask>=3.1.2
 Requires-Dist: flask-migrate>=4.1.0
 Requires-Dist: flask-sqlalchemy>=3.1.1
-Requires-Dist: google-auth>=2.42.0
+Requires-Dist: google-auth==2.41.1
 Requires-Dist: google-api-python-client>=2.185.0
 Requires-Dist: google-cloud-bigquery>=3.38.0
 Requires-Dist: google-cloud-pubsub>=2.32.0

{nci_cidc_api_modules-1.2.15 → nci_cidc_api_modules-1.2.16}/nci_cidc_api_modules.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ cidc_api/shared/file_handling.py
 cidc_api/shared/gcloud_client.py
 cidc_api/shared/jose.py
 cidc_api/shared/rest_utils.py
+cidc_api/shared/utils.py
 nci_cidc_api_modules.egg-info/PKG-INFO
 nci_cidc_api_modules.egg-info/SOURCES.txt
 nci_cidc_api_modules.egg-info/dependency_links.txt

{nci_cidc_api_modules-1.2.15 → nci_cidc_api_modules-1.2.16}/nci_cidc_api_modules.egg-info/requires.txt

@@ -3,7 +3,7 @@ cloud-sql-python-connector[pg8000]>=1.18.5
 flask>=3.1.2
 flask-migrate>=4.1.0
 flask-sqlalchemy>=3.1.1
-google-auth>=2.42.0
+google-auth==2.41.1
 google-api-python-client>=2.185.0
 google-cloud-bigquery>=3.38.0
 google-cloud-pubsub>=2.32.0

{nci_cidc_api_modules-1.2.15 → nci_cidc_api_modules-1.2.16}/requirements.modules.txt

@@ -3,7 +3,7 @@ cloud-sql-python-connector[pg8000]>=1.18.5
 flask>=3.1.2
 flask-migrate>=4.1.0
 flask-sqlalchemy>=3.1.1
-google-auth>=2.42.0
+google-auth==2.41.1 # There is a bug in 2.42.X that causes local to fail when connecting to dev
 google-api-python-client>=2.185.0
 google-cloud-bigquery>=3.38.0
 google-cloud-pubsub>=2.32.0