nci-cidc-api-modules 1.1.39__tar.gz → 1.1.41__tar.gz

{nci_cidc_api_modules-1.1.39/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.1.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.39
+Version: 1.1.41
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -29,7 +29,7 @@ Requires-Dist: requests==2.32.4
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: certifi==2024.7.4
 Requires-Dist: cloud-sql-python-connector[pg8000]==1.18.3
-Requires-Dist: nci-cidc-schemas==0.27.27
+Requires-Dist: nci-cidc-schemas==0.27.28
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

{nci_cidc_api_modules-1.1.39 → nci_cidc_api_modules-1.1.41}/cidc_api/config/db.py

@@ -45,7 +45,34 @@ def get_sqlalchemy_database_uri(testing: bool = False) -> str:
         # Connect to the test database
         db_uri = environ.get("TEST_POSTGRES_URI", "fake-conn-string")
     elif not db_uri:
-        db_uri = f"postgresql+pg8000://{environ.get('CLOUD_SQL_DB_USER')}:xxx@/{environ.get('CLOUD_SQL_DB_NAME')}"
+        # TODO: figure out why IAM authentication causes an issue with the huge filter query
+        # db_uri = f"postgresql+pg8000://{environ.get('CLOUD_SQL_DB_USER')}:xxx@/{environ.get('CLOUD_SQL_DB_NAME')}"
+
+        secrets = get_secrets_manager(testing)
+
+        # If POSTGRES_URI env variable is not set,
+        # we're connecting to a Cloud SQL instance.
+
+        config: dict = {
+            "drivername": "postgresql",
+            "username": environ.get("CLOUD_SQL_DB_USER"),
+            "password": secrets.get(environ.get("CLOUD_SQL_DB_PASS_ID")),
+            "database": environ.get("CLOUD_SQL_DB_NAME"),
+        }
+
+        if environ.get("CLOUD_SQL_INSTANCE_NAME"):
+            socket_dir = environ.get("CLOUD_SQL_SOCKET_DIR", "/cloudsql/")
+
+            # If CLOUD_SQL_INSTANCE_NAME is defined, we're connecting
+            # via a unix socket from inside App Engine.
+            config["query"] = {"host": f'{socket_dir}{environ.get("CLOUD_SQL_INSTANCE_NAME")}'}
+        else:
+            raise RuntimeError(
+                "Either POSTGRES_URI or CLOUD_SQL_INSTANCE_NAME must be defined to connect " + "to a database."
+            )
+
+        db_uri = str(URL.create(**config).render_as_string(hide_password=False))
+
     assert db_uri
 
     return db_uri
@@ -53,7 +80,9 @@ def get_sqlalchemy_database_uri(testing: bool = False) -> str:
 
 # Use SQLALCHEMY_ENGINE_OPTIONS to connect to the cloud but use uri for local db
 def cloud_connector(testing: bool = False):
-    if not testing and not environ.get("POSTGRES_URI"):
-        return {"creator": getconn}
-    else:
-        return {}
+    return {}
+    # TODO: figure out IAM authentication
+    # if not testing and not environ.get("POSTGRES_URI"):
+    #    return {"creator": getconn}
+    # else:
+    #    return {}

{nci_cidc_api_modules-1.1.39 → nci_cidc_api_modules-1.1.41}/cidc_api/models/models.py

@@ -23,6 +23,7 @@ __all__ = [
     "ValidationMultiError",
     "with_default_session",
     "PreprocessedFiles",
+    "FileValidationErrors",
     "IngestionJobs",
     "JobFileCategories",
     "TRIAL_APPENDIX_A",
@@ -3261,6 +3262,7 @@ class PreprocessedFiles(CommonColumns):
     status = Column(String)
     version = Column(Integer)
     released_version = Column(String)
+    error_status = Column(String)
 
     @staticmethod
     @with_default_session
@@ -3457,19 +3459,24 @@ class IngestionJobs(CommonColumns):
                     categories.append(cell)
             elif cell == "PATIENT-LEVEL DATA":
                 headers_ended = True
+        if "Data_Dictionary" not in categories:
+            # Ensure Data_Dictionary is always a required file category
+            categories.append("Data_Dictionary")
         return categories
 
     @classmethod
     @with_default_session
     def atomic_set_job_as_pending(cls, job_id: int, session: Session) -> Boolean:
         # Preventing rare race condition where multiple people try and submit a job for validation
-        return bool(
+        result = bool(
             session.execute(
                 update(IngestionJobs)
                 .where(and_(IngestionJobs.id == job_id, IngestionJobs.pending == False))
                 .values(pending=True)
             ).rowcount
         )
+        session.commit()
+        return result
 
     @classmethod
     @with_default_session
@@ -3536,3 +3543,31 @@ class JobFileCategories(CommonColumns):
     def categories_for_job(cls, job_id: int, type: str, session: Session = None):
         categories = session.query(cls).filter(cls.job_id == job_id, cls.type == type).all()
         return [c.category for c in categories]
+
+
+class FileValidationErrors(CommonColumns):
+    __tablename__ = "file_validation_errors"
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ["job_id"],
+            ["ingestion_jobs.id"],
+            ondelete="CASCADE",
+        ),
+        ForeignKeyConstraint(
+            ["file_id"],
+            ["preprocessed_files.id"],
+            ondelete="CASCADE",
+        ),
+    )
+
+    error_message = Column(String, nullable=False)
+    location = Column(String, nullable=True)
+    job_id = Column(Integer, nullable=False)
+    file_id = Column(Integer, nullable=False)
+
+    @staticmethod
+    @with_default_session
+    def create(file_id: int, job_id: int, error_message: str, location: str = None, session: Session = None):
+        new_error = FileValidationErrors(file_id=file_id, job_id=job_id, error_message=error_message, location=location)
+        new_error.insert(session=session)
+        return new_error

{nci_cidc_api_modules-1.1.39 → nci_cidc_api_modules-1.1.41}/cidc_api/shared/gcloud_client.py

@@ -421,9 +421,8 @@ def upload_xlsx_to_intake_bucket(user_email: str, trial_id: str, upload_type: st
 
 
 def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
-    """Reads an XLSX file from Google Cloud Storage into a Pandas DataFrame."""
-    sheet_data = storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
-    temp_file = io.BytesIO(sheet_data)
+    """Reads an XLSX or CSV file from Google Cloud Storage into a Pandas DataFrame."""
+    temp_file = get_file_bytes_from_gcs(bucket_name, blob_name)
 
     # TODO: specify sheet in xlsx file and/or accept tsv and xls files
     if blob_name[-3:] == "csv":
@@ -434,6 +433,12 @@ def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
         raise Exception("Can only read csv or xlsx files")
 
 
+def get_file_bytes_from_gcs(bucket_name: str, blob_name: str) -> io.BytesIO:
+    """Reads a file from Google Cloud Storage and returns it as BytesIO."""
+    sheet_data = storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
+    return io.BytesIO(sheet_data)
+
+
 def _execute_multiblob_acl_change(
     user_email_list: List[str],
     blob_list: List[storage.Blob],

{nci_cidc_api_modules-1.1.39 → nci_cidc_api_modules-1.1.41/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.39
+Version: 1.1.41
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -29,7 +29,7 @@ Requires-Dist: requests==2.32.4
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: certifi==2024.7.4
 Requires-Dist: cloud-sql-python-connector[pg8000]==1.18.3
-Requires-Dist: nci-cidc-schemas==0.27.27
+Requires-Dist: nci-cidc-schemas==0.27.28
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page

{nci_cidc_api_modules-1.1.39 → nci_cidc_api_modules-1.1.41}/nci_cidc_api_modules.egg-info/requires.txt

@@ -20,4 +20,4 @@ requests==2.32.4
 jinja2==3.1.6
 certifi==2024.7.4
 cloud-sql-python-connector[pg8000]==1.18.3
-nci-cidc-schemas==0.27.27
+nci-cidc-schemas==0.27.28

{nci_cidc_api_modules-1.1.39 → nci_cidc_api_modules-1.1.41}/requirements.modules.txt

@@ -20,4 +20,4 @@ requests==2.32.4
 jinja2==3.1.6
 certifi==2024.7.4
 cloud-sql-python-connector[pg8000]==1.18.3
-nci-cidc-schemas==0.27.27
+nci-cidc-schemas==0.27.28