nci-cidc-api-modules 1.1.40__tar.gz → 1.1.41__tar.gz

{nci_cidc_api_modules-1.1.40/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.1.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.40
+Version: 1.1.41
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.1.41}/cidc_api/models/models.py

@@ -23,6 +23,7 @@ __all__ = [
     "ValidationMultiError",
     "with_default_session",
     "PreprocessedFiles",
+    "FileValidationErrors",
     "IngestionJobs",
     "JobFileCategories",
     "TRIAL_APPENDIX_A",
@@ -3261,6 +3262,7 @@ class PreprocessedFiles(CommonColumns):
     status = Column(String)
     version = Column(Integer)
     released_version = Column(String)
+    error_status = Column(String)
 
     @staticmethod
     @with_default_session
@@ -3457,6 +3459,9 @@ class IngestionJobs(CommonColumns):
                     categories.append(cell)
             elif cell == "PATIENT-LEVEL DATA":
                 headers_ended = True
+        if "Data_Dictionary" not in categories:
+            # Ensure Data_Dictionary is always a required file category
+            categories.append("Data_Dictionary")
         return categories
 
     @classmethod
@@ -3538,3 +3543,31 @@ class JobFileCategories(CommonColumns):
     def categories_for_job(cls, job_id: int, type: str, session: Session = None):
         categories = session.query(cls).filter(cls.job_id == job_id, cls.type == type).all()
         return [c.category for c in categories]
+
+
+class FileValidationErrors(CommonColumns):
+    __tablename__ = "file_validation_errors"
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ["job_id"],
+            ["ingestion_jobs.id"],
+            ondelete="CASCADE",
+        ),
+        ForeignKeyConstraint(
+            ["file_id"],
+            ["preprocessed_files.id"],
+            ondelete="CASCADE",
+        ),
+    )
+
+    error_message = Column(String, nullable=False)
+    location = Column(String, nullable=True)
+    job_id = Column(Integer, nullable=False)
+    file_id = Column(Integer, nullable=False)
+
+    @staticmethod
+    @with_default_session
+    def create(file_id: int, job_id: int, error_message: str, location: str = None, session: Session = None):
+        new_error = FileValidationErrors(file_id=file_id, job_id=job_id, error_message=error_message, location=location)
+        new_error.insert(session=session)
+        return new_error

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.1.41}/cidc_api/shared/gcloud_client.py

@@ -421,9 +421,8 @@ def upload_xlsx_to_intake_bucket(user_email: str, trial_id: str, upload_type: st
 
 
 def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
-    """Reads an XLSX file from Google Cloud Storage into a Pandas DataFrame."""
-    sheet_data = storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
-    temp_file = io.BytesIO(sheet_data)
+    """Reads an XLSX or CSV file from Google Cloud Storage into a Pandas DataFrame."""
+    temp_file = get_file_bytes_from_gcs(bucket_name, blob_name)
 
     # TODO: specify sheet in xlsx file and/or accept tsv and xls files
     if blob_name[-3:] == "csv":
@@ -434,6 +433,12 @@ def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
         raise Exception("Can only read csv or xlsx files")
 
 
+def get_file_bytes_from_gcs(bucket_name: str, blob_name: str) -> io.BytesIO:
+    """Reads a file from Google Cloud Storage and returns it as BytesIO."""
+    sheet_data = storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
+    return io.BytesIO(sheet_data)
+
+
 def _execute_multiblob_acl_change(
     user_email_list: List[str],
     blob_list: List[storage.Blob],

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.1.41/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.40
+Version: 1.1.41
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license