nci-cidc-api-modules 1.1.40__tar.gz → 1.2.0__tar.gz

{nci_cidc_api_modules-1.1.40/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.0}/PKG-INFO

@@ -1,35 +1,35 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.40
+Version: 1.2.0
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
-Requires-Python: >=3.9
+Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: werkzeug==3.0.6
-Requires-Dist: flask==3.0.3
-Requires-Dist: flask-migrate==3.1.0
-Requires-Dist: flask-sqlalchemy==3.1.1
-Requires-Dist: sqlalchemy==2.0.41
-Requires-Dist: marshmallow==3.19.0
-Requires-Dist: marshmallow-sqlalchemy==1.4.2
-Requires-Dist: google-cloud-storage==2.18.0
-Requires-Dist: google-cloud-secret-manager==2.20.1
-Requires-Dist: google-cloud-pubsub==2.22.0
-Requires-Dist: google-cloud-bigquery==3.18.0
-Requires-Dist: google-api-python-client==2.64.0
-Requires-Dist: google-auth==2.32.0
-Requires-Dist: packaging>=20.0.0
-Requires-Dist: pyarrow==14.0.1
-Requires-Dist: numpy<2,>=1.16.5
-Requires-Dist: pandas==1.5.3
-Requires-Dist: python-dotenv==0.10.3
-Requires-Dist: requests==2.32.4
-Requires-Dist: jinja2==3.1.6
-Requires-Dist: certifi==2024.7.4
-Requires-Dist: cloud-sql-python-connector[pg8000]==1.18.3
-Requires-Dist: nci-cidc-schemas==0.27.28
+Requires-Dist: werkzeug~=3.1.3
+Requires-Dist: flask~=3.1.2
+Requires-Dist: flask-migrate~=4.1.0
+Requires-Dist: flask-sqlalchemy~=3.1.1
+Requires-Dist: sqlalchemy~=2.0.43
+Requires-Dist: marshmallow~=4.0.0
+Requires-Dist: marshmallow-sqlalchemy~=1.4.2
+Requires-Dist: google-cloud-storage~=3.3.0
+Requires-Dist: google-cloud-secret-manager~=2.24.0
+Requires-Dist: google-cloud-pubsub~=2.31.1
+Requires-Dist: google-cloud-bigquery~=3.36.0
+Requires-Dist: google-api-python-client~=2.179.0
+Requires-Dist: google-auth~=2.40.3
+Requires-Dist: packaging~=25.0
+Requires-Dist: pyarrow~=21.0.0
+Requires-Dist: numpy~=2.3.2
+Requires-Dist: pandas~=2.3.1
+Requires-Dist: python-dotenv~=1.1.1
+Requires-Dist: requests~=2.32.5
+Requires-Dist: jinja2~=3.1.6
+Requires-Dist: certifi~=2025.8.3
+Requires-Dist: cloud-sql-python-connector[pg8000]~=1.18.4
+Requires-Dist: nci-cidc-schemas==0.28.2
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
@@ -62,7 +62,7 @@ The next generation of the CIDC API, reworked to use Google Cloud-managed servic
 
 ## Install Python dependencies
 
-Python versions tested include 3.9 and 3.10. The current App Engine is using version 3.9 (see [app.prod.yaml](./app.prod.yaml)). You can use https://github.com/pyenv/pyenv to manage your python versions. Homebrew will also work, but you will have to be specific when you install packages with pip outside of virtual environments. On that note, it is recommended that you install your python dependencies in an isolated environment. For example,
+Use Python version 3.13
 
 ```bash
 # make a virtual environment in the current direcory called "venv"

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.2.0}/README.md

@@ -21,7 +21,7 @@ The next generation of the CIDC API, reworked to use Google Cloud-managed servic
 
 ## Install Python dependencies
 
-Python versions tested include 3.9 and 3.10. The current App Engine is using version 3.9 (see [app.prod.yaml](./app.prod.yaml)). You can use https://github.com/pyenv/pyenv to manage your python versions. Homebrew will also work, but you will have to be specific when you install packages with pip outside of virtual environments. On that note, it is recommended that you install your python dependencies in an isolated environment. For example,
+Use Python version 3.13
 
 ```bash
 # make a virtual environment in the current direcory called "venv"

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.2.0}/cidc_api/models/models.py

@@ -23,6 +23,7 @@ __all__ = [
     "ValidationMultiError",
     "with_default_session",
     "PreprocessedFiles",
+    "FileValidationErrors",
     "IngestionJobs",
     "JobFileCategories",
     "TRIAL_APPENDIX_A",
@@ -1344,14 +1345,14 @@ class TrialMetadata(CommonColumns):
                     if not trial.file_bundle[assay]:
                         del trial.file_bundle[assay]
             # Check if trial is ready for submission
-            setattr(trial, "ready_for_submission", trial.ready_for_submission())
+            setattr(trial, "ready_for_submission", trial.determine_ready_for_submission())
 
             trials.append(trial)
 
         return trials
 
     @with_default_session
-    def ready_for_submission(self, session: Session) -> Boolean:
+    def determine_ready_for_submission(self, session: Session) -> Boolean:
         open_job = IngestionJobs.get_open_job_by_trial(self.trial_id)
         if not open_job:
             return False
@@ -3261,6 +3262,7 @@ class PreprocessedFiles(CommonColumns):
     status = Column(String)
     version = Column(Integer)
     released_version = Column(String)
+    error_status = Column(String)
 
     @staticmethod
     @with_default_session
@@ -3457,6 +3459,9 @@ class IngestionJobs(CommonColumns):
                     categories.append(cell)
             elif cell == "PATIENT-LEVEL DATA":
                 headers_ended = True
+        if "Data_Dictionary" not in categories:
+            # Ensure Data_Dictionary is always a required file category
+            categories.append("Data_Dictionary")
         return categories
 
     @classmethod
@@ -3538,3 +3543,31 @@ class JobFileCategories(CommonColumns):
     def categories_for_job(cls, job_id: int, type: str, session: Session = None):
         categories = session.query(cls).filter(cls.job_id == job_id, cls.type == type).all()
         return [c.category for c in categories]
+
+
+class FileValidationErrors(CommonColumns):
+    __tablename__ = "file_validation_errors"
+    __table_args__ = (
+        ForeignKeyConstraint(
+            ["job_id"],
+            ["ingestion_jobs.id"],
+            ondelete="CASCADE",
+        ),
+        ForeignKeyConstraint(
+            ["file_id"],
+            ["preprocessed_files.id"],
+            ondelete="CASCADE",
+        ),
+    )
+
+    error_message = Column(String, nullable=False)
+    location = Column(String, nullable=True)
+    job_id = Column(Integer, nullable=False)
+    file_id = Column(Integer, nullable=False)
+
+    @staticmethod
+    @with_default_session
+    def create(file_id: int, job_id: int, error_message: str, location: str = None, session: Session = None):
+        new_error = FileValidationErrors(file_id=file_id, job_id=job_id, error_message=error_message, location=location)
+        new_error.insert(session=session)
+        return new_error

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.2.0}/cidc_api/shared/gcloud_client.py

@@ -421,9 +421,8 @@ def upload_xlsx_to_intake_bucket(user_email: str, trial_id: str, upload_type: st
 
 
 def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
-    """Reads an XLSX file from Google Cloud Storage into a Pandas DataFrame."""
-    sheet_data = storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
-    temp_file = io.BytesIO(sheet_data)
+    """Reads an XLSX or CSV file from Google Cloud Storage into a Pandas DataFrame."""
+    temp_file = get_file_bytes_from_gcs(bucket_name, blob_name)
 
     # TODO: specify sheet in xlsx file and/or accept tsv and xls files
     if blob_name[-3:] == "csv":
@@ -434,6 +433,12 @@ def gcs_xlsx_or_csv_file_to_pandas_dataframe(bucket_name: str, blob_name: str):
         raise Exception("Can only read csv or xlsx files")
 
 
+def get_file_bytes_from_gcs(bucket_name: str, blob_name: str) -> io.BytesIO:
+    """Reads a file from Google Cloud Storage and returns it as BytesIO."""
+    sheet_data = storage.Client().bucket(bucket_name).blob(blob_name).download_as_bytes()
+    return io.BytesIO(sheet_data)
+
+
 def _execute_multiblob_acl_change(
     user_email_list: List[str],
     blob_list: List[storage.Blob],

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.2.0/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,35 +1,35 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.40
+Version: 1.2.0
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
-Requires-Python: >=3.9
+Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: werkzeug==3.0.6
-Requires-Dist: flask==3.0.3
-Requires-Dist: flask-migrate==3.1.0
-Requires-Dist: flask-sqlalchemy==3.1.1
-Requires-Dist: sqlalchemy==2.0.41
-Requires-Dist: marshmallow==3.19.0
-Requires-Dist: marshmallow-sqlalchemy==1.4.2
-Requires-Dist: google-cloud-storage==2.18.0
-Requires-Dist: google-cloud-secret-manager==2.20.1
-Requires-Dist: google-cloud-pubsub==2.22.0
-Requires-Dist: google-cloud-bigquery==3.18.0
-Requires-Dist: google-api-python-client==2.64.0
-Requires-Dist: google-auth==2.32.0
-Requires-Dist: packaging>=20.0.0
-Requires-Dist: pyarrow==14.0.1
-Requires-Dist: numpy<2,>=1.16.5
-Requires-Dist: pandas==1.5.3
-Requires-Dist: python-dotenv==0.10.3
-Requires-Dist: requests==2.32.4
-Requires-Dist: jinja2==3.1.6
-Requires-Dist: certifi==2024.7.4
-Requires-Dist: cloud-sql-python-connector[pg8000]==1.18.3
-Requires-Dist: nci-cidc-schemas==0.27.28
+Requires-Dist: werkzeug~=3.1.3
+Requires-Dist: flask~=3.1.2
+Requires-Dist: flask-migrate~=4.1.0
+Requires-Dist: flask-sqlalchemy~=3.1.1
+Requires-Dist: sqlalchemy~=2.0.43
+Requires-Dist: marshmallow~=4.0.0
+Requires-Dist: marshmallow-sqlalchemy~=1.4.2
+Requires-Dist: google-cloud-storage~=3.3.0
+Requires-Dist: google-cloud-secret-manager~=2.24.0
+Requires-Dist: google-cloud-pubsub~=2.31.1
+Requires-Dist: google-cloud-bigquery~=3.36.0
+Requires-Dist: google-api-python-client~=2.179.0
+Requires-Dist: google-auth~=2.40.3
+Requires-Dist: packaging~=25.0
+Requires-Dist: pyarrow~=21.0.0
+Requires-Dist: numpy~=2.3.2
+Requires-Dist: pandas~=2.3.1
+Requires-Dist: python-dotenv~=1.1.1
+Requires-Dist: requests~=2.32.5
+Requires-Dist: jinja2~=3.1.6
+Requires-Dist: certifi~=2025.8.3
+Requires-Dist: cloud-sql-python-connector[pg8000]~=1.18.4
+Requires-Dist: nci-cidc-schemas==0.28.2
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
@@ -62,7 +62,7 @@ The next generation of the CIDC API, reworked to use Google Cloud-managed servic
 
 ## Install Python dependencies
 
-Python versions tested include 3.9 and 3.10. The current App Engine is using version 3.9 (see [app.prod.yaml](./app.prod.yaml)). You can use https://github.com/pyenv/pyenv to manage your python versions. Homebrew will also work, but you will have to be specific when you install packages with pip outside of virtual environments. On that note, it is recommended that you install your python dependencies in an isolated environment. For example,
+Use Python version 3.13
 
 ```bash
 # make a virtual environment in the current direcory called "venv"

nci_cidc_api_modules-1.2.0/nci_cidc_api_modules.egg-info/requires.txt

@@ -0,0 +1,23 @@
+werkzeug~=3.1.3
+flask~=3.1.2
+flask-migrate~=4.1.0
+flask-sqlalchemy~=3.1.1
+sqlalchemy~=2.0.43
+marshmallow~=4.0.0
+marshmallow-sqlalchemy~=1.4.2
+google-cloud-storage~=3.3.0
+google-cloud-secret-manager~=2.24.0
+google-cloud-pubsub~=2.31.1
+google-cloud-bigquery~=3.36.0
+google-api-python-client~=2.179.0
+google-auth~=2.40.3
+packaging~=25.0
+pyarrow~=21.0.0
+numpy~=2.3.2
+pandas~=2.3.1
+python-dotenv~=1.1.1
+requests~=2.32.5
+jinja2~=3.1.6
+certifi~=2025.8.3
+cloud-sql-python-connector[pg8000]~=1.18.4
+nci-cidc-schemas==0.28.2

nci_cidc_api_modules-1.2.0/requirements.modules.txt

@@ -0,0 +1,23 @@
+werkzeug~=3.1.3
+flask~=3.1.2
+flask-migrate~=4.1.0
+flask-sqlalchemy~=3.1.1
+sqlalchemy~=2.0.43
+marshmallow~=4.0.0
+marshmallow-sqlalchemy~=1.4.2
+google-cloud-storage~=3.3.0
+google-cloud-secret-manager~=2.24.0
+google-cloud-pubsub~=2.31.1
+google-cloud-bigquery~=3.36.0
+google-api-python-client~=2.179.0
+google-auth~=2.40.3
+packaging~=25.0
+pyarrow~=21.0.0
+numpy~=2.3.2
+pandas~=2.3.1
+python-dotenv~=1.1.1
+requests~=2.32.5
+jinja2~=3.1.6
+certifi~=2025.8.3
+cloud-sql-python-connector[pg8000]~=1.18.4
+nci-cidc-schemas==0.28.2

{nci_cidc_api_modules-1.1.40 → nci_cidc_api_modules-1.2.0}/setup.py

@@ -13,7 +13,7 @@ from cidc_api import __version__
 setup(
     name="nci_cidc_api_modules",
     description="SQLAlchemy data models and configuration tools used in the NCI CIDC API",
-    python_requires=">=3.9",
+    python_requires=">=3.13",
     install_requires=requirements,
     license="MIT license",
     packages=[

nci_cidc_api_modules-1.1.40/nci_cidc_api_modules.egg-info/requires.txt

@@ -1,23 +0,0 @@
-werkzeug==3.0.6
-flask==3.0.3
-flask-migrate==3.1.0
-flask-sqlalchemy==3.1.1
-sqlalchemy==2.0.41
-marshmallow==3.19.0
-marshmallow-sqlalchemy==1.4.2
-google-cloud-storage==2.18.0
-google-cloud-secret-manager==2.20.1
-google-cloud-pubsub==2.22.0
-google-cloud-bigquery==3.18.0
-google-api-python-client==2.64.0
-google-auth==2.32.0
-packaging>=20.0.0
-pyarrow==14.0.1
-numpy<2,>=1.16.5
-pandas==1.5.3
-python-dotenv==0.10.3
-requests==2.32.4
-jinja2==3.1.6
-certifi==2024.7.4
-cloud-sql-python-connector[pg8000]==1.18.3
-nci-cidc-schemas==0.27.28

nci_cidc_api_modules-1.1.40/requirements.modules.txt

@@ -1,23 +0,0 @@
-werkzeug==3.0.6
-flask==3.0.3
-flask-migrate==3.1.0
-flask-sqlalchemy==3.1.1
-sqlalchemy==2.0.41
-marshmallow==3.19.0
-marshmallow-sqlalchemy==1.4.2
-google-cloud-storage==2.18.0
-google-cloud-secret-manager==2.20.1
-google-cloud-pubsub==2.22.0
-google-cloud-bigquery==3.18.0
-google-api-python-client==2.64.0
-google-auth==2.32.0
-packaging>=20.0.0
-pyarrow==14.0.1
-numpy>=1.16.5,<2
-pandas==1.5.3
-python-dotenv==0.10.3
-requests==2.32.4
-jinja2==3.1.6
-certifi==2024.7.4
-cloud-sql-python-connector[pg8000]==1.18.3
-nci-cidc-schemas==0.27.28