nci-cidc-api-modules 1.2.16__tar.gz → 1.2.18__tar.gz

{nci_cidc_api_modules-1.2.16/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.16
+Version: 1.2.18
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license

{nci_cidc_api_modules-1.2.16 → nci_cidc_api_modules-1.2.18}/cidc_api/config/db.py

@@ -10,8 +10,7 @@ from google.cloud.sql.connector import Connector, IPTypes
 from .secrets import get_secrets_manager
 
 db = SQLAlchemy()
-BaseModel = declarative_base()
-db.Model = BaseModel
+BaseModel = db.Model
 
 connector = Connector()
 
@@ -31,7 +30,6 @@ def getconn():
 def init_db(app: Flask):
     """Connect `app` to the database and run migrations"""
     db.init_app(app)
-    db.Model = BaseModel
     Migrate(app, db, app.config["MIGRATIONS_PATH"])
     with app.app_context():
         upgrade(app.config["MIGRATIONS_PATH"])

{nci_cidc_api_modules-1.2.16 → nci_cidc_api_modules-1.2.18}/cidc_api/models/models.py

@@ -96,7 +96,7 @@ from sqlalchemy import (
     String,
     Table,
 )
-from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.dialects.postgresql import JSONB, UUID, CITEXT
 from sqlalchemy.engine import ResultProxy
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.hybrid import hybrid_property
@@ -381,7 +381,7 @@ class Users(CommonColumns):
     last_n = Column(String)
     organization = Column(Enum(*ORGS, name="orgs"))
     approval_date = Column(DateTime)
-    role = Column(Enum(*ROLES, name="role"))
+    role = Column(Enum(*ROLES, name="roles"))
     disabled = Column(Boolean, default=False, server_default="false")
 
     @validates("approval_date")
@@ -606,6 +606,22 @@ class Permissions(CommonColumns):
             unique=True,
             postgresql_where=file_group_id.isnot(None),
         ),
+        Index(
+            "unique_trial_id_upload_type_is_null_perms",
+            "granted_to_user",
+            "trial_id",
+            literal_column("(upload_type IS NULL)"),
+            unique=True,
+            postgresql_where="(upload_type IS NULL)",
+        ),
+        Index(
+            "unique_upload_type_trial_id_is_null_perms",
+            "granted_to_user",
+            literal_column("(trial_id IS NULL)"),
+            "upload_type",
+            unique=True,
+            postgresql_where="(trial_id IS NULL)",
+        ),
     )
 
     # Shorthand to make code related to trial- and upload-type-level permissions
@@ -2293,7 +2309,7 @@ class DownloadableFiles(CommonColumns):
     additional_metadata = Column(JSONB, nullable=False)
     # TODO rename upload_type, because we store manifests in there too.
     # NOTE: this column actually has type CITEXT.
-    upload_type = Column(String, nullable=False)
+    upload_type = Column(CITEXT, nullable=False)
     md5_hash = Column(String, nullable=True)
     crc32c_hash = Column(String, nullable=True)
     trial_id = Column(String, nullable=False)
@@ -2313,7 +2329,7 @@ class DownloadableFiles(CommonColumns):
     #   used instead of data_format.
     # The columns are left as optional for short term backwards compatibility.
     file_name = Column(String, nullable=True)
-    data_format = Column(String, nullable=True)
+    data_format = Column(CITEXT, nullable=True)
 
     file_groups = relationship(
         "FileGroups",
@@ -3261,11 +3277,11 @@ class PreprocessedFiles(CommonColumns):
         ),
     )
 
-    file_name = Column(String)
-    object_url = Column(String)
+    file_name = Column(String, nullable=False)
+    object_url = Column(String, nullable=False)
     job_id = Column(Integer)
-    file_category = Column(String)
-    uploader_email = Column(String)
+    file_category = Column(String, nullable=False)
+    uploader_email = Column(String, nullable=False)
     status = Column(String)
     version = Column(Integer)
     released_version = Column(String)
@@ -3433,7 +3449,7 @@ class IngestionJobs(CommonColumns):
         ),
     )
 
-    status = Column("status", Enum(*INGESTION_JOB_STATUSES, name="status"), nullable=False)
+    status = Column("status", Enum(*INGESTION_JOB_STATUSES, name="ingestion_job_status"), nullable=False)
     trial_id = Column(String, nullable=False)
     version = Column(Integer, nullable=False)
     pending = Column(Boolean, nullable=False, default=False)
@@ -3522,16 +3538,17 @@ class JobFileCategories(CommonColumns):
             ["ingestion_jobs.id"],
         ),
         Index(
-            "idx_categories_job_id" "job_id",
+            "idx_categories_job_id",
+            "job_id",
             "category",
             unique=True,
         ),
     )
 
     category = Column(String)
-    job_id = Column(Integer)
-    type = Column(Enum("required", "optional", name="type"))
-    is_custom = Column(Boolean, default=False, server_default="false")
+    job_id = Column(Integer, nullable=False)
+    type = Column(Enum("required", "optional", name="type"), nullable=False)
+    is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
 
     @staticmethod
     @with_default_session
@@ -3557,6 +3574,11 @@ class JobFileCategories(CommonColumns):
         categories = session.query(cls).filter(cls.job_id == job_id, cls.type == type).all()
         return [c.category for c in categories]
 
+    @classmethod
+    @with_default_session
+    def full_categories_for_job(cls, job_id: int, session: Session = None):
+        return session.query(cls).filter_by(job_id=job_id).all()
+
 
 class CategoryDataElements(CommonColumns):
     __tablename__ = "category_data_elements"
@@ -3567,7 +3589,8 @@ class CategoryDataElements(CommonColumns):
             ondelete="CASCADE",
         ),
         Index(
-            "idx_elements_category_id" "category_id",
+            "idx_elements_category_id",
+            "category_id",
             "name",
             unique=True,
         ),
@@ -3579,6 +3602,22 @@ class CategoryDataElements(CommonColumns):
     element_type = Column(String, nullable=False)
     cardinality = Column(String, nullable=True)
 
+    @classmethod
+    @with_default_session
+    def elements_for_category(cls, category_id: int, session: Session = None):
+        return session.query(cls).filter_by(category_id=category_id).all()
+
+    @classmethod
+    @with_default_session
+    def elements_by_category_for_job(cls, job_id: int, session: Session = None):
+        """Fetch all CategoryDataElements for a job, along with the category name."""
+        return (
+            session.query(JobFileCategories.category, cls)
+            .join(cls, cls.category_id == JobFileCategories.id)
+            .filter(JobFileCategories.job_id == job_id)
+            .all()
+        )
+
 
 class FileValidationErrors(CommonColumns):
     __tablename__ = "file_validation_errors"

{nci_cidc_api_modules-1.2.16 → nci_cidc_api_modules-1.2.18}/cidc_api/shared/file_handling.py

@@ -1,14 +1,15 @@
 from pathlib import Path
 
+from pandas import Series, DataFrame
+from sqlalchemy.orm.session import Session
 from werkzeug.datastructures import FileStorage
 from werkzeug.exceptions import BadRequest, InternalServerError
 
 from ..config.logging import get_logger
 from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
-from ..models import PreprocessedFiles
+from ..models import PreprocessedFiles, TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER
 from ..shared.auth import get_current_user
 from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
-from sqlalchemy.orm.session import Session
 
 logger = get_logger(__name__)
 
@@ -104,3 +105,37 @@ def strip_filename_and_pending_folder(path_str):
     if path.parent.name != "pending":
         raise ValueError("Expected 'pending' folder above file")
     return str(path.parent.parent)
+
+
+def get_row_at_condition(df: DataFrame, condition):
+    condition_met_index = df[condition].index[0]
+    row_at_condition_series = df.iloc[condition_met_index]
+
+    return row_at_condition_series
+
+
+def get_column(header_row_series: Series, header_name: str, use_raw_header_val: bool = False):
+    for idx, raw_header in enumerate(header_row_series):
+        if str(raw_header).lower() == header_name.lower():
+            return raw_header if use_raw_header_val else header_row_series.index[idx]
+    return None
+
+
+def get_column_from_appendix_a(appendix_a_df: DataFrame, header_name: str):
+    category_column = appendix_a_df.columns[0]
+    aa_header_condition = appendix_a_df[category_column] == TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER
+    header_row_series = get_row_at_condition(appendix_a_df, aa_header_condition)
+    return get_column(header_row_series, header_name)
+
+
+def get_column_from_first_row(df: DataFrame, header_name: str):
+    use_raw_header_val = False
+    if df.columns.inferred_type == "integer":
+        # If columns are integers (i.e. file was read without headers), treat the first row as header values.
+        header_row_series = df.iloc[0]
+    else:
+        # Otherwise columns already are headers
+        header_row_series = Series(df.columns)
+        use_raw_header_val = True
+
+    return get_column(header_row_series, header_name, use_raw_header_val=use_raw_header_val)

{nci_cidc_api_modules-1.2.16 → nci_cidc_api_modules-1.2.18/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.16
+Version: 1.2.18
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license