nci-cidc-api-modules 1.2.16__py3-none-any.whl → 1.2.17__py3-none-any.whl

cidc_api/config/db.py

@@ -10,8 +10,7 @@ from google.cloud.sql.connector import Connector, IPTypes
 from .secrets import get_secrets_manager
 
 db = SQLAlchemy()
-BaseModel = declarative_base()
-db.Model = BaseModel
+BaseModel = db.Model
 
 connector = Connector()
 
@@ -31,7 +30,6 @@ def getconn():
 def init_db(app: Flask):
     """Connect `app` to the database and run migrations"""
     db.init_app(app)
-    db.Model = BaseModel
     Migrate(app, db, app.config["MIGRATIONS_PATH"])
     with app.app_context():
         upgrade(app.config["MIGRATIONS_PATH"])

cidc_api/models/models.py

@@ -96,7 +96,7 @@ from sqlalchemy import (
     String,
     Table,
 )
-from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.dialects.postgresql import JSONB, UUID, CITEXT
 from sqlalchemy.engine import ResultProxy
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.hybrid import hybrid_property
@@ -381,7 +381,7 @@ class Users(CommonColumns):
     last_n = Column(String)
     organization = Column(Enum(*ORGS, name="orgs"))
     approval_date = Column(DateTime)
-    role = Column(Enum(*ROLES, name="role"))
+    role = Column(Enum(*ROLES, name="roles"))
     disabled = Column(Boolean, default=False, server_default="false")
 
     @validates("approval_date")
@@ -606,6 +606,22 @@ class Permissions(CommonColumns):
             unique=True,
             postgresql_where=file_group_id.isnot(None),
         ),
+        Index(
+            "unique_trial_id_upload_type_is_null_perms",
+            "granted_to_user",
+            "trial_id",
+            literal_column("(upload_type IS NULL)"),
+            unique=True,
+            postgresql_where="(upload_type IS NULL)",
+        ),
+        Index(
+            "unique_upload_type_trial_id_is_null_perms",
+            "granted_to_user",
+            literal_column("(trial_id IS NULL)"),
+            "upload_type",
+            unique=True,
+            postgresql_where="(trial_id IS NULL)",
+        ),
     )
 
     # Shorthand to make code related to trial- and upload-type-level permissions
@@ -2293,7 +2309,7 @@ class DownloadableFiles(CommonColumns):
     additional_metadata = Column(JSONB, nullable=False)
     # TODO rename upload_type, because we store manifests in there too.
     # NOTE: this column actually has type CITEXT.
-    upload_type = Column(String, nullable=False)
+    upload_type = Column(CITEXT, nullable=False)
     md5_hash = Column(String, nullable=True)
     crc32c_hash = Column(String, nullable=True)
     trial_id = Column(String, nullable=False)
@@ -2313,7 +2329,7 @@ class DownloadableFiles(CommonColumns):
     #   used instead of data_format.
     # The columns are left as optional for short term backwards compatibility.
     file_name = Column(String, nullable=True)
-    data_format = Column(String, nullable=True)
+    data_format = Column(CITEXT, nullable=True)
 
     file_groups = relationship(
         "FileGroups",
@@ -3261,11 +3277,11 @@ class PreprocessedFiles(CommonColumns):
         ),
     )
 
-    file_name = Column(String)
-    object_url = Column(String)
+    file_name = Column(String, nullable=False)
+    object_url = Column(String, nullable=False)
     job_id = Column(Integer)
-    file_category = Column(String)
-    uploader_email = Column(String)
+    file_category = Column(String, nullable=False)
+    uploader_email = Column(String, nullable=False)
     status = Column(String)
     version = Column(Integer)
     released_version = Column(String)
@@ -3433,7 +3449,7 @@ class IngestionJobs(CommonColumns):
         ),
     )
 
-    status = Column("status", Enum(*INGESTION_JOB_STATUSES, name="status"), nullable=False)
+    status = Column("status", Enum(*INGESTION_JOB_STATUSES, name="ingestion_job_status"), nullable=False)
     trial_id = Column(String, nullable=False)
     version = Column(Integer, nullable=False)
     pending = Column(Boolean, nullable=False, default=False)
@@ -3522,16 +3538,17 @@ class JobFileCategories(CommonColumns):
             ["ingestion_jobs.id"],
         ),
         Index(
-            "idx_categories_job_id" "job_id",
+            "idx_categories_job_id",
+            "job_id",
             "category",
             unique=True,
         ),
     )
 
     category = Column(String)
-    job_id = Column(Integer)
-    type = Column(Enum("required", "optional", name="type"))
-    is_custom = Column(Boolean, default=False, server_default="false")
+    job_id = Column(Integer, nullable=False)
+    type = Column(Enum("required", "optional", name="type"), nullable=False)
+    is_custom = Column(Boolean, nullable=False, default=False, server_default="false")
 
     @staticmethod
     @with_default_session
@@ -3557,6 +3574,11 @@ class JobFileCategories(CommonColumns):
         categories = session.query(cls).filter(cls.job_id == job_id, cls.type == type).all()
         return [c.category for c in categories]
 
+    @classmethod
+    @with_default_session
+    def full_categories_for_job(cls, job_id: int, session: Session = None):
+        return session.query(cls).filter_by(job_id=job_id).all()
+
 
 class CategoryDataElements(CommonColumns):
     __tablename__ = "category_data_elements"
@@ -3567,7 +3589,8 @@ class CategoryDataElements(CommonColumns):
             ondelete="CASCADE",
         ),
         Index(
-            "idx_elements_category_id" "category_id",
+            "idx_elements_category_id",
+            "category_id",
             "name",
             unique=True,
         ),
@@ -3579,6 +3602,11 @@ class CategoryDataElements(CommonColumns):
     element_type = Column(String, nullable=False)
     cardinality = Column(String, nullable=True)
 
+    @classmethod
+    @with_default_session
+    def elements_for_category(cls, category_id: int, session: Session = None):
+        return session.query(cls).filter_by(category_id=category_id).all()
+
 
 class FileValidationErrors(CommonColumns):
     __tablename__ = "file_validation_errors"

cidc_api/shared/file_handling.py

@@ -1,14 +1,15 @@
 from pathlib import Path
 
+from pandas import Series, DataFrame
+from sqlalchemy.orm.session import Session
 from werkzeug.datastructures import FileStorage
 from werkzeug.exceptions import BadRequest, InternalServerError
 
 from ..config.logging import get_logger
 from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
-from ..models import PreprocessedFiles
+from ..models import PreprocessedFiles, TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER
 from ..shared.auth import get_current_user
 from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
-from sqlalchemy.orm.session import Session
 
 logger = get_logger(__name__)
 
@@ -104,3 +105,37 @@ def strip_filename_and_pending_folder(path_str):
     if path.parent.name != "pending":
         raise ValueError("Expected 'pending' folder above file")
     return str(path.parent.parent)
+
+
+def get_row_at_condition(df: DataFrame, condition):
+    condition_met_index = df[condition].index[0]
+    row_at_condition_series = df.iloc[condition_met_index]
+
+    return row_at_condition_series
+
+
+def get_column(header_row_series: Series, header_name: str, use_raw_header_val: bool = False):
+    for idx, raw_header in enumerate(header_row_series):
+        if str(raw_header).lower() == header_name.lower():
+            return raw_header if use_raw_header_val else header_row_series.index[idx]
+    return None
+
+
+def get_column_from_appendix_a(appendix_a_df: DataFrame, header_name: str):
+    category_column = appendix_a_df.columns[0]
+    aa_header_condition = appendix_a_df[category_column] == TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER
+    header_row_series = get_row_at_condition(appendix_a_df, aa_header_condition)
+    return get_column(header_row_series, header_name)
+
+
+def get_column_from_first_row(df: DataFrame, header_name: str):
+    use_raw_header_val = False
+    if df.columns.inferred_type == "integer":
+        # If columns are integers (i.e. file was read without headers), treat the first row as header values.
+        header_row_series = df.iloc[0]
+    else:
+        # Otherwise columns already are headers
+        header_row_series = Series(df.columns)
+        use_raw_header_val = True
+
+    return get_column(header_row_series, header_name, use_raw_header_val=use_raw_header_val)

{nci_cidc_api_modules-1.2.16.dist-info → nci_cidc_api_modules-1.2.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.16
+Version: 1.2.17
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license

{nci_cidc_api_modules-1.2.16.dist-info → nci_cidc_api_modules-1.2.17.dist-info}/RECORD

@@ -1,11 +1,11 @@
 cidc_api/config/__init__.py,sha256=5mX8GAPxUKV84iS-aGOoE-4m68LsOCGCDptXNdlgvj0,148
-cidc_api/config/db.py,sha256=eFCkJDeykRIJZ25kePVjRNrZcrcKTc1s2K0yErr46LI,1683
+cidc_api/config/db.py,sha256=CRgpyw7uVP9v7CTAa7_1dcXURqrfjRcLNjGgZC7iPQE,1627
 cidc_api/config/logging.py,sha256=abhVYtn8lfhIt0tyV2WHFgSmp_s2eeJh7kodB6LH4J0,1149
 cidc_api/config/secrets.py,sha256=jRFj7W43pWuPf9DZQLCKF7WPXf5cUv-BAaS3ASqhV_Q,1481
 cidc_api/config/settings.py,sha256=ttOGvk_6zVMn4dtxIZ2-0w3wF2fpAUVfGpVZbKJ2b6s,4653
 cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
 cidc_api/models/migrations.py,sha256=gp9vtkYbA9FFy2s-7woelAmsvQbJ41LO2_DY-YkFIrQ,11464
-cidc_api/models/models.py,sha256=Id0BjccYd0aj6C-WCGXtYDxdpzK5KLjzub0HLVsnrxk,146517
+cidc_api/models/models.py,sha256=8WoCXeRebLpnQ70cGGRw_URdoGNS0sAfFeIMppwQp0s,147598
 cidc_api/models/schemas.py,sha256=6IE2dJoEMcMbi0Vr1V3cYKnPKU0hv9vRKBixOZHe88s,2766
 cidc_api/models/files/__init__.py,sha256=8BMTnUSHzUbz0lBeEQY6NvApxDD3GMWMduoVMos2g4Y,213
 cidc_api/models/files/details.py,sha256=sZkGM7iEV4-J6IDQCdiMV6KBDLbPxCOqUMaU3aY9rX8,65153
@@ -14,13 +14,13 @@ cidc_api/shared/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cidc_api/shared/auth.py,sha256=PHqmVGkqDjbmUofytVFwD_9ssgUomESl3fFtFHPwZYQ,9062
 cidc_api/shared/email_layout.html,sha256=pBoTNw3ACHH-ncZFaNvcy5bXMqPwizR78usb0uCYtIc,7670
 cidc_api/shared/emails.py,sha256=8kNFEaSnKpY-GX_iE59QUhSp3c4_uzy3SpHYt2QjuqI,6121
-cidc_api/shared/file_handling.py,sha256=4xkekdrzxWsfmfBtWpylyDMT98ICjQwx1meqqh1FAis,4103
+cidc_api/shared/file_handling.py,sha256=UP3KZ61Km5RNmNfk126_upKv-jGkDzpc5Pvk1s7cU2Q,5585
 cidc_api/shared/gcloud_client.py,sha256=ovXGS2ynaBgB_23prj23H10GNN4fectiVF7Hj4LJXQk,37302
 cidc_api/shared/jose.py,sha256=-qzGzEDAlokEp9E7WtBtQkXyyfPWTYXlwYpCqVJWmqM,1830
 cidc_api/shared/rest_utils.py,sha256=RwR30WOUAYCxL7V-i2totEyeriG30GbBDvBcpLXhM9w,6594
 cidc_api/shared/utils.py,sha256=oDGC8MHxEf7MDuzWynZuE66OfNUnRZE8z7Yn2Q9kYO8,178
-nci_cidc_api_modules-1.2.16.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
-nci_cidc_api_modules-1.2.16.dist-info/METADATA,sha256=0AG4rjFYqfR1q7wd44iGY7RCqqLyCylA4aHGv5zmkOI,39540
-nci_cidc_api_modules-1.2.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nci_cidc_api_modules-1.2.16.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
-nci_cidc_api_modules-1.2.16.dist-info/RECORD,,
+nci_cidc_api_modules-1.2.17.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
+nci_cidc_api_modules-1.2.17.dist-info/METADATA,sha256=cfEKZgJxtFXHOBDexrc-R04ceMLvGhXmbRjiofj3gBw,39540
+nci_cidc_api_modules-1.2.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nci_cidc_api_modules-1.2.17.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
+nci_cidc_api_modules-1.2.17.dist-info/RECORD,,