nci-cidc-api-modules 1.1.9__py3-none-any.whl → 1.1.11__py3-none-any.whl

cidc_api/models/models.py

@@ -24,10 +24,10 @@ __all__ = [
     "with_default_session",
 ]
 
-from collections import defaultdict
-import re
 import hashlib
 import os
+import re
+from collections import defaultdict
 from datetime import datetime, timedelta
 from enum import Enum as EnumBaseClass
 from functools import wraps
@@ -46,8 +46,10 @@ from typing import (
 )
 
 import pandas as pd
+from cidc_schemas import prism, unprism, json_validation
 from flask import current_app as app
 from google.cloud.storage import Blob
+from jsonschema.exceptions import ValidationError
 from sqlalchemy import (
     and_,
     Column,
@@ -76,11 +78,15 @@ from sqlalchemy import (
     Table,
     MetaData,
 )
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.engine import ResultProxy
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, validates
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.orm.exc import NoResultFound
-from sqlalchemy.orm.session import Session
 from sqlalchemy.orm.query import Query
+from sqlalchemy.orm.session import Session
 from sqlalchemy.sql import (
     # This is unfortunate but other code in this file relies on sqlalchemy.and_, or_, etc
     # instead of the sqlalchemy.sql versions we are importing here.  The solution is to
@@ -91,12 +97,7 @@ from sqlalchemy.sql import (
     text,
 )
 from sqlalchemy.sql.functions import coalesce
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy.dialects.postgresql import JSONB, UUID
-from sqlalchemy.engine import ResultProxy
-
-from cidc_schemas import prism, unprism, json_validation
+from werkzeug.exceptions import BadRequest
 
 from .files import (
     build_trial_facets,
@@ -107,8 +108,8 @@ from .files import (
     FilePurpose,
     FACET_NAME_DELIM,
 )
-
 from ..config.db import BaseModel
+from ..config.logging import get_logger
 from ..config.settings import (
     PAGINATION_PAGE_SIZE,
     MAX_PAGINATION_PAGE_SIZE,
@@ -121,13 +122,13 @@ from ..shared.gcloud_client import (
     grant_lister_access,
     grant_download_access,
     publish_artifact_upload,
+    publish_patient_sample_update,
     refresh_intake_access,
     revoke_download_access,
     revoke_intake_access,
     revoke_lister_access,
     revoke_bigquery_access,
 )
-from ..config.logging import get_logger
 
 os.environ["TZ"] = "UTC"
 logger = get_logger(__name__)
@@ -149,7 +150,7 @@ def with_default_session(f):
     @wraps(f)
     def wrapped(*args, **kwargs):
         if "session" not in kwargs:
-            kwargs["session"] = app.extensions["sqlalchemy"].db.session
+            kwargs["session"] = app.extensions["sqlalchemy"].session
         return f(*args, **kwargs)
 
     return wrapped
@@ -1329,6 +1330,7 @@ class TrialMetadata(CommonColumns):
     def list(
         cls,
         session: Session,
+        user: Users,
         include_file_bundles: bool = False,
         include_counts: bool = False,
         **pagination_args,
@@ -1350,10 +1352,25 @@ class TrialMetadata(CommonColumns):
 
         # Add other subqueries/columns to include in the query
         subqueries = []
+
         if include_file_bundles:
-            file_bundle_query = DownloadableFiles.build_file_bundle_query()
+            allowed_upload_types = []
+            if user and not user.is_admin() and not user.is_nci_user():
+                permissions = Permissions.find_for_user(user.id)
+                # An 'empty' upload_type means full trial-level access
+                allowed_upload_types = [
+                    p.upload_type for p in permissions if p.upload_type
+                ]
+                logger.info(
+                    f"Restricting file bundle for user {user.id} to {allowed_upload_types=}"
+                )
+
+            file_bundle_query = DownloadableFiles.build_file_bundle_query(
+                allowed_upload_types
+            )
             columns.append(file_bundle_query.c.file_bundle)
             subqueries.append(file_bundle_query)
+
         if include_counts:
             trial_summaries: List[dict] = cls.get_summaries()
 
@@ -1369,6 +1386,7 @@ class TrialMetadata(CommonColumns):
         for subquery in subqueries:
             # Each subquery will have a trial_id column and one record per trial id
             query = query.outerjoin(subquery, cls.trial_id == subquery.c.trial_id)
+
         query = cls._add_pagination_filters(query, **pagination_args)
 
         trials = []
@@ -2167,7 +2185,7 @@ class UploadJobs(CommonColumns):
         job.insert(session=session, commit=commit)
 
         if send_email:
-            trial = TrialMetadata.find_by_trial_id(trial_id)
+            trial = TrialMetadata.find_by_trial_id(trial_id, session=session)
             job.alert_upload_success(trial)
 
         return job
@@ -3136,7 +3154,9 @@ class DownloadableFiles(CommonColumns):
         return [r[0] for r in query.all()]
 
     @classmethod
-    def build_file_bundle_query(cls) -> Query:
+    def build_file_bundle_query(
+        cls, allowed_upload_types: Optional[List[str]]
+    ) -> Query:
         """
         Build a query that selects nested file bundles from the downloadable files table.
         The `file_bundles` query below should produce one bundle per unique `trial_id` that
@@ -3151,6 +3171,8 @@ class DownloadableFiles(CommonColumns):
           }
         ```
         where "type" is something like `"Olink"` or `"Participants Info"` and "purpose" is a `FilePurpose` string.
+
+        If `allowed_upload_types` is provided, the query will filter by files that only have an `upload_type` that appear in the list.
         """
         tid_col, type_col, purp_col, ids_col, purps_col = (
             literal_column("trial_id"),
@@ -3160,18 +3182,20 @@ class DownloadableFiles(CommonColumns):
             literal_column("purposes"),
         )
 
-        id_bundles = (
-            select(
-                [
-                    cls.trial_id,
-                    cls.data_category_prefix.label(type_col.key),
-                    cls.file_purpose.label(purp_col.key),
-                    func.json_agg(cls.id).label(ids_col.key),
-                ]
-            )
-            .group_by(cls.trial_id, cls.data_category_prefix, cls.file_purpose)
-            .alias("id_bundles")
-        )
+        id_bundles = select(
+            [
+                cls.trial_id,
+                cls.data_category_prefix.label(type_col.key),
+                cls.file_purpose.label(purp_col.key),
+                func.json_agg(cls.id).label(ids_col.key),
+            ]
+        ).group_by(cls.trial_id, cls.data_category_prefix, cls.file_purpose)
+
+        # Restrict files from appearing in the file bundle if the user doesn't have permissions for them
+        if allowed_upload_types:
+            id_bundles = id_bundles.filter(cls.upload_type.in_(allowed_upload_types))
+        id_bundles = id_bundles.alias("id_bundles")
+
         purpose_bundles = (
             select(
                 [
@@ -3272,3 +3296,41 @@ def result_proxy_to_models(
 ) -> List[BaseModel]:
     """Materialize a sqlalchemy `result_proxy` iterable as a list of `model` instances"""
     return [model(**dict(row_proxy)) for row_proxy in result_proxy.all()]
+
+
+@with_default_session
+def upload_manifest_json(
+    uploader_email: str,
+    trial_id: str,
+    template_type: str,
+    md_patch: dict,
+    session: Session,
+):
+    """
+    Ingest manifest data from JSON.
+
+    * Tries to load existing trial metadata blob (if fails, merge request fails; nothing saved).
+    * Merges the request JSON into the trial metadata (if fails, merge request fails; nothing saved).
+    * The merge request JSON is saved to `UploadJobs`.
+    * The updated trial metadata object is updated in the `TrialMetadata` table.
+    """
+    try:
+        TrialMetadata.patch_manifest(trial_id, md_patch, session=session, commit=False)
+    except ValidationError as e:
+        raise BadRequest(json_validation.format_validation_error(e)) from e
+    except ValidationMultiError as e:
+        raise BadRequest({"errors": e.args[0]}) from e
+
+    manifest_upload = UploadJobs.create(
+        upload_type=template_type,
+        uploader_email=uploader_email,
+        metadata=md_patch,
+        gcs_xlsx_uri="",  # not saving xlsx so we won't have phi-ish stuff in it
+        gcs_file_map=None,
+        session=session,
+        send_email=True,
+        status=UploadJobStatus.MERGE_COMPLETED.value,
+    )
+    # Publish that a manifest upload has been received
+    publish_patient_sample_update(manifest_upload.id)
+    return manifest_upload.id

{nci_cidc_api_modules-1.1.9.dist-info → nci_cidc_api_modules-1.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.1.9
+Version: 1.1.11
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -209,19 +209,27 @@ FLASK_APP=cidc_api.app:app flask db upgrade
 Install the [Cloud SQL Proxy](https://cloud.google.com/sql/docs/mysql/quickstart-proxy-test):
 
 ```bash
-curl -o /usr/local/bin/cloud_sql_proxy https://dl.google.com/cloudsql/cloud_sql_proxy.darwin.amd64
-chmod +x /usr/local/bin/cloud_sql_proxy
+sudo curl -o /usr/local/bin/cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.darwin.amd64
+sudo chmod +x /usr/local/bin/cloud-sql-proxy
 mkdir ~/.cloudsql
-chmod 777 ~/.cloudsql
+chmod 770 ~/.cloudsql
 ```
 
 Proxy to the dev Cloud SQL instance:
 
 ```bash
-cloud-sql-proxy --unix-socket ~/.cloudsql/ nih-nci-cimac-cidc-dev2:us-east4:cidc-postgresql-dev2 &
+cloud-sql-proxy --auto-iam-authn --address 127.0.0.1 --port 5432 nih-nci-cimac-cidc-dev2:us-east4:cidc-postgresql-dev2 &
 ```
 
-In your `.env` file, comment out `POSTGRES_URI` and uncomment all environment variables prefixed with `CLOUD_SQL_`. Change CLOUD_SQL_SOCKET_DIR to contain a reference to your home directory. Restart your local API instance, and it will connect to the staging Cloud SQL instance via the local proxy.
+If you want to run the proxy alongside a postgres instance on localhost listening on 5432, change the port for the proxy to another port instead like 5433.
+If you experience auth errors, make sure your google cloud sdk is authenticated.
+
+```bash
+gcloud auth login
+gcloud auth application-default login
+```
+
+To point an API running on localhost to the remote Postgres database, edit your `.env` file and comment out `POSTGRES_URI` and uncomment all environment variables prefixed with `CLOUD_SQL_`. Change CLOUD_SQL_SOCKET_DIR to contain a reference to your home directory. Restart your local API instance, and it will connect to the staging Cloud SQL instance via the local proxy.
 
 If you wish to connect to the staging Cloud SQL instance via the postgres REPL, download and run the CIDC sql proxy tool (a wrapper for `cloud_sql_proxy`):
 

{nci_cidc_api_modules-1.1.9.dist-info → nci_cidc_api_modules-1.1.11.dist-info}/RECORD

@@ -8,7 +8,7 @@ cidc_api/csms/auth.py,sha256=25Yma2Kz3KLENAPSeBYacFuSZXng-EDgmgInKBsRyP0,3191
 cidc_api/models/__init__.py,sha256=bl445G8Zic9YbhZ8ZBni07wtBMhLJRMBA-JqjLxx2bw,66
 cidc_api/models/csms_api.py,sha256=_uB9ZoxCFxKO8ZDTxCjS0CpeQg14EdlkEqnwyAFyYFQ,31377
 cidc_api/models/migrations.py,sha256=gp9vtkYbA9FFy2s-7woelAmsvQbJ41LO2_DY-YkFIrQ,11464
-cidc_api/models/models.py,sha256=kszT6ApBOIDsRJRHiYQvWo8283Jox_x0nPuetXBeK74,126680
+cidc_api/models/models.py,sha256=C0s28yCozvZ6K5xpSiVgURTci8fjQ2_wJlxU4OAQz-I,129135
 cidc_api/models/schemas.py,sha256=7tDYtmULuzTt2kg7RorWhte06ffalgpQKrFiDRGcPEQ,2711
 cidc_api/models/files/__init__.py,sha256=8BMTnUSHzUbz0lBeEQY6NvApxDD3GMWMduoVMos2g4Y,213
 cidc_api/models/files/details.py,sha256=h6R0p_hi-ukHsO7HV-3Wukccp0zRLJ1Oie_JNA_7Pl0,62274
@@ -19,8 +19,8 @@ cidc_api/shared/emails.py,sha256=FXW9UfI2bCus350SQuL7ZQYq1Vg-vGXaGWmRfA6z2nM,440
 cidc_api/shared/gcloud_client.py,sha256=7dDs0crLMJKdIp4IDSfrZBMB3h-zvWNieB81azoeLO4,33746
 cidc_api/shared/jose.py,sha256=QO30uIhbYDwzPEWWJXz0PfyV7E1AZHReEZJUVT70UJY,1844
 cidc_api/shared/rest_utils.py,sha256=LMfBpvJRjkfQjCzVXuhTTe4Foz4wlvaKg6QntyR-Hkc,6648
-nci_cidc_api_modules-1.1.9.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
-nci_cidc_api_modules-1.1.9.dist-info/METADATA,sha256=nePLp9yaC9CqWH-S3tTjDTb6eCYO-5ffp_1dt4ImZJg,40824
-nci_cidc_api_modules-1.1.9.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-nci_cidc_api_modules-1.1.9.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
-nci_cidc_api_modules-1.1.9.dist-info/RECORD,,
+nci_cidc_api_modules-1.1.11.dist-info/licenses/LICENSE,sha256=pNYWVTHaYonnmJyplmeAp7tQAjosmDpAWjb34jjv7Xs,1102
+nci_cidc_api_modules-1.1.11.dist-info/METADATA,sha256=b72D5JEyi8PEsRLq9c5T-AnKQvVB-LKVD_n8gbbbXhM,41284
+nci_cidc_api_modules-1.1.11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+nci_cidc_api_modules-1.1.11.dist-info/top_level.txt,sha256=rNiRzL0lJGi5Q9tY9uSoMdTbJ-7u5c_D2E86KA94yRA,9
+nci_cidc_api_modules-1.1.11.dist-info/RECORD,,