nci-cidc-api-modules 1.2.35__tar.gz → 1.2.38__tar.gz

{nci_cidc_api_modules-1.2.35/nci_cidc_api_modules.egg-info → nci_cidc_api_modules-1.2.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.35
+Version: 1.2.38
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -33,6 +33,14 @@ Requires-Dist: requests>=2.32.5
 Requires-Dist: sqlalchemy>=2.0.44
 Requires-Dist: sqlalchemy-mixins~=2.0.5
 Requires-Dist: werkzeug>=3.1.4
+Requires-Dist: opentelemetry-api>=1.38.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.38.0
+Requires-Dist: opentelemetry-sdk>=1.38.0
+Requires-Dist: opentelemetry-instrumentation-flask>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
+Requires-Dist: opentelemetry-exporter-gcp-trace>=1.11.0
+Requires-Dist: opentelemetry-propagator-gcp>=1.11.0
 Requires-Dist: nci-cidc-schemas==0.28.10
 Dynamic: description
 Dynamic: description-content-type

nci_cidc_api_modules-1.2.38/cidc_api/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.2.38"

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/cidc_api/models/models.py

@@ -35,7 +35,8 @@ __all__ = [
     "ADMIN_FILE_CATEGORIES",
     "FINAL_JOB_STATUS",
     "INGESTION_JOB_STATUSES",
-    "INGESTION_JOB_COLORS",
+    "ASSAY_JOB_COLORS",
+    "CLINICAL_JOB_COLORS",
 ]
 
 import hashlib
@@ -3436,7 +3437,7 @@ INGESTION_JOB_STATUSES = [
 ]
 
 # Business decision to pass hex codes from the backend though that should be done by the front end...
-INGESTION_JOB_COLORS = {
+CLINICAL_JOB_COLORS = {
     "DRAFT": "",
     "INITIAL SUBMISSION": "#ACCAD7",
     "VALIDATION REVIEW": "#DABE90",
@@ -3444,6 +3445,13 @@ INGESTION_JOB_COLORS = {
     "INGESTION": "#8FCEC7",
     "PUBLISHED": "#90D9E6",
 }
+ASSAY_JOB_COLORS = {
+    "INITIAL SUBMISSION": "#43807E",
+    "VALIDATION REVIEW": "#906F3F",
+    "REVISION SUBMISSION": "#95358A",
+    "INGESTION": "#542C88",
+    "PUBLISHED": "#1C81A0",
+}
 # TODO If have "CANCELLED" concept or other final status, add here
 FINAL_JOB_STATUS = ["PUBLISHED"]
 TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER = "Data Category"

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/cidc_api/shared/assay_handling.py

@@ -53,7 +53,7 @@ def prepare_assay_job(trial_id: str, assay_type: str, batch_id: str) -> tuple[st
 
     # Create or retrieve intake bucket corresponding to the trial
     intake_bucket = gcloud_client.create_intake_bucket(get_current_user().email, trial_id=trial_id)
-    gcs_path = f"{intake_bucket.name}/{trial_id}/{assay_type}"
+    gcs_path = f"{intake_bucket.name}/{assay_type}/{submission_id}"
 
     return submission_id, job_status, error_status, start_date, version, gcs_path
 

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/cidc_api/shared/auth.py

@@ -1,16 +1,14 @@
 from functools import wraps
 from typing import List
 
-from packaging import version
-
 from flask import g, request, current_app as app, Flask
+from packaging import version
 from werkzeug.exceptions import Unauthorized, BadRequest, PreconditionFailed
 
-from ..models import Users, UserSchema
-
 from ..config.logging import get_logger
-
+from ..models import Users, UserSchema
 from ..shared.jose import decode_id_token
+from ..telemetry import trace_
 
 logger = get_logger(__name__)
 
@@ -144,6 +142,7 @@ def get_current_user() -> Users:
 _user_schema = UserSchema()
 
 
+@trace_()
 def authenticate() -> Users:
     id_token = _extract_token()
     token_payload = decode_id_token(id_token)
@@ -172,6 +171,7 @@ def _extract_token() -> str:
 
 
 ### Authorization logic ###
+@trace_()
 def authorize(user: Users, allowed_roles: List[str], resource: str, method: str) -> bool:
     """Check if the current user is authorized to act on the current request's resource.
     Raises Unauthorized

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/cidc_api/shared/file_handling.py

@@ -10,10 +10,12 @@ from ..config.settings import GOOGLE_CLINICAL_DATA_BUCKET
 from ..models import PreprocessedFiles, TRIAL_APPENDIX_A_CELL_THAT_ENDS_THE_HEADER
 from ..shared.auth import get_current_user
 from ..shared.gcloud_client import upload_file_to_gcs, move_gcs_file
+from ..telemetry import trace_
 
 logger = get_logger(__name__)
 
 
+@trace_()
 def set_current_file(
     file: FileStorage, file_category: str, gcs_folder: str, session: Session, uploader_email: str, job_id: int = None
 ) -> PreprocessedFiles:
@@ -26,6 +28,7 @@ def set_current_file(
     return latest_file
 
 
+@trace_()
 def create_file(
     file: FileStorage,
     gcs_folder: str,

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/cidc_api/shared/gcloud_client.py

@@ -1,6 +1,6 @@
 """Utilities for interacting with the Google Cloud Platform APIs."""
 
-# pylint: disable=logging-fstring-interpolation,too-many-lines
+# pylint: disable=logging-fstring-interpolation,too-many-lines, broad-exception-raised
 
 import base64
 import datetime
@@ -634,6 +634,7 @@ def _build_trial_upload_prefixes(
     trial_set: Set[str] = set()
     upload_set: Set[str] = set()
     if not trial_id:
+        # import is here becasue of circular import
         from ..models.models import TrialMetadata
 
         trial_set = {str(t.trial_id) for t in session.query(TrialMetadata).add_columns(TrialMetadata.trial_id)}
@@ -940,7 +941,8 @@ def get_signed_url(
 
 def _encode_and_publish(content: str, topic: str) -> Future:
     """Convert `content` to bytes and publish it to `topic`."""
-    pubsub_publisher = pubsub.PublisherClient()
+    publisher_options = pubsub.types.PublisherOptions(enable_open_telemetry_tracing=ENV == "dev-int")
+    pubsub_publisher = pubsub.PublisherClient(publisher_options=publisher_options)
     topic = pubsub_publisher.topic_path(GOOGLE_CLOUD_PROJECT, topic)
     data = bytes(content, "utf-8")
 

nci_cidc_api_modules-1.2.38/cidc_api/shared/utils.py

@@ -0,0 +1,19 @@
+from cidc_api.telemetry import trace_
+
+
+def _stripper(x):
+    if x and isinstance(x, str):
+        return x.strip()
+    else:
+        return x
+
+
+@trace_("sheet")
+def strip_whitespaces(df, sheet=None):
+    if sheet:
+        df = df[sheet]
+
+    df.rename(columns=_stripper, inplace=True)
+    df = df.map(_stripper)
+
+    return df

nci_cidc_api_modules-1.2.38/cidc_api/telemetry.py

@@ -0,0 +1,101 @@
+# standard modules
+from functools import wraps
+
+# external modules
+from opentelemetry import trace
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+# local modules
+from .config.settings import ENV, TESTING
+
+# pylint: disable=import-outside-toplevel
+
+
+def instrument_flask(app):
+    from opentelemetry.instrumentation.flask import FlaskInstrumentor
+    from opentelemetry.propagate import set_global_textmap
+    from opentelemetry.propagators.cloud_trace_propagator import CloudTraceFormatPropagator
+
+    FlaskInstrumentor().instrument_app(app)
+
+    # use the X-Cloud-Trace-Context header
+    set_global_textmap(CloudTraceFormatPropagator())
+
+
+def instrument_requests():
+    from opentelemetry.instrumentation.requests import RequestsInstrumentor
+
+    def _request_hook(span, request_obj):
+        span.update_name(f"requests {request_obj.method}")
+
+    RequestsInstrumentor().instrument(request_hook=_request_hook)
+
+
+def instrument_sqlachemy(engine):
+    from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+
+    SQLAlchemyInstrumentor().instrument(engine=engine)
+
+
+resource = Resource(attributes={"service.name": f"CIDC-{ENV}"})
+provider = TracerProvider(resource=resource)
+
+
+if ENV == "dev" and not TESTING:
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+    COLLECTOR_ENDPOINT = "127.0.0.1"
+    COLLECTOR_GRPC_PORT = 6004
+
+    # send spans to local exporter
+    # 1. download latest version from https://github.com/open-telemetry/opentelemetry-collector-releases/releases (otelcol-contrib_0.140.1_darwin_arm64)
+    # 2. start exporter from otel folder with `./otelcol-contrib --config=config.yaml`
+    # 3. download and start Jeager (all-in-one image)  - https://www.jaegertracing.io/download/
+    exporter = OTLPSpanExporter(endpoint=f"http://{COLLECTOR_ENDPOINT}:{COLLECTOR_GRPC_PORT}", insecure=True)
+    processor = BatchSpanProcessor(exporter)
+    provider.add_span_processor(processor)
+
+if ENV == "dev-int":
+    from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
+
+    # send span to Cloud Trace service - https://console.cloud.google.com/traces/explorer
+    exporter = CloudTraceSpanExporter()
+    processor = BatchSpanProcessor(exporter)
+    provider.add_span_processor(processor)
+
+
+# NOTE: we don't run telemetry in upper tiers; no span processor is noop
+
+trace.set_tracer_provider(provider)
+tracer = trace.get_tracer(__name__)
+
+
+def trace_(*args):
+    def decorator_factory(func):
+
+        @wraps(func)
+        def wrapper(*args_, **kwargs_):
+            func_name = f"{func.__module__.split(".")[-1]}.{func.__name__}"
+
+            with tracer.start_as_current_span(func_name) as span:
+                for arg in args:
+                    value = kwargs_.get(arg)
+
+                    # track id of argument if exists
+                    if hasattr(value, "id"):
+                        value = getattr(value, "id")
+
+                    span.set_attributes({arg: value})
+
+                result = func(*args_, **kwargs_)
+
+                if isinstance(result, (str, int, float, bool)):
+                    span.set_attribute("result", result)
+
+            return result
+
+        return wrapper
+
+    return decorator_factory

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38/nci_cidc_api_modules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nci_cidc_api_modules
-Version: 1.2.35
+Version: 1.2.38
 Summary: SQLAlchemy data models and configuration tools used in the NCI CIDC API
 Home-page: https://github.com/NCI-CIDC/cidc-api-gae
 License: MIT license
@@ -33,6 +33,14 @@ Requires-Dist: requests>=2.32.5
 Requires-Dist: sqlalchemy>=2.0.44
 Requires-Dist: sqlalchemy-mixins~=2.0.5
 Requires-Dist: werkzeug>=3.1.4
+Requires-Dist: opentelemetry-api>=1.38.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.38.0
+Requires-Dist: opentelemetry-sdk>=1.38.0
+Requires-Dist: opentelemetry-instrumentation-flask>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-requests>=0.59b0
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy>=0.59b0
+Requires-Dist: opentelemetry-exporter-gcp-trace>=1.11.0
+Requires-Dist: opentelemetry-propagator-gcp>=1.11.0
 Requires-Dist: nci-cidc-schemas==0.28.10
 Dynamic: description
 Dynamic: description-content-type

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/nci_cidc_api_modules.egg-info/SOURCES.txt

@@ -4,6 +4,8 @@ README.md
 pyproject.toml
 requirements.modules.txt
 setup.py
+cidc_api/__init__.py
+cidc_api/telemetry.py
 cidc_api/config/__init__.py
 cidc_api/config/db.py
 cidc_api/config/logging.py

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/nci_cidc_api_modules.egg-info/requires.txt

@@ -24,4 +24,12 @@ requests>=2.32.5
 sqlalchemy>=2.0.44
 sqlalchemy-mixins~=2.0.5
 werkzeug>=3.1.4
+opentelemetry-api>=1.38.0
+opentelemetry-exporter-otlp-proto-grpc>=1.38.0
+opentelemetry-sdk>=1.38.0
+opentelemetry-instrumentation-flask>=0.59b0
+opentelemetry-instrumentation-requests>=0.59b0
+opentelemetry-instrumentation-sqlalchemy>=0.59b0
+opentelemetry-exporter-gcp-trace>=1.11.0
+opentelemetry-propagator-gcp>=1.11.0
 nci-cidc-schemas==0.28.10

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/requirements.modules.txt

@@ -25,5 +25,19 @@ sqlalchemy>=2.0.44
 sqlalchemy-mixins~=2.0.5
 werkzeug>=3.1.4
 
+# open telemetry sdk
+opentelemetry-api >= 1.38.0
+opentelemetry-exporter-otlp-proto-grpc >= 1.38.0
+opentelemetry-sdk >= 1.38.0
+
+# open telemetry instrumentation
+opentelemetry-instrumentation-flask >= 0.59b0
+opentelemetry-instrumentation-requests >= 0.59b0
+opentelemetry-instrumentation-sqlalchemy >= 0.59b0
+
+# open telemetry gcp
+opentelemetry-exporter-gcp-trace >= 1.11.0
+opentelemetry-propagator-gcp >= 1.11.0
+
 # cidc deps
 nci-cidc-schemas==0.28.10

{nci_cidc_api_modules-1.2.35 → nci_cidc_api_modules-1.2.38}/setup.py

@@ -13,15 +13,17 @@ from cidc_api import __version__
 packages = [
     "cidc_api.config",
     "cidc_api.models",
-    "cidc_api.shared",
     "cidc_api.reference",
+    "cidc_api.shared",
 ]
 packages += find_namespace_packages(include=["cidc_api.models.*"])
 print(packages)
+
 setup(
     name="nci_cidc_api_modules",
     description="SQLAlchemy data models and configuration tools used in the NCI CIDC API",
     python_requires=">=3.13",
+    py_modules=["cidc_api.telemetry"],
     install_requires=requirements,
     license="MIT license",
     include_package_data=True,

nci_cidc_api_modules-1.2.35/cidc_api/shared/utils.py

@@ -1,11 +0,0 @@
-def strip_whitespaces(df):
-    def stripper(x):
-        if x and isinstance(x, str):
-            return x.strip()
-        else:
-            return x
-
-    df.rename(columns=stripper, inplace=True)
-    df = df.map(stripper)
-
-    return df