aind-data-transfer-service 1.14.0__py3-none-any.whl → 1.15.0__py3-none-any.whl

aind_data_transfer_service/__init__.py

@@ -1,7 +1,7 @@
 """Init package"""
 import os
 
-__version__ = "1.14.0"
+__version__ = "1.15.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

aind_data_transfer_service/configs/job_upload_template.py

@@ -2,7 +2,7 @@
 
 import datetime
 from io import BytesIO
-from typing import Any, Dict, List
+from typing import Any, ClassVar, Dict, List
 
 from aind_data_schema_models.modalities import Modality
 from aind_data_schema_models.platforms import Platform
@@ -10,16 +10,16 @@ from openpyxl import Workbook
 from openpyxl.styles import Font
 from openpyxl.utils import get_column_letter
 from openpyxl.worksheet.datavalidation import DataValidation
+from pydantic import BaseModel
 
 
-# TODO: convert to pydantic model
-class JobUploadTemplate:
+class JobUploadTemplate(BaseModel):
     """Class to configure and create xlsx job upload template"""
 
-    FILE_NAME = "job_upload_template.xlsx"
-    NUM_TEMPLATE_ROWS = 20
-    XLSX_DATETIME_FORMAT = "YYYY-MM-DDTHH:mm:ss"
-    HEADERS = [
+    FILE_NAME: ClassVar[str] = "job_upload_template.xlsx"
+    _NUM_TEMPLATE_ROWS: ClassVar[int] = 20
+    _XLSX_DATETIME_FORMAT: ClassVar[str] = "YYYY-MM-DDTHH:mm:ss"
+    _HEADERS: ClassVar[List[str]] = [
         "job_type",
         "project_name",
         "platform",
@@ -31,7 +31,7 @@ class JobUploadTemplate:
         "modality1",
         "modality1.input_source",
     ]
-    SAMPLE_JOBS = [
+    _SAMPLE_JOBS: ClassVar[List[List[Any]]] = [
         [
             "default",
             "Behavior Platform",
@@ -68,8 +68,8 @@ class JobUploadTemplate:
         ],
     ]
 
-    @property
-    def validators(self) -> List[Dict[str, Any]]:
+    @classmethod
+    def _get_validators(cls) -> List[Dict[str, Any]]:
         """
         Returns
         -------
@@ -82,36 +82,36 @@ class JobUploadTemplate:
                 "name": "platform",
                 "type": "list",
                 "options": list(Platform.abbreviation_map.keys()),
-                "column_indexes": [self.HEADERS.index("platform")],
+                "column_indexes": [cls._HEADERS.index("platform")],
             },
             {
                 "name": "modality",
                 "type": "list",
                 "options": list(Modality.abbreviation_map.keys()),
                 "column_indexes": [
-                    self.HEADERS.index("modality0"),
-                    self.HEADERS.index("modality1"),
+                    cls._HEADERS.index("modality0"),
+                    cls._HEADERS.index("modality1"),
                 ],
             },
             {
                 "name": "datetime",
                 "type": "date",
-                "column_indexes": [self.HEADERS.index("acq_datetime")],
+                "column_indexes": [cls._HEADERS.index("acq_datetime")],
             },
         ]
 
-    @property
-    def excel_sheet_filestream(self) -> BytesIO:
+    @classmethod
+    def create_excel_sheet_filestream(cls) -> BytesIO:
         """Create job template as xlsx filestream"""
         xl_io = BytesIO()
         workbook = Workbook()
         workbook.iso_dates = True
         worksheet = workbook.active
-        worksheet.append(self.HEADERS)
-        for job in self.SAMPLE_JOBS:
+        worksheet.append(cls._HEADERS)
+        for job in cls._SAMPLE_JOBS:
             worksheet.append(job)
         # data validators
-        for validator in self.validators:
+        for validator in cls._get_validators():
             dv_type = validator["type"]
             dv_name = validator["name"]
             dv_params = {
@@ -127,17 +127,17 @@ class JobUploadTemplate:
                 dv_params["prompt"] = f"Select a {dv_name} from the dropdown"
             elif dv_type == "date":
                 dv_params["prompt"] = "Provide a {} using {}".format(
-                    dv_name, self.XLSX_DATETIME_FORMAT
+                    dv_name, cls._XLSX_DATETIME_FORMAT
                 )
             dv = DataValidation(**dv_params)
             for i in validator["column_indexes"]:
                 col = get_column_letter(i + 1)
-                col_range = f"{col}2:{col}{self.NUM_TEMPLATE_ROWS}"
+                col_range = f"{col}2:{col}{cls._NUM_TEMPLATE_ROWS}"
                 dv.add(col_range)
                 if dv_type != "date":
                     continue
                 for (cell,) in worksheet[col_range]:
-                    cell.number_format = self.XLSX_DATETIME_FORMAT
+                    cell.number_format = cls._XLSX_DATETIME_FORMAT
             worksheet.add_data_validation(dv)
         # formatting
         bold = Font(bold=True)

aind_data_transfer_service/models/core.py

@@ -290,11 +290,15 @@ class SubmitJobRequestV2(BaseSettings):
         # check against any jobs in the context
         current_jobs = (info.context or dict()).get("current_jobs", list())
         for job in current_jobs:
-            prefix = job.get("s3_prefix")
-            if (
-                prefix is not None
-                and prefix in jobs_map
-                and json.dumps(job, sort_keys=True) in jobs_map[prefix]
-            ):
-                raise ValueError(f"Job is already running/queued for {prefix}")
+            jobs_to_check = job.get("upload_jobs", [job])
+            for j in jobs_to_check:
+                prefix = j.get("s3_prefix")
+                if (
+                    prefix is not None
+                    and prefix in jobs_map
+                    and json.dumps(j, sort_keys=True) in jobs_map[prefix]
+                ):
+                    raise ValueError(
+                        f"Job is already running/queued for {prefix}"
+                    )
         return self

aind_data_transfer_service/server.py

@@ -15,6 +15,7 @@ from aind_data_transfer_models import (
     __version__ as aind_data_transfer_models_version,
 )
 from aind_data_transfer_models.core import SubmitJobRequest, validation_context
+from authlib.integrations.starlette_client import OAuth
 from botocore.exceptions import ClientError
 from fastapi import Request
 from fastapi.responses import JSONResponse, StreamingResponse
@@ -23,6 +24,9 @@ from httpx import AsyncClient
 from openpyxl import load_workbook
 from pydantic import SecretStr, ValidationError
 from starlette.applications import Starlette
+from starlette.config import Config
+from starlette.middleware.sessions import SessionMiddleware
+from starlette.responses import RedirectResponse
 from starlette.routing import Route
 
 from aind_data_transfer_service import OPEN_DATA_BUCKET_NAME
@@ -95,6 +99,27 @@ def get_project_names() -> List[str]:
     return project_names
 
 
+def set_oauth() -> OAuth:
+    """Set up OAuth for the service"""
+    secrets_client = boto3.client("secretsmanager")
+    secret_response = secrets_client.get_secret_value(
+        SecretId=os.getenv("AIND_SSO_SECRET_NAME")
+    )
+    secret_value = json.loads(secret_response["SecretString"])
+    for secrets in secret_value:
+        os.environ[secrets] = secret_value[secrets]
+    config = Config()
+    oauth = OAuth(config)
+    oauth.register(
+        name="azure",
+        client_id=config("CLIENT_ID"),
+        client_secret=config("CLIENT_SECRET"),
+        server_metadata_url=config("AUTHORITY"),
+        client_kwargs={"scope": "openid email profile"},
+    )
+    return oauth
+
+
 def get_job_types(version: Optional[str] = None) -> List[str]:
     """Get a list of job_types"""
     params = get_parameter_infos(version)
@@ -228,7 +253,7 @@ async def validate_csv(request: Request):
                 data = csv_io.getvalue()
             csv_reader = csv.DictReader(io.StringIO(data))
             params = AirflowDagRunsRequestParameters(
-                dag_ids=["transform_and_upload_v2"],
+                dag_ids=["transform_and_upload_v2", "run_list_of_jobs"],
                 states=["running", "queued"],
             )
             _, current_jobs = await get_airflow_jobs(
@@ -324,7 +349,8 @@ async def validate_json_v2(request: Request):
     content = await request.json()
     try:
         params = AirflowDagRunsRequestParameters(
-            dag_ids=["transform_and_upload_v2"], states=["running", "queued"]
+            dag_ids=["transform_and_upload_v2", "run_list_of_jobs"],
+            states=["running", "queued"],
         )
         _, current_jobs = await get_airflow_jobs(params=params, get_confs=True)
         context = {
@@ -439,7 +465,8 @@ async def submit_jobs_v2(request: Request):
     content = await request.json()
     try:
         params = AirflowDagRunsRequestParameters(
-            dag_ids=["transform_and_upload_v2"], states=["running", "queued"]
+            dag_ids=["transform_and_upload_v2", "run_list_of_jobs"],
+            states=["running", "queued"],
         )
         _, current_jobs = await get_airflow_jobs(params=params, get_confs=True)
         context = {
@@ -987,8 +1014,7 @@ async def download_job_template(_: Request):
     """Get job template as xlsx filestream for download"""
 
     try:
-        job_template = JobUploadTemplate()
-        xl_io = job_template.excel_sheet_filestream
+        xl_io = JobUploadTemplate.create_excel_sheet_filestream()
         return StreamingResponse(
             io.BytesIO(xl_io.getvalue()),
             media_type=(
@@ -997,7 +1023,7 @@ async def download_job_template(_: Request):
             ),
             headers={
                 "Content-Disposition": (
-                    f"attachment; filename={job_template.FILE_NAME}"
+                    f"attachment; filename={JobUploadTemplate.FILE_NAME}"
                 )
             },
             status_code=200,
@@ -1089,6 +1115,60 @@ def get_parameter(request: Request):
         )
 
 
+async def admin(request: Request):
+    """Get admin page if authenticated, else redirect to login."""
+    user = request.session.get("user")
+    if os.getenv("ENV_NAME") == "local":
+        user = {"name": "local user"}
+    if user:
+        return templates.TemplateResponse(
+            name="admin.html",
+            context=(
+                {
+                    "request": request,
+                    "project_names_url": project_names_url,
+                    "user_name": user.get("name", "unknown"),
+                    "user_email": user.get("email", "unknown"),
+                }
+            ),
+        )
+    return RedirectResponse(url="/login")
+
+
+async def login(request: Request):
+    """Redirect to Azure login page"""
+    oauth = set_oauth()
+    redirect_uri = request.url_for("auth")
+    response = await oauth.azure.authorize_redirect(request, redirect_uri)
+    return response
+
+
+async def logout(request: Request):
+    """Logout user and clear session"""
+    request.session.pop("user", None)
+    return RedirectResponse(url="/")
+
+
+async def auth(request: Request):
+    """Authenticate user and store user info in session"""
+    oauth = set_oauth()
+    try:
+        token = await oauth.azure.authorize_access_token(request)
+        user = token.get("userinfo")
+        if not user:
+            raise ValueError("User info not found in access token.")
+        request.session["user"] = dict(user)
+    except Exception as error:
+        return JSONResponse(
+            content={
+                "message": "Error Logging In",
+                "data": {"error": f"{error.__class__.__name__}{error.args}"},
+            },
+            status_code=500,
+        )
+    return RedirectResponse(url="/admin")
+
+
 routes = [
     Route("/", endpoint=index, methods=["GET", "POST"]),
     Route("/api/validate_csv", endpoint=validate_csv_legacy, methods=["POST"]),
@@ -1131,6 +1211,11 @@ routes = [
         endpoint=download_job_template,
         methods=["GET"],
     ),
+    Route("/login", login, methods=["GET"]),
+    Route("/logout", logout, methods=["GET"]),
+    Route("/auth", auth, methods=["GET"]),
+    Route("/admin", admin, methods=["GET"]),
 ]
 
 app = Starlette(routes=routes)
+app.add_middleware(SessionMiddleware, secret_key=None)

aind_data_transfer_service/templates/admin.html

@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="UTF-8">
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet">
+  <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"></script>
+  <title>{% block title %} {% endblock %} AIND Data Transfer Service Admin</title>
+  <style>
+    body {
+      margin: 20px;
+      font-family: arial, sans-serif;
+    }
+    nav {
+      height: 40px;
+    }
+  </style>
+</head>
+<body>
+  <nav>
+    <a href="/">Submit Jobs</a> |
+    <a href="/jobs">Job Status</a> |
+    <a href="/job_params">Job Parameters</a> |
+    <a title="Download job template as .xslx" href="/api/job_upload_template" download>Job Submit Template</a> |
+    <a title="List of project names" href="{{ project_names_url }}" target="_blank">Project Names</a> |
+    <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"
+      target="_blank">Help</a> |
+    <a href="/admin">Admin</a> |
+    <a href="/logout">Log out</a>
+  </nav>
+  <div>
+    <h3>Admin</h3>
+    <div>Hello {{user_name}}, welcome to the admin page</div>
+    <div>Email: {{user_email}}</div>
+  </div>
+</body>
+</html>

aind_data_transfer_service/templates/index.html

@@ -49,7 +49,8 @@
         <a href="/job_params">Job Parameters</a> |
         <a title="Download job template as .xslx" href= "/api/job_upload_template" download>Job Submit Template</a> |
         <a title="List of project names" href= "{{ project_names_url }}" target="_blank" >Project Names</a> |
-        <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"  target="_blank" >Help</a>
+        <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"  target="_blank" >Help</a> |
+        <a href="/admin">Admin</a>
     </nav>
     <br>
     <div>

aind_data_transfer_service/templates/job_params.html

@@ -34,7 +34,8 @@
         <a href="/job_params">Job Parameters</a> |
         <a title="Download job template as .xslx" href= "/api/job_upload_template" download>Job Submit Template</a> |
         <a title="List of project names" href= "{{ project_names_url }}" target="_blank" >Project Names</a> |
-        <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"  target="_blank" >Help</a>
+        <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"  target="_blank" >Help</a> |
+        <a href="/admin">Admin</a>
     </nav>
     <div class="content">
         <h4 class="mb-2">

aind_data_transfer_service/templates/job_status.html

@@ -32,7 +32,8 @@
         <a href="/job_params">Job Parameters</a> |
         <a title="Download job template as .xslx" href= "/api/job_upload_template" download>Job Submit Template</a> |
         <a title="List of project names" href= "{{ project_names_url }}" target="_blank" >Project Names</a> |
-        <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"  target="_blank" >Help</a>
+        <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"  target="_blank" >Help</a> |
+        <a href="/admin">Admin</a>
     </nav>
     <div class="content">
         <!-- display total entries -->

{aind_data_transfer_service-1.14.0.dist-info → aind_data_transfer_service-1.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.14.0
+Version: 1.15.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
@@ -38,6 +38,7 @@ Requires-Dist: wtforms; extra == "server"
 Requires-Dist: requests==2.25.0; extra == "server"
 Requires-Dist: openpyxl; extra == "server"
 Requires-Dist: python-logging-loki; extra == "server"
+Requires-Dist: authlib; extra == "server"
 Dynamic: license-file
 
 # aind-data-transfer-service

{aind_data_transfer_service-1.14.0.dist-info → aind_data_transfer_service-1.15.0.dist-info}/RECORD

@@ -1,23 +1,24 @@
-aind_data_transfer_service/__init__.py,sha256=02hESRKUPpXuMD22H3psRTP20D33fQMG08afGio0d9k,272
+aind_data_transfer_service/__init__.py,sha256=lho7ClOyDOm8TOnoEsiTmbuidZwwCwry8o_Vu_mJ5qI,272
 aind_data_transfer_service/log_handler.py,sha256=c7a-gLmZeRpeCUBwCz6XsTszWXQeQdR7eKZtas4llXM,1700
-aind_data_transfer_service/server.py,sha256=ozavVyxMpZEwDAQ0OwRAwy_9CHXoLcUZ2rriSX1qH04,41643
+aind_data_transfer_service/server.py,sha256=8TRxybpk8hkqPng_6mEZc9wv8ofGErgYf0BSPk-8VAU,44545
 aind_data_transfer_service/configs/__init__.py,sha256=9W5GTuso9Is1B9X16RXcdb_GxasZvj6qDzOBDv0AbTc,36
 aind_data_transfer_service/configs/csv_handler.py,sha256=9jM0fUlWCzmqTC7ubAeFCl0eEIX5BQvHcPPPTPngcog,4374
 aind_data_transfer_service/configs/job_configs.py,sha256=T-h5N6lyY9xTZ_xg_5FxkyYuMdagApbE6xalxFQ-bqA,18848
-aind_data_transfer_service/configs/job_upload_template.py,sha256=dJo_nuGIjHNzkGtZmJd7-qqRz8s8R8RcWnYV3Hi_8QE,4990
+aind_data_transfer_service/configs/job_upload_template.py,sha256=aC5m1uD_YcpbggFQ-yZ7ZJSUUGX1yQqQLF3SwmljrLk,5127
 aind_data_transfer_service/hpc/__init__.py,sha256=YNc68YNlmXwKIPFMIViz_K4XzVVHkLPEBOFyO5DKMKI,53
 aind_data_transfer_service/hpc/client.py,sha256=-JSxAWn96_XOIDwhsXAHK3TZAdckddUhtcCzRHnaTqA,4700
 aind_data_transfer_service/hpc/models.py,sha256=-7HhV16s_MUyKPy0x0FGIbnq8DPL2qJAzJO5G7003AE,16184
 aind_data_transfer_service/models/__init__.py,sha256=Meym73bEZ9nQr4QoeyhQmV3nRTYtd_4kWKPNygsBfJg,25
-aind_data_transfer_service/models/core.py,sha256=_DHYFQL8kgXWgjvE21mWkRIVDCrmReskgwFAaTb5KQI,9971
+aind_data_transfer_service/models/core.py,sha256=uXtPUqjxKalg-sE8MxaJr11w_T_KKBRBSJuUgwoMZlQ,10135
 aind_data_transfer_service/models/internal.py,sha256=MGQrPuHrR21nn4toqdTCIEDW6MG7pWRajoPqD3j-ST0,9706
-aind_data_transfer_service/templates/index.html,sha256=KoqedswLWOiqgtkk2Z3HrDfEJycS_SJ7ueiuYGhL2Yo,11289
-aind_data_transfer_service/templates/job_params.html,sha256=vqIdNQsZTM0kq3Wa9u-VjmmMa0UzBTpK02WpOSatXBQ,8817
-aind_data_transfer_service/templates/job_status.html,sha256=vIOaJGJM78hOWTLTAzMfHjG9sNqPvS-muAyXYQtpnYI,16901
+aind_data_transfer_service/templates/admin.html,sha256=KvQB54-mD8LL8wnDd3G9a8lDYAT8BWK13_MhpIJAiSY,1200
+aind_data_transfer_service/templates/index.html,sha256=TDmmHlhWFPnQrk6nk1OhNjC3SapZtX0TXeuUonoCO7g,11326
+aind_data_transfer_service/templates/job_params.html,sha256=ivofS1CjSnC87T5J2BTsNtVzq6xnUqlPZePdGt_fc4U,8854
+aind_data_transfer_service/templates/job_status.html,sha256=5lUUGZL-5urppG610qDOgpfIE-OcQH57gFWvRA5pBNM,16938
 aind_data_transfer_service/templates/job_tasks_table.html,sha256=rWFukhjZ4dhPyabe372tmi4lbQS2fyELZ7Awbn5Un4g,6181
 aind_data_transfer_service/templates/task_logs.html,sha256=y1GnQft0S50ghPb2xJDjAlefymB9a4zYdMikUFV7Tl4,918
-aind_data_transfer_service-1.14.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
-aind_data_transfer_service-1.14.0.dist-info/METADATA,sha256=N7A2du3anL11r5AHh8NXI-yPtB8Ilsm1saCJ-_o-200,2410
-aind_data_transfer_service-1.14.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aind_data_transfer_service-1.14.0.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
-aind_data_transfer_service-1.14.0.dist-info/RECORD,,
+aind_data_transfer_service-1.15.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
+aind_data_transfer_service-1.15.0.dist-info/METADATA,sha256=XNDcKsS_NFZIZMBOip1nBheDFo8lirOA7a9P14SVtNo,2452
+aind_data_transfer_service-1.15.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aind_data_transfer_service-1.15.0.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
+aind_data_transfer_service-1.15.0.dist-info/RECORD,,