aind-data-transfer-service 1.16.0__tar.gz → 1.17.0__tar.gz

{aind_data_transfer_service-1.16.0/src/aind_data_transfer_service.egg-info → aind_data_transfer_service-1.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.16.0
+Version: 1.17.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
@@ -25,7 +25,7 @@ Requires-Dist: furo; extra == "docs"
 Provides-Extra: server
 Requires-Dist: aind-data-schema<2.0,>=1.0.0; extra == "server"
 Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
-Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
+Requires-Dist: aind-metadata-mapper>=0.23.0; extra == "server"
 Requires-Dist: boto3; extra == "server"
 Requires-Dist: boto3-stubs[ssm]; extra == "server"
 Requires-Dist: fastapi>=0.115.13; extra == "server"

aind_data_transfer_service-1.17.0/docs/examples/behavior_videos_compression.csv

@@ -0,0 +1,2 @@
+project_name, modality0, modality0.input_source, modality0.compression_requested.compression_enum, subject-id, platform, acq-datetime, job_type
+Ephys Platform, behavior-videos, dir/data_set_1, gamma fix colorspace, 123454, ecephys, 2020-10-10 14:10:10, ecephys

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/docs/source/Contributing.rst

@@ -62,6 +62,7 @@ To run uvicorn locally:
 
 .. code:: bash
 
+   # export MSYS_NO_PATHCONV=1 # Uncomment this line if running on Windows
    export AIND_METADATA_SERVICE_PROJECT_NAMES_URL='http://aind-metadata-service-dev/project_names'
    export AIND_AIRFLOW_SERVICE_URL='http://aind-airflow-service-dev:8080/api/v1/dags/run_list_of_jobs/dagRuns'
    # export AIND_AIRFLOW_SERVICE_JOBS_URL='http://aind-airflow-service-dev:8080/api/v1/dags'

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/docs/source/UserGuideV1.rst

@@ -520,6 +520,7 @@ Available job types and their configurations can be viewed at:
 http://aind-data-transfer-service/job_params
 
 To request a new job type, please reach out to Scientific Computing.
+Admins can manage job types directly in the Job Parameters page.
 
 Reporting bugs or making feature requests
 -----------------------------------------

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/docs/source/UserGuideV2.rst

@@ -186,6 +186,7 @@ Available job types and their configurations can be viewed at:
 http://aind-data-transfer-service/job_params
 
 To request a new job type, please reach out to Scientific Computing.
+Admins can manage job types directly in the Job Parameters page.
 
 Reporting bugs or making feature requests
 -----------------------------------------

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/pyproject.toml

@@ -41,7 +41,7 @@ docs = [
 server = [
     'aind-data-schema>=1.0.0,<2.0',
     'aind-data-transfer-models==0.17.0',
-    'aind-metadata-mapper==0.23.0',
+    'aind-metadata-mapper>=0.23.0',
     'boto3',
     'boto3-stubs[ssm]',
     'fastapi>=0.115.13', 

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/__init__.py

@@ -1,7 +1,7 @@
 """Init package"""
 import os
 
-__version__ = "1.16.0"
+__version__ = "1.17.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/models/internal.py

@@ -3,8 +3,9 @@
 import ast
 import os
 from datetime import datetime, timedelta, timezone
-from typing import List, Optional, Union
+from typing import ClassVar, List, Optional, Union
 
+from aind_data_schema_models.modalities import Modality
 from mypy_boto3_ssm.type_defs import ParameterMetadataTypeDef
 from pydantic import AwareDatetime, BaseModel, Field, field_validator
 from starlette.datastructures import QueryParams
@@ -223,11 +224,30 @@ class JobTasks(BaseModel):
 class JobParamInfo(BaseModel):
     """Model for job parameter info from AWS Parameter Store"""
 
+    _MODALITIES_LIST: ClassVar[list[str]] = list(
+        Modality.abbreviation_map.keys()
+    )
+    _MODALITY_TASKS: ClassVar[list[str]] = [
+        "modality_transformation_settings",
+        "codeocean_pipeline_settings",
+    ]
+
     name: Optional[str]
     last_modified: Optional[datetime]
-    job_type: str
-    task_id: str
+    job_type: str = Field(..., pattern=r"^[^\s/]+$")
+    task_id: str = Field(..., pattern=r"^[^\s/]+$")
     modality: Optional[str]
+    version: Optional[str] = Field(..., pattern=r"^(v1|v2)?$")
+
+    @field_validator("modality", mode="after")
+    def validate_modality(cls, v):
+        """Check that modality is one of aind-data-schema modalities"""
+        if v is not None and v not in JobParamInfo._MODALITIES_LIST:
+            raise ValueError(
+                "Invalid modality: modality must be one of "
+                f"{JobParamInfo._MODALITIES_LIST}"
+            )
+        return v
 
     @classmethod
     def from_aws_describe_parameter(
@@ -236,6 +256,7 @@ class JobParamInfo(BaseModel):
         job_type: str,
         task_id: str,
         modality: Optional[str],
+        version: Optional[str],
     ):
         """Map the parameter to the model"""
         return cls(
@@ -244,13 +265,14 @@ class JobParamInfo(BaseModel):
             job_type=job_type,
             task_id=task_id,
             modality=modality,
+            version=version,
         )
 
     @staticmethod
     def get_parameter_prefix(version: Optional[str] = None) -> str:
         """Get the prefix for job_type parameters"""
         prefix = os.getenv("AIND_AIRFLOW_PARAM_PREFIX")
-        if version is None:
+        if version is None or version == "v1":
             return prefix
         return f"{prefix}/{version}"
 
@@ -262,16 +284,19 @@ class JobParamInfo(BaseModel):
             "(?P<job_type>[^/]+)/tasks/(?P<task_id>[^/]+)"
             "(?:/(?P<modality>[^/]+))?"
         )
-        if version is None:
+        if version is None or version == "v1":
             return f"{prefix}/{regex}"
         return f"{prefix}/{version}/{regex}"
 
     @staticmethod
     def get_parameter_name(
-        job_type: str, task_id: str, version: Optional[str] = None
+        job_type: str,
+        task_id: str,
+        modality: Optional[str],
+        version: Optional[str] = None,
     ) -> str:
         """Create the parameter name from job_type and task_id"""
-        prefix = os.getenv("AIND_AIRFLOW_PARAM_PREFIX")
-        if version is None:
-            return f"{prefix}/{job_type}/tasks/{task_id}"
-        return f"{prefix}/{version}/{job_type}/tasks/{task_id}"
+        prefix = JobParamInfo.get_parameter_prefix(version)
+        if modality:
+            return f"{prefix}/{job_type}/tasks/{task_id}/{modality}"
+        return f"{prefix}/{job_type}/tasks/{task_id}"

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/server.py

@@ -7,7 +7,7 @@ import os
 import re
 from asyncio import gather, sleep
 from pathlib import PurePosixPath
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
 import boto3
 import requests
@@ -156,6 +156,7 @@ def get_parameter_infos(version: Optional[str] = None) -> List[JobParamInfo]:
                     job_type=match.group("job_type"),
                     task_id=match.group("task_id"),
                     modality=match.group("modality"),
+                    version=version,
                 )
                 params.append(param_info)
             else:
@@ -173,6 +174,19 @@ def get_parameter_value(param_name: str) -> dict:
     return param_value
 
 
+def put_parameter_value(param_name: str, param_value: dict) -> Any:
+    """Set a parameter value in AWS param store based on parameter name"""
+    param_value_str = json.dumps(param_value)
+    ssm_client = boto3.client("ssm")
+    result = ssm_client.put_parameter(
+        Name=param_name,
+        Value=param_value_str,
+        Type="String",
+        Overwrite=True,
+    )
+    return result
+
+
 async def get_airflow_jobs(
     params: AirflowDagRunsRequestParameters, get_confs: bool = False
 ) -> tuple[int, Union[List[JobStatus], List[dict]]]:
@@ -1001,16 +1015,20 @@ async def jobs(request: Request):
 
 async def job_params(request: Request):
     """Get Job Parameters page"""
+    user = request.session.get("user")
     return templates.TemplateResponse(
         request=request,
         name="job_params.html",
         context=(
             {
+                "user_signed_in": user is not None,
                 "project_names_url": os.getenv(
                     "AIND_METADATA_SERVICE_PROJECT_NAMES_URL"
                 ),
                 "versions": ["v1", "v2"],
                 "default_version": "v1",
+                "modalities": JobParamInfo._MODALITIES_LIST,
+                "modality_tasks": JobParamInfo._MODALITY_TASKS,
             }
         ),
     )
@@ -1074,7 +1092,10 @@ def get_parameter_v2(request: Request):
     # path params are auto validated
     job_type = request.path_params.get("job_type")
     task_id = request.path_params.get("task_id")
-    param_name = JobParamInfo.get_parameter_name(job_type, task_id, "v2")
+    modality = request.path_params.get("modality")
+    param_name = JobParamInfo.get_parameter_name(
+        job_type=job_type, task_id=task_id, modality=modality, version="v2"
+    )
     try:
         param_value = get_parameter_value(param_name)
         return JSONResponse(
@@ -1095,12 +1116,79 @@ def get_parameter_v2(request: Request):
         )
 
 
+async def put_parameter(request: Request):
+    """Set v1/v2 parameter in AWS param store based on job_type and task_id"""
+    # User must be signed in
+    user = request.session.get("user")
+    if not user:
+        return JSONResponse(
+            content={
+                "message": "User not authenticated",
+                "data": {"error": "User not authenticated"},
+            },
+            status_code=401,
+        )
+    try:
+        # path params
+        param_info = JobParamInfo(
+            name=None,
+            last_modified=None,
+            job_type=request.path_params.get("job_type"),
+            task_id=request.path_params.get("task_id"),
+            modality=request.path_params.get("modality"),
+            version=request.path_params.get("version"),
+        )
+        param_name = JobParamInfo.get_parameter_name(
+            job_type=param_info.job_type,
+            task_id=param_info.task_id,
+            modality=param_info.modality,
+            version=param_info.version,
+        )
+        # update param store
+        logger.info(
+            f"Received request from {user} to set parameter {param_name}"
+        )
+        param_value = await request.json()
+        logger.info(f"Setting parameter {param_name} to {param_value}")
+        result = put_parameter_value(
+            param_name=param_name, param_value=param_value
+        )
+        logger.info(result)
+        return JSONResponse(
+            content={
+                "message": f"Set parameter for {param_name}",
+                "data": param_value,
+            },
+            status_code=200,
+        )
+    except ValidationError as error:
+        return JSONResponse(
+            content={
+                "message": "Invalid parameter",
+                "data": {"errors": json.loads(error.json())},
+            },
+            status_code=400,
+        )
+    except Exception as e:
+        logger.exception(f"Error setting parameter {param_name}: {e}")
+        return JSONResponse(
+            content={
+                "message": f"Error setting parameter {param_name}",
+                "data": {"error": f"{e.__class__.__name__}{e.args}"},
+            },
+            status_code=500,
+        )
+
+
 def get_parameter(request: Request):
     """Get parameter from AWS parameter store based on job_type and task_id"""
     # path params are auto validated
     job_type = request.path_params.get("job_type")
     task_id = request.path_params.get("task_id")
-    param_name = JobParamInfo.get_parameter_name(job_type, task_id)
+    modality = request.path_params.get("modality")
+    param_name = JobParamInfo.get_parameter_name(
+        job_type=job_type, task_id=task_id, modality=modality
+    )
     try:
         param_value = get_parameter_value(param_name)
         return JSONResponse(
@@ -1124,8 +1212,6 @@ def get_parameter(request: Request):
 async def admin(request: Request):
     """Get admin page if authenticated, else redirect to login."""
     user = request.session.get("user")
-    if os.getenv("ENV_NAME") == "local":
-        user = {"name": "local user"}
     if user:
         return templates.TemplateResponse(
             request=request,
@@ -1143,6 +1229,9 @@ async def admin(request: Request):
 
 async def login(request: Request):
     """Redirect to Azure login page"""
+    if os.getenv("ENV_NAME") == "local":
+        request.session["user"] = {"name": "local user"}
+        return RedirectResponse(url="/admin")
     oauth = set_oauth()
     redirect_uri = request.url_for("auth")
     response = await oauth.azure.authorize_redirect(request, redirect_uri)
@@ -1193,7 +1282,13 @@ routes = [
     Route("/api/v1/get_task_logs", endpoint=get_task_logs, methods=["GET"]),
     Route("/api/v1/parameters", endpoint=list_parameters, methods=["GET"]),
     Route(
-        "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:path}",
+        "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:str}",
+        endpoint=get_parameter,
+        methods=["GET"],
+    ),
+    Route(
+        "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:str}"
+        "/{modality:str}",
         endpoint=get_parameter,
         methods=["GET"],
     ),
@@ -1204,10 +1299,28 @@ routes = [
     Route("/api/v2/submit_jobs", endpoint=submit_jobs_v2, methods=["POST"]),
     Route("/api/v2/parameters", endpoint=list_parameters_v2, methods=["GET"]),
     Route(
-        "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:path}",
+        "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:str}",
+        endpoint=get_parameter_v2,
+        methods=["GET"],
+    ),
+    Route(
+        "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:str}"
+        "/{modality:str}",
         endpoint=get_parameter_v2,
         methods=["GET"],
     ),
+    Route(
+        "/api/{version:str}/parameters/job_types/{job_type:str}"
+        "/tasks/{task_id:str}",
+        endpoint=put_parameter,
+        methods=["PUT"],
+    ),
+    Route(
+        "/api/{version:str}/parameters/job_types/{job_type:str}"
+        "/tasks/{task_id:str}/{modality:str}",
+        endpoint=put_parameter,
+        methods=["PUT"],
+    ),
     Route("/jobs", endpoint=jobs, methods=["GET"]),
     Route("/job_tasks_table", endpoint=job_tasks_table, methods=["GET"]),
     Route("/task_logs", endpoint=task_logs, methods=["GET"]),

{aind_data_transfer_service-1.16.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/templates/admin.html

@@ -24,13 +24,22 @@
     <a title="List of project names" href="{{ project_names_url }}" target="_blank">Project Names</a> |
     <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"
       target="_blank">Help</a> |
-    <a href="/admin">Admin</a> |
-    <a href="/logout">Log out</a>
+    <a href="/admin">Admin</a>
+    <a href="/logout" class="float-end">Log out</a>
   </nav>
   <div>
     <h3>Admin</h3>
     <div>Hello {{user_name}}, welcome to the admin page</div>
     <div>Email: {{user_email}}</div>
+    <h4 class="mt-4">Job Type Parameters</h4>
+     <div class="mb-2">
+      <div class="alert alert-info mb-0 p-4" role="alert">
+        To manage job type parameters, go to the <a href="/job_params" class="alert-link">Job Parameters</a> page.
+        Select an existing parameter, or click the
+        <button type="button" class="btn btn-success btn-sm" disabled><i class="bi bi-plus-circle"></i> Add New Parameter</button>
+        button.
+      </div>
+    </div>
   </div>
 </body>
 </html>