aind-data-transfer-service 1.15.0__py3-none-any.whl → 1.17.0__py3-none-any.whl

aind_data_transfer_service/__init__.py

@@ -1,7 +1,7 @@
 """Init package"""
 import os
 
-__version__ = "1.15.0"
+__version__ = "1.17.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

aind_data_transfer_service/configs/csv_handler.py

@@ -1,7 +1,10 @@
 """Module to handle processing legacy csv files"""
 
 import re
+from collections.abc import Mapping
+from copy import deepcopy
 from datetime import datetime
+from typing import Any, Dict
 
 from aind_data_schema_models.modalities import Modality
 from aind_data_schema_models.platforms import Platform
@@ -13,6 +16,45 @@ DATETIME_PATTERN2 = re.compile(
 )
 
 
+def nested_update(dict_to_update: Dict[str, Any], updates: Mapping):
+    """
+    Update a nested dictionary in-place.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    updates : Mapping
+
+    """
+    for k, v in updates.items():
+        if isinstance(v, Mapping):
+            dict_to_update[k] = nested_update(dict_to_update.get(k, {}), v)
+        else:
+            dict_to_update[k] = v
+    return dict_to_update
+
+
+def create_nested_dict(
+    dict_to_update: Dict[str, Any], key_string: str, value: Any
+):
+    """
+    Updates in-place a nested dictionary with a period delimited key and value.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    key_string : str
+    value : Any
+
+    """
+    keys = key_string.split(".", 1)
+    current_key = keys[0]
+    if len(keys) == 1:
+        dict_to_update[current_key] = value
+    else:
+        if current_key not in dict_to_update:
+            dict_to_update[current_key] = dict()
+        create_nested_dict(dict_to_update[current_key], keys[1], value)
+
+
 def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     """
     Maps csv row into a UploadJobConfigsV2 model. This attempts to be somewhat
@@ -29,7 +71,6 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     modality_configs = dict()
     job_configs = dict()
     check_s3_folder_exists_task = None
-    final_check_s3_folder_exist = None
     codeocean_tasks = dict()
     for key, value in row.items():
         # Strip white spaces and replace dashes with underscores
@@ -42,7 +83,9 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
             modality_parts = clean_key.split(".")
             modality_key = modality_parts[0]
             sub_key = (
-                "modality" if len(modality_parts) == 1 else modality_parts[1]
+                "modality"
+                if len(modality_parts) == 1
+                else ".".join(modality_parts[1:])
             )
             modality_configs.setdefault(modality_key, dict())
             # Temp backwards compatibility check
@@ -66,13 +109,22 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
                     job_settings=codeocean_pipeline_monitor_settings,
                 )
             else:
-                modality_configs[modality_key].update({sub_key: clean_val})
+                nested_val = dict()
+                create_nested_dict(
+                    dict_to_update=nested_val,
+                    key_string=sub_key,
+                    value=clean_val,
+                )
+                current_dict = deepcopy(
+                    modality_configs.get(modality_key, dict())
+                )
+                nested_update(current_dict, nested_val)
+                modality_configs[modality_key] = current_dict
         elif clean_key == "force_cloud_sync" and clean_val.upper() in [
             "TRUE",
             "T",
         ]:
             check_s3_folder_exists_task = {"skip_task": True}
-            final_check_s3_folder_exist = {"skip_task": True}
         else:
             job_configs[clean_key] = clean_val
     # Rename codeocean config keys with correct modality
@@ -93,8 +145,7 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     )
     tasks = {
         "gather_preliminary_metadata": metadata_task,
-        "check_s3_folder_exists_task": check_s3_folder_exists_task,
-        "final_check_s3_folder_exist": final_check_s3_folder_exist,
+        "check_s3_folder_exists": check_s3_folder_exists_task,
         "modality_transformation_settings": modality_tasks,
         "codeocean_pipeline_settings": None
         if codeocean_tasks == dict()

aind_data_transfer_service/models/internal.py

@@ -3,8 +3,9 @@
 import ast
 import os
 from datetime import datetime, timedelta, timezone
-from typing import List, Optional, Union
+from typing import ClassVar, List, Optional, Union
 
+from aind_data_schema_models.modalities import Modality
 from mypy_boto3_ssm.type_defs import ParameterMetadataTypeDef
 from pydantic import AwareDatetime, BaseModel, Field, field_validator
 from starlette.datastructures import QueryParams
@@ -223,11 +224,30 @@ class JobTasks(BaseModel):
 class JobParamInfo(BaseModel):
     """Model for job parameter info from AWS Parameter Store"""
 
+    _MODALITIES_LIST: ClassVar[list[str]] = list(
+        Modality.abbreviation_map.keys()
+    )
+    _MODALITY_TASKS: ClassVar[list[str]] = [
+        "modality_transformation_settings",
+        "codeocean_pipeline_settings",
+    ]
+
     name: Optional[str]
     last_modified: Optional[datetime]
-    job_type: str
-    task_id: str
+    job_type: str = Field(..., pattern=r"^[^\s/]+$")
+    task_id: str = Field(..., pattern=r"^[^\s/]+$")
     modality: Optional[str]
+    version: Optional[str] = Field(..., pattern=r"^(v1|v2)?$")
+
+    @field_validator("modality", mode="after")
+    def validate_modality(cls, v):
+        """Check that modality is one of aind-data-schema modalities"""
+        if v is not None and v not in JobParamInfo._MODALITIES_LIST:
+            raise ValueError(
+                "Invalid modality: modality must be one of "
+                f"{JobParamInfo._MODALITIES_LIST}"
+            )
+        return v
 
     @classmethod
     def from_aws_describe_parameter(
@@ -236,6 +256,7 @@ class JobParamInfo(BaseModel):
         job_type: str,
         task_id: str,
         modality: Optional[str],
+        version: Optional[str],
     ):
         """Map the parameter to the model"""
         return cls(
@@ -244,13 +265,14 @@ class JobParamInfo(BaseModel):
             job_type=job_type,
             task_id=task_id,
             modality=modality,
+            version=version,
         )
 
     @staticmethod
     def get_parameter_prefix(version: Optional[str] = None) -> str:
         """Get the prefix for job_type parameters"""
         prefix = os.getenv("AIND_AIRFLOW_PARAM_PREFIX")
-        if version is None:
+        if version is None or version == "v1":
             return prefix
         return f"{prefix}/{version}"
 
@@ -262,16 +284,19 @@ class JobParamInfo(BaseModel):
             "(?P<job_type>[^/]+)/tasks/(?P<task_id>[^/]+)"
             "(?:/(?P<modality>[^/]+))?"
         )
-        if version is None:
+        if version is None or version == "v1":
             return f"{prefix}/{regex}"
         return f"{prefix}/{version}/{regex}"
 
     @staticmethod
     def get_parameter_name(
-        job_type: str, task_id: str, version: Optional[str] = None
+        job_type: str,
+        task_id: str,
+        modality: Optional[str],
+        version: Optional[str] = None,
     ) -> str:
         """Create the parameter name from job_type and task_id"""
-        prefix = os.getenv("AIND_AIRFLOW_PARAM_PREFIX")
-        if version is None:
-            return f"{prefix}/{job_type}/tasks/{task_id}"
-        return f"{prefix}/{version}/{job_type}/tasks/{task_id}"
+        prefix = JobParamInfo.get_parameter_prefix(version)
+        if modality:
+            return f"{prefix}/{job_type}/tasks/{task_id}/{modality}"
+        return f"{prefix}/{job_type}/tasks/{task_id}"

aind_data_transfer_service/server.py

@@ -7,7 +7,7 @@ import os
 import re
 from asyncio import gather, sleep
 from pathlib import PurePosixPath
-from typing import List, Optional, Union
+from typing import Any, List, Optional, Union
 
 import boto3
 import requests
@@ -29,7 +29,9 @@ from starlette.middleware.sessions import SessionMiddleware
 from starlette.responses import RedirectResponse
 from starlette.routing import Route
 
-from aind_data_transfer_service import OPEN_DATA_BUCKET_NAME
+from aind_data_transfer_service import (
+    OPEN_DATA_BUCKET_NAME,
+)
 from aind_data_transfer_service import (
     __version__ as aind_data_transfer_service_version,
 )
@@ -37,14 +39,18 @@ from aind_data_transfer_service.configs.csv_handler import map_csv_row_to_job
 from aind_data_transfer_service.configs.job_configs import (
     BasicUploadJobConfigs as LegacyBasicUploadJobConfigs,
 )
-from aind_data_transfer_service.configs.job_configs import HpcJobConfigs
+from aind_data_transfer_service.configs.job_configs import (
+    HpcJobConfigs,
+)
 from aind_data_transfer_service.configs.job_upload_template import (
     JobUploadTemplate,
 )
 from aind_data_transfer_service.hpc.client import HpcClient, HpcClientConfigs
 from aind_data_transfer_service.hpc.models import HpcJobSubmitSettings
 from aind_data_transfer_service.log_handler import LoggingConfigs, get_logger
-from aind_data_transfer_service.models.core import SubmitJobRequestV2
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+)
 from aind_data_transfer_service.models.core import (
     validation_context as validation_context_v2,
 )
@@ -150,6 +156,7 @@ def get_parameter_infos(version: Optional[str] = None) -> List[JobParamInfo]:
                     job_type=match.group("job_type"),
                     task_id=match.group("task_id"),
                     modality=match.group("modality"),
+                    version=version,
                 )
                 params.append(param_info)
             else:
@@ -167,6 +174,19 @@ def get_parameter_value(param_name: str) -> dict:
     return param_value
 
 
+def put_parameter_value(param_name: str, param_value: dict) -> Any:
+    """Set a parameter value in AWS param store based on parameter name"""
+    param_value_str = json.dumps(param_value)
+    ssm_client = boto3.client("ssm")
+    result = ssm_client.put_parameter(
+        Name=param_name,
+        Value=param_value_str,
+        Type="String",
+        Overwrite=True,
+    )
+    return result
+
+
 async def get_airflow_jobs(
     params: AirflowDagRunsRequestParameters, get_confs: bool = False
 ) -> tuple[int, Union[List[JobStatus], List[dict]]]:
@@ -929,10 +949,10 @@ async def get_task_logs(request: Request):
 async def index(request: Request):
     """GET|POST /: form handler"""
     return templates.TemplateResponse(
+        request=request,
         name="index.html",
         context=(
             {
-                "request": request,
                 "project_names_url": project_names_url,
             }
         ),
@@ -945,10 +965,10 @@ async def job_tasks_table(request: Request):
     response_tasks_json = json.loads(response_tasks.body)
     data = response_tasks_json.get("data")
     return templates.TemplateResponse(
+        request=request,
         name="job_tasks_table.html",
         context=(
             {
-                "request": request,
                 "status_code": response_tasks.status_code,
                 "message": response_tasks_json.get("message"),
                 "errors": data.get("errors", []),
@@ -965,10 +985,10 @@ async def task_logs(request: Request):
     response_tasks_json = json.loads(response_tasks.body)
     data = response_tasks_json.get("data")
     return templates.TemplateResponse(
+        request=request,
         name="task_logs.html",
         context=(
             {
-                "request": request,
                 "status_code": response_tasks.status_code,
                 "message": response_tasks_json.get("message"),
                 "errors": data.get("errors", []),
@@ -982,10 +1002,10 @@ async def jobs(request: Request):
     """Get Job Status page with pagination"""
     dag_ids = AirflowDagRunsRequestParameters.model_fields["dag_ids"].default
     return templates.TemplateResponse(
+        request=request,
         name="job_status.html",
         context=(
             {
-                "request": request,
                 "project_names_url": project_names_url,
                 "dag_ids": dag_ids,
             }
@@ -995,16 +1015,20 @@ async def jobs(request: Request):
 
 async def job_params(request: Request):
     """Get Job Parameters page"""
+    user = request.session.get("user")
     return templates.TemplateResponse(
+        request=request,
         name="job_params.html",
         context=(
             {
-                "request": request,
+                "user_signed_in": user is not None,
                 "project_names_url": os.getenv(
                     "AIND_METADATA_SERVICE_PROJECT_NAMES_URL"
                 ),
                 "versions": ["v1", "v2"],
                 "default_version": "v1",
+                "modalities": JobParamInfo._MODALITIES_LIST,
+                "modality_tasks": JobParamInfo._MODALITY_TASKS,
             }
         ),
     )
@@ -1068,7 +1092,10 @@ def get_parameter_v2(request: Request):
     # path params are auto validated
     job_type = request.path_params.get("job_type")
     task_id = request.path_params.get("task_id")
-    param_name = JobParamInfo.get_parameter_name(job_type, task_id, "v2")
+    modality = request.path_params.get("modality")
+    param_name = JobParamInfo.get_parameter_name(
+        job_type=job_type, task_id=task_id, modality=modality, version="v2"
+    )
     try:
         param_value = get_parameter_value(param_name)
         return JSONResponse(
@@ -1089,12 +1116,79 @@ def get_parameter_v2(request: Request):
         )
 
 
+async def put_parameter(request: Request):
+    """Set v1/v2 parameter in AWS param store based on job_type and task_id"""
+    # User must be signed in
+    user = request.session.get("user")
+    if not user:
+        return JSONResponse(
+            content={
+                "message": "User not authenticated",
+                "data": {"error": "User not authenticated"},
+            },
+            status_code=401,
+        )
+    try:
+        # path params
+        param_info = JobParamInfo(
+            name=None,
+            last_modified=None,
+            job_type=request.path_params.get("job_type"),
+            task_id=request.path_params.get("task_id"),
+            modality=request.path_params.get("modality"),
+            version=request.path_params.get("version"),
+        )
+        param_name = JobParamInfo.get_parameter_name(
+            job_type=param_info.job_type,
+            task_id=param_info.task_id,
+            modality=param_info.modality,
+            version=param_info.version,
+        )
+        # update param store
+        logger.info(
+            f"Received request from {user} to set parameter {param_name}"
+        )
+        param_value = await request.json()
+        logger.info(f"Setting parameter {param_name} to {param_value}")
+        result = put_parameter_value(
+            param_name=param_name, param_value=param_value
+        )
+        logger.info(result)
+        return JSONResponse(
+            content={
+                "message": f"Set parameter for {param_name}",
+                "data": param_value,
+            },
+            status_code=200,
+        )
+    except ValidationError as error:
+        return JSONResponse(
+            content={
+                "message": "Invalid parameter",
+                "data": {"errors": json.loads(error.json())},
+            },
+            status_code=400,
+        )
+    except Exception as e:
+        logger.exception(f"Error setting parameter {param_name}: {e}")
+        return JSONResponse(
+            content={
+                "message": f"Error setting parameter {param_name}",
+                "data": {"error": f"{e.__class__.__name__}{e.args}"},
+            },
+            status_code=500,
+        )
+
+
 def get_parameter(request: Request):
     """Get parameter from AWS parameter store based on job_type and task_id"""
     # path params are auto validated
     job_type = request.path_params.get("job_type")
     task_id = request.path_params.get("task_id")
-    param_name = JobParamInfo.get_parameter_name(job_type, task_id)
+    modality = request.path_params.get("modality")
+    param_name = JobParamInfo.get_parameter_name(
+        job_type=job_type, task_id=task_id, modality=modality
+    )
     try:
         param_value = get_parameter_value(param_name)
         return JSONResponse(
@@ -1118,14 +1212,12 @@ def get_parameter(request: Request):
 async def admin(request: Request):
     """Get admin page if authenticated, else redirect to login."""
     user = request.session.get("user")
-    if os.getenv("ENV_NAME") == "local":
-        user = {"name": "local user"}
     if user:
         return templates.TemplateResponse(
+            request=request,
             name="admin.html",
             context=(
                 {
-                    "request": request,
                     "project_names_url": project_names_url,
                     "user_name": user.get("name", "unknown"),
                     "user_email": user.get("email", "unknown"),
@@ -1137,6 +1229,9 @@ async def admin(request: Request):
 
 async def login(request: Request):
     """Redirect to Azure login page"""
+    if os.getenv("ENV_NAME") == "local":
+        request.session["user"] = {"name": "local user"}
+        return RedirectResponse(url="/admin")
     oauth = set_oauth()
     redirect_uri = request.url_for("auth")
     response = await oauth.azure.authorize_redirect(request, redirect_uri)
@@ -1187,7 +1282,13 @@ routes = [
     Route("/api/v1/get_task_logs", endpoint=get_task_logs, methods=["GET"]),
     Route("/api/v1/parameters", endpoint=list_parameters, methods=["GET"]),
     Route(
-        "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:path}",
+        "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:str}",
+        endpoint=get_parameter,
+        methods=["GET"],
+    ),
+    Route(
+        "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:str}"
+        "/{modality:str}",
         endpoint=get_parameter,
         methods=["GET"],
     ),
@@ -1198,10 +1299,28 @@ routes = [
     Route("/api/v2/submit_jobs", endpoint=submit_jobs_v2, methods=["POST"]),
     Route("/api/v2/parameters", endpoint=list_parameters_v2, methods=["GET"]),
     Route(
-        "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:path}",
+        "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:str}",
+        endpoint=get_parameter_v2,
+        methods=["GET"],
+    ),
+    Route(
+        "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:str}"
+        "/{modality:str}",
         endpoint=get_parameter_v2,
         methods=["GET"],
     ),
+    Route(
+        "/api/{version:str}/parameters/job_types/{job_type:str}"
+        "/tasks/{task_id:str}",
+        endpoint=put_parameter,
+        methods=["PUT"],
+    ),
+    Route(
+        "/api/{version:str}/parameters/job_types/{job_type:str}"
+        "/tasks/{task_id:str}/{modality:str}",
+        endpoint=put_parameter,
+        methods=["PUT"],
+    ),
     Route("/jobs", endpoint=jobs, methods=["GET"]),
     Route("/job_tasks_table", endpoint=job_tasks_table, methods=["GET"]),
     Route("/task_logs", endpoint=task_logs, methods=["GET"]),

aind_data_transfer_service/templates/admin.html

@@ -24,13 +24,22 @@
     <a title="List of project names" href="{{ project_names_url }}" target="_blank">Project Names</a> |
     <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"
       target="_blank">Help</a> |
-    <a href="/admin">Admin</a> |
-    <a href="/logout">Log out</a>
+    <a href="/admin">Admin</a>
+    <a href="/logout" class="float-end">Log out</a>
   </nav>
   <div>
     <h3>Admin</h3>
     <div>Hello {{user_name}}, welcome to the admin page</div>
     <div>Email: {{user_email}}</div>
+    <h4 class="mt-4">Job Type Parameters</h4>
+     <div class="mb-2">
+      <div class="alert alert-info mb-0 p-4" role="alert">
+        To manage job type parameters, go to the <a href="/job_params" class="alert-link">Job Parameters</a> page.
+        Select an existing parameter, or click the
+        <button type="button" class="btn btn-success btn-sm" disabled><i class="bi bi-plus-circle"></i> Add New Parameter</button>
+        button.
+      </div>
+    </div>
   </div>
 </body>
 </html>

aind_data_transfer_service/templates/job_params.html

@@ -36,6 +36,11 @@
         <a title="List of project names" href= "{{ project_names_url }}" target="_blank" >Project Names</a> |
         <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"  target="_blank" >Help</a> |
         <a href="/admin">Admin</a>
+        {% if user_signed_in %}
+            <a href="/logout" class="float-end">Log out</a>
+        {% else %}
+            <a href="/login" class="float-end">Log in</a>
+        {% endif %}
     </nav>
     <div class="content">
         <h4 class="mb-2">
@@ -52,6 +57,14 @@
             </div>
             <span>Job Parameters</span>
         </h4>
+        <!-- button and modal for adding new parameters-->
+        {% if user_signed_in %}
+        <div class="mb-2">
+            <button type="button" class="btn btn-success btn-sm" data-bs-toggle="modal" data-bs-target="#param-modal" data-bs-action="new">
+                <i class="bi bi-plus-circle"></i> Add New Parameter
+            </button>
+        </div>
+        {% endif %}
         <!-- job params table -->
         <div>
             <table id="job-params-table" class="display compact table table-bordered table-sm" style="font-size: small">
@@ -66,59 +79,128 @@
                 </thead>
             </table>
             <!-- modal for displaying param value as json -->
+            <!-- if user is signed in, the textarea will be editable and the footer will display the Submit button -->
             <div class="modal fade" id="param-modal" tabindex="-1" aria-labelledby="param-modal-label" aria-hidden="true">
                 <div class="modal-dialog modal-xl">
                     <div class="modal-content">
                         <div class="modal-header">
                             <h5 class="modal-title" id="param-modal-label"></h5>
+                            <span class="badge bg-primary ms-2" id="param-modal-version"></span>
                             <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
                         </div>
                         <div class="modal-body">
-                            <pre id="param-modal-content" class="bg-light p-1 border rounded"></pre>
+                            <!-- dropdowns for Add New Parameter -->
+                            <div class="row mb-2" id="param-modal-dropdowns" style="display:none;">
+                                <div class="col-md-4">
+                                    <label for="param-modal-job-type-select" class="form-label">Job Type</label>
+                                    <select id="param-modal-job-type-select" class="form-select form-select-sm mb-1"></select>
+                                    <input type="text" id="param-modal-job-type-input" class="form-control form-control-sm mt-1" placeholder="Enter new Job Type" style="display:none;" />
+                                </div>
+                                <div class="col-md-4">
+                                    <label for="param-modal-task-id-select" class="form-label">Task ID</label>
+                                    <select id="param-modal-task-id-select" class="form-select form-select-sm"></select>
+                                    <input type="text" id="param-modal-task-id-input" class="form-control form-control-sm mt-1" placeholder="Enter new Task ID" style="display:none;" />
+                                </div>
+                                <div class="col-md-4" style="display:none;">
+                                    <label for="param-modal-modality-select" class="form-label">Modality</label>
+                                    <select id="param-modal-modality-select" class="form-select form-select-sm">
+                                        <option value="">Select Modality</option>
+                                        {% for modality in modalities %}
+                                        <option value="{{ modality }}">{{ modality }}</option>
+                                        {% endfor %}
+                                    </select>
+                                </div>
+                            </div>
+                            <!-- textarea for parameter value -->
+                            <textarea
+                                id="param-modal-content" class="bg-light form-control form-control-sm font-monospace" rows="15"
+                                placeholder='{"skip_task": false}' {% if not user_signed_in %}readonly{% endif %}
+                            ></textarea>
+                            <!-- message if parameter already exists -->
+                            <div id="param-modal-param-exists-alert" class="alert alert-info" role="alert" style="display:none;">
+                                This parameter already exists! To edit, please click on the parameter from the Job Parameters table.
+                            </div>
+                        </div>
+                        {% if user_signed_in %}
+                        <div class="modal-footer">
+                            <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+                            <button type="button" class="btn btn-secondary" id="param-modal-reset-btn">Reset</button>
+                            <button type="button" class="btn btn-primary" id="param-modal-submit-btn">Submit</button>
                         </div>
+                        {% endif %}
                     </div>
                 </div>
             </div>
         </div>
     </div>
     <script>
+        const MODALITY_TASKS = {{ modality_tasks | tojson }}; // from Jinja context
+        const MODAL_ID = 'param-modal';
         $(document).ready(function() {
             createJobParamsTable();
-            // Event listeners for param value modal
-            $('#param-modal').on('show.bs.modal', function(event) {
-                var button = $(event.relatedTarget);
-                var paramName = button.data('bs-param-name');
-                var jobType = button.data('bs-job-type');
-                var taskId = button.data('bs-task-id');
-                var modality = button.data('bs-modality');
-                // update the modal label and contents
-                var modal = $(this);
-                modal.find('#param-modal-label').text(paramName);
-                var version = $('#version-button').text().trim();
-                var getParameterUrl = `/api/${version}/parameters/job_types/${jobType}/tasks/${taskId}`;
-                if (modality) {
-                    getParameterUrl += `/${modality}`;
+            // Event listeners for modal to display/edit params
+            $(`#${MODAL_ID}`).on('show.bs.modal', function(event) {
+                // Add New Parameter: set label and dropdown options
+                // Edit Existing Parameter: set label and load intital value
+                const eventTarget = $(event.relatedTarget);
+                const action = eventTarget.data('bs-action');
+                const modal = $(`#${MODAL_ID}`);
+                modal.data('bs-action', action); // save action type
+                const isNew = action === 'new';
+                modal.find(`#${MODAL_ID}-label`).text(isNew ? 'Add New Parameter' : eventTarget.data('bs-param-name'));
+                modal.find(`#${MODAL_ID}-version`).text(getCurrentVersion()).toggle(isNew);
+                modal.find(`#${MODAL_ID}-dropdowns`).toggle(isNew);
+                if (isNew) {
+                    ['Job Type', 'Task ID'].forEach(field => {
+                        const select = modal.find(`#${MODAL_ID}-${field.toLowerCase().replace(' ', '-')}-select`);
+                        select.empty()
+                            .append(`<option value="">Select ${field}</option>`)
+                            .append(getUniqueColumnValues(field).map(val => `<option value="${val}">${val}</option>`))
+                            .append(`<option value="__new__">Create new ${field}</option>`);
+                    });
+                } else {
+                    loadParameterValue(getParamUrlFromParamName(eventTarget.data('bs-param-name')));
                 }
-                $.ajax({
-                    url: getParameterUrl,
-                    type: 'GET',
-                    success: function(response) {
-                        jsonStr = JSON.stringify(response.data, null, 3);
-                        modal.find('#param-modal-content').text(jsonStr);
-                    },
-                    error: function(xhr, status, error) {
-                        modal.find('#param-modal-content').text(error);
-                    }
-                });
             });
-            $('#param-modal').on('hidden.bs.modal', function() {
-                $(this).find('#param-modal-label').text('');
-                $(this).find('#param-modal-content').text('');
+            $(`#${MODAL_ID}`).on('hidden.bs.modal', function() {
+                onResetModal();
+            });
+            $(`#${MODAL_ID}`).on('click', `#${MODAL_ID}-reset-btn`, function() {
+                onResetModal(true);
+            });
+            $(`#${MODAL_ID}`).on('click', `#${MODAL_ID}-submit-btn`, function() {
+                const modal = $(`#${MODAL_ID}`);
+                const action = modal.data('bs-action');
+                let url;
+                if (action === "new") {
+                    const version = getCurrentVersion();
+                    const jobType = getValidatedInputValue('Job Type', modal);
+                    const taskId = getValidatedInputValue('Task ID', modal);
+                    const modality = MODALITY_TASKS.includes(taskId) ? getValidatedInputValue('Modality', modal) : null;
+                    url = getParamUrlFromParamInfo(version, jobType, taskId, modality);
+                } else {
+                    const paramName = modal.find(`#${MODAL_ID}-label`).text();
+                    url = getParamUrlFromParamName(paramName);
+                }
+                const paramValue = modal.find(`#${MODAL_ID}-content`).val();
+                submitParameterValue(url, paramValue);
+            });
+            $(`#${MODAL_ID}-job-type-select`).on('change', function() {
+                $(`#${MODAL_ID}-job-type-input`).toggle($(this).val() === '__new__').focus();
+                handleExistingParam();
+            });
+            $(`#${MODAL_ID}-task-id-select`).on('change', function() {
+                $(`#${MODAL_ID}-task-id-input`).toggle($(this).val() === '__new__').focus();
+                $(`#${MODAL_ID}-modality-select`).val('').parent().toggle(MODALITY_TASKS.includes($(this).val()));
+                handleExistingParam();
+            });
+            $(`#${MODAL_ID}-modality-select`).on('change', function() {
+                handleExistingParam();
             });
             // Event listener for version dropdown
             $('#version-dropdown .dropdown-item').on('click', function() {
                 var version = $(this).text();
-                if (version != $('#version-button').text().trim()) {
+                if (version != getCurrentVersion()) {
                     $('#version-button').text(version);
                     $('#job-params-table').DataTable().ajax.url(`/api/${version}/parameters`).load();
                 }
@@ -179,7 +261,8 @@
                 return (
                     `<button type="button" class="btn btn-link btn-sm" 
                         data-bs-toggle="modal" 
-                        data-bs-target="#param-modal" 
+                        data-bs-target="#${MODAL_ID}"
+                        data-bs-action="edit"
                         data-bs-param-name=${data} 
                         data-bs-job-type=${row.job_type} 
                         data-bs-task-id=${row.task_id}
@@ -190,6 +273,133 @@
             }
             return data;
         }
+        // Methods to load/submit param values in the modal
+        function getParamUrlFromParamInfo(version, jobType, taskId, modality) {
+            var baseUrl = `/api/${version}/parameters/job_types/${jobType}/tasks/${taskId}`;
+            return modality ? `${baseUrl}/${modality}` : baseUrl;
+        }
+        function getParamUrlFromParamName(paramName) {
+            const match = paramName.match(/job_types\/(v\d+)?\/?(.*)/);
+            const version = match && match[1] ? match[1] : 'v1';
+            const cleanedParamName = match ? match[2] : paramName;
+            return `/api/${version}/parameters/job_types/${cleanedParamName}`;
+        }
+        function loadParameterValue(paramUrl) {
+            const modal = $(`#${MODAL_ID}`);
+            $.ajax({
+                url: paramUrl,
+                type: 'GET',
+                success: function (response) {
+                    jsonStr = JSON.stringify(response.data, null, 3);
+                    modal.find(`#${MODAL_ID}-content`).val(jsonStr);
+                },
+                error: function (xhr, status, error) {
+                    console.error(`Error fetching ${paramUrl}: ${error}`);
+                    modal.find(`#${MODAL_ID}-content`).val(error);
+                }
+            });
+        }
+        function submitParameterValue(paramUrl, paramValue) {
+            try {
+                JSON.parse(paramValue);
+            } catch (e) {
+                alert('Parameter value must be valid JSON:\n' + e.message);
+                return;
+            }
+            $.ajax({
+                url: paramUrl,
+                type: 'PUT',
+                contentType: 'application/json',
+                data: paramValue,
+                success: function (response) {
+                    alert('Parameter updated successfully');
+                    // reload the content to verify and reformat changes
+                    loadParameterValue(paramUrl);
+                },
+                error: function (xhr, status, error) {
+                    var msg = `Error submitting parameter: ${error}`;
+                    try {
+                        msg += `\n\n${JSON.stringify(JSON.parse(xhr.responseText), null, 3)}`;
+                    } catch (e) {
+                        console.error('Failed to parse error response:', e.message);
+                    }
+                    console.error(msg);
+                    alert(msg);
+                }
+            });
+        }
+        // Methods for param modal updates
+        function toggleParamTextArea(isEnabled){
+            const modal = $(`#${MODAL_ID}`);
+            modal.find(`#${MODAL_ID}-content`).toggle(isEnabled);
+            modal.find(`#${MODAL_ID}-param-exists-alert`).toggle(!isEnabled);
+            modal.find(`#${MODAL_ID}-submit-btn`).prop('disabled', !isEnabled);
+        }
+        function handleExistingParam() {
+            let exists = false;
+            const modal = $(`#${MODAL_ID}`);
+            const version = getCurrentVersion();
+            const jobType = getInputValues('Job Type', modal).input;
+            const taskId = getInputValues('Task ID', modal).input;
+            const modality = MODALITY_TASKS.includes(taskId) ? getInputValues('Modality', modal).input : null;
+            if (jobType && taskId && (!MODALITY_TASKS.includes(taskId) || modality)) {
+                const table = $('#job-params-table').DataTable();
+                const searchStr = `/${jobType}/tasks/${taskId}` + (modality ? `/${modality}` : '');
+                exists = table.column('Parameter Name:title').data().toArray().some(val => val.endsWith(searchStr));
+            }
+            toggleParamTextArea(!exists);
+        }
+        function onResetModal(reload = false) {
+            const modal = $(`#${MODAL_ID}`);
+            const action = modal.data('bs-action');
+            if (action === "new") {
+                modal.find(`#${MODAL_ID}-dropdowns input`).val('').hide();
+                modal.find(`#${MODAL_ID}-dropdowns select`).val('');
+                modal.find(`#${MODAL_ID}-modality-select`).parent().hide();
+                modal.find(`#${MODAL_ID}-content`).val('');
+                toggleParamTextArea(true);
+            } else if (action === "edit") {
+                if (reload) {
+                    const paramName = modal.find(`#${MODAL_ID}-label`).text();
+                    loadParameterValue(getParamUrlFromParamName(paramName));
+                } else {
+                    modal.find(`#${MODAL_ID}-label, #${MODAL_ID}-content`).text('').val('');
+                }
+            }
+        }
+        // Helper methods to get current values
+        function getCurrentVersion() {
+            return $('#version-button').text().trim();
+        }
+        function getUniqueColumnValues(columnTitle) {
+            return Array.from(new Set($('#job-params-table').DataTable().column(`${columnTitle}:title`).data().toArray()));
+        }
+        function getInputValues(inputField, modal) {
+            const selector = `#${MODAL_ID}-${inputField.toLowerCase().replace(' ', '-')}`;
+            const dropdown = modal.find(`${selector}-select`).val()?.trim() || '';
+            const input = (dropdown === '__new__') ? (modal.find(`${selector}-input`).val()?.trim() || '') : dropdown;
+            return { dropdown, input };
+        }
+        function getValidatedInputValue(inputField, modal) {
+            // Check for empty string, spaces, slashes, or existing values
+            const { dropdown, input } = getInputValues(inputField, modal);
+            var error = false;
+            if (!input) {
+                error = `${inputField} cannot be empty`;
+            } else if (input.includes(' ') || input.includes('/')) {
+                error = `${inputField} cannot contain spaces or slashes`;
+            } else if (
+                dropdown === '__new__' &&
+                getUniqueColumnValues(inputField).some(val => val.toLowerCase() === input.toLowerCase())
+            ) {
+                error = `${inputField} "${input}" already exists. Please select it from the dropdown.`;
+            }
+            if (error) {
+                alert(error);
+                throw new Error(error);
+            }
+            return input;
+        }
     </script>
-    </body>
+</body>
 </html>

{aind_data_transfer_service-1.15.0.dist-info → aind_data_transfer_service-1.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.15.0
+Version: 1.17.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
@@ -25,13 +25,13 @@ Requires-Dist: furo; extra == "docs"
 Provides-Extra: server
 Requires-Dist: aind-data-schema<2.0,>=1.0.0; extra == "server"
 Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
-Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
+Requires-Dist: aind-metadata-mapper>=0.23.0; extra == "server"
 Requires-Dist: boto3; extra == "server"
 Requires-Dist: boto3-stubs[ssm]; extra == "server"
-Requires-Dist: fastapi; extra == "server"
+Requires-Dist: fastapi>=0.115.13; extra == "server"
 Requires-Dist: httpx; extra == "server"
 Requires-Dist: jinja2; extra == "server"
-Requires-Dist: starlette; extra == "server"
+Requires-Dist: starlette<0.47.0,>=0.40.0; extra == "server"
 Requires-Dist: starlette_wtf; extra == "server"
 Requires-Dist: uvicorn[standard]; extra == "server"
 Requires-Dist: wtforms; extra == "server"

{aind_data_transfer_service-1.15.0.dist-info → aind_data_transfer_service-1.17.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-aind_data_transfer_service/__init__.py,sha256=lho7ClOyDOm8TOnoEsiTmbuidZwwCwry8o_Vu_mJ5qI,272
+aind_data_transfer_service/__init__.py,sha256=3zeB4HVxFXx2bXdNaQXcSZw0Irq7ZulbJJ5bI0ixIYo,272
 aind_data_transfer_service/log_handler.py,sha256=c7a-gLmZeRpeCUBwCz6XsTszWXQeQdR7eKZtas4llXM,1700
-aind_data_transfer_service/server.py,sha256=8TRxybpk8hkqPng_6mEZc9wv8ofGErgYf0BSPk-8VAU,44545
+aind_data_transfer_service/server.py,sha256=1vtmqMKF_7mv2FWrRktvjPmi-slYwt4pfUy51P3WRVc,48341
 aind_data_transfer_service/configs/__init__.py,sha256=9W5GTuso9Is1B9X16RXcdb_GxasZvj6qDzOBDv0AbTc,36
-aind_data_transfer_service/configs/csv_handler.py,sha256=9jM0fUlWCzmqTC7ubAeFCl0eEIX5BQvHcPPPTPngcog,4374
+aind_data_transfer_service/configs/csv_handler.py,sha256=hCdfAYZW_49-l1rbua5On2Tw2ks674Z-MgB_NJlIkU4,5746
 aind_data_transfer_service/configs/job_configs.py,sha256=T-h5N6lyY9xTZ_xg_5FxkyYuMdagApbE6xalxFQ-bqA,18848
 aind_data_transfer_service/configs/job_upload_template.py,sha256=aC5m1uD_YcpbggFQ-yZ7ZJSUUGX1yQqQLF3SwmljrLk,5127
 aind_data_transfer_service/hpc/__init__.py,sha256=YNc68YNlmXwKIPFMIViz_K4XzVVHkLPEBOFyO5DKMKI,53
@@ -10,15 +10,15 @@ aind_data_transfer_service/hpc/client.py,sha256=-JSxAWn96_XOIDwhsXAHK3TZAdckddUh
 aind_data_transfer_service/hpc/models.py,sha256=-7HhV16s_MUyKPy0x0FGIbnq8DPL2qJAzJO5G7003AE,16184
 aind_data_transfer_service/models/__init__.py,sha256=Meym73bEZ9nQr4QoeyhQmV3nRTYtd_4kWKPNygsBfJg,25
 aind_data_transfer_service/models/core.py,sha256=uXtPUqjxKalg-sE8MxaJr11w_T_KKBRBSJuUgwoMZlQ,10135
-aind_data_transfer_service/models/internal.py,sha256=MGQrPuHrR21nn4toqdTCIEDW6MG7pWRajoPqD3j-ST0,9706
-aind_data_transfer_service/templates/admin.html,sha256=KvQB54-mD8LL8wnDd3G9a8lDYAT8BWK13_MhpIJAiSY,1200
+aind_data_transfer_service/models/internal.py,sha256=tWO1yRMu9hHLv0mt7QhOwTPWGbKApWmCf1wwajx5qjI,10681
+aind_data_transfer_service/templates/admin.html,sha256=owmWgcTFzfWh5S2L82OT-0r5pOt1bl8Fh-C0zyVo1wU,1682
 aind_data_transfer_service/templates/index.html,sha256=TDmmHlhWFPnQrk6nk1OhNjC3SapZtX0TXeuUonoCO7g,11326
-aind_data_transfer_service/templates/job_params.html,sha256=ivofS1CjSnC87T5J2BTsNtVzq6xnUqlPZePdGt_fc4U,8854
+aind_data_transfer_service/templates/job_params.html,sha256=Q5oAsuQqBiTvOOS_7BejcqPJ4rskg_ZtG1E0NAjcGJw,21063
 aind_data_transfer_service/templates/job_status.html,sha256=5lUUGZL-5urppG610qDOgpfIE-OcQH57gFWvRA5pBNM,16938
 aind_data_transfer_service/templates/job_tasks_table.html,sha256=rWFukhjZ4dhPyabe372tmi4lbQS2fyELZ7Awbn5Un4g,6181
 aind_data_transfer_service/templates/task_logs.html,sha256=y1GnQft0S50ghPb2xJDjAlefymB9a4zYdMikUFV7Tl4,918
-aind_data_transfer_service-1.15.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
-aind_data_transfer_service-1.15.0.dist-info/METADATA,sha256=XNDcKsS_NFZIZMBOip1nBheDFo8lirOA7a9P14SVtNo,2452
-aind_data_transfer_service-1.15.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aind_data_transfer_service-1.15.0.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
-aind_data_transfer_service-1.15.0.dist-info/RECORD,,
+aind_data_transfer_service-1.17.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
+aind_data_transfer_service-1.17.0.dist-info/METADATA,sha256=pBXxvTxzxZRam6dubbJn7oPNQfBClr1TPUvwrPB3W7U,2478
+aind_data_transfer_service-1.17.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aind_data_transfer_service-1.17.0.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
+aind_data_transfer_service-1.17.0.dist-info/RECORD,,