aind-data-transfer-service 1.15.0__py3-none-any.whl → 1.16.0__py3-none-any.whl

aind_data_transfer_service/__init__.py

@@ -1,7 +1,7 @@
 """Init package"""
 import os
 
-__version__ = "1.15.0"
+__version__ = "1.16.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

aind_data_transfer_service/configs/csv_handler.py

@@ -1,7 +1,10 @@
 """Module to handle processing legacy csv files"""
 
 import re
+from collections.abc import Mapping
+from copy import deepcopy
 from datetime import datetime
+from typing import Any, Dict
 
 from aind_data_schema_models.modalities import Modality
 from aind_data_schema_models.platforms import Platform
@@ -13,6 +16,45 @@ DATETIME_PATTERN2 = re.compile(
 )
 
 
+def nested_update(dict_to_update: Dict[str, Any], updates: Mapping):
+    """
+    Update a nested dictionary in-place.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    updates : Mapping
+
+    """
+    for k, v in updates.items():
+        if isinstance(v, Mapping):
+            dict_to_update[k] = nested_update(dict_to_update.get(k, {}), v)
+        else:
+            dict_to_update[k] = v
+    return dict_to_update
+
+
+def create_nested_dict(
+    dict_to_update: Dict[str, Any], key_string: str, value: Any
+):
+    """
+    Updates in-place a nested dictionary with a period delimited key and value.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    key_string : str
+    value : Any
+
+    """
+    keys = key_string.split(".", 1)
+    current_key = keys[0]
+    if len(keys) == 1:
+        dict_to_update[current_key] = value
+    else:
+        if current_key not in dict_to_update:
+            dict_to_update[current_key] = dict()
+        create_nested_dict(dict_to_update[current_key], keys[1], value)
+
+
 def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     """
     Maps csv row into a UploadJobConfigsV2 model. This attempts to be somewhat
@@ -29,7 +71,6 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     modality_configs = dict()
     job_configs = dict()
     check_s3_folder_exists_task = None
-    final_check_s3_folder_exist = None
     codeocean_tasks = dict()
     for key, value in row.items():
         # Strip white spaces and replace dashes with underscores
@@ -42,7 +83,9 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
             modality_parts = clean_key.split(".")
             modality_key = modality_parts[0]
             sub_key = (
-                "modality" if len(modality_parts) == 1 else modality_parts[1]
+                "modality"
+                if len(modality_parts) == 1
+                else ".".join(modality_parts[1:])
             )
             modality_configs.setdefault(modality_key, dict())
             # Temp backwards compatibility check
@@ -66,13 +109,22 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
                     job_settings=codeocean_pipeline_monitor_settings,
                 )
             else:
-                modality_configs[modality_key].update({sub_key: clean_val})
+                nested_val = dict()
+                create_nested_dict(
+                    dict_to_update=nested_val,
+                    key_string=sub_key,
+                    value=clean_val,
+                )
+                current_dict = deepcopy(
+                    modality_configs.get(modality_key, dict())
+                )
+                nested_update(current_dict, nested_val)
+                modality_configs[modality_key] = current_dict
         elif clean_key == "force_cloud_sync" and clean_val.upper() in [
             "TRUE",
             "T",
         ]:
             check_s3_folder_exists_task = {"skip_task": True}
-            final_check_s3_folder_exist = {"skip_task": True}
         else:
             job_configs[clean_key] = clean_val
     # Rename codeocean config keys with correct modality
@@ -93,8 +145,7 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     )
     tasks = {
         "gather_preliminary_metadata": metadata_task,
-        "check_s3_folder_exists_task": check_s3_folder_exists_task,
-        "final_check_s3_folder_exist": final_check_s3_folder_exist,
+        "check_s3_folder_exists": check_s3_folder_exists_task,
         "modality_transformation_settings": modality_tasks,
         "codeocean_pipeline_settings": None
         if codeocean_tasks == dict()

aind_data_transfer_service/server.py

@@ -29,7 +29,9 @@ from starlette.middleware.sessions import SessionMiddleware
 from starlette.responses import RedirectResponse
 from starlette.routing import Route
 
-from aind_data_transfer_service import OPEN_DATA_BUCKET_NAME
+from aind_data_transfer_service import (
+    OPEN_DATA_BUCKET_NAME,
+)
 from aind_data_transfer_service import (
     __version__ as aind_data_transfer_service_version,
 )
@@ -37,14 +39,18 @@ from aind_data_transfer_service.configs.csv_handler import map_csv_row_to_job
 from aind_data_transfer_service.configs.job_configs import (
     BasicUploadJobConfigs as LegacyBasicUploadJobConfigs,
 )
-from aind_data_transfer_service.configs.job_configs import HpcJobConfigs
+from aind_data_transfer_service.configs.job_configs import (
+    HpcJobConfigs,
+)
 from aind_data_transfer_service.configs.job_upload_template import (
     JobUploadTemplate,
 )
 from aind_data_transfer_service.hpc.client import HpcClient, HpcClientConfigs
 from aind_data_transfer_service.hpc.models import HpcJobSubmitSettings
 from aind_data_transfer_service.log_handler import LoggingConfigs, get_logger
-from aind_data_transfer_service.models.core import SubmitJobRequestV2
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+)
 from aind_data_transfer_service.models.core import (
     validation_context as validation_context_v2,
 )
@@ -929,10 +935,10 @@ async def get_task_logs(request: Request):
 async def index(request: Request):
     """GET|POST /: form handler"""
     return templates.TemplateResponse(
+        request=request,
         name="index.html",
         context=(
             {
-                "request": request,
                 "project_names_url": project_names_url,
             }
         ),
@@ -945,10 +951,10 @@ async def job_tasks_table(request: Request):
     response_tasks_json = json.loads(response_tasks.body)
     data = response_tasks_json.get("data")
     return templates.TemplateResponse(
+        request=request,
         name="job_tasks_table.html",
         context=(
             {
-                "request": request,
                 "status_code": response_tasks.status_code,
                 "message": response_tasks_json.get("message"),
                 "errors": data.get("errors", []),
@@ -965,10 +971,10 @@ async def task_logs(request: Request):
     response_tasks_json = json.loads(response_tasks.body)
     data = response_tasks_json.get("data")
     return templates.TemplateResponse(
+        request=request,
         name="task_logs.html",
         context=(
             {
-                "request": request,
                 "status_code": response_tasks.status_code,
                 "message": response_tasks_json.get("message"),
                 "errors": data.get("errors", []),
@@ -982,10 +988,10 @@ async def jobs(request: Request):
     """Get Job Status page with pagination"""
     dag_ids = AirflowDagRunsRequestParameters.model_fields["dag_ids"].default
     return templates.TemplateResponse(
+        request=request,
         name="job_status.html",
         context=(
             {
-                "request": request,
                 "project_names_url": project_names_url,
                 "dag_ids": dag_ids,
             }
@@ -996,10 +1002,10 @@ async def jobs(request: Request):
 async def job_params(request: Request):
     """Get Job Parameters page"""
     return templates.TemplateResponse(
+        request=request,
         name="job_params.html",
         context=(
             {
-                "request": request,
                 "project_names_url": os.getenv(
                     "AIND_METADATA_SERVICE_PROJECT_NAMES_URL"
                 ),
@@ -1122,10 +1128,10 @@ async def admin(request: Request):
         user = {"name": "local user"}
     if user:
         return templates.TemplateResponse(
+            request=request,
             name="admin.html",
             context=(
                 {
-                    "request": request,
                     "project_names_url": project_names_url,
                     "user_name": user.get("name", "unknown"),
                     "user_email": user.get("email", "unknown"),

{aind_data_transfer_service-1.15.0.dist-info → aind_data_transfer_service-1.16.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.15.0
+Version: 1.16.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
@@ -28,10 +28,10 @@ Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
 Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
 Requires-Dist: boto3; extra == "server"
 Requires-Dist: boto3-stubs[ssm]; extra == "server"
-Requires-Dist: fastapi; extra == "server"
+Requires-Dist: fastapi>=0.115.13; extra == "server"
 Requires-Dist: httpx; extra == "server"
 Requires-Dist: jinja2; extra == "server"
-Requires-Dist: starlette; extra == "server"
+Requires-Dist: starlette<0.47.0,>=0.40.0; extra == "server"
 Requires-Dist: starlette_wtf; extra == "server"
 Requires-Dist: uvicorn[standard]; extra == "server"
 Requires-Dist: wtforms; extra == "server"

{aind_data_transfer_service-1.15.0.dist-info → aind_data_transfer_service-1.16.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-aind_data_transfer_service/__init__.py,sha256=lho7ClOyDOm8TOnoEsiTmbuidZwwCwry8o_Vu_mJ5qI,272
+aind_data_transfer_service/__init__.py,sha256=BHEL9NBC6zix81jxrsxrEp-bqoWgrWqKd2qrtmDv1fg,272
 aind_data_transfer_service/log_handler.py,sha256=c7a-gLmZeRpeCUBwCz6XsTszWXQeQdR7eKZtas4llXM,1700
-aind_data_transfer_service/server.py,sha256=8TRxybpk8hkqPng_6mEZc9wv8ofGErgYf0BSPk-8VAU,44545
+aind_data_transfer_service/server.py,sha256=cJf57MXuzKQb9oBK6n0z7gksHMSbssQ0f409-rOqoXI,44506
 aind_data_transfer_service/configs/__init__.py,sha256=9W5GTuso9Is1B9X16RXcdb_GxasZvj6qDzOBDv0AbTc,36
-aind_data_transfer_service/configs/csv_handler.py,sha256=9jM0fUlWCzmqTC7ubAeFCl0eEIX5BQvHcPPPTPngcog,4374
+aind_data_transfer_service/configs/csv_handler.py,sha256=hCdfAYZW_49-l1rbua5On2Tw2ks674Z-MgB_NJlIkU4,5746
 aind_data_transfer_service/configs/job_configs.py,sha256=T-h5N6lyY9xTZ_xg_5FxkyYuMdagApbE6xalxFQ-bqA,18848
 aind_data_transfer_service/configs/job_upload_template.py,sha256=aC5m1uD_YcpbggFQ-yZ7ZJSUUGX1yQqQLF3SwmljrLk,5127
 aind_data_transfer_service/hpc/__init__.py,sha256=YNc68YNlmXwKIPFMIViz_K4XzVVHkLPEBOFyO5DKMKI,53
@@ -17,8 +17,8 @@ aind_data_transfer_service/templates/job_params.html,sha256=ivofS1CjSnC87T5J2BTs
 aind_data_transfer_service/templates/job_status.html,sha256=5lUUGZL-5urppG610qDOgpfIE-OcQH57gFWvRA5pBNM,16938
 aind_data_transfer_service/templates/job_tasks_table.html,sha256=rWFukhjZ4dhPyabe372tmi4lbQS2fyELZ7Awbn5Un4g,6181
 aind_data_transfer_service/templates/task_logs.html,sha256=y1GnQft0S50ghPb2xJDjAlefymB9a4zYdMikUFV7Tl4,918
-aind_data_transfer_service-1.15.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
-aind_data_transfer_service-1.15.0.dist-info/METADATA,sha256=XNDcKsS_NFZIZMBOip1nBheDFo8lirOA7a9P14SVtNo,2452
-aind_data_transfer_service-1.15.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-aind_data_transfer_service-1.15.0.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
-aind_data_transfer_service-1.15.0.dist-info/RECORD,,
+aind_data_transfer_service-1.16.0.dist-info/licenses/LICENSE,sha256=U0Y7B3gZJHXpjJVLgTQjM8e_c8w4JJpLgGhIdsoFR1Y,1092
+aind_data_transfer_service-1.16.0.dist-info/METADATA,sha256=YkT9jGjZKxpa2O5qzekouiwzsuviOT_2w-7kg3511Ck,2478
+aind_data_transfer_service-1.16.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+aind_data_transfer_service-1.16.0.dist-info/top_level.txt,sha256=XmxH0q27Jholj2-VYh-6WMrh9Lw6kkuCX_fdsj3SaFE,27
+aind_data_transfer_service-1.16.0.dist-info/RECORD,,