aind-data-transfer-service 1.15.0__tar.gz → 1.16.0__tar.gz

{aind_data_transfer_service-1.15.0/src/aind_data_transfer_service.egg-info → aind_data_transfer_service-1.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.15.0
+Version: 1.16.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
@@ -28,10 +28,10 @@ Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
 Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
 Requires-Dist: boto3; extra == "server"
 Requires-Dist: boto3-stubs[ssm]; extra == "server"
-Requires-Dist: fastapi; extra == "server"
+Requires-Dist: fastapi>=0.115.13; extra == "server"
 Requires-Dist: httpx; extra == "server"
 Requires-Dist: jinja2; extra == "server"
-Requires-Dist: starlette; extra == "server"
+Requires-Dist: starlette<0.47.0,>=0.40.0; extra == "server"
 Requires-Dist: starlette_wtf; extra == "server"
 Requires-Dist: uvicorn[standard]; extra == "server"
 Requires-Dist: wtforms; extra == "server"

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/docs/examples/remove_source_folders.py

@@ -55,7 +55,7 @@ upload_job_configs_v2 = UploadJobConfigsV2(
     tasks={
         "modality_transformation_settings": modality_transformation_settings,
         "gather_preliminary_metadata": gather_preliminary_metadata,
-        "remove_source_folders": remove_source_folders
+        "remove_source_folders": remove_source_folders,
     },
 )
 

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/pyproject.toml

@@ -44,10 +44,10 @@ server = [
     'aind-metadata-mapper==0.23.0',
     'boto3',
     'boto3-stubs[ssm]',
-    'fastapi',
+    'fastapi>=0.115.13', 
     'httpx',
     'jinja2',
-    'starlette',
+    'starlette>=0.40.0,<0.47.0',
     'starlette_wtf',
     'uvicorn[standard]',
     'wtforms',

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/src/aind_data_transfer_service/__init__.py

@@ -1,7 +1,7 @@
 """Init package"""
 import os
 
-__version__ = "1.15.0"
+__version__ = "1.16.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/src/aind_data_transfer_service/configs/csv_handler.py

@@ -1,7 +1,10 @@
 """Module to handle processing legacy csv files"""
 
 import re
+from collections.abc import Mapping
+from copy import deepcopy
 from datetime import datetime
+from typing import Any, Dict
 
 from aind_data_schema_models.modalities import Modality
 from aind_data_schema_models.platforms import Platform
@@ -13,6 +16,45 @@ DATETIME_PATTERN2 = re.compile(
 )
 
 
+def nested_update(dict_to_update: Dict[str, Any], updates: Mapping):
+    """
+    Update a nested dictionary in-place.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    updates : Mapping
+
+    """
+    for k, v in updates.items():
+        if isinstance(v, Mapping):
+            dict_to_update[k] = nested_update(dict_to_update.get(k, {}), v)
+        else:
+            dict_to_update[k] = v
+    return dict_to_update
+
+
+def create_nested_dict(
+    dict_to_update: Dict[str, Any], key_string: str, value: Any
+):
+    """
+    Updates in-place a nested dictionary with a period delimited key and value.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    key_string : str
+    value : Any
+
+    """
+    keys = key_string.split(".", 1)
+    current_key = keys[0]
+    if len(keys) == 1:
+        dict_to_update[current_key] = value
+    else:
+        if current_key not in dict_to_update:
+            dict_to_update[current_key] = dict()
+        create_nested_dict(dict_to_update[current_key], keys[1], value)
+
+
 def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     """
     Maps csv row into a UploadJobConfigsV2 model. This attempts to be somewhat
@@ -29,7 +71,6 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     modality_configs = dict()
     job_configs = dict()
     check_s3_folder_exists_task = None
-    final_check_s3_folder_exist = None
     codeocean_tasks = dict()
     for key, value in row.items():
         # Strip white spaces and replace dashes with underscores
@@ -42,7 +83,9 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
             modality_parts = clean_key.split(".")
             modality_key = modality_parts[0]
             sub_key = (
-                "modality" if len(modality_parts) == 1 else modality_parts[1]
+                "modality"
+                if len(modality_parts) == 1
+                else ".".join(modality_parts[1:])
             )
             modality_configs.setdefault(modality_key, dict())
             # Temp backwards compatibility check
@@ -66,13 +109,22 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
                     job_settings=codeocean_pipeline_monitor_settings,
                 )
             else:
-                modality_configs[modality_key].update({sub_key: clean_val})
+                nested_val = dict()
+                create_nested_dict(
+                    dict_to_update=nested_val,
+                    key_string=sub_key,
+                    value=clean_val,
+                )
+                current_dict = deepcopy(
+                    modality_configs.get(modality_key, dict())
+                )
+                nested_update(current_dict, nested_val)
+                modality_configs[modality_key] = current_dict
         elif clean_key == "force_cloud_sync" and clean_val.upper() in [
             "TRUE",
             "T",
         ]:
             check_s3_folder_exists_task = {"skip_task": True}
-            final_check_s3_folder_exist = {"skip_task": True}
         else:
             job_configs[clean_key] = clean_val
     # Rename codeocean config keys with correct modality
@@ -93,8 +145,7 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     )
     tasks = {
         "gather_preliminary_metadata": metadata_task,
-        "check_s3_folder_exists_task": check_s3_folder_exists_task,
-        "final_check_s3_folder_exist": final_check_s3_folder_exist,
+        "check_s3_folder_exists": check_s3_folder_exists_task,
         "modality_transformation_settings": modality_tasks,
         "codeocean_pipeline_settings": None
         if codeocean_tasks == dict()

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/src/aind_data_transfer_service/server.py

@@ -29,7 +29,9 @@ from starlette.middleware.sessions import SessionMiddleware
 from starlette.responses import RedirectResponse
 from starlette.routing import Route
 
-from aind_data_transfer_service import OPEN_DATA_BUCKET_NAME
+from aind_data_transfer_service import (
+    OPEN_DATA_BUCKET_NAME,
+)
 from aind_data_transfer_service import (
     __version__ as aind_data_transfer_service_version,
 )
@@ -37,14 +39,18 @@ from aind_data_transfer_service.configs.csv_handler import map_csv_row_to_job
 from aind_data_transfer_service.configs.job_configs import (
     BasicUploadJobConfigs as LegacyBasicUploadJobConfigs,
 )
-from aind_data_transfer_service.configs.job_configs import HpcJobConfigs
+from aind_data_transfer_service.configs.job_configs import (
+    HpcJobConfigs,
+)
 from aind_data_transfer_service.configs.job_upload_template import (
     JobUploadTemplate,
 )
 from aind_data_transfer_service.hpc.client import HpcClient, HpcClientConfigs
 from aind_data_transfer_service.hpc.models import HpcJobSubmitSettings
 from aind_data_transfer_service.log_handler import LoggingConfigs, get_logger
-from aind_data_transfer_service.models.core import SubmitJobRequestV2
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+)
 from aind_data_transfer_service.models.core import (
     validation_context as validation_context_v2,
 )
@@ -929,10 +935,10 @@ async def get_task_logs(request: Request):
 async def index(request: Request):
     """GET|POST /: form handler"""
     return templates.TemplateResponse(
+        request=request,
         name="index.html",
         context=(
             {
-                "request": request,
                 "project_names_url": project_names_url,
             }
         ),
@@ -945,10 +951,10 @@ async def job_tasks_table(request: Request):
     response_tasks_json = json.loads(response_tasks.body)
     data = response_tasks_json.get("data")
     return templates.TemplateResponse(
+        request=request,
         name="job_tasks_table.html",
         context=(
             {
-                "request": request,
                 "status_code": response_tasks.status_code,
                 "message": response_tasks_json.get("message"),
                 "errors": data.get("errors", []),
@@ -965,10 +971,10 @@ async def task_logs(request: Request):
     response_tasks_json = json.loads(response_tasks.body)
     data = response_tasks_json.get("data")
     return templates.TemplateResponse(
+        request=request,
         name="task_logs.html",
         context=(
             {
-                "request": request,
                 "status_code": response_tasks.status_code,
                 "message": response_tasks_json.get("message"),
                 "errors": data.get("errors", []),
@@ -982,10 +988,10 @@ async def jobs(request: Request):
     """Get Job Status page with pagination"""
     dag_ids = AirflowDagRunsRequestParameters.model_fields["dag_ids"].default
     return templates.TemplateResponse(
+        request=request,
         name="job_status.html",
         context=(
             {
-                "request": request,
                 "project_names_url": project_names_url,
                 "dag_ids": dag_ids,
             }
@@ -996,10 +1002,10 @@ async def jobs(request: Request):
 async def job_params(request: Request):
     """Get Job Parameters page"""
     return templates.TemplateResponse(
+        request=request,
         name="job_params.html",
         context=(
             {
-                "request": request,
                 "project_names_url": os.getenv(
                     "AIND_METADATA_SERVICE_PROJECT_NAMES_URL"
                 ),
@@ -1122,10 +1128,10 @@ async def admin(request: Request):
         user = {"name": "local user"}
     if user:
         return templates.TemplateResponse(
+            request=request,
             name="admin.html",
             context=(
                 {
-                    "request": request,
                     "project_names_url": project_names_url,
                     "user_name": user.get("name", "unknown"),
                     "user_email": user.get("email", "unknown"),

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0/src/aind_data_transfer_service.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.15.0
+Version: 1.16.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
@@ -28,10 +28,10 @@ Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
 Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
 Requires-Dist: boto3; extra == "server"
 Requires-Dist: boto3-stubs[ssm]; extra == "server"
-Requires-Dist: fastapi; extra == "server"
+Requires-Dist: fastapi>=0.115.13; extra == "server"
 Requires-Dist: httpx; extra == "server"
 Requires-Dist: jinja2; extra == "server"
-Requires-Dist: starlette; extra == "server"
+Requires-Dist: starlette<0.47.0,>=0.40.0; extra == "server"
 Requires-Dist: starlette_wtf; extra == "server"
 Requires-Dist: uvicorn[standard]; extra == "server"
 Requires-Dist: wtforms; extra == "server"

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/src/aind_data_transfer_service.egg-info/SOURCES.txt

@@ -87,6 +87,7 @@ tests/resources/get_secrets_response.json
 tests/resources/job_upload_template.xlsx
 tests/resources/legacy_sample.csv
 tests/resources/legacy_sample2.csv
+tests/resources/nested_sample.csv
 tests/resources/new_sample.csv
 tests/resources/sample.csv
 tests/resources/sample.xlsx

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/src/aind_data_transfer_service.egg-info/requires.txt

@@ -21,10 +21,10 @@ aind-data-transfer-models==0.17.0
 aind-metadata-mapper==0.23.0
 boto3
 boto3-stubs[ssm]
-fastapi
+fastapi>=0.115.13
 httpx
 jinja2
-starlette
+starlette<0.47.0,>=0.40.0
 starlette_wtf
 uvicorn[standard]
 wtforms

aind_data_transfer_service-1.16.0/tests/resources/nested_sample.csv

@@ -0,0 +1,2 @@
+project_name, modality0, modality0.input_source, modality0.compression_requested.compression_enum, subject-id, platform, acq-datetime, job_type
+Ephys Platform, ecephys, dir/data_set_1, gamma fix colorspace, 123454, ecephys, 2020-10-10 14:10:10, ecephys

{aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.16.0}/tests/test_csv_handler.py

@@ -9,11 +9,16 @@ from pathlib import Path
 from aind_data_schema_models.modalities import Modality
 from aind_data_schema_models.platforms import Platform
 
-from aind_data_transfer_service.configs.csv_handler import map_csv_row_to_job
+from aind_data_transfer_service.configs.csv_handler import (
+    create_nested_dict,
+    map_csv_row_to_job,
+    nested_update,
+)
 from aind_data_transfer_service.models.core import Task, UploadJobConfigsV2
 
 RESOURCES_DIR = Path(os.path.dirname(os.path.realpath(__file__))) / "resources"
 SAMPLE_FILE = RESOURCES_DIR / "new_sample.csv"
+NESTED_SAMPLE_FILE = RESOURCES_DIR / "nested_sample.csv"
 LEGACY_FILE = RESOURCES_DIR / "legacy_sample.csv"
 LEGACY_FILE_2 = RESOURCES_DIR / "legacy_sample2.csv"
 
@@ -21,6 +26,26 @@ LEGACY_FILE_2 = RESOURCES_DIR / "legacy_sample2.csv"
 class TestCsvHandler(unittest.TestCase):
     """Tests methods in csv_handler module"""
 
+    def test_create_nested_dict(self):
+        """Tests create_nested_dict method"""
+
+        key = "abc.def.ghi"
+        value = "my_val"
+        nested_dict = dict()
+        create_nested_dict(
+            dict_to_update=nested_dict, key_string=key, value=value
+        )
+        expected_dict = {"abc": {"def": {"ghi": "my_val"}}}
+        self.assertEqual(expected_dict, nested_dict)
+
+    def test_nested_update(self):
+        """Tests nested_update method."""
+        current_dict = {"abc": {"def": {"ghi": "my_val"}}}
+        updates = {"abc": {"def": {"jkl": 123}}}
+        nested_update(current_dict, updates)
+        expected_dict = {"abc": {"def": {"ghi": "my_val", "jkl": 123}}}
+        self.assertEqual(expected_dict, current_dict)
+
     def test_map_csv_row_to_job(self):
         """Tests map_csv_row_to_job method"""
 
@@ -87,6 +112,42 @@ class TestCsvHandler(unittest.TestCase):
         ]
         self.assertEqual(expected_jobs, jobs)
 
+    def test_map_nested_csv_row_to_job(self):
+        """Tests map_csv_row_to_job method with nested keys"""
+
+        jobs = []
+        with open(NESTED_SAMPLE_FILE, newline="") as csvfile:
+            reader = csv.DictReader(csvfile, skipinitialspace=True)
+            for row in reader:
+                jobs.append(map_csv_row_to_job(row))
+        expected_jobs = [
+            UploadJobConfigsV2(
+                job_type="ecephys",
+                s3_bucket="default",
+                project_name="Ephys Platform",
+                platform=Platform.ECEPHYS,
+                modalities=[Modality.ECEPHYS],
+                subject_id="123454",
+                acq_datetime=datetime(2020, 10, 10, 14, 10, 10),
+                tasks={
+                    "modality_transformation_settings": {
+                        "ecephys": Task(
+                            skip_task=False,
+                            job_settings={
+                                "input_source": "dir/data_set_1",
+                                "compression_requested": {
+                                    "compression_enum": "gamma fix colorspace"
+                                },
+                            },
+                        )
+                    }
+                },
+                s3_prefix="ecephys_123454_2020-10-10_14-10-10",
+            )
+        ]
+
+        self.assertEqual(expected_jobs, jobs)
+
     def test_map_legacy_csv_row_to_job(self):
         """Tests map_csv_row_to_job method"""
 
@@ -105,10 +166,7 @@ class TestCsvHandler(unittest.TestCase):
                 subject_id="123454",
                 acq_datetime=datetime(2020, 10, 10, 14, 10, 10),
                 tasks={
-                    "check_s3_folder_exists_task": Task(
-                        skip_task=True,
-                    ),
-                    "final_check_s3_folder_exist": Task(
+                    "check_s3_folder_exists": Task(
                         skip_task=True,
                     ),
                     "modality_transformation_settings": {