aind-data-transfer-service 1.14.0__tar.gz → 1.20.0__tar.gz

{aind_data_transfer_service-1.14.0 → aind_data_transfer_service-1.20.0}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.14.0
+Version: 1.20.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aind-data-schema-models<1.0.0,>=0.3.2
+Requires-Dist: aind-data-schema-models>=0.3.2
 Requires-Dist: email-validator
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pydantic-settings>=2.0
@@ -23,21 +23,20 @@ Provides-Extra: docs
 Requires-Dist: Sphinx; extra == "docs"
 Requires-Dist: furo; extra == "docs"
 Provides-Extra: server
-Requires-Dist: aind-data-schema<2.0,>=1.0.0; extra == "server"
-Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
-Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
+Requires-Dist: aind-data-schema==2.0.0; extra == "server"
 Requires-Dist: boto3; extra == "server"
 Requires-Dist: boto3-stubs[ssm]; extra == "server"
-Requires-Dist: fastapi; extra == "server"
+Requires-Dist: fastapi>=0.115.13; extra == "server"
 Requires-Dist: httpx; extra == "server"
 Requires-Dist: jinja2; extra == "server"
-Requires-Dist: starlette; extra == "server"
+Requires-Dist: starlette<0.47.0,>=0.40.0; extra == "server"
 Requires-Dist: starlette_wtf; extra == "server"
 Requires-Dist: uvicorn[standard]; extra == "server"
 Requires-Dist: wtforms; extra == "server"
 Requires-Dist: requests==2.25.0; extra == "server"
 Requires-Dist: openpyxl; extra == "server"
 Requires-Dist: python-logging-loki; extra == "server"
+Requires-Dist: authlib; extra == "server"
 Dynamic: license-file
 
 # aind-data-transfer-service

{aind_data_transfer_service-1.14.0 → aind_data_transfer_service-1.20.0}/pyproject.toml

@@ -1,12 +1,12 @@
 [build-system]
-requires = ["setuptools", "setuptools-scm"]
+requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "aind-data-transfer-service"
 description = "Service that handles requests to upload data to the cloud"
 license = {text = "MIT"}
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 authors = [
     {name = "Allen Institute for Neural Dynamics"}
 ]
@@ -17,7 +17,7 @@ readme = "README.md"
 dynamic = ["version"]
 
 dependencies = [
-    'aind-data-schema-models>=0.3.2,<1.0.0',
+    'aind-data-schema-models>=0.3.2',
     'email-validator',
     'pydantic>=2.0',
     'pydantic-settings>=2.0',
@@ -39,21 +39,20 @@ docs = [
 ]
 
 server = [
-    'aind-data-schema>=1.0.0,<2.0',
-    'aind-data-transfer-models==0.17.0',
-    'aind-metadata-mapper==0.23.0',
+    'aind-data-schema==2.0.0',
     'boto3',
     'boto3-stubs[ssm]',
-    'fastapi',
+    'fastapi>=0.115.13', 
     'httpx',
     'jinja2',
-    'starlette',
+    'starlette>=0.40.0,<0.47.0',
     'starlette_wtf',
     'uvicorn[standard]',
     'wtforms',
     'requests==2.25.0',
     'openpyxl',
-    'python-logging-loki'
+    'python-logging-loki',
+    'authlib'
 ]
 
 [tool.setuptools.packages.find]

{aind_data_transfer_service-1.14.0 → aind_data_transfer_service-1.20.0}/src/aind_data_transfer_service/__init__.py

@@ -1,7 +1,8 @@
 """Init package"""
+
 import os
 
-__version__ = "1.14.0"
+__version__ = "1.20.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

{aind_data_transfer_service-1.14.0 → aind_data_transfer_service-1.20.0}/src/aind_data_transfer_service/configs/csv_handler.py

@@ -1,11 +1,14 @@
 """Module to handle processing legacy csv files"""
 
 import re
+from collections.abc import Mapping
+from copy import deepcopy
 from datetime import datetime
+from typing import Any, Dict
 
 from aind_data_schema_models.modalities import Modality
-from aind_data_schema_models.platforms import Platform
 
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 from aind_data_transfer_service.models.core import Task, UploadJobConfigsV2
 
 DATETIME_PATTERN2 = re.compile(
@@ -13,6 +16,45 @@ DATETIME_PATTERN2 = re.compile(
 )
 
 
+def nested_update(dict_to_update: Dict[str, Any], updates: Mapping):
+    """
+    Update a nested dictionary in-place.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    updates : Mapping
+
+    """
+    for k, v in updates.items():
+        if isinstance(v, Mapping):
+            dict_to_update[k] = nested_update(dict_to_update.get(k, {}), v)
+        else:
+            dict_to_update[k] = v
+    return dict_to_update
+
+
+def create_nested_dict(
+    dict_to_update: Dict[str, Any], key_string: str, value: Any
+):
+    """
+    Updates in-place a nested dictionary with a period delimited key and value.
+    Parameters
+    ----------
+    dict_to_update : Dict[str, Any]
+    key_string : str
+    value : Any
+
+    """
+    keys = key_string.split(".", 1)
+    current_key = keys[0]
+    if len(keys) == 1:
+        dict_to_update[current_key] = value
+    else:
+        if current_key not in dict_to_update:
+            dict_to_update[current_key] = dict()
+        create_nested_dict(dict_to_update[current_key], keys[1], value)
+
+
 def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     """
     Maps csv row into a UploadJobConfigsV2 model. This attempts to be somewhat
@@ -29,7 +71,6 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     modality_configs = dict()
     job_configs = dict()
     check_s3_folder_exists_task = None
-    final_check_s3_folder_exist = None
     codeocean_tasks = dict()
     for key, value in row.items():
         # Strip white spaces and replace dashes with underscores
@@ -42,7 +83,9 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
             modality_parts = clean_key.split(".")
             modality_key = modality_parts[0]
             sub_key = (
-                "modality" if len(modality_parts) == 1 else modality_parts[1]
+                "modality"
+                if len(modality_parts) == 1
+                else ".".join(modality_parts[1:])
             )
             modality_configs.setdefault(modality_key, dict())
             # Temp backwards compatibility check
@@ -66,13 +109,22 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
                     job_settings=codeocean_pipeline_monitor_settings,
                 )
             else:
-                modality_configs[modality_key].update({sub_key: clean_val})
+                nested_val = dict()
+                create_nested_dict(
+                    dict_to_update=nested_val,
+                    key_string=sub_key,
+                    value=clean_val,
+                )
+                current_dict = deepcopy(
+                    modality_configs.get(modality_key, dict())
+                )
+                nested_update(current_dict, nested_val)
+                modality_configs[modality_key] = current_dict
         elif clean_key == "force_cloud_sync" and clean_val.upper() in [
             "TRUE",
             "T",
         ]:
             check_s3_folder_exists_task = {"skip_task": True}
-            final_check_s3_folder_exist = {"skip_task": True}
         else:
             job_configs[clean_key] = clean_val
     # Rename codeocean config keys with correct modality
@@ -93,16 +145,20 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
     )
     tasks = {
         "gather_preliminary_metadata": metadata_task,
-        "check_s3_folder_exists_task": check_s3_folder_exists_task,
-        "final_check_s3_folder_exist": final_check_s3_folder_exist,
+        "check_s3_folder_exists": check_s3_folder_exists_task,
         "modality_transformation_settings": modality_tasks,
-        "codeocean_pipeline_settings": None
-        if codeocean_tasks == dict()
-        else codeocean_tasks,
+        "codeocean_pipeline_settings": (
+            None if codeocean_tasks == dict() else codeocean_tasks
+        ),
     }
+    platform = (
+        None
+        if job_configs.get("platform") is None
+        else Platform.from_abbreviation(job_configs["platform"])
+    )
     job_configs.update(
         {
-            "platform": Platform.from_abbreviation(job_configs["platform"]),
+            "platform": platform,
             "modalities": [
                 Modality.from_abbreviation(m) for m in modality_tasks.keys()
             ],

{aind_data_transfer_service-1.14.0 → aind_data_transfer_service-1.20.0}/src/aind_data_transfer_service/configs/job_upload_template.py

@@ -2,24 +2,25 @@
 
 import datetime
 from io import BytesIO
-from typing import Any, Dict, List
+from typing import Any, ClassVar, Dict, List
 
 from aind_data_schema_models.modalities import Modality
-from aind_data_schema_models.platforms import Platform
 from openpyxl import Workbook
 from openpyxl.styles import Font
 from openpyxl.utils import get_column_letter
 from openpyxl.worksheet.datavalidation import DataValidation
+from pydantic import BaseModel
 
+from aind_data_transfer_service.configs.platforms_v1 import Platform
 
-# TODO: convert to pydantic model
-class JobUploadTemplate:
+
+class JobUploadTemplate(BaseModel):
     """Class to configure and create xlsx job upload template"""
 
-    FILE_NAME = "job_upload_template.xlsx"
-    NUM_TEMPLATE_ROWS = 20
-    XLSX_DATETIME_FORMAT = "YYYY-MM-DDTHH:mm:ss"
-    HEADERS = [
+    FILE_NAME: ClassVar[str] = "job_upload_template.xlsx"
+    _NUM_TEMPLATE_ROWS: ClassVar[int] = 20
+    _XLSX_DATETIME_FORMAT: ClassVar[str] = "YYYY-MM-DDTHH:mm:ss"
+    _HEADERS: ClassVar[List[str]] = [
         "job_type",
         "project_name",
         "platform",
@@ -31,7 +32,7 @@ class JobUploadTemplate:
         "modality1",
         "modality1.input_source",
     ]
-    SAMPLE_JOBS = [
+    _SAMPLE_JOBS: ClassVar[List[List[Any]]] = [
         [
             "default",
             "Behavior Platform",
@@ -68,8 +69,8 @@ class JobUploadTemplate:
         ],
     ]
 
-    @property
-    def validators(self) -> List[Dict[str, Any]]:
+    @classmethod
+    def _get_validators(cls) -> List[Dict[str, Any]]:
         """
         Returns
         -------
@@ -82,36 +83,36 @@ class JobUploadTemplate:
                 "name": "platform",
                 "type": "list",
                 "options": list(Platform.abbreviation_map.keys()),
-                "column_indexes": [self.HEADERS.index("platform")],
+                "column_indexes": [cls._HEADERS.index("platform")],
             },
             {
                 "name": "modality",
                 "type": "list",
                 "options": list(Modality.abbreviation_map.keys()),
                 "column_indexes": [
-                    self.HEADERS.index("modality0"),
-                    self.HEADERS.index("modality1"),
+                    cls._HEADERS.index("modality0"),
+                    cls._HEADERS.index("modality1"),
                 ],
             },
             {
                 "name": "datetime",
                 "type": "date",
-                "column_indexes": [self.HEADERS.index("acq_datetime")],
+                "column_indexes": [cls._HEADERS.index("acq_datetime")],
             },
         ]
 
-    @property
-    def excel_sheet_filestream(self) -> BytesIO:
+    @classmethod
+    def create_excel_sheet_filestream(cls) -> BytesIO:
         """Create job template as xlsx filestream"""
         xl_io = BytesIO()
         workbook = Workbook()
         workbook.iso_dates = True
         worksheet = workbook.active
-        worksheet.append(self.HEADERS)
-        for job in self.SAMPLE_JOBS:
+        worksheet.append(cls._HEADERS)
+        for job in cls._SAMPLE_JOBS:
             worksheet.append(job)
         # data validators
-        for validator in self.validators:
+        for validator in cls._get_validators():
             dv_type = validator["type"]
             dv_name = validator["name"]
             dv_params = {
@@ -127,17 +128,17 @@ class JobUploadTemplate:
                 dv_params["prompt"] = f"Select a {dv_name} from the dropdown"
             elif dv_type == "date":
                 dv_params["prompt"] = "Provide a {} using {}".format(
-                    dv_name, self.XLSX_DATETIME_FORMAT
+                    dv_name, cls._XLSX_DATETIME_FORMAT
                 )
             dv = DataValidation(**dv_params)
             for i in validator["column_indexes"]:
                 col = get_column_letter(i + 1)
-                col_range = f"{col}2:{col}{self.NUM_TEMPLATE_ROWS}"
+                col_range = f"{col}2:{col}{cls._NUM_TEMPLATE_ROWS}"
                 dv.add(col_range)
                 if dv_type != "date":
                     continue
                 for (cell,) in worksheet[col_range]:
-                    cell.number_format = self.XLSX_DATETIME_FORMAT
+                    cell.number_format = cls._XLSX_DATETIME_FORMAT
             worksheet.add_data_validation(dv)
         # formatting
         bold = Font(bold=True)

aind_data_transfer_service-1.20.0/src/aind_data_transfer_service/configs/platforms_v1.py

@@ -0,0 +1,177 @@
+"""Platforms"""
+
+from typing import Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+from typing_extensions import Annotated
+
+
+class _PlatformModel(BaseModel):
+    """Base model for platform"""
+
+    model_config = ConfigDict(frozen=True)
+    name: str
+    abbreviation: str
+
+
+class _Behavior(_PlatformModel):
+    """Model behavior"""
+
+    name: Literal["Behavior platform"] = "Behavior platform"
+    abbreviation: Literal["behavior"] = "behavior"
+
+
+class _Confocal(_PlatformModel):
+    """Model confocal"""
+
+    name: Literal["Confocal microscopy platform"] = (
+        "Confocal microscopy platform"
+    )
+    abbreviation: Literal["confocal"] = "confocal"
+
+
+class _Ecephys(_PlatformModel):
+    """Model ecephys"""
+
+    name: Literal["Electrophysiology platform"] = "Electrophysiology platform"
+    abbreviation: Literal["ecephys"] = "ecephys"
+
+
+class _Exaspim(_PlatformModel):
+    """Model exaSPIM"""
+
+    name: Literal["ExaSPIM platform"] = "ExaSPIM platform"
+    abbreviation: Literal["exaSPIM"] = "exaSPIM"
+
+
+class _Fip(_PlatformModel):
+    """Model FIP"""
+
+    name: Literal["Frame-projected independent-fiber photometry platform"] = (
+        "Frame-projected independent-fiber photometry platform"
+    )
+    abbreviation: Literal["FIP"] = "FIP"
+
+
+class _Hcr(_PlatformModel):
+    """Model HCR"""
+
+    name: Literal["Hybridization chain reaction platform"] = (
+        "Hybridization chain reaction platform"
+    )
+    abbreviation: Literal["HCR"] = "HCR"
+
+
+class _Hsfp(_PlatformModel):
+    """Model HSFP"""
+
+    name: Literal["Hyperspectral fiber photometry platform"] = (
+        "Hyperspectral fiber photometry platform"
+    )
+    abbreviation: Literal["HSFP"] = "HSFP"
+
+
+class _Isi(_PlatformModel):
+    """Model ISI"""
+
+    name: Literal["Intrinsic signal imaging platform"] = (
+        "Intrinsic signal imaging platform"
+    )
+    abbreviation: Literal["ISI"] = "ISI"
+
+
+class _Merfish(_PlatformModel):
+    """Model MERFISH"""
+
+    name: Literal["MERFISH platform"] = "MERFISH platform"
+    abbreviation: Literal["MERFISH"] = "MERFISH"
+
+
+class _Mri(_PlatformModel):
+    """Model MRI"""
+
+    name: Literal["Magnetic resonance imaging platform"] = (
+        "Magnetic resonance imaging platform"
+    )
+    abbreviation: Literal["MRI"] = "MRI"
+
+
+class _Mesospim(_PlatformModel):
+    """Model mesoSPIM"""
+
+    name: Literal["MesoSPIM platform"] = "MesoSPIM platform"
+    abbreviation: Literal["mesoSPIM"] = "mesoSPIM"
+
+
+class _Motor_Observatory(_PlatformModel):
+    """Model motor-observatory"""
+
+    name: Literal["Motor observatory platform"] = "Motor observatory platform"
+    abbreviation: Literal["motor-observatory"] = "motor-observatory"
+
+
+class _Multiplane_Ophys(_PlatformModel):
+    """Model multiplane-ophys"""
+
+    name: Literal["Multiplane optical physiology platform"] = (
+        "Multiplane optical physiology platform"
+    )
+    abbreviation: Literal["multiplane-ophys"] = "multiplane-ophys"
+
+
+class _Slap2(_PlatformModel):
+    """Model SLAP2"""
+
+    name: Literal["SLAP2 platform"] = "SLAP2 platform"
+    abbreviation: Literal["SLAP2"] = "SLAP2"
+
+
+class _Single_Plane_Ophys(_PlatformModel):
+    """Model single-plane-ophys"""
+
+    name: Literal["Single-plane optical physiology platform"] = (
+        "Single-plane optical physiology platform"
+    )
+    abbreviation: Literal["single-plane-ophys"] = "single-plane-ophys"
+
+
+class _Smartspim(_PlatformModel):
+    """Model SmartSPIM"""
+
+    name: Literal["SmartSPIM platform"] = "SmartSPIM platform"
+    abbreviation: Literal["SmartSPIM"] = "SmartSPIM"
+
+
+class Platform:
+    """Platforms"""
+
+    BEHAVIOR = _Behavior()
+    CONFOCAL = _Confocal()
+    ECEPHYS = _Ecephys()
+    EXASPIM = _Exaspim()
+    FIP = _Fip()
+    HCR = _Hcr()
+    HSFP = _Hsfp()
+    ISI = _Isi()
+    MERFISH = _Merfish()
+    MRI = _Mri()
+    MESOSPIM = _Mesospim()
+    MOTOR_OBSERVATORY = _Motor_Observatory()
+    MULTIPLANE_OPHYS = _Multiplane_Ophys()
+    SLAP2 = _Slap2()
+    SINGLE_PLANE_OPHYS = _Single_Plane_Ophys()
+    SMARTSPIM = _Smartspim()
+
+    ALL = tuple(_PlatformModel.__subclasses__())
+
+    ONE_OF = Annotated[
+        Union[tuple(_PlatformModel.__subclasses__())],
+        Field(discriminator="name"),
+    ]
+
+    abbreviation_map = {m().abbreviation: m() for m in ALL}
+
+    @classmethod
+    def from_abbreviation(cls, abbreviation: str):
+        """Get platform from abbreviation"""
+        return cls.abbreviation_map.get(abbreviation, None)

{aind_data_transfer_service-1.14.0 → aind_data_transfer_service-1.20.0}/src/aind_data_transfer_service/log_handler.py

@@ -17,9 +17,9 @@ class LoggingConfigs(BaseSettings):
     loki_uri: Optional[str] = Field(
         default=None, description="URI of Loki logging server."
     )
-    log_level: Literal[
-        "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
-    ] = Field(default="INFO", description="Log level")
+    log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = (
+        Field(default="DEBUG", description="Log level")
+    )
 
     @property
     def app_name(self):

{aind_data_transfer_service-1.14.0 → aind_data_transfer_service-1.20.0}/src/aind_data_transfer_service/models/core.py

@@ -8,7 +8,6 @@ from typing import Any, Dict, List, Literal, Optional, Set, Union
 
 from aind_data_schema_models.data_name_patterns import build_data_name
 from aind_data_schema_models.modalities import Modality
-from aind_data_schema_models.platforms import Platform
 from pydantic import (
     BaseModel,
     ConfigDict,
@@ -21,6 +20,8 @@ from pydantic import (
 )
 from pydantic_settings import BaseSettings
 
+from aind_data_transfer_service.configs.platforms_v1 import Platform
+
 _validation_context: ContextVar[Union[Dict[str, Any], None]] = ContextVar(
     "_validation_context", default=None
 )
@@ -163,8 +164,13 @@ class UploadJobConfigsV2(BaseSettings):
     project_name: str = Field(
         ..., description="Name of project", title="Project Name"
     )
-    platform: Platform.ONE_OF = Field(
-        ..., description="Platform", title="Platform"
+    platform: Optional[Platform.ONE_OF] = Field(
+        default=None,
+        title="Platform",
+        description=(
+            "Legacy field required for aind-data-schema v1. Will be removed"
+            " in future versions."
+        ),
     )
     modalities: List[Modality.ONE_OF] = Field(
         ...,
@@ -190,11 +196,26 @@ class UploadJobConfigsV2(BaseSettings):
     @computed_field
     def s3_prefix(self) -> str:
         """Construct s3_prefix from configs."""
+        if self.platform is not None:
+            label = f"{self.platform.abbreviation}_{self.subject_id}"
+        else:
+            label = self.subject_id
         return build_data_name(
-            label=f"{self.platform.abbreviation}_{self.subject_id}",
+            label=label,
             creation_datetime=self.acq_datetime,
         )
 
+    @field_validator("platform", mode="before")
+    def validate_platform(cls, v):
+        """
+        For backwards compatibility, allow a user to input an
+        aind-data-schema-model platform and then convert it.
+        """
+        if type(v).__module__ == "aind_data_schema_models.platforms":
+            return v.model_dump()
+        else:
+            return v
+
     @field_validator("job_type", "project_name", mode="before")
     def validate_with_context(cls, v: str, info: ValidationInfo) -> str:
         """
@@ -290,11 +311,15 @@ class SubmitJobRequestV2(BaseSettings):
         # check against any jobs in the context
         current_jobs = (info.context or dict()).get("current_jobs", list())
         for job in current_jobs:
-            prefix = job.get("s3_prefix")
-            if (
-                prefix is not None
-                and prefix in jobs_map
-                and json.dumps(job, sort_keys=True) in jobs_map[prefix]
-            ):
-                raise ValueError(f"Job is already running/queued for {prefix}")
+            jobs_to_check = job.get("upload_jobs", [job])
+            for j in jobs_to_check:
+                prefix = j.get("s3_prefix")
+                if (
+                    prefix is not None
+                    and prefix in jobs_map
+                    and json.dumps(j, sort_keys=True) in jobs_map[prefix]
+                ):
+                    raise ValueError(
+                        f"Job is already running/queued for {prefix}"
+                    )
         return self