aind-data-transfer-service 1.13.0__tar.gz → 1.14.0__tar.gz

{aind_data_transfer_service-1.13.0/src/aind_data_transfer_service.egg-info → aind_data_transfer_service-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.13.0
+Version: 1.14.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/docs/source/UserGuideV2.rst

@@ -45,13 +45,7 @@ job_types.
       container here.
    -  gather_final_metadata: Automatically generates a processing.json and
       metadata.nd.json file.
-   -  final_check_s3_folder_exist: The current behavior is to upload data to
-      the private data bucket if the metadata is invalid and the s3_bucket is
-      set to default. If the metadata is valid and the s3_bucket is set to
-      default, then the data is uploaded to the open bucket. Hence, another
-      check is added once the final s3_location is resolved. We plan to change
-      this behavior soon to just raise an error if the metadata is incomplete
-      and upload to the open bucket by default.
+   -  check_metadata_files: Checks that the metadata files exist and are json.
    -  upload_data_to_s3: Uploads the data to S3.
    -  register_data_asset_to_codeocean: Registers the data asset to Code Ocean.
    -  update_docdb_record: Updates the DocDB record with the Code Ocean Data
@@ -118,6 +112,11 @@ portal can accessed at
 
 -  Required fields
 
+   -  job_type: We store pre-compiled default configurations in AWS Parameter
+      Store (e.g. modality transformation settings, Code Ocean pipeline
+      settings). This field determines which preset to use when
+      running the upload job. A list of job types can be seen by clicking the
+      ``Job Parameters`` link.
    -  project_name: A list of project names can be seen by clicking the
       ``Project Names`` link
    -  subject_id: The LabTracks ID of the mouse
@@ -141,11 +140,6 @@ portal can accessed at
 
 -  Optional fields
 
-   -  job_type: We store pre-compiled default configurations in AWS Parameter
-      Store (e.g. modality transformation settings, Code Ocean pipeline
-      settings). If you set this field, then we will use this preset when
-      running the upload job. A list of job types can be seen by clicking the
-      ``Job Parameters`` link.
    -  metadata_dir: If metadata files are pre-compiled and saved to a
       directory, you can add the Posix style path to the directory under
       this column

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/src/aind_data_transfer_service/__init__.py

@@ -1,7 +1,7 @@
 """Init package"""
 import os
 
-__version__ = "1.13.0"
+__version__ = "1.14.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/src/aind_data_transfer_service/models/core.py

@@ -129,7 +129,7 @@ class UploadJobConfigsV2(BaseSettings):
     model_config = ConfigDict(use_enum_values=True, extra="ignore")
 
     job_type: str = Field(
-        default="default",
+        ...,
         description=(
             "Job type for the upload job. Tasks will be run based on the "
             "job_type unless otherwise specified in task_overrides."

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0/src/aind_data_transfer_service.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.13.0
+Version: 1.14.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/tests/resources/legacy_sample.csv

@@ -1,4 +1,4 @@
-project_name, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime, force_cloud_sync, modality0.process_capsule_id
-Ephys Platform, ecephys, dir/data_set_1, ,, private, 123454, ecephys, 2020-10-10 14:10:10, True,
-Behavior Platform, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, behavior, 10/13/2020 1:10:10 PM,,1f999652-00a0-4c4b-99b5-64c2985ad070
-Behavior Platform, behavior-videos, dir/data_set_2, behavior-videos, dir/data_set_3, default, 123456, behavior, 10/13/2020 1:10:10 PM,,
+project_name, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime, force_cloud_sync, modality0.process_capsule_id, job_type
+Ephys Platform, ecephys, dir/data_set_1, ,, private, 123454, ecephys, 2020-10-10 14:10:10, True,, default
+Behavior Platform, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, behavior, 10/13/2020 1:10:10 PM,,1f999652-00a0-4c4b-99b5-64c2985ad070, default
+Behavior Platform, behavior-videos, dir/data_set_2, behavior-videos, dir/data_set_3, default, 123456, behavior, 10/13/2020 1:10:10 PM,,, default

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/tests/resources/legacy_sample2.csv

@@ -1,2 +1,2 @@
-project_name, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime, modality0.pipeline_id
-Behavior Platform, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, behavior, 10/13/2020 1:10:10 PM,1f999652-00a0-4c4b-99b5-64c2985ad070
+project_name, modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime, modality0.pipeline_id, job_type
+Behavior Platform, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, behavior, 10/13/2020 1:10:10 PM,1f999652-00a0-4c4b-99b5-64c2985ad070, default

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/tests/resources/new_sample.csv

@@ -1,4 +1,4 @@
 project_name, modality0, modality0.input_source, modality1, modality1.input_source, s3-bucket, subject-id, platform, acq-datetime, job_type, metadata_dir
-Ephys Platform, ecephys, dir/data_set_1, , , default,123454, ecephys, 2020-10-10 14:10:10, , dir/metadata
+Ephys Platform, ecephys, dir/data_set_1, , , default,123454, ecephys, 2020-10-10 14:10:10, ecephys, dir/metadata
 Behavior Platform, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open,123456, behavior, 2020-10-13T13:10:10, custom,
-Behavior Platform, behavior-videos, dir/data_set_2, , , ,123456, behavior, 2020-10-13 13:10:10, ,
+Behavior Platform, behavior-videos, dir/data_set_2, , , ,123456, behavior, 2020-10-13 13:10:10, default,

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/tests/resources/sample_empty_rows_2.csv

@@ -1,7 +1,7 @@
 project_name, modality0, modality0.input_source, modality1, modality1.input_source, s3-bucket, subject-id, platform, acq-datetime, job_type, metadata_dir
-Ephys Platform, ecephys, dir/data_set_1, , , default,123454, ecephys, 2020-10-10 14:10:10, , dir/metadata
+Ephys Platform, ecephys, dir/data_set_1, , , default,123454, ecephys, 2020-10-10 14:10:10, ecephys, dir/metadata
 Behavior Platform, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open,123456, behavior, 2020-10-13T13:10:10, custom,
-Behavior Platform, behavior-videos, dir/data_set_2, , , ,123456, behavior, 2020-10-13 13:10:10, ,
+Behavior Platform, behavior-videos, dir/data_set_2, , , ,123456, behavior, 2020-10-13 13:10:10, default,
 ,,,,,,,,,,
 ,,,,,,,,,,
 ,,,,,,,,,,

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/tests/test_core.py

@@ -86,6 +86,7 @@ class TestUploadJobConfigsV2(unittest.TestCase):
     def setUpClass(cls) -> None:
         """Set up test class"""
         example_configs = UploadJobConfigsV2(
+            job_type="default",
             project_name="Behavior Platform",
             platform=Platform.BEHAVIOR,
             modalities=[Modality.BEHAVIOR_VIDEOS],
@@ -102,7 +103,7 @@ class TestUploadJobConfigsV2(unittest.TestCase):
 
         cls.example_configs = example_configs
         cls.base_configs = example_configs.model_dump(
-            exclude={"job_type": True, "s3_bucket": True, "s3_prefix": True}
+            exclude={"s3_bucket": True, "s3_prefix": True}
         )
 
     def test_s3_prefix(self):
@@ -118,21 +119,27 @@ class TestUploadJobConfigsV2(unittest.TestCase):
 
     def test_job_type_validation(self):
         """Test job_type is validated against list context provided."""
+        base_configs = self.example_configs.model_dump(
+            exclude={"job_type": True, "s3_bucket": True, "s3_prefix": True}
+        )
         with validation_context({"job_types": ["default", "ecephys"]}):
             round_trip_model = UploadJobConfigsV2(
                 job_type="ecephys",
-                **self.base_configs,
+                **base_configs,
             )
         self.assertEqual("ecephys", round_trip_model.job_type)
 
     def test_job_type_validation_fail(self):
         """Test job_type is validated against list context provided and
         fails validation."""
+        base_configs = self.example_configs.model_dump(
+            exclude={"job_type": True, "s3_bucket": True, "s3_prefix": True}
+        )
         with self.assertRaises(ValidationError) as err:
             with validation_context({"job_types": ["default", "ecephys"]}):
                 UploadJobConfigsV2(
                     job_type="random_string",
-                    **self.base_configs,
+                    **base_configs,
                 )
         err_msg = json.loads(err.exception.json())[0]["msg"]
         self.assertEqual(
@@ -244,6 +251,7 @@ class TestSubmitJobRequestV2(unittest.TestCase):
     def setUpClass(cls) -> None:
         """Set up example configs to be used in tests"""
         example_upload_config = UploadJobConfigsV2(
+            job_type="default",
             project_name="Behavior Platform",
             platform=Platform.BEHAVIOR,
             modalities=[

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/tests/test_csv_handler.py

@@ -48,6 +48,7 @@ class TestCsvHandler(unittest.TestCase):
                 },
                 subject_id="123454",
                 acq_datetime=datetime(2020, 10, 10, 14, 10, 10),
+                job_type="ecephys",
             ),
             UploadJobConfigsV2(
                 project_name="Behavior Platform",
@@ -81,6 +82,7 @@ class TestCsvHandler(unittest.TestCase):
                 },
                 subject_id="123456",
                 acq_datetime=datetime(2020, 10, 13, 13, 10, 10),
+                job_type="default",
             ),
         ]
         self.assertEqual(expected_jobs, jobs)

{aind_data_transfer_service-1.13.0 → aind_data_transfer_service-1.14.0}/tests/test_server.py

@@ -1593,7 +1593,7 @@ class TestServer(unittest.TestCase):
             "Ephys Platform",
             "Behavior Platform",
         ]
-        mock_get_job_types.return_value = ["default", "custom"]
+        mock_get_job_types.return_value = ["default", "ecephys", "custom"]
         mock_get_airflow_jobs.return_value = (0, list())
         with TestClient(app) as client:
             with open(NEW_SAMPLE_CSV, "rb") as f:
@@ -1665,7 +1665,7 @@ class TestServer(unittest.TestCase):
             "Ephys Platform",
             "Behavior Platform",
         ]
-        mock_get_job_types.return_value = ["default", "custom"]
+        mock_get_job_types.return_value = ["default", "ecephys", "custom"]
         mock_get_airflow_jobs.return_value = (0, list())
         with TestClient(app) as client:
             with open(SAMPLE_CSV_EMPTY_ROWS_2, "rb") as f: