aind-data-transfer-service 1.12.0__tar.gz → 1.13.0__tar.gz

aind_data_transfer_service-1.13.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,133 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+ScientificComputing@AllenInstitute.onmicrosoft.com.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

{aind_data_transfer_service-1.12.0/src/aind_data_transfer_service.egg-info → aind_data_transfer_service-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aind-data-transfer-service
-Version: 1.12.0
+Version: 1.13.0
 Summary: Service that handles requests to upload data to the cloud
 Author: Allen Institute for Neural Dynamics
 License: MIT
@@ -8,13 +8,10 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3
-Requires-Dist: boto3-stubs[ssm]
-Requires-Dist: pydantic<2.9,>=2.7
+Requires-Dist: aind-data-schema-models<1.0.0,>=0.3.2
+Requires-Dist: email-validator
+Requires-Dist: pydantic>=2.0
 Requires-Dist: pydantic-settings>=2.0
-Requires-Dist: aind-data-schema<2.0,>=1.0.0
-Requires-Dist: aind-data-transfer-models==0.17.0
-Requires-Dist: aind-metadata-mapper==0.23.0
 Provides-Extra: dev
 Requires-Dist: aind-data-transfer-service[server]; extra == "dev"
 Requires-Dist: black; extra == "dev"
@@ -26,6 +23,11 @@ Provides-Extra: docs
 Requires-Dist: Sphinx; extra == "docs"
 Requires-Dist: furo; extra == "docs"
 Provides-Extra: server
+Requires-Dist: aind-data-schema<2.0,>=1.0.0; extra == "server"
+Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
+Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
+Requires-Dist: boto3; extra == "server"
+Requires-Dist: boto3-stubs[ssm]; extra == "server"
 Requires-Dist: fastapi; extra == "server"
 Requires-Dist: httpx; extra == "server"
 Requires-Dist: jinja2; extra == "server"

aind_data_transfer_service-1.13.0/docs/examples/basic_upload.py

@@ -0,0 +1,76 @@
+"""
+This example demonstrates how to set a modality and metadata directory. It
+assumes that the job_type has been defined.
+"""
+
+from datetime import datetime
+
+import requests
+from aind_data_schema_models.modalities import Modality
+from aind_data_schema_models.platforms import Platform
+
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+    Task,
+    UploadJobConfigsV2,
+)
+
+# The job_type contains the default settings for compression and Code Ocean
+# pipelines.
+job_type = "ecephys"
+
+acq_datetime = datetime(2023, 4, 3, 18, 17, 7)
+
+ecephys_task = Task(
+    job_settings={
+        "input_source": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "ecephys/655019_2023-04-03_18-17-07"
+        )
+    }
+)
+
+modality_transformation_settings = {"ecephys": ecephys_task}
+
+gather_preliminary_metadata = Task(
+    job_settings={
+        "metadata_dir": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "ecephys/655019_2023-04-03_18-17-07"
+        )
+    }
+)
+
+
+upload_job_configs_v2 = UploadJobConfigsV2(
+    job_type=job_type,
+    project_name="Ephys Platform",
+    platform=Platform.ECEPHYS,
+    modalities=[Modality.ECEPHYS],
+    subject_id="655019",
+    acq_datetime=acq_datetime,
+    tasks={
+        "modality_transformation_settings": modality_transformation_settings,
+        "gather_preliminary_metadata": gather_preliminary_metadata,
+    },
+)
+
+submit_request_v2 = SubmitJobRequestV2(
+    upload_jobs=[upload_job_configs_v2],
+)
+
+post_request_content = submit_request_v2.model_dump(
+    mode="json", exclude_none=True
+)
+
+# Please use the production endpoint for submitting jobs and the dev endpoint
+# for running tests.
+# endpoint = "http://aind-data-transfer-service"
+endpoint = "http://aind-data-transfer-service-dev"  # For testing
+
+submit_job_response = requests.post(
+    url=f"{endpoint}/api/v2/submit_jobs",
+    json=post_request_content,
+)
+print(submit_job_response.status_code)
+print(submit_job_response.json())

aind_data_transfer_service-1.13.0/docs/examples/behavior_videos_compression.py

@@ -0,0 +1,123 @@
+"""
+This example demonstrates how to perform behavior video compression when a
+predefined job_type is not available. However, it might be better to request a
+job_type to be stored.
+"""
+
+from datetime import datetime
+
+import requests
+from aind_data_schema_models.modalities import Modality
+from aind_data_schema_models.platforms import Platform
+
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+    Task,
+    UploadJobConfigsV2,
+)
+
+# As default, behavior videos are not compressed during upload.
+# This example demonstrates how to override the default behavior.
+job_type = "default"
+
+acq_datetime = datetime(2024, 5, 9, 12, 31, 22)
+
+behavior_task = Task(
+    job_settings={
+        "input_source": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "behavior_711256_2024-05-09_12-31-22/behavior"
+        )
+    }
+)
+
+# Using a predefined job_type would simplify this model greatly.
+# Alternatively, this model can be validated against the JobSettings
+# pydantic class in the aind-behavior-video-transformation package.
+behavior_video_job_settings = {
+    "input_source": (
+        "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+        "behavior_711256_2024-05-09_12-31-22/behavior-videos"
+    ),
+    "output_directory": "%OUTPUT_LOCATION",  # This will be replaced by svc
+    "compression_requested": {"compression_enum": "gamma fix colorspace"},
+}
+behavior_videos_task = {
+    "skip_task": False,
+    "image": "ghcr.io/allenneuraldynamics/aind-behavior-video-transformation",
+    "image_version": "0.1.7",
+    "image_resources": {
+        "memory_per_cpu": {"set": True, "number": 250},
+        "minimum_cpus_per_node": 32,
+        "partition": "aind",
+        "qos": "dev",
+        "standard_error": (
+            "/allen/aind/scratch/svc_aind_airflow/dev/logs/%x_%j_error.out"
+        ),
+        "tasks": 1,
+        "time_limit": {"set": True, "number": 720},
+        "standard_output": (
+            "/allen/aind/scratch/svc_aind_airflow/dev/logs/%x_%j.out"
+        ),
+        "environment": [
+            "PATH=/bin:/usr/bin/:/usr/local/bin/",
+            "LD_LIBRARY_PATH=/lib/:/lib64/:/usr/local/lib",
+        ],
+        "current_working_directory": ".",
+    },
+    "job_settings": behavior_video_job_settings,
+    "command_script": (
+        "#!/bin/bash \nsingularity exec --cleanenv "
+        "docker://%IMAGE:%IMAGE_VERSION python -m "
+        "aind_behavior_video_transformation.etl "
+        "--job-settings ' %JOB_SETTINGS '"
+    ),
+}
+
+modality_transformation_settings = {
+    "behavior": behavior_task,
+    "behavior-videos": behavior_videos_task,
+}
+
+gather_preliminary_metadata = Task(
+    job_settings={
+        "metadata_dir": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "behavior_711256_2024-05-09_12-31-22/metadata-dir"
+        )
+    }
+)
+
+
+upload_job_configs_v2 = UploadJobConfigsV2(
+    job_type=job_type,
+    project_name="Behavior Platform",
+    platform=Platform.BEHAVIOR,
+    modalities=[Modality.BEHAVIOR, Modality.BEHAVIOR_VIDEOS],
+    subject_id="711256",
+    acq_datetime=acq_datetime,
+    tasks={
+        "modality_transformation_settings": modality_transformation_settings,
+        "gather_preliminary_metadata": gather_preliminary_metadata,
+    },
+)
+
+submit_request_v2 = SubmitJobRequestV2(
+    upload_jobs=[upload_job_configs_v2],
+)
+
+post_request_content = submit_request_v2.model_dump(
+    mode="json", exclude_none=True
+)
+
+# Please use the production endpoint for submitting jobs and the dev endpoint
+# for running tests.
+# endpoint = "http://aind-data-transfer-service"
+endpoint = "http://aind-data-transfer-service-dev"  # For testing
+
+submit_job_response = requests.post(
+    url=f"{endpoint}/api/v2/submit_jobs",
+    json=post_request_content,
+)
+print(submit_job_response.status_code)
+print(submit_job_response.json())

aind_data_transfer_service-1.13.0/docs/examples/custom_codeocean_pipeline_settings.py

@@ -0,0 +1,107 @@
+"""
+This example demonstrates how to set custom configs for the code ocean
+pipeline monitor capsule. It is recommended to specify a job_type to set the
+code ocean settings.
+"""
+
+from datetime import datetime
+
+import requests
+from aind_data_schema_models.modalities import Modality
+from aind_data_schema_models.platforms import Platform
+
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+    Task,
+    UploadJobConfigsV2,
+)
+
+# The job_type contains the default settings for compression and Code Ocean
+# pipelines.
+job_type = "default"
+
+acq_datetime = datetime(2023, 4, 3, 18, 17, 7)
+
+# As default, compression won't be run.
+ecephys_task = Task(
+    job_settings={
+        "input_source": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "ecephys/655019_2023-04-03_18-17-07"
+        )
+    }
+)
+
+modality_transformation_settings = {"ecephys": ecephys_task}
+
+gather_preliminary_metadata = Task(
+    job_settings={
+        "metadata_dir": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "ecephys/655019_2023-04-03_18-17-07"
+        )
+    }
+)
+
+# For extra validation on the pipeline_monitor_settings field, you can import
+# the pydantic model by pip installing aind-codeocean-pipeline-monitor
+# and importing aind_codeocean_pipeline_monitor.models.PipelineMonitorSettings.
+# We provide a default pipeline monitor capsule id, but you can
+# override it by uncommenting the pipeline_monitor_capsule_id key.
+ecephys_codeocean_pipeline_settings = Task(
+    skip_task=False,
+    job_settings={
+        # "pipeline_monitor_capsule_id": ...,
+        "pipeline_monitor_settings": {
+            "run_params": {
+                "capsule_id": "87cbe6ce-9b38-4266-8d4a-62f0e23ba2d6",
+            },
+            "capture_settings": {
+                "permissions": {"everyone": "viewer"},
+                "tags": ["derived", "test"],
+                "custom_metadata": {
+                    "data level": "derived",
+                },
+                "process_name_suffix": "test",
+            },
+        },
+    },
+)
+
+# You can specify one pipeline per modality:
+codeocean_pipeline_settings = {"ecephys": ecephys_codeocean_pipeline_settings}
+
+
+upload_job_configs_v2 = UploadJobConfigsV2(
+    job_type=job_type,
+    project_name="Ephys Platform",
+    platform=Platform.ECEPHYS,
+    modalities=[Modality.ECEPHYS],
+    subject_id="655019",
+    acq_datetime=acq_datetime,
+    tasks={
+        "modality_transformation_settings": modality_transformation_settings,
+        "gather_preliminary_metadata": gather_preliminary_metadata,
+        "codeocean_pipeline_settings": codeocean_pipeline_settings,
+    },
+)
+
+submit_request_v2 = SubmitJobRequestV2(
+    upload_jobs=[upload_job_configs_v2],
+)
+
+post_request_content = submit_request_v2.model_dump(
+    mode="json", exclude_none=True
+)
+
+# Please use the production endpoint for submitting jobs and the dev endpoint
+# for running tests.
+# endpoint = "http://aind-data-transfer-service"
+endpoint = "http://aind-data-transfer-service-dev"  # For testing
+
+submit_job_response = requests.post(
+    url=f"{endpoint}/api/v2/submit_jobs",
+    json=post_request_content,
+)
+print(submit_job_response.status_code)
+print(submit_job_response.json())

aind_data_transfer_service-1.13.0/docs/examples/custom_metadata_mapper_settings.py

@@ -0,0 +1,113 @@
+"""
+This example demonstrates how to pass in custom job_settings into the metadata
+mapper package. It assumes a user has pip installed the aind-metadata-mapper
+package. If there are dependency conflicts it may also be possible to use raw
+dictionaries.
+"""
+
+from datetime import datetime
+
+import requests
+from aind_data_schema_models.modalities import Modality
+from aind_data_schema_models.platforms import Platform
+from aind_metadata_mapper.bergamo.models import (
+    JobSettings as BergamoSessionSettings,
+)
+from aind_metadata_mapper.models import (
+    JobSettings as GatherMetadataJobSettings,
+)
+from aind_metadata_mapper.models import SessionSettings
+
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+    Task,
+    UploadJobConfigsV2,
+)
+
+# This job_type contains the default settings for compression and Code Ocean
+# pipelines. We recommend creating a custom one for your experiments. Reach
+# out to Scientific Computing for more information.
+job_type = "default"
+
+# acq_datetime = datetime(2025, 4, 25, 16, 41, 23)
+acq_datetime = datetime(2020, 4, 25, 16, 41, 0)
+
+# Compression settings. As job_type default, no compression will be performed.
+pophys_task = Task(
+    job_settings={
+        "input_source": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/bci/"
+            "042525/042525/pophys"
+        )
+    }
+)
+
+modality_transformation_settings = {"pophys": pophys_task}
+
+# 1. Define the JobSettings for the GatherMetadataJob in aind-metadata-mapper
+bergamo_session_settings = BergamoSessionSettings(
+    input_source=(
+        "/allen/aind/scratch/svc_aind_upload/test_data_sets/bci/"
+        "042525/042525/pophys"
+    ),
+    experimenter_full_name=["John Apple"],
+    subject_id="784746",
+    imaging_laser_wavelength=920,
+    fov_imaging_depth=100,
+    fov_targeted_structure="Primary Motor Cortex",
+    notes="test upload",
+)
+
+# 2. Define SessionSettings object with defined job settings
+session_settings = SessionSettings(job_settings=bergamo_session_settings)
+
+# 3. Define GatherMetadataJobSettings with session_settings.
+# %OUTPUT_LOCATION is a special string that will be replaced by the service
+# with the correct staging location.
+metadata_job_settings = GatherMetadataJobSettings(
+    directory_to_write_to="%OUTPUT_LOCATION",
+    session_settings=session_settings,
+    metadata_dir=(
+        "/allen/aind/scratch/svc_aind_upload/test_data_sets/bci/042525/042525"
+    ),
+)
+
+gather_preliminary_metadata = Task(
+    job_settings=metadata_job_settings.model_dump(
+        mode="json", exclude_none=True
+    )
+)
+
+
+upload_job_configs_v2 = UploadJobConfigsV2(
+    job_type=job_type,
+    project_name="Brain Computer Interface",
+    platform=Platform.SINGLE_PLANE_OPHYS,
+    modalities=[Modality.POPHYS],
+    subject_id="784746",
+    acq_datetime=acq_datetime,
+    tasks={
+        "modality_transformation_settings": modality_transformation_settings,
+        "gather_preliminary_metadata": gather_preliminary_metadata,
+    },
+)
+
+submit_request_v2 = SubmitJobRequestV2(
+    upload_jobs=[upload_job_configs_v2],
+)
+
+post_request_content = submit_request_v2.model_dump(
+    mode="json", exclude_none=True
+)
+
+# Please use the production endpoint for submitting jobs and the dev endpoint
+# for running tests.
+# endpoint = "http://aind-data-transfer-service"
+endpoint = "http://aind-data-transfer-service-dev"  # For testing
+
+submit_job_response = requests.post(
+    url=f"{endpoint}/api/v2/submit_jobs",
+    json=post_request_content,
+)
+print(submit_job_response.status_code)
+print(submit_job_response.json())

aind_data_transfer_service-1.13.0/docs/examples/example1.csv

@@ -0,0 +1,4 @@
+project_name, modality0.capsule_id, modality0, modality0.input_source, modality1, modality1.input_source, s3_bucket, subject_id, platform, acq_datetime, job_type
+Ephys Platform, , ecephys, dir/data_set_1, ,, default, 123454, ecephys, 2020-10-10 14:10:10, ecephys
+Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, behavior, 10/13/2020 1:10:10 PM,
+Behavior Platform, , behavior-videos, dir/data_set_2, behavior, dir/data_set_3, private, 123456, behavior, 10/13/2020 1:10:10 PM,

aind_data_transfer_service-1.13.0/docs/examples/hcr_example.py

@@ -0,0 +1,81 @@
+"""This example demonstrates how to submit an HCR job."""
+
+from datetime import datetime
+
+import requests
+from aind_data_schema_models.modalities import Modality
+from aind_data_schema_models.platforms import Platform
+
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+    Task,
+    UploadJobConfigsV2,
+)
+
+job_type = "HCR"
+
+acq_datetime = datetime(2025, 3, 10, 12, 0, 9)
+
+num_of_partitions: int = 4
+
+# Add custom job_settings for the hcr image python command.
+spim_task = Task(
+    image_resources={"array": f"0-{num_of_partitions - 1}"},
+    job_settings={
+        "input_source": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "HCR_772646_2025-03-10_12-00-00"
+        ),
+        "num_of_partitions": num_of_partitions,
+    },
+)
+
+modality_settings = {
+    Modality.SPIM.abbreviation: spim_task,
+}
+
+gather_preliminary_metadata = Task(
+    job_settings={
+        "metadata_dir": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "HCR_772646_2025-03-10_12-00-00"
+        )
+    }
+)
+
+# The job_type loads defaults settings from AWS Parameter Store
+upload_job_configs_v2 = UploadJobConfigsV2(
+    job_type=job_type,
+    s3_bucket="open",
+    project_name="MSMA Platform",
+    platform=Platform.HCR,
+    modalities=[Modality.SPIM],
+    subject_id="772646",
+    acq_datetime=acq_datetime,
+    tasks={
+        "modality_transformation_settings": modality_settings,
+        "gather_preliminary_metadata": gather_preliminary_metadata,
+    },
+)
+
+upload_jobs = [upload_job_configs_v2]
+
+submit_request = SubmitJobRequestV2(
+    upload_jobs=upload_jobs,
+)
+
+post_request_content = submit_request.model_dump(
+    mode="json", exclude_none=True
+)
+
+# Please use the production endpoint for submitting jobs and the dev endpoint
+# for running tests.
+# endpoint = "http://aind-data-transfer-service"
+endpoint = "http://aind-data-transfer-service-dev"  # For testing
+
+submit_job_response = requests.post(
+    url=f"{endpoint}/api/v2/submit_jobs",
+    json=post_request_content,
+)
+print(submit_job_response.status_code)
+print(submit_job_response.json())