aind-data-transfer-service 1.15.0__tar.gz → 1.17.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of aind-data-transfer-service might be problematic. Click here for more details.

Files changed (108) hide show
  1. {aind_data_transfer_service-1.15.0/src/aind_data_transfer_service.egg-info → aind_data_transfer_service-1.17.0}/PKG-INFO +4 -4
  2. aind_data_transfer_service-1.17.0/docs/examples/behavior_videos_compression.csv +2 -0
  3. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/remove_source_folders.py +1 -1
  4. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/Contributing.rst +1 -0
  5. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/UserGuideV1.rst +1 -0
  6. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/UserGuideV2.rst +1 -0
  7. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/pyproject.toml +3 -3
  8. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/__init__.py +1 -1
  9. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/configs/csv_handler.py +57 -6
  10. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/models/internal.py +35 -10
  11. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/server.py +135 -16
  12. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/templates/admin.html +11 -2
  13. aind_data_transfer_service-1.17.0/src/aind_data_transfer_service/templates/job_params.html +405 -0
  14. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0/src/aind_data_transfer_service.egg-info}/PKG-INFO +4 -4
  15. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service.egg-info/SOURCES.txt +3 -0
  16. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service.egg-info/requires.txt +3 -3
  17. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/describe_parameters_response.json +1 -1
  18. aind_data_transfer_service-1.17.0/tests/resources/nested_sample.csv +2 -0
  19. aind_data_transfer_service-1.17.0/tests/resources/put_parameter_response.json +17 -0
  20. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_csv_handler.py +63 -5
  21. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_server.py +203 -5
  22. aind_data_transfer_service-1.15.0/src/aind_data_transfer_service/templates/job_params.html +0 -195
  23. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.flake8 +0 -0
  24. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  25. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  26. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/ISSUE_TEMPLATE/user-story.md +0 -0
  27. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/workflows/add_issue_to_project_board.yml +0 -0
  28. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/workflows/publish_dev.yml +0 -0
  29. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/workflows/publish_main.yml +0 -0
  30. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/workflows/run_dev_tests.yml +0 -0
  31. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.github/workflows/run_main_tests.yml +0 -0
  32. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.gitignore +0 -0
  33. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/.readthedocs.yaml +0 -0
  34. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/CODE_OF_CONDUCT.md +0 -0
  35. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/Dockerfile +0 -0
  36. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/LICENSE +0 -0
  37. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/README.md +0 -0
  38. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/Makefile +0 -0
  39. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/diagrams/system_container.png +0 -0
  40. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/diagrams/system_container.puml +0 -0
  41. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/diagrams/system_context.png +0 -0
  42. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/diagrams/system_context.puml +0 -0
  43. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/basic_upload.py +0 -0
  44. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/behavior_videos_compression.py +0 -0
  45. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/custom_codeocean_pipeline_settings.py +0 -0
  46. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/custom_metadata_mapper_settings.py +0 -0
  47. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/example1.csv +0 -0
  48. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/hcr_example.py +0 -0
  49. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/skip_s3_check.py +0 -0
  50. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/upload_with_custom_slurm_settings.py +0 -0
  51. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/examples/upload_with_notification.py +0 -0
  52. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/make.bat +0 -0
  53. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/_static/dark-logo.svg +0 -0
  54. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/_static/favicon.ico +0 -0
  55. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/_static/light-logo.svg +0 -0
  56. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/aind_data_transfer_service.configs.rst +0 -0
  57. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/aind_data_transfer_service.hpc.rst +0 -0
  58. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/aind_data_transfer_service.models.rst +0 -0
  59. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/aind_data_transfer_service.rst +0 -0
  60. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/conf.py +0 -0
  61. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/index.rst +0 -0
  62. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/docs/source/modules.rst +0 -0
  63. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/setup.cfg +0 -0
  64. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/setup.py +0 -0
  65. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/configs/__init__.py +0 -0
  66. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/configs/job_configs.py +0 -0
  67. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/configs/job_upload_template.py +0 -0
  68. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/hpc/__init__.py +0 -0
  69. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/hpc/client.py +0 -0
  70. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/hpc/models.py +0 -0
  71. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/log_handler.py +0 -0
  72. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/models/__init__.py +0 -0
  73. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/models/core.py +0 -0
  74. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/templates/index.html +0 -0
  75. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/templates/job_status.html +0 -0
  76. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/templates/job_tasks_table.html +0 -0
  77. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service/templates/task_logs.html +0 -0
  78. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service.egg-info/dependency_links.txt +0 -0
  79. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/src/aind_data_transfer_service.egg-info/top_level.txt +0 -0
  80. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/__init__.py +0 -0
  81. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/airflow_dag_run_response.json +0 -0
  82. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/airflow_dag_runs_response.json +0 -0
  83. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/airflow_task_instances_response.json +0 -0
  84. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/get_parameter_response.json +0 -0
  85. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/get_secrets_response.json +0 -0
  86. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/job_upload_template.xlsx +0 -0
  87. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/legacy_sample.csv +0 -0
  88. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/legacy_sample2.csv +0 -0
  89. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/new_sample.csv +0 -0
  90. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample.csv +0 -0
  91. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample.xlsx +0 -0
  92. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_alt_modality_case.csv +0 -0
  93. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_empty_rows.csv +0 -0
  94. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_empty_rows.xlsx +0 -0
  95. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_empty_rows_2.csv +0 -0
  96. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_invalid_ext.txt +0 -0
  97. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_malformed.csv +0 -0
  98. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_malformed.xlsx +0 -0
  99. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/resources/sample_malformed_2.csv +0 -0
  100. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_configs.py +0 -0
  101. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_core.py +0 -0
  102. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_hpc_client.py +0 -0
  103. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_hpc_models.py +0 -0
  104. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_internal.py +0 -0
  105. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_job_upload_template.py +0 -0
  106. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_log_handler.py +0 -0
  107. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_server/Dockerfile +0 -0
  108. {aind_data_transfer_service-1.15.0 → aind_data_transfer_service-1.17.0}/tests/test_server/db.json +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: aind-data-transfer-service
3
- Version: 1.15.0
3
+ Version: 1.17.0
4
4
  Summary: Service that handles requests to upload data to the cloud
5
5
  Author: Allen Institute for Neural Dynamics
6
6
  License: MIT
@@ -25,13 +25,13 @@ Requires-Dist: furo; extra == "docs"
25
25
  Provides-Extra: server
26
26
  Requires-Dist: aind-data-schema<2.0,>=1.0.0; extra == "server"
27
27
  Requires-Dist: aind-data-transfer-models==0.17.0; extra == "server"
28
- Requires-Dist: aind-metadata-mapper==0.23.0; extra == "server"
28
+ Requires-Dist: aind-metadata-mapper>=0.23.0; extra == "server"
29
29
  Requires-Dist: boto3; extra == "server"
30
30
  Requires-Dist: boto3-stubs[ssm]; extra == "server"
31
- Requires-Dist: fastapi; extra == "server"
31
+ Requires-Dist: fastapi>=0.115.13; extra == "server"
32
32
  Requires-Dist: httpx; extra == "server"
33
33
  Requires-Dist: jinja2; extra == "server"
34
- Requires-Dist: starlette; extra == "server"
34
+ Requires-Dist: starlette<0.47.0,>=0.40.0; extra == "server"
35
35
  Requires-Dist: starlette_wtf; extra == "server"
36
36
  Requires-Dist: uvicorn[standard]; extra == "server"
37
37
  Requires-Dist: wtforms; extra == "server"
@@ -0,0 +1,2 @@
1
+ project_name, modality0, modality0.input_source, modality0.compression_requested.compression_enum, subject-id, platform, acq-datetime, job_type
2
+ Ephys Platform, behavior-videos, dir/data_set_1, gamma fix colorspace, 123454, ecephys, 2020-10-10 14:10:10, ecephys
@@ -55,7 +55,7 @@ upload_job_configs_v2 = UploadJobConfigsV2(
55
55
  tasks={
56
56
  "modality_transformation_settings": modality_transformation_settings,
57
57
  "gather_preliminary_metadata": gather_preliminary_metadata,
58
- "remove_source_folders": remove_source_folders
58
+ "remove_source_folders": remove_source_folders,
59
59
  },
60
60
  )
61
61
 
@@ -62,6 +62,7 @@ To run uvicorn locally:
62
62
 
63
63
  .. code:: bash
64
64
 
65
+ # export MSYS_NO_PATHCONV=1 # Uncomment this line if running on Windows
65
66
  export AIND_METADATA_SERVICE_PROJECT_NAMES_URL='http://aind-metadata-service-dev/project_names'
66
67
  export AIND_AIRFLOW_SERVICE_URL='http://aind-airflow-service-dev:8080/api/v1/dags/run_list_of_jobs/dagRuns'
67
68
  # export AIND_AIRFLOW_SERVICE_JOBS_URL='http://aind-airflow-service-dev:8080/api/v1/dags'
@@ -520,6 +520,7 @@ Available job types and their configurations can be viewed at:
520
520
  http://aind-data-transfer-service/job_params
521
521
 
522
522
  To request a new job type, please reach out to Scientific Computing.
523
+ Admins can manage job types directly in the Job Parameters page.
523
524
 
524
525
  Reporting bugs or making feature requests
525
526
  -----------------------------------------
@@ -186,6 +186,7 @@ Available job types and their configurations can be viewed at:
186
186
  http://aind-data-transfer-service/job_params
187
187
 
188
188
  To request a new job type, please reach out to Scientific Computing.
189
+ Admins can manage job types directly in the Job Parameters page.
189
190
 
190
191
  Reporting bugs or making feature requests
191
192
  -----------------------------------------
@@ -41,13 +41,13 @@ docs = [
41
41
  server = [
42
42
  'aind-data-schema>=1.0.0,<2.0',
43
43
  'aind-data-transfer-models==0.17.0',
44
- 'aind-metadata-mapper==0.23.0',
44
+ 'aind-metadata-mapper>=0.23.0',
45
45
  'boto3',
46
46
  'boto3-stubs[ssm]',
47
- 'fastapi',
47
+ 'fastapi>=0.115.13',
48
48
  'httpx',
49
49
  'jinja2',
50
- 'starlette',
50
+ 'starlette>=0.40.0,<0.47.0',
51
51
  'starlette_wtf',
52
52
  'uvicorn[standard]',
53
53
  'wtforms',
@@ -1,7 +1,7 @@
1
1
  """Init package"""
2
2
  import os
3
3
 
4
- __version__ = "1.15.0"
4
+ __version__ = "1.17.0"
5
5
 
6
6
  # Global constants
7
7
  OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")
@@ -1,7 +1,10 @@
1
1
  """Module to handle processing legacy csv files"""
2
2
 
3
3
  import re
4
+ from collections.abc import Mapping
5
+ from copy import deepcopy
4
6
  from datetime import datetime
7
+ from typing import Any, Dict
5
8
 
6
9
  from aind_data_schema_models.modalities import Modality
7
10
  from aind_data_schema_models.platforms import Platform
@@ -13,6 +16,45 @@ DATETIME_PATTERN2 = re.compile(
13
16
  )
14
17
 
15
18
 
19
+ def nested_update(dict_to_update: Dict[str, Any], updates: Mapping):
20
+ """
21
+ Update a nested dictionary in-place.
22
+ Parameters
23
+ ----------
24
+ dict_to_update : Dict[str, Any]
25
+ updates : Mapping
26
+
27
+ """
28
+ for k, v in updates.items():
29
+ if isinstance(v, Mapping):
30
+ dict_to_update[k] = nested_update(dict_to_update.get(k, {}), v)
31
+ else:
32
+ dict_to_update[k] = v
33
+ return dict_to_update
34
+
35
+
36
+ def create_nested_dict(
37
+ dict_to_update: Dict[str, Any], key_string: str, value: Any
38
+ ):
39
+ """
40
+ Updates in-place a nested dictionary with a period delimited key and value.
41
+ Parameters
42
+ ----------
43
+ dict_to_update : Dict[str, Any]
44
+ key_string : str
45
+ value : Any
46
+
47
+ """
48
+ keys = key_string.split(".", 1)
49
+ current_key = keys[0]
50
+ if len(keys) == 1:
51
+ dict_to_update[current_key] = value
52
+ else:
53
+ if current_key not in dict_to_update:
54
+ dict_to_update[current_key] = dict()
55
+ create_nested_dict(dict_to_update[current_key], keys[1], value)
56
+
57
+
16
58
  def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
17
59
  """
18
60
  Maps csv row into a UploadJobConfigsV2 model. This attempts to be somewhat
@@ -29,7 +71,6 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
29
71
  modality_configs = dict()
30
72
  job_configs = dict()
31
73
  check_s3_folder_exists_task = None
32
- final_check_s3_folder_exist = None
33
74
  codeocean_tasks = dict()
34
75
  for key, value in row.items():
35
76
  # Strip white spaces and replace dashes with underscores
@@ -42,7 +83,9 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
42
83
  modality_parts = clean_key.split(".")
43
84
  modality_key = modality_parts[0]
44
85
  sub_key = (
45
- "modality" if len(modality_parts) == 1 else modality_parts[1]
86
+ "modality"
87
+ if len(modality_parts) == 1
88
+ else ".".join(modality_parts[1:])
46
89
  )
47
90
  modality_configs.setdefault(modality_key, dict())
48
91
  # Temp backwards compatibility check
@@ -66,13 +109,22 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
66
109
  job_settings=codeocean_pipeline_monitor_settings,
67
110
  )
68
111
  else:
69
- modality_configs[modality_key].update({sub_key: clean_val})
112
+ nested_val = dict()
113
+ create_nested_dict(
114
+ dict_to_update=nested_val,
115
+ key_string=sub_key,
116
+ value=clean_val,
117
+ )
118
+ current_dict = deepcopy(
119
+ modality_configs.get(modality_key, dict())
120
+ )
121
+ nested_update(current_dict, nested_val)
122
+ modality_configs[modality_key] = current_dict
70
123
  elif clean_key == "force_cloud_sync" and clean_val.upper() in [
71
124
  "TRUE",
72
125
  "T",
73
126
  ]:
74
127
  check_s3_folder_exists_task = {"skip_task": True}
75
- final_check_s3_folder_exist = {"skip_task": True}
76
128
  else:
77
129
  job_configs[clean_key] = clean_val
78
130
  # Rename codeocean config keys with correct modality
@@ -93,8 +145,7 @@ def map_csv_row_to_job(row: dict) -> UploadJobConfigsV2:
93
145
  )
94
146
  tasks = {
95
147
  "gather_preliminary_metadata": metadata_task,
96
- "check_s3_folder_exists_task": check_s3_folder_exists_task,
97
- "final_check_s3_folder_exist": final_check_s3_folder_exist,
148
+ "check_s3_folder_exists": check_s3_folder_exists_task,
98
149
  "modality_transformation_settings": modality_tasks,
99
150
  "codeocean_pipeline_settings": None
100
151
  if codeocean_tasks == dict()
@@ -3,8 +3,9 @@
3
3
  import ast
4
4
  import os
5
5
  from datetime import datetime, timedelta, timezone
6
- from typing import List, Optional, Union
6
+ from typing import ClassVar, List, Optional, Union
7
7
 
8
+ from aind_data_schema_models.modalities import Modality
8
9
  from mypy_boto3_ssm.type_defs import ParameterMetadataTypeDef
9
10
  from pydantic import AwareDatetime, BaseModel, Field, field_validator
10
11
  from starlette.datastructures import QueryParams
@@ -223,11 +224,30 @@ class JobTasks(BaseModel):
223
224
  class JobParamInfo(BaseModel):
224
225
  """Model for job parameter info from AWS Parameter Store"""
225
226
 
227
+ _MODALITIES_LIST: ClassVar[list[str]] = list(
228
+ Modality.abbreviation_map.keys()
229
+ )
230
+ _MODALITY_TASKS: ClassVar[list[str]] = [
231
+ "modality_transformation_settings",
232
+ "codeocean_pipeline_settings",
233
+ ]
234
+
226
235
  name: Optional[str]
227
236
  last_modified: Optional[datetime]
228
- job_type: str
229
- task_id: str
237
+ job_type: str = Field(..., pattern=r"^[^\s/]+$")
238
+ task_id: str = Field(..., pattern=r"^[^\s/]+$")
230
239
  modality: Optional[str]
240
+ version: Optional[str] = Field(..., pattern=r"^(v1|v2)?$")
241
+
242
+ @field_validator("modality", mode="after")
243
+ def validate_modality(cls, v):
244
+ """Check that modality is one of aind-data-schema modalities"""
245
+ if v is not None and v not in JobParamInfo._MODALITIES_LIST:
246
+ raise ValueError(
247
+ "Invalid modality: modality must be one of "
248
+ f"{JobParamInfo._MODALITIES_LIST}"
249
+ )
250
+ return v
231
251
 
232
252
  @classmethod
233
253
  def from_aws_describe_parameter(
@@ -236,6 +256,7 @@ class JobParamInfo(BaseModel):
236
256
  job_type: str,
237
257
  task_id: str,
238
258
  modality: Optional[str],
259
+ version: Optional[str],
239
260
  ):
240
261
  """Map the parameter to the model"""
241
262
  return cls(
@@ -244,13 +265,14 @@ class JobParamInfo(BaseModel):
244
265
  job_type=job_type,
245
266
  task_id=task_id,
246
267
  modality=modality,
268
+ version=version,
247
269
  )
248
270
 
249
271
  @staticmethod
250
272
  def get_parameter_prefix(version: Optional[str] = None) -> str:
251
273
  """Get the prefix for job_type parameters"""
252
274
  prefix = os.getenv("AIND_AIRFLOW_PARAM_PREFIX")
253
- if version is None:
275
+ if version is None or version == "v1":
254
276
  return prefix
255
277
  return f"{prefix}/{version}"
256
278
 
@@ -262,16 +284,19 @@ class JobParamInfo(BaseModel):
262
284
  "(?P<job_type>[^/]+)/tasks/(?P<task_id>[^/]+)"
263
285
  "(?:/(?P<modality>[^/]+))?"
264
286
  )
265
- if version is None:
287
+ if version is None or version == "v1":
266
288
  return f"{prefix}/{regex}"
267
289
  return f"{prefix}/{version}/{regex}"
268
290
 
269
291
  @staticmethod
270
292
  def get_parameter_name(
271
- job_type: str, task_id: str, version: Optional[str] = None
293
+ job_type: str,
294
+ task_id: str,
295
+ modality: Optional[str],
296
+ version: Optional[str] = None,
272
297
  ) -> str:
273
298
  """Create the parameter name from job_type and task_id"""
274
- prefix = os.getenv("AIND_AIRFLOW_PARAM_PREFIX")
275
- if version is None:
276
- return f"{prefix}/{job_type}/tasks/{task_id}"
277
- return f"{prefix}/{version}/{job_type}/tasks/{task_id}"
299
+ prefix = JobParamInfo.get_parameter_prefix(version)
300
+ if modality:
301
+ return f"{prefix}/{job_type}/tasks/{task_id}/{modality}"
302
+ return f"{prefix}/{job_type}/tasks/{task_id}"
@@ -7,7 +7,7 @@ import os
7
7
  import re
8
8
  from asyncio import gather, sleep
9
9
  from pathlib import PurePosixPath
10
- from typing import List, Optional, Union
10
+ from typing import Any, List, Optional, Union
11
11
 
12
12
  import boto3
13
13
  import requests
@@ -29,7 +29,9 @@ from starlette.middleware.sessions import SessionMiddleware
29
29
  from starlette.responses import RedirectResponse
30
30
  from starlette.routing import Route
31
31
 
32
- from aind_data_transfer_service import OPEN_DATA_BUCKET_NAME
32
+ from aind_data_transfer_service import (
33
+ OPEN_DATA_BUCKET_NAME,
34
+ )
33
35
  from aind_data_transfer_service import (
34
36
  __version__ as aind_data_transfer_service_version,
35
37
  )
@@ -37,14 +39,18 @@ from aind_data_transfer_service.configs.csv_handler import map_csv_row_to_job
37
39
  from aind_data_transfer_service.configs.job_configs import (
38
40
  BasicUploadJobConfigs as LegacyBasicUploadJobConfigs,
39
41
  )
40
- from aind_data_transfer_service.configs.job_configs import HpcJobConfigs
42
+ from aind_data_transfer_service.configs.job_configs import (
43
+ HpcJobConfigs,
44
+ )
41
45
  from aind_data_transfer_service.configs.job_upload_template import (
42
46
  JobUploadTemplate,
43
47
  )
44
48
  from aind_data_transfer_service.hpc.client import HpcClient, HpcClientConfigs
45
49
  from aind_data_transfer_service.hpc.models import HpcJobSubmitSettings
46
50
  from aind_data_transfer_service.log_handler import LoggingConfigs, get_logger
47
- from aind_data_transfer_service.models.core import SubmitJobRequestV2
51
+ from aind_data_transfer_service.models.core import (
52
+ SubmitJobRequestV2,
53
+ )
48
54
  from aind_data_transfer_service.models.core import (
49
55
  validation_context as validation_context_v2,
50
56
  )
@@ -150,6 +156,7 @@ def get_parameter_infos(version: Optional[str] = None) -> List[JobParamInfo]:
150
156
  job_type=match.group("job_type"),
151
157
  task_id=match.group("task_id"),
152
158
  modality=match.group("modality"),
159
+ version=version,
153
160
  )
154
161
  params.append(param_info)
155
162
  else:
@@ -167,6 +174,19 @@ def get_parameter_value(param_name: str) -> dict:
167
174
  return param_value
168
175
 
169
176
 
177
+ def put_parameter_value(param_name: str, param_value: dict) -> Any:
178
+ """Set a parameter value in AWS param store based on parameter name"""
179
+ param_value_str = json.dumps(param_value)
180
+ ssm_client = boto3.client("ssm")
181
+ result = ssm_client.put_parameter(
182
+ Name=param_name,
183
+ Value=param_value_str,
184
+ Type="String",
185
+ Overwrite=True,
186
+ )
187
+ return result
188
+
189
+
170
190
  async def get_airflow_jobs(
171
191
  params: AirflowDagRunsRequestParameters, get_confs: bool = False
172
192
  ) -> tuple[int, Union[List[JobStatus], List[dict]]]:
@@ -929,10 +949,10 @@ async def get_task_logs(request: Request):
929
949
  async def index(request: Request):
930
950
  """GET|POST /: form handler"""
931
951
  return templates.TemplateResponse(
952
+ request=request,
932
953
  name="index.html",
933
954
  context=(
934
955
  {
935
- "request": request,
936
956
  "project_names_url": project_names_url,
937
957
  }
938
958
  ),
@@ -945,10 +965,10 @@ async def job_tasks_table(request: Request):
945
965
  response_tasks_json = json.loads(response_tasks.body)
946
966
  data = response_tasks_json.get("data")
947
967
  return templates.TemplateResponse(
968
+ request=request,
948
969
  name="job_tasks_table.html",
949
970
  context=(
950
971
  {
951
- "request": request,
952
972
  "status_code": response_tasks.status_code,
953
973
  "message": response_tasks_json.get("message"),
954
974
  "errors": data.get("errors", []),
@@ -965,10 +985,10 @@ async def task_logs(request: Request):
965
985
  response_tasks_json = json.loads(response_tasks.body)
966
986
  data = response_tasks_json.get("data")
967
987
  return templates.TemplateResponse(
988
+ request=request,
968
989
  name="task_logs.html",
969
990
  context=(
970
991
  {
971
- "request": request,
972
992
  "status_code": response_tasks.status_code,
973
993
  "message": response_tasks_json.get("message"),
974
994
  "errors": data.get("errors", []),
@@ -982,10 +1002,10 @@ async def jobs(request: Request):
982
1002
  """Get Job Status page with pagination"""
983
1003
  dag_ids = AirflowDagRunsRequestParameters.model_fields["dag_ids"].default
984
1004
  return templates.TemplateResponse(
1005
+ request=request,
985
1006
  name="job_status.html",
986
1007
  context=(
987
1008
  {
988
- "request": request,
989
1009
  "project_names_url": project_names_url,
990
1010
  "dag_ids": dag_ids,
991
1011
  }
@@ -995,16 +1015,20 @@ async def jobs(request: Request):
995
1015
 
996
1016
  async def job_params(request: Request):
997
1017
  """Get Job Parameters page"""
1018
+ user = request.session.get("user")
998
1019
  return templates.TemplateResponse(
1020
+ request=request,
999
1021
  name="job_params.html",
1000
1022
  context=(
1001
1023
  {
1002
- "request": request,
1024
+ "user_signed_in": user is not None,
1003
1025
  "project_names_url": os.getenv(
1004
1026
  "AIND_METADATA_SERVICE_PROJECT_NAMES_URL"
1005
1027
  ),
1006
1028
  "versions": ["v1", "v2"],
1007
1029
  "default_version": "v1",
1030
+ "modalities": JobParamInfo._MODALITIES_LIST,
1031
+ "modality_tasks": JobParamInfo._MODALITY_TASKS,
1008
1032
  }
1009
1033
  ),
1010
1034
  )
@@ -1068,7 +1092,10 @@ def get_parameter_v2(request: Request):
1068
1092
  # path params are auto validated
1069
1093
  job_type = request.path_params.get("job_type")
1070
1094
  task_id = request.path_params.get("task_id")
1071
- param_name = JobParamInfo.get_parameter_name(job_type, task_id, "v2")
1095
+ modality = request.path_params.get("modality")
1096
+ param_name = JobParamInfo.get_parameter_name(
1097
+ job_type=job_type, task_id=task_id, modality=modality, version="v2"
1098
+ )
1072
1099
  try:
1073
1100
  param_value = get_parameter_value(param_name)
1074
1101
  return JSONResponse(
@@ -1089,12 +1116,79 @@ def get_parameter_v2(request: Request):
1089
1116
  )
1090
1117
 
1091
1118
 
1119
+ async def put_parameter(request: Request):
1120
+ """Set v1/v2 parameter in AWS param store based on job_type and task_id"""
1121
+ # User must be signed in
1122
+ user = request.session.get("user")
1123
+ if not user:
1124
+ return JSONResponse(
1125
+ content={
1126
+ "message": "User not authenticated",
1127
+ "data": {"error": "User not authenticated"},
1128
+ },
1129
+ status_code=401,
1130
+ )
1131
+ try:
1132
+ # path params
1133
+ param_info = JobParamInfo(
1134
+ name=None,
1135
+ last_modified=None,
1136
+ job_type=request.path_params.get("job_type"),
1137
+ task_id=request.path_params.get("task_id"),
1138
+ modality=request.path_params.get("modality"),
1139
+ version=request.path_params.get("version"),
1140
+ )
1141
+ param_name = JobParamInfo.get_parameter_name(
1142
+ job_type=param_info.job_type,
1143
+ task_id=param_info.task_id,
1144
+ modality=param_info.modality,
1145
+ version=param_info.version,
1146
+ )
1147
+ # update param store
1148
+ logger.info(
1149
+ f"Received request from {user} to set parameter {param_name}"
1150
+ )
1151
+ param_value = await request.json()
1152
+ logger.info(f"Setting parameter {param_name} to {param_value}")
1153
+ result = put_parameter_value(
1154
+ param_name=param_name, param_value=param_value
1155
+ )
1156
+ logger.info(result)
1157
+ return JSONResponse(
1158
+ content={
1159
+ "message": f"Set parameter for {param_name}",
1160
+ "data": param_value,
1161
+ },
1162
+ status_code=200,
1163
+ )
1164
+ except ValidationError as error:
1165
+ return JSONResponse(
1166
+ content={
1167
+ "message": "Invalid parameter",
1168
+ "data": {"errors": json.loads(error.json())},
1169
+ },
1170
+ status_code=400,
1171
+ )
1172
+ except Exception as e:
1173
+ logger.exception(f"Error setting parameter {param_name}: {e}")
1174
+ return JSONResponse(
1175
+ content={
1176
+ "message": f"Error setting parameter {param_name}",
1177
+ "data": {"error": f"{e.__class__.__name__}{e.args}"},
1178
+ },
1179
+ status_code=500,
1180
+ )
1181
+
1182
+
1092
1183
  def get_parameter(request: Request):
1093
1184
  """Get parameter from AWS parameter store based on job_type and task_id"""
1094
1185
  # path params are auto validated
1095
1186
  job_type = request.path_params.get("job_type")
1096
1187
  task_id = request.path_params.get("task_id")
1097
- param_name = JobParamInfo.get_parameter_name(job_type, task_id)
1188
+ modality = request.path_params.get("modality")
1189
+ param_name = JobParamInfo.get_parameter_name(
1190
+ job_type=job_type, task_id=task_id, modality=modality
1191
+ )
1098
1192
  try:
1099
1193
  param_value = get_parameter_value(param_name)
1100
1194
  return JSONResponse(
@@ -1118,14 +1212,12 @@ def get_parameter(request: Request):
1118
1212
  async def admin(request: Request):
1119
1213
  """Get admin page if authenticated, else redirect to login."""
1120
1214
  user = request.session.get("user")
1121
- if os.getenv("ENV_NAME") == "local":
1122
- user = {"name": "local user"}
1123
1215
  if user:
1124
1216
  return templates.TemplateResponse(
1217
+ request=request,
1125
1218
  name="admin.html",
1126
1219
  context=(
1127
1220
  {
1128
- "request": request,
1129
1221
  "project_names_url": project_names_url,
1130
1222
  "user_name": user.get("name", "unknown"),
1131
1223
  "user_email": user.get("email", "unknown"),
@@ -1137,6 +1229,9 @@ async def admin(request: Request):
1137
1229
 
1138
1230
  async def login(request: Request):
1139
1231
  """Redirect to Azure login page"""
1232
+ if os.getenv("ENV_NAME") == "local":
1233
+ request.session["user"] = {"name": "local user"}
1234
+ return RedirectResponse(url="/admin")
1140
1235
  oauth = set_oauth()
1141
1236
  redirect_uri = request.url_for("auth")
1142
1237
  response = await oauth.azure.authorize_redirect(request, redirect_uri)
@@ -1187,7 +1282,13 @@ routes = [
1187
1282
  Route("/api/v1/get_task_logs", endpoint=get_task_logs, methods=["GET"]),
1188
1283
  Route("/api/v1/parameters", endpoint=list_parameters, methods=["GET"]),
1189
1284
  Route(
1190
- "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:path}",
1285
+ "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:str}",
1286
+ endpoint=get_parameter,
1287
+ methods=["GET"],
1288
+ ),
1289
+ Route(
1290
+ "/api/v1/parameters/job_types/{job_type:str}/tasks/{task_id:str}"
1291
+ "/{modality:str}",
1191
1292
  endpoint=get_parameter,
1192
1293
  methods=["GET"],
1193
1294
  ),
@@ -1198,10 +1299,28 @@ routes = [
1198
1299
  Route("/api/v2/submit_jobs", endpoint=submit_jobs_v2, methods=["POST"]),
1199
1300
  Route("/api/v2/parameters", endpoint=list_parameters_v2, methods=["GET"]),
1200
1301
  Route(
1201
- "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:path}",
1302
+ "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:str}",
1303
+ endpoint=get_parameter_v2,
1304
+ methods=["GET"],
1305
+ ),
1306
+ Route(
1307
+ "/api/v2/parameters/job_types/{job_type:str}/tasks/{task_id:str}"
1308
+ "/{modality:str}",
1202
1309
  endpoint=get_parameter_v2,
1203
1310
  methods=["GET"],
1204
1311
  ),
1312
+ Route(
1313
+ "/api/{version:str}/parameters/job_types/{job_type:str}"
1314
+ "/tasks/{task_id:str}",
1315
+ endpoint=put_parameter,
1316
+ methods=["PUT"],
1317
+ ),
1318
+ Route(
1319
+ "/api/{version:str}/parameters/job_types/{job_type:str}"
1320
+ "/tasks/{task_id:str}/{modality:str}",
1321
+ endpoint=put_parameter,
1322
+ methods=["PUT"],
1323
+ ),
1205
1324
  Route("/jobs", endpoint=jobs, methods=["GET"]),
1206
1325
  Route("/job_tasks_table", endpoint=job_tasks_table, methods=["GET"]),
1207
1326
  Route("/task_logs", endpoint=task_logs, methods=["GET"]),
@@ -24,13 +24,22 @@
24
24
  <a title="List of project names" href="{{ project_names_url }}" target="_blank">Project Names</a> |
25
25
  <a title="For more information click here" href="https://aind-data-transfer-service.readthedocs.io"
26
26
  target="_blank">Help</a> |
27
- <a href="/admin">Admin</a> |
28
- <a href="/logout">Log out</a>
27
+ <a href="/admin">Admin</a>
28
+ <a href="/logout" class="float-end">Log out</a>
29
29
  </nav>
30
30
  <div>
31
31
  <h3>Admin</h3>
32
32
  <div>Hello {{user_name}}, welcome to the admin page</div>
33
33
  <div>Email: {{user_email}}</div>
34
+ <h4 class="mt-4">Job Type Parameters</h4>
35
+ <div class="mb-2">
36
+ <div class="alert alert-info mb-0 p-4" role="alert">
37
+ To manage job type parameters, go to the <a href="/job_params" class="alert-link">Job Parameters</a> page.
38
+ Select an existing parameter, or click the
39
+ <button type="button" class="btn btn-success btn-sm" disabled><i class="bi bi-plus-circle"></i> Add New Parameter</button>
40
+ button.
41
+ </div>
42
+ </div>
34
43
  </div>
35
44
  </body>
36
45
  </html>