dkist-processing-common 14.7.0__tar.gz → 14.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. dkist_processing_common-14.8.0/.snyk +24 -0
  2. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/CHANGELOG.rst +17 -0
  3. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/PKG-INFO +1 -1
  4. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/_util/scratch.py +3 -1
  5. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/graphql.py +1 -0
  6. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/__init__.py +1 -0
  7. dkist_processing_common-14.8.0/dkist_processing_common/tasks/delete_input_frames.py +49 -0
  8. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/transfer_input_data.py +16 -3
  9. dkist_processing_common-14.8.0/dkist_processing_common/tests/test_delete_input_frames.py +97 -0
  10. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_transfer_input_data.py +9 -6
  11. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common.egg-info/PKG-INFO +1 -1
  12. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common.egg-info/SOURCES.txt +2 -0
  13. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common.egg-info/scm_file_list.json +2 -0
  14. dkist_processing_common-14.8.0/dkist_processing_common.egg-info/scm_version.json +8 -0
  15. dkist_processing_common-14.7.0/.snyk +0 -13
  16. dkist_processing_common-14.7.0/dkist_processing_common.egg-info/scm_version.json +0 -8
  17. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/.gitignore +0 -0
  18. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/.pre-commit-config.yaml +0 -0
  19. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/.readthedocs.yml +0 -0
  20. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/.rovodev/.review-agent.md +0 -0
  21. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/README.rst +0 -0
  22. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/bitbucket-pipelines.yml +0 -0
  23. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/changelog/.gitempty +0 -0
  24. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/__init__.py +0 -0
  25. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/_util/__init__.py +0 -0
  26. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/_util/constants.py +0 -0
  27. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/_util/graphql.py +0 -0
  28. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/_util/tags.py +0 -0
  29. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/__init__.py +0 -0
  30. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/array.py +0 -0
  31. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/asdf.py +0 -0
  32. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/basemodel.py +0 -0
  33. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/bytes.py +0 -0
  34. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/fits.py +0 -0
  35. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/iobase.py +0 -0
  36. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/json.py +0 -0
  37. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/path.py +0 -0
  38. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/quality.py +0 -0
  39. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/codecs/str.py +0 -0
  40. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/config.py +0 -0
  41. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/fonts/Lato-Regular.ttf +0 -0
  42. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/fonts/__init__.py +0 -0
  43. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/manual.py +0 -0
  44. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/__init__.py +0 -0
  45. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/constants.py +0 -0
  46. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/dkist_location.py +0 -0
  47. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/extras.py +0 -0
  48. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/fits_access.py +0 -0
  49. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/flower_pot.py +0 -0
  50. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/fried_parameter.py +0 -0
  51. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/input_dataset.py +0 -0
  52. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/message.py +0 -0
  53. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/message_queue_binding.py +0 -0
  54. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/metric_code.py +0 -0
  55. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/parameters.py +0 -0
  56. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/quality.py +0 -0
  57. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/tags.py +0 -0
  58. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/task_name.py +0 -0
  59. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/telemetry.py +0 -0
  60. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/models/wavelength.py +0 -0
  61. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/__init__.py +0 -0
  62. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/average_bud.py +0 -0
  63. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/cs_step.py +0 -0
  64. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/dsps_repeat.py +0 -0
  65. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/experiment_id_bud.py +0 -0
  66. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/id_bud.py +0 -0
  67. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/l0_fits_access.py +0 -0
  68. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/l1_fits_access.py +0 -0
  69. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/lookup_bud.py +0 -0
  70. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/near_bud.py +0 -0
  71. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/observing_program_id_bud.py +0 -0
  72. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/proposal_id_bud.py +0 -0
  73. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/quality.py +0 -0
  74. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/retarder.py +0 -0
  75. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/single_value_single_key_flower.py +0 -0
  76. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/task.py +0 -0
  77. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/time.py +0 -0
  78. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/unique_bud.py +0 -0
  79. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/parsers/wavelength.py +0 -0
  80. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/assemble_movie.py +0 -0
  81. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/base.py +0 -0
  82. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/l1_output_data.py +0 -0
  83. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/mixin/__init__.py +0 -0
  84. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/mixin/globus.py +0 -0
  85. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/mixin/interservice_bus.py +0 -0
  86. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/mixin/metadata_store.py +0 -0
  87. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/mixin/object_store.py +0 -0
  88. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/mixin/quality.py +0 -0
  89. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/output_data_base.py +0 -0
  90. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/parse_l0_input_data.py +0 -0
  91. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/quality_metrics.py +0 -0
  92. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/teardown.py +0 -0
  93. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/trial_catalog.py +0 -0
  94. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/trial_output_data.py +0 -0
  95. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/write_extra.py +0 -0
  96. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/write_l1.py +0 -0
  97. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tasks/write_l1_base.py +0 -0
  98. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/__init__.py +0 -0
  99. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/conftest.py +0 -0
  100. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/mock_metadata_store.py +0 -0
  101. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_assemble_movie.py +0 -0
  102. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_assemble_quality.py +0 -0
  103. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_base.py +0 -0
  104. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_codecs.py +0 -0
  105. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_constants.py +0 -0
  106. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_construct_dataset_extras.py +0 -0
  107. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_cs_step.py +0 -0
  108. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_dkist_location.py +0 -0
  109. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_fits_access.py +0 -0
  110. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_flower_pot.py +0 -0
  111. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_fried_parameter.py +0 -0
  112. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_input_dataset.py +0 -0
  113. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_interservice_bus.py +0 -0
  114. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_interservice_bus_mixin.py +0 -0
  115. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_manual_processing.py +0 -0
  116. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_output_data_base.py +0 -0
  117. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_parameters.py +0 -0
  118. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_parse_l0_input_data.py +0 -0
  119. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_publish_catalog_messages.py +0 -0
  120. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_quality.py +0 -0
  121. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_quality_mixin.py +0 -0
  122. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_scratch.py +0 -0
  123. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_stems.py +0 -0
  124. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_submit_dataset_metadata.py +0 -0
  125. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_tags.py +0 -0
  126. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_task_name.py +0 -0
  127. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_task_parsing.py +0 -0
  128. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_teardown.py +0 -0
  129. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_transfer_l1_output_data.py +0 -0
  130. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_trial_catalog.py +0 -0
  131. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_trial_output_data.py +0 -0
  132. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_workflow_task_base.py +0 -0
  133. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common/tests/test_write_l1.py +0 -0
  134. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common.egg-info/dependency_links.txt +0 -0
  135. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common.egg-info/requires.txt +0 -0
  136. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/dkist_processing_common.egg-info/top_level.txt +0 -0
  137. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/docs/Makefile +0 -0
  138. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/docs/changelog.rst +0 -0
  139. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/docs/conf.py +0 -0
  140. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/docs/index.rst +0 -0
  141. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/docs/landing_page.rst +0 -0
  142. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/docs/make.bat +0 -0
  143. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/docs/requirements.txt +0 -0
  144. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/licenses/LICENSE.rst +0 -0
  145. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/pyproject.toml +0 -0
  146. {dkist_processing_common-14.7.0 → dkist_processing_common-14.8.0}/setup.cfg +0 -0
@@ -0,0 +1,24 @@
1
+ # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities
2
+ # https://docs.snyk.io/manage-risk/policies/the-.snyk-file#syntax-of-the-.snyk-file
3
+ version: v1.25.0
4
+ ignore:
5
+ SNYK-PYTHON-PILLOW-21277783:
6
+ - '*':
7
+ reason: 'Pillow upgrade blocked by version incompatibility with moviepy'
8
+ expires: '2027-10-20T00:00:00.000Z'
9
+ SNYK-PYTHON-PILLOW-21277801:
10
+ - '*':
11
+ reason: 'Pillow upgrade blocked by version incompatibility with moviepy'
12
+ expires: '2027-10-20T00:00:00.000Z'
13
+ SNYK-PYTHON-PILLOW-21277802:
14
+ - '*':
15
+ reason: 'Pillow upgrade blocked by version incompatibility with moviepy'
16
+ expires: '2027-10-20T00:00:00.000Z'
17
+ SNYK-PYTHON-PILLOW-21279300:
18
+ - '*':
19
+ reason: 'Pillow upgrade blocked by version incompatibility with moviepy'
20
+ expires: '2027-10-20T00:00:00.000Z'
21
+ SNYK-PYTHON-PILLOW-21431493:
22
+ - '*':
23
+ reason: 'Pillow upgrade blocked by version incompatibility with moviepy'
24
+ expires: '2027-10-20T00:00:00.000Z'
@@ -1,3 +1,20 @@
1
+ v14.8.0 (2026-10-22)
2
+ ====================
3
+
4
+ Features
5
+ --------
6
+
7
+ - Add the `DeleteInputFrames` task to remove scratch input frames after they are no longer needed, enabled by the recipe-run configuration flag `delete_input_frames` while preserving file-tag references and leaving recipe-run status unchanged. (`#357 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/357>`__)
8
+
9
+
10
+ Misc
11
+ ----
12
+
13
+ - Transfer input frames into `input_frames/<object_key>` to provide a consistent location for cleanup using DeleteInputFrames. (`#357 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/357>`__)
14
+ - Ignore four Pillow vulnerabilities while moviepy version incompatibility blocks upgrades.
15
+ Remove ignores for resolved Apache Airflow vulnerabilities.
16
+
17
+
1
18
  v14.7.0 (2026-10-19)
2
19
  ====================
3
20
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dkist-processing-common
3
- Version: 14.7.0
3
+ Version: 14.8.0
4
4
  Summary: Common task classes used by the DKIST science data processing pipelines
5
5
  Author-email: NSO / AURA <dkistdc@nso.edu>
6
6
  License: BSD-3-Clause
@@ -12,6 +12,8 @@ from typing import Iterable
12
12
  from dkist_processing_common._util.tags import TagDB
13
13
  from dkist_processing_common.config import common_configurations
14
14
 
15
+ INPUT_FRAMES_DIRECTORY = "input_frames"
16
+
15
17
  logger = logging.getLogger(__name__)
16
18
 
17
19
 
@@ -131,7 +133,7 @@ class WorkflowFileSystem:
131
133
  f.write(file_obj)
132
134
  self.tag(path, tags)
133
135
 
134
- def delete(self, path: Path | str):
136
+ def delete(self, path: Path | str) -> None:
135
137
  """
136
138
  Delete the file or path.
137
139
 
@@ -112,6 +112,7 @@ class RecipeRunConfiguration(GraphqlBaseModel):
112
112
  trial_directory_name: str | None = None
113
113
  trial_root_directory_name: str | None = None
114
114
  teardown_enabled: bool = True
115
+ delete_input_frames: bool = False
115
116
  trial_exclusive_transfer_tag_lists: list[list[str]] | None = None
116
117
 
117
118
 
@@ -2,6 +2,7 @@
2
2
 
3
3
  from dkist_processing_common.tasks.base import * # isort: skip
4
4
  from dkist_processing_common.tasks.assemble_movie import *
5
+ from dkist_processing_common.tasks.delete_input_frames import *
5
6
  from dkist_processing_common.tasks.l1_output_data import *
6
7
  from dkist_processing_common.tasks.parse_l0_input_data import *
7
8
  from dkist_processing_common.tasks.quality_metrics import *
@@ -0,0 +1,49 @@
1
+ """Delete scratch input frames after they are no longer needed."""
2
+
3
+ import logging
4
+ from shutil import rmtree
5
+
6
+ from dkist_processing_common._util.scratch import INPUT_FRAMES_DIRECTORY
7
+ from dkist_processing_common.tasks.base import WorkflowTaskBase
8
+
9
+ __all__ = ["DeleteInputFrames"]
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ class DeleteInputFrames(WorkflowTaskBase):
15
+ """Delete scratch input frames while preserving downstream processing data.
16
+
17
+ Instrument workflows must make this task depend on all consumers of the
18
+ input frames, including quality metrics, and make final teardown wait for
19
+ cleanup. This task does not infer readiness from product tags or change the
20
+ recipe run status.
21
+
22
+ The `input_frames` directory created by TransferL0Data is removed. All
23
+ file-tag references are preserved. Cleanup can be retried if interrupted;
24
+ an already-removed input directory is treated as complete.
25
+ """
26
+
27
+ def run(self) -> None:
28
+ """Remove scratch input frame files when explicitly enabled, preserving tags."""
29
+ configuration = self.metadata_store_recipe_run.configuration
30
+ if not configuration.delete_input_frames:
31
+ logger.info("Keeping scratch input frames because delete_input_frames is disabled")
32
+ return
33
+ input_frames_path = self.scratch.absolute_path(INPUT_FRAMES_DIRECTORY)
34
+ logger.info("Deleting scratch input frame directory")
35
+ with self.telemetry_span("Delete input frame files"):
36
+ try:
37
+ rmtree(input_frames_path)
38
+ except FileNotFoundError:
39
+ logger.info("Scratch input frame directory is already absent")
40
+ return
41
+ logger.info("Deleted scratch input frame directory")
42
+
43
+ def rollback(self) -> None:
44
+ """Roll back task bookkeeping; deleted input files cannot be restored."""
45
+ super().rollback()
46
+ logger.warning(
47
+ "Deleted input frames were not restored. Restore them from "
48
+ "the source before rerunning tasks that require raw inputs."
49
+ )
@@ -3,6 +3,7 @@
3
3
  import logging
4
4
  from pathlib import Path
5
5
 
6
+ from dkist_processing_common._util.scratch import INPUT_FRAMES_DIRECTORY
6
7
  from dkist_processing_common.codecs.basemodel import basemodel_decoder
7
8
  from dkist_processing_common.codecs.basemodel import basemodel_encoder
8
9
  from dkist_processing_common.models.input_dataset import InputDatasetObject
@@ -18,7 +19,10 @@ logger = logging.getLogger(__name__)
18
19
 
19
20
 
20
21
  class TransferL0Data(WorkflowTaskBase, GlobusMixin):
21
- """Transfers Level 0 data and required parameter files to the scratch store."""
22
+ """Transfer Level 0 frames and required parameter files to scratch.
23
+
24
+ Frames are stored under `input_frames/<object_key>`. Parameter files are stored at the top level.
25
+ """
22
26
 
23
27
  def download_input_dataset(self):
24
28
  """Write the input dataset part documents to scratch with appropriate tags."""
@@ -48,7 +52,7 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin):
48
52
  transfer_items = []
49
53
  for obj in input_dataset_objects:
50
54
  source_path = Path("/", obj.bucket, obj.object_key)
51
- destination_path = self.scratch.absolute_path(obj.object_key)
55
+ destination_path = self._input_dataset_object_path(obj)
52
56
  transfer_items.append(
53
57
  GlobusTransferItem(
54
58
  source_path=source_path,
@@ -58,6 +62,15 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin):
58
62
  )
59
63
  return transfer_items
60
64
 
65
+ def _input_dataset_object_path(self, obj: InputDatasetObject) -> Path:
66
+ """Locate frames in the input directory and tagged parameter files outside it."""
67
+ relative_path = Path(obj.object_key)
68
+ if not obj.tag:
69
+ # This is a raw input frame
70
+ relative_path = Path(INPUT_FRAMES_DIRECTORY) / relative_path
71
+ # This is a parameter file
72
+ return self.scratch.absolute_path(relative_path)
73
+
61
74
  def build_transfer_list(self, doc_tag: str) -> list[InputDatasetObject]:
62
75
  """Format the list of frames as transfer items to be used by globus."""
63
76
  doc = next(
@@ -73,7 +86,7 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin):
73
86
  def tag_transfer_objects(self, input_dataset_objects: list[InputDatasetObject]) -> None:
74
87
  """Tag all the transferred input files."""
75
88
  for obj in input_dataset_objects:
76
- obj_path = self.scratch.absolute_path(obj.object_key)
89
+ obj_path = self._input_dataset_object_path(obj)
77
90
  if obj.tag:
78
91
  self.tag(obj_path, tags=obj.tag)
79
92
  else:
@@ -0,0 +1,97 @@
1
+ """Test deletion of input frames."""
2
+
3
+ import json
4
+ from collections.abc import Generator
5
+ from pathlib import Path
6
+
7
+ import pytest
8
+
9
+ from dkist_processing_common._util.scratch import INPUT_FRAMES_DIRECTORY
10
+ from dkist_processing_common._util.scratch import WorkflowFileSystem
11
+ from dkist_processing_common.models.tags import Tag
12
+ from dkist_processing_common.tasks import DeleteInputFrames
13
+ from dkist_processing_common.tests.mock_metadata_store import RecipeRunResponseMapping
14
+ from dkist_processing_common.tests.mock_metadata_store import fake_gql_client_factory
15
+ from dkist_processing_common.tests.mock_metadata_store import make_default_recipe_run_response
16
+
17
+
18
+ @pytest.fixture
19
+ def fake_gql_client_input_cleanup_configuration(
20
+ rr_config_dict: dict[str, bool] | None,
21
+ ) -> type[object]:
22
+ """Build a fake GraphQL client with the requested recipe-run configuration."""
23
+ response = make_default_recipe_run_response()
24
+ if rr_config_dict is None:
25
+ response.configuration = None
26
+ else:
27
+ configuration = response.configuration.model_dump(exclude_unset=True)
28
+ configuration.update(rr_config_dict)
29
+ response.configuration = json.dumps(configuration)
30
+ return fake_gql_client_factory(
31
+ response_mapping_override=RecipeRunResponseMapping(response=response)
32
+ )
33
+
34
+
35
+ @pytest.fixture
36
+ def input_cleanup_task(
37
+ tmp_path: Path,
38
+ recipe_run_id: int,
39
+ mocker,
40
+ fake_gql_client_input_cleanup_configuration: type[object],
41
+ ) -> Generator[DeleteInputFrames, None, None]:
42
+ """Create the task with temporary scratch and purge its data after each test."""
43
+ mocker.patch(
44
+ "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
45
+ new=fake_gql_client_input_cleanup_configuration,
46
+ )
47
+ with DeleteInputFrames(
48
+ recipe_run_id=recipe_run_id,
49
+ workflow_name="test_cleanup",
50
+ workflow_version="test",
51
+ ) as task:
52
+ task.scratch.close()
53
+ task.scratch = WorkflowFileSystem(
54
+ recipe_run_id=recipe_run_id,
55
+ task_name=task.task_name,
56
+ scratch_base_path=tmp_path,
57
+ )
58
+ yield task
59
+ task._purge()
60
+
61
+
62
+ @pytest.mark.parametrize(
63
+ "rr_config_dict, cleanup_expected",
64
+ [
65
+ pytest.param(None, False, id="configuration-unavailable"),
66
+ pytest.param({}, False, id="flag-omitted"),
67
+ pytest.param({"delete_input_frames": False}, False, id="flag-disabled"),
68
+ pytest.param({"delete_input_frames": True}, True, id="flag-enabled"),
69
+ ],
70
+ )
71
+ def test_selective_cleanup_and_retention(
72
+ input_cleanup_task: DeleteInputFrames,
73
+ cleanup_expected: bool,
74
+ ) -> None:
75
+ """
76
+ Given: Input frames in their dedicated directory and recipe-run configuration.
77
+ When: The cleanup task runs.
78
+ Then: Input files are deleted only when cleanup is enabled, and all tags are preserved.
79
+ """
80
+ task = input_cleanup_task
81
+ input_paths = [
82
+ task.write(
83
+ data=b"input",
84
+ tags=[Tag.input(), Tag.frame()],
85
+ relative_path=Path(INPUT_FRAMES_DIRECTORY) / f"frame-{index}.fits",
86
+ )
87
+ for index in range(2)
88
+ ]
89
+ original_tags = {path: set(task.tags(task.scratch.absolute_path(path))) for path in input_paths}
90
+
91
+ task()
92
+
93
+ assert task.scratch.absolute_path(INPUT_FRAMES_DIRECTORY).exists() == (not cleanup_expected)
94
+ for path in input_paths:
95
+ absolute_path = task.scratch.absolute_path(path)
96
+ assert absolute_path.exists() == (not cleanup_expected)
97
+ assert set(task.tags(absolute_path)) == original_tags[path]
@@ -4,6 +4,7 @@ from pathlib import Path
4
4
 
5
5
  import pytest
6
6
 
7
+ from dkist_processing_common._util.scratch import INPUT_FRAMES_DIRECTORY
7
8
  from dkist_processing_common._util.scratch import WorkflowFileSystem
8
9
  from dkist_processing_common.codecs.basemodel import basemodel_decoder
9
10
  from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
@@ -52,7 +53,7 @@ def create_input_frames(
52
53
  """
53
54
  for frame in input_frame_docs:
54
55
  for object_key in frame["object_keys"]:
55
- file_path = task.scratch.workflow_base_path / Path(object_key)
56
+ file_path = task.scratch.workflow_base_path / INPUT_FRAMES_DIRECTORY / object_key
56
57
  if not file_path.parent.exists():
57
58
  file_path.parent.mkdir(parents=True, exist_ok=True)
58
59
  file_path.write_text(data="")
@@ -237,11 +238,13 @@ def test_build_frame_transfer_list_formatted(request, task_name):
237
238
  for frame_set in expected_frames:
238
239
  for key in frame_set["object_keys"]:
239
240
  source_filenames.append(os.path.join("/", frame_set["bucket"], key))
240
- destination_filenames.append(Path(key).name)
241
+ destination_filenames.append(
242
+ task.scratch.workflow_base_path / INPUT_FRAMES_DIRECTORY / key
243
+ )
241
244
  assert len(formatted_transfer_items) == len(source_filenames)
242
245
  for item in formatted_transfer_items:
243
246
  assert item.source_path.as_posix() in source_filenames
244
- assert item.destination_path.name in destination_filenames
247
+ assert item.destination_path in destination_filenames
245
248
  assert not item.recursive
246
249
 
247
250
 
@@ -268,11 +271,11 @@ def test_build_parameter_file_transfer_items(transfer_l0_data_task):
268
271
  bucket = value_dict["__file__"]["bucket"]
269
272
  object_key = value_dict["__file__"]["objectKey"]
270
273
  source_filenames.append(os.path.join("/", bucket, object_key))
271
- destination_filenames.append(Path(object_key).name)
274
+ destination_filenames.append(task.scratch.workflow_base_path / object_key)
272
275
  assert len(formatted_transfer_items) == len(source_filenames)
273
276
  for transfer_item in formatted_transfer_items:
274
277
  assert transfer_item.source_path.as_posix() in source_filenames
275
- assert transfer_item.destination_path.name in destination_filenames
278
+ assert transfer_item.destination_path in destination_filenames
276
279
  assert str(transfer_item.destination_path).startswith(str(task.scratch.workflow_base_path))
277
280
  assert not transfer_item.recursive
278
281
 
@@ -301,7 +304,7 @@ def test_tag_transfer_items(transfer_l0_data_task):
301
304
  input_tags = [Tag.input(), Tag.frame()]
302
305
  input_frames_on_disk = list(task.scratch.find_all(tags=input_tags))
303
306
  for obj in frame_transfer_objects:
304
- destination_path = task.scratch.absolute_path(obj.object_key)
307
+ destination_path = task.scratch.absolute_path(Path(INPUT_FRAMES_DIRECTORY) / obj.object_key)
305
308
  assert destination_path in input_frames_on_disk
306
309
  assert len(input_frames_on_disk) == len(frame_transfer_objects)
307
310
  for obj in parameter_transfer_objects:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dkist-processing-common
3
- Version: 14.7.0
3
+ Version: 14.8.0
4
4
  Summary: Common task classes used by the DKIST science data processing pipelines
5
5
  Author-email: NSO / AURA <dkistdc@nso.edu>
6
6
  License: BSD-3-Clause
@@ -76,6 +76,7 @@ dkist_processing_common/parsers/wavelength.py
76
76
  dkist_processing_common/tasks/__init__.py
77
77
  dkist_processing_common/tasks/assemble_movie.py
78
78
  dkist_processing_common/tasks/base.py
79
+ dkist_processing_common/tasks/delete_input_frames.py
79
80
  dkist_processing_common/tasks/l1_output_data.py
80
81
  dkist_processing_common/tasks/output_data_base.py
81
82
  dkist_processing_common/tasks/parse_l0_input_data.py
@@ -103,6 +104,7 @@ dkist_processing_common/tests/test_codecs.py
103
104
  dkist_processing_common/tests/test_constants.py
104
105
  dkist_processing_common/tests/test_construct_dataset_extras.py
105
106
  dkist_processing_common/tests/test_cs_step.py
107
+ dkist_processing_common/tests/test_delete_input_frames.py
106
108
  dkist_processing_common/tests/test_dkist_location.py
107
109
  dkist_processing_common/tests/test_fits_access.py
108
110
  dkist_processing_common/tests/test_flower_pot.py
@@ -70,6 +70,7 @@
70
70
  "dkist_processing_common/tasks/__init__.py",
71
71
  "dkist_processing_common/tasks/assemble_movie.py",
72
72
  "dkist_processing_common/tasks/base.py",
73
+ "dkist_processing_common/tasks/delete_input_frames.py",
73
74
  "dkist_processing_common/tasks/l1_output_data.py",
74
75
  "dkist_processing_common/tasks/mixin/__init__.py",
75
76
  "dkist_processing_common/tasks/mixin/globus.py",
@@ -97,6 +98,7 @@
97
98
  "dkist_processing_common/tests/test_constants.py",
98
99
  "dkist_processing_common/tests/test_construct_dataset_extras.py",
99
100
  "dkist_processing_common/tests/test_cs_step.py",
101
+ "dkist_processing_common/tests/test_delete_input_frames.py",
100
102
  "dkist_processing_common/tests/test_dkist_location.py",
101
103
  "dkist_processing_common/tests/test_fits_access.py",
102
104
  "dkist_processing_common/tests/test_flower_pot.py",
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "14.8.0",
3
+ "distance": 0,
4
+ "node": "gda2fdd21ce7484d7a12275c6ebf1dda787033d76",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-10-22"
8
+ }
@@ -1,13 +0,0 @@
1
- # Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities
2
- # https://docs.snyk.io/manage-risk/policies/the-.snyk-file#syntax-of-the-.snyk-file
3
- version: v1.25.0
4
- ignore:
5
- SNYK-PYTHON-APACHEAIRFLOWCORE-17879182:
6
- - "*" :
7
- reason: 'Patched in Airflow 3.3.0, but we are not ready to upgrade till 3.3.1 or later'
8
- expires: '2027-07-07T00:00:00.000Z'
9
- SNYK-PYTHON-APACHEAIRFLOWCORE-17933331:
10
- - '*':
11
- reason: 'Ignored by bulk CI/CD Snyk policy update'
12
- expires: '2099-12-31T23:59:59.999Z'
13
- created: '2026-07-10T15:53:27.931Z'
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "14.7.0",
3
- "distance": 0,
4
- "node": "gc385c7fdd2f8402b4d9a2e307b217d529b3b1540",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-10-19"
8
- }