dkist-processing-cryonirsp 1.3.5__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dkist-processing-cryonirsp might be problematic. Click here for more details.
- dkist_processing_cryonirsp/tasks/__init__.py +0 -1
- dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py +2 -18
- dkist_processing_cryonirsp/workflows/trial_workflows.py +5 -5
- {dkist_processing_cryonirsp-1.3.5.dist-info → dkist_processing_cryonirsp-1.4.0.dist-info}/METADATA +2 -2
- {dkist_processing_cryonirsp-1.3.5.dist-info → dkist_processing_cryonirsp-1.4.0.dist-info}/RECORD +7 -9
- {dkist_processing_cryonirsp-1.3.5.dist-info → dkist_processing_cryonirsp-1.4.0.dist-info}/WHEEL +1 -1
- dkist_processing_cryonirsp/tasks/trial_output_data.py +0 -61
- dkist_processing_cryonirsp/tests/test_trial_output_data.py +0 -251
- {dkist_processing_cryonirsp-1.3.5.dist-info → dkist_processing_cryonirsp-1.4.0.dist-info}/top_level.txt +0 -0
|
@@ -15,5 +15,4 @@ from dkist_processing_cryonirsp.tasks.sp_dispersion_axis_correction import *
|
|
|
15
15
|
from dkist_processing_cryonirsp.tasks.sp_geometric import *
|
|
16
16
|
from dkist_processing_cryonirsp.tasks.sp_science import *
|
|
17
17
|
from dkist_processing_cryonirsp.tasks.sp_solar_gain import *
|
|
18
|
-
from dkist_processing_cryonirsp.tasks.trial_output_data import *
|
|
19
18
|
from dkist_processing_cryonirsp.tasks.write_l1 import *
|
|
@@ -9,13 +9,13 @@ import asdf
|
|
|
9
9
|
from astropy.io import fits
|
|
10
10
|
from dkist_header_validator import spec122_validator
|
|
11
11
|
from dkist_processing_common.models.task_name import TaskName
|
|
12
|
+
from dkist_processing_common.tasks import TransferTrialData
|
|
12
13
|
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
13
14
|
from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
|
|
14
15
|
from dkist_service_configuration.logging import logger
|
|
15
16
|
|
|
16
17
|
from dkist_processing_cryonirsp.models.tags import CryonirspTag
|
|
17
18
|
from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
|
|
18
|
-
from dkist_processing_cryonirsp.tasks import TransferCryoTrialData
|
|
19
19
|
from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
|
|
20
20
|
from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
|
|
21
21
|
|
|
@@ -550,27 +550,11 @@ def transfer_trial_data_locally_task(
|
|
|
550
550
|
output_swtich: bool = True,
|
|
551
551
|
tag_lists: list | None = None,
|
|
552
552
|
):
|
|
553
|
-
class LocalTrialData(
|
|
553
|
+
class LocalTrialData(TransferTrialData):
|
|
554
554
|
@property
|
|
555
555
|
def destination_folder(self) -> Path:
|
|
556
556
|
return Path(trial_dir)
|
|
557
557
|
|
|
558
|
-
@property
|
|
559
|
-
def debug_frame_switch(self) -> bool:
|
|
560
|
-
return debug_switch
|
|
561
|
-
|
|
562
|
-
@property
|
|
563
|
-
def intermediate_frame_switch(self) -> bool:
|
|
564
|
-
return intermediate_switch
|
|
565
|
-
|
|
566
|
-
@property
|
|
567
|
-
def output_frame_switch(self) -> bool:
|
|
568
|
-
return output_swtich
|
|
569
|
-
|
|
570
|
-
@property
|
|
571
|
-
def specific_frame_tag_lists(self) -> list:
|
|
572
|
-
return tag_lists or []
|
|
573
|
-
|
|
574
558
|
def remove_folder_objects(self):
|
|
575
559
|
logger.info("Would have removed folder objects here")
|
|
576
560
|
|
|
@@ -4,6 +4,7 @@ from dkist_processing_common.tasks import CreateTrialDatasetInventory
|
|
|
4
4
|
from dkist_processing_common.tasks import CreateTrialQualityReport
|
|
5
5
|
from dkist_processing_common.tasks import QualityL1Metrics
|
|
6
6
|
from dkist_processing_common.tasks import TransferL0Data
|
|
7
|
+
from dkist_processing_common.tasks import TransferTrialData
|
|
7
8
|
from dkist_processing_common.tasks import TrialTeardown
|
|
8
9
|
from dkist_processing_core import ResourceQueue
|
|
9
10
|
from dkist_processing_core import Workflow
|
|
@@ -33,7 +34,6 @@ from dkist_processing_cryonirsp.tasks import (
|
|
|
33
34
|
from dkist_processing_cryonirsp.tasks import SPScienceCalibration
|
|
34
35
|
from dkist_processing_cryonirsp.tasks import SPSolarGainCalibration
|
|
35
36
|
from dkist_processing_cryonirsp.tasks import SPWriteL1Frame
|
|
36
|
-
from dkist_processing_cryonirsp.tasks import TransferCryoTrialData
|
|
37
37
|
from dkist_processing_cryonirsp.tasks.l1_output_data import CIAssembleQualityData
|
|
38
38
|
from dkist_processing_cryonirsp.tasks.l1_output_data import SPAssembleQualityData
|
|
39
39
|
from dkist_processing_cryonirsp.tasks.parse import ParseL0CryonirspCILinearizedData
|
|
@@ -99,7 +99,7 @@ full_trial_ci_pipeline.add_node(
|
|
|
99
99
|
|
|
100
100
|
# Output flow
|
|
101
101
|
full_trial_ci_pipeline.add_node(
|
|
102
|
-
task=
|
|
102
|
+
task=TransferTrialData,
|
|
103
103
|
upstreams=[
|
|
104
104
|
CreateTrialDatasetInventory,
|
|
105
105
|
CreateTrialAsdf,
|
|
@@ -109,7 +109,7 @@ full_trial_ci_pipeline.add_node(
|
|
|
109
109
|
)
|
|
110
110
|
|
|
111
111
|
# goodbye
|
|
112
|
-
full_trial_ci_pipeline.add_node(task=TrialTeardown, upstreams=
|
|
112
|
+
full_trial_ci_pipeline.add_node(task=TrialTeardown, upstreams=TransferTrialData)
|
|
113
113
|
|
|
114
114
|
#######################
|
|
115
115
|
#######################
|
|
@@ -177,7 +177,7 @@ full_trial_sp_pipeline.add_node(
|
|
|
177
177
|
|
|
178
178
|
# Output flow
|
|
179
179
|
full_trial_sp_pipeline.add_node(
|
|
180
|
-
task=
|
|
180
|
+
task=TransferTrialData,
|
|
181
181
|
upstreams=[
|
|
182
182
|
CreateTrialDatasetInventory,
|
|
183
183
|
CreateTrialAsdf,
|
|
@@ -187,4 +187,4 @@ full_trial_sp_pipeline.add_node(
|
|
|
187
187
|
)
|
|
188
188
|
|
|
189
189
|
# goodbye
|
|
190
|
-
full_trial_sp_pipeline.add_node(task=TrialTeardown, upstreams=
|
|
190
|
+
full_trial_sp_pipeline.add_node(task=TrialTeardown, upstreams=TransferTrialData)
|
{dkist_processing_cryonirsp-1.3.5.dist-info → dkist_processing_cryonirsp-1.4.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dkist-processing-cryonirsp
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
|
|
5
5
|
Author-email: NSO / AURA <dkistdc@nso.edu>
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -17,7 +17,7 @@ Requires-Dist: Pillow ==10.3.0
|
|
|
17
17
|
Requires-Dist: astropy ==6.1.0
|
|
18
18
|
Requires-Dist: dkist-fits-specifications ==4.7.0
|
|
19
19
|
Requires-Dist: dkist-header-validator ==5.1.1
|
|
20
|
-
Requires-Dist: dkist-processing-common ==10.
|
|
20
|
+
Requires-Dist: dkist-processing-common ==10.4.0
|
|
21
21
|
Requires-Dist: dkist-processing-math ==2.1.1
|
|
22
22
|
Requires-Dist: dkist-processing-pac ==3.1.0
|
|
23
23
|
Requires-Dist: dkist-spectral-lines ==3.0.0
|
{dkist_processing_cryonirsp-1.3.5.dist-info → dkist_processing_cryonirsp-1.4.0.dist-info}/RECORD
RENAMED
|
@@ -19,7 +19,7 @@ dkist_processing_cryonirsp/parsers/polarimetric_check.py,sha256=HBzg9Zi9-M1M2jh4
|
|
|
19
19
|
dkist_processing_cryonirsp/parsers/scan_step.py,sha256=7XNUq48qbdpV8GeaBeG9XFRDkYlgw8Fjd6ByFZ24OAc,18972
|
|
20
20
|
dkist_processing_cryonirsp/parsers/time.py,sha256=dpajm3tfrih-8pjlOWOvZVJVP2FihR87HFQSjR_KPYI,2543
|
|
21
21
|
dkist_processing_cryonirsp/parsers/wavelength.py,sha256=Ay5hZiDNV25-N_QXZINTCgn1ToiP2jvwumkbirBiJGk,883
|
|
22
|
-
dkist_processing_cryonirsp/tasks/__init__.py,sha256=
|
|
22
|
+
dkist_processing_cryonirsp/tasks/__init__.py,sha256=JyM9oA1A6t5sPsbaNmnK9_e6t00kUYzDZzqmSg0nVjE,1061
|
|
23
23
|
dkist_processing_cryonirsp/tasks/assemble_movie.py,sha256=Brj3zdTKQiwgclIKoW2TSxuEHlkST8WjxvtBZONyGKM,7337
|
|
24
24
|
dkist_processing_cryonirsp/tasks/bad_pixel_map.py,sha256=QZzCngleitLDjq_2zRBuUorMMkmzMEo11Xwvu6dEudc,3713
|
|
25
25
|
dkist_processing_cryonirsp/tasks/beam_boundaries_base.py,sha256=N3zk08sRcczoNGIqRd8vzGtYLQUUiH52-WOc7fIw7t4,9960
|
|
@@ -40,7 +40,6 @@ dkist_processing_cryonirsp/tasks/sp_dispersion_axis_correction.py,sha256=y9TKJCt
|
|
|
40
40
|
dkist_processing_cryonirsp/tasks/sp_geometric.py,sha256=Ym5M1DJR_ZYGqjD1Cy5IQiult3GN5dp0hnJ9sLfMDQM,21860
|
|
41
41
|
dkist_processing_cryonirsp/tasks/sp_science.py,sha256=FubmQSg_ZrVy08TcT6dRK2nuA2cmGMrMYMmjT6_bYQM,12091
|
|
42
42
|
dkist_processing_cryonirsp/tasks/sp_solar_gain.py,sha256=WJKenRlqFlsP3zb3IPqRFdxCGcmJZMVlM9Me-myyvn0,18541
|
|
43
|
-
dkist_processing_cryonirsp/tasks/trial_output_data.py,sha256=wQP-zHxw_gJ3KKOF_9AZ_nTyRRdsaM4Ekv0JeXwxxJg,2481
|
|
44
43
|
dkist_processing_cryonirsp/tasks/write_l1.py,sha256=7xM4Uy66BCZzFhNPNHkiov6zIcfkZH45SadsJJr6QKc,41023
|
|
45
44
|
dkist_processing_cryonirsp/tasks/mixin/__init__.py,sha256=m6UekKftTahNJ3W5K3mZSz4Y4ZZpHRxF_ZAxuaKYL7o,12
|
|
46
45
|
dkist_processing_cryonirsp/tasks/mixin/beam_access.py,sha256=DiPRbsAyfhX-juHL0slypcXi5C8PJT-DNMX6ck1IH1w,1627
|
|
@@ -74,18 +73,17 @@ dkist_processing_cryonirsp/tests/test_sp_make_movie_frames.py,sha256=Kn8U_HzaaHW
|
|
|
74
73
|
dkist_processing_cryonirsp/tests/test_sp_science.py,sha256=bjUtZNlRiCaDK0ql0iMdZ6YF8f1dGAhz5HmOxWX1XBE,19416
|
|
75
74
|
dkist_processing_cryonirsp/tests/test_sp_solar.py,sha256=NHHE-PPa2bU0JK0fuezZakFvAi1f_2z7AHM9vxInF-U,8913
|
|
76
75
|
dkist_processing_cryonirsp/tests/test_trial_create_quality_report.py,sha256=x8dRAMyawQVYZQGsuO36DMZxK4L13hZBUn86qbhJiNk,2805
|
|
77
|
-
dkist_processing_cryonirsp/tests/test_trial_output_data.py,sha256=BTjBK4nHtOK_c5yQf9r4VbDqt1jyaWzdO0rxRccPeFc,9105
|
|
78
76
|
dkist_processing_cryonirsp/tests/test_workflows.py,sha256=2a-TUMbVEyNjHYNyzo2UQMLg5PSaEx6iV9hOsGp-Mck,291
|
|
79
77
|
dkist_processing_cryonirsp/tests/test_write_l1.py,sha256=TIizk5mViC6Km8Ee-CfHwRFTqJmSa3Ey_GfM09iGBRc,17597
|
|
80
78
|
dkist_processing_cryonirsp/tests/local_trial_workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
81
79
|
dkist_processing_cryonirsp/tests/local_trial_workflows/l0_cals_only.py,sha256=jJ9g2muTBfSHP-RgUInnMzsfqmMY9O0Qu_N9SVH0nq4,20420
|
|
82
80
|
dkist_processing_cryonirsp/tests/local_trial_workflows/l0_to_l1.py,sha256=f00YZl9ew1IX5p-a6hisXjOcA8WvM6cJ_h_jKXCFZb8,24061
|
|
83
81
|
dkist_processing_cryonirsp/tests/local_trial_workflows/linearize_only.py,sha256=kl9yrE1EbgxNn__x4q0WYoZIvS8zefcOPbQQdxB4_ZE,3280
|
|
84
|
-
dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py,sha256=
|
|
82
|
+
dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py,sha256=14XonDUbMWh6n1AwtPaDLCeSvoHCzn-MNHCkHLB4VUc,19330
|
|
85
83
|
dkist_processing_cryonirsp/workflows/__init__.py,sha256=iEbV3oiKmkfmdZjeNJxqw08IIZnYSroq-2b-ARpkLFQ,112
|
|
86
84
|
dkist_processing_cryonirsp/workflows/ci_l0_processing.py,sha256=6yw_mn8mxZ551ubJSHUyWd8l_Nwx1uL6yW1y_qBL5a8,3866
|
|
87
85
|
dkist_processing_cryonirsp/workflows/sp_l0_processing.py,sha256=A74av8E8mL9WzI4vWI2GbhVrRFQ7JCMlGAioEjXkkCU,4219
|
|
88
|
-
dkist_processing_cryonirsp/workflows/trial_workflows.py,sha256=
|
|
86
|
+
dkist_processing_cryonirsp/workflows/trial_workflows.py,sha256=x7odG41cz6a2Em0PQIDh3k1RaCl-pR0WC2-v6o2M9FE,8177
|
|
89
87
|
docs/Makefile,sha256=qnlVz6PuBqE39NfHWuUnHhNEA-EFgT2-WJNNNy9ttfk,4598
|
|
90
88
|
docs/bad_pixel_calibration.rst,sha256=bHRclUItBA1QbGlek36B1yN5WHN53WkfjKUvZaq_j2I,2452
|
|
91
89
|
docs/beam_angle_calculation.rst,sha256=-yUPFRl6SbPBIj-wOwylEymQDMQnkay7kAIAiubpQT8,2941
|
|
@@ -105,7 +103,7 @@ docs/requirements_table.rst,sha256=FaqSag9kPi77gWPhzeo_tFEhRFjb3qUuNqqQe1K76NM,2
|
|
|
105
103
|
docs/scientific_changelog.rst,sha256=01AWBSHg8zElnodCgAq-hMxhk9CkX5rtEENx4iz0sjI,300
|
|
106
104
|
docs/sp_science_calibration.rst,sha256=fHBOZ2cqySxLjNi737KfynlmHZy9W4EwvuzxnyjDNvk,2597
|
|
107
105
|
licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
|
|
108
|
-
dkist_processing_cryonirsp-1.
|
|
109
|
-
dkist_processing_cryonirsp-1.
|
|
110
|
-
dkist_processing_cryonirsp-1.
|
|
111
|
-
dkist_processing_cryonirsp-1.
|
|
106
|
+
dkist_processing_cryonirsp-1.4.0.dist-info/METADATA,sha256=8yUf3G9VeyZX03aPkcDrXZjIvqEv_GovulJjgloLqq0,8450
|
|
107
|
+
dkist_processing_cryonirsp-1.4.0.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
|
|
108
|
+
dkist_processing_cryonirsp-1.4.0.dist-info/top_level.txt,sha256=Sm9b1ddKnsF9Bh3mqDOct1Sm7k8I9aN7vGHgpmu-MlQ,51
|
|
109
|
+
dkist_processing_cryonirsp-1.4.0.dist-info/RECORD,,
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
"""Tasks for transferring scratch data to custom location for post-pipeline analysis."""
|
|
2
|
-
from dkist_processing_common.models.task_name import TaskName
|
|
3
|
-
from dkist_processing_common.tasks.mixin.globus import GlobusTransferItem
|
|
4
|
-
from dkist_processing_common.tasks.trial_output_data import TransferTrialDataBase
|
|
5
|
-
|
|
6
|
-
__all__ = ["TransferCryoTrialData"]
|
|
7
|
-
|
|
8
|
-
from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class TransferCryoTrialData(TransferTrialDataBase):
|
|
12
|
-
"""Transfer DEBUG, Intermediate, and/or output data to the trial location."""
|
|
13
|
-
|
|
14
|
-
@property
|
|
15
|
-
def intermediate_task_names(self) -> list[str]:
|
|
16
|
-
"""Grab all the Calibration products used to calibrate science data."""
|
|
17
|
-
return [
|
|
18
|
-
TaskName.dark.value,
|
|
19
|
-
TaskName.lamp_gain.value,
|
|
20
|
-
TaskName.geometric_angle.value,
|
|
21
|
-
TaskName.geometric_offsets.value,
|
|
22
|
-
TaskName.geometric_spectral_shifts.value,
|
|
23
|
-
TaskName.solar_gain.value,
|
|
24
|
-
TaskName.demodulation_matrices.value,
|
|
25
|
-
CryonirspTaskName.spectral_corrected_solar_array.value,
|
|
26
|
-
]
|
|
27
|
-
|
|
28
|
-
def build_transfer_list(self) -> list[GlobusTransferItem]:
|
|
29
|
-
"""
|
|
30
|
-
Build a list containing all files we want to transfer to the trial environment.
|
|
31
|
-
|
|
32
|
-
The classes of/specific files to transfer are defined in the switches that look at the recipe run configuration.
|
|
33
|
-
"""
|
|
34
|
-
transfer_list = []
|
|
35
|
-
|
|
36
|
-
if self.debug_frame_switch:
|
|
37
|
-
transfer_list += self.build_debug_frame_transfer_list()
|
|
38
|
-
|
|
39
|
-
if self.intermediate_frame_switch:
|
|
40
|
-
transfer_list += self.build_intermediate_frame_transfer_list()
|
|
41
|
-
|
|
42
|
-
if self.output_frame_switch:
|
|
43
|
-
transfer_list += self.build_output_frame_transfer_list()
|
|
44
|
-
transfer_list += self.build_output_movie_transfer_list()
|
|
45
|
-
|
|
46
|
-
if self.specific_frame_tag_lists:
|
|
47
|
-
transfer_list += self.build_transfer_list_from_tag_lists(self.specific_frame_tag_lists)
|
|
48
|
-
|
|
49
|
-
if self.output_dataset_inventory_switch:
|
|
50
|
-
transfer_list += self.build_output_dataset_inventory_transfer_list()
|
|
51
|
-
|
|
52
|
-
if self.output_asdf_switch:
|
|
53
|
-
transfer_list += self.build_output_asdf_transfer_list()
|
|
54
|
-
|
|
55
|
-
if self.output_quality_report_switch:
|
|
56
|
-
transfer_list += self.build_output_quality_report_transfer_list()
|
|
57
|
-
|
|
58
|
-
if self.output_quality_data_switch:
|
|
59
|
-
transfer_list += self.build_output_quality_data_transfer_list()
|
|
60
|
-
|
|
61
|
-
return transfer_list
|
|
@@ -1,251 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from uuid import uuid4
|
|
3
|
-
|
|
4
|
-
import pytest
|
|
5
|
-
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
6
|
-
from dkist_processing_common.codecs.json import json_encoder
|
|
7
|
-
from dkist_processing_common.codecs.quality import quality_data_encoder
|
|
8
|
-
from dkist_processing_common.codecs.str import str_encoder
|
|
9
|
-
from dkist_processing_common.models.task_name import TaskName
|
|
10
|
-
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
11
|
-
from pydantic import BaseModel
|
|
12
|
-
|
|
13
|
-
from dkist_processing_cryonirsp.models.tags import CryonirspTag
|
|
14
|
-
from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
|
|
15
|
-
from dkist_processing_cryonirsp.tasks import TransferCryoTrialData
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@pytest.fixture
|
|
19
|
-
def recipe_run_configuration(
|
|
20
|
-
debug_switch,
|
|
21
|
-
intermediate_switch,
|
|
22
|
-
output_switch,
|
|
23
|
-
dataset_inventory_switch,
|
|
24
|
-
asdf_switch,
|
|
25
|
-
quality_report_switch,
|
|
26
|
-
quality_data_switch,
|
|
27
|
-
tag_lists,
|
|
28
|
-
):
|
|
29
|
-
"""Mock Recipe Run endpoint for trial output configuration flags"""
|
|
30
|
-
|
|
31
|
-
class GQLClientWithConfiguration(FakeGQLClient):
|
|
32
|
-
def execute_gql_query(self, **kwargs):
|
|
33
|
-
response = super().execute_gql_query(**kwargs)
|
|
34
|
-
response[0].configuration = json.dumps(
|
|
35
|
-
{
|
|
36
|
-
"trial_transfer_debug_frames": bool(debug_switch),
|
|
37
|
-
"trial_transfer_intermediate_frames": bool(intermediate_switch),
|
|
38
|
-
"trial_transfer_output_frames": bool(output_switch),
|
|
39
|
-
"trial_transfer_output_dataset_inventory": bool(dataset_inventory_switch),
|
|
40
|
-
"trial_transfer_output_asdf": bool(asdf_switch),
|
|
41
|
-
"trial_transfer_output_quality_report": bool(quality_report_switch),
|
|
42
|
-
"trial_transfer_output_quality_data": bool(quality_data_switch),
|
|
43
|
-
"trial_transfer_tag_lists": tag_lists,
|
|
44
|
-
}
|
|
45
|
-
)
|
|
46
|
-
return response
|
|
47
|
-
|
|
48
|
-
return GQLClientWithConfiguration
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
intermediate_task_names = [
|
|
52
|
-
TaskName.dark.value,
|
|
53
|
-
TaskName.lamp_gain.value,
|
|
54
|
-
TaskName.geometric_angle.value,
|
|
55
|
-
TaskName.geometric_offsets.value,
|
|
56
|
-
TaskName.geometric_spectral_shifts.value,
|
|
57
|
-
TaskName.solar_gain.value,
|
|
58
|
-
TaskName.demodulation_matrices.value,
|
|
59
|
-
CryonirspTaskName.spectral_corrected_solar_array.value,
|
|
60
|
-
]
|
|
61
|
-
|
|
62
|
-
tag_lists = [[CryonirspTag.movie()], ["FOO", "BAR"]]
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def write_debug_frames_to_task(task: TransferCryoTrialData) -> int:
|
|
66
|
-
num_debug = 3
|
|
67
|
-
for _ in range(num_debug):
|
|
68
|
-
task.write(
|
|
69
|
-
data="123", encoder=str_encoder, tags=[CryonirspTag.frame(), CryonirspTag.debug()]
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
return num_debug
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def write_intermediate_frames_to_task(task: TransferCryoTrialData) -> int:
|
|
76
|
-
for task_name in intermediate_task_names:
|
|
77
|
-
task.write(
|
|
78
|
-
data=task_name,
|
|
79
|
-
encoder=str_encoder,
|
|
80
|
-
tags=[CryonirspTag.frame(), CryonirspTag.intermediate(), CryonirspTag.task(task_name)],
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
return len(intermediate_task_names)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def write_dummy_output_frames_to_task(task: TransferCryoTrialData) -> int:
|
|
87
|
-
num_output = 2
|
|
88
|
-
for i in range(num_output):
|
|
89
|
-
task.write(
|
|
90
|
-
data=f"output_{i}",
|
|
91
|
-
encoder=str_encoder,
|
|
92
|
-
tags=[CryonirspTag.frame(), CryonirspTag.output()],
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
return num_output
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def write_specific_tags_to_task(task: TransferCryoTrialData) -> int:
|
|
99
|
-
for tags in tag_lists:
|
|
100
|
-
task.write(data="foo", encoder=str_encoder, tags=tags)
|
|
101
|
-
|
|
102
|
-
return len(tag_lists)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def write_dataset_inventory_to_task(task: TransferCryoTrialData) -> int:
|
|
106
|
-
dataset_inventory_obj: dict = {f"dsi_key": uuid4().hex}
|
|
107
|
-
task.write(
|
|
108
|
-
data=dataset_inventory_obj,
|
|
109
|
-
encoder=json_encoder,
|
|
110
|
-
tags=[CryonirspTag.output(), CryonirspTag.dataset_inventory()],
|
|
111
|
-
)
|
|
112
|
-
return 1
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def write_asdf_to_task(task: TransferCryoTrialData) -> int:
|
|
116
|
-
asdf_obj = uuid4().hex
|
|
117
|
-
task.write(
|
|
118
|
-
data=asdf_obj, encoder=str_encoder, tags=[CryonirspTag.output(), CryonirspTag.asdf()]
|
|
119
|
-
)
|
|
120
|
-
return 1
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def write_quality_report_to_task(task: TransferCryoTrialData) -> int:
|
|
124
|
-
quality_report_obj = uuid4().hex.encode("utf-8")
|
|
125
|
-
task.write(data=quality_report_obj, tags=[CryonirspTag.output(), CryonirspTag.quality_report()])
|
|
126
|
-
return 1
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
def write_quality_data_to_task(task: TransferCryoTrialData) -> int:
|
|
130
|
-
quality_data_obj: list[dict] = [{f"quality_key": uuid4().hex}]
|
|
131
|
-
task.write(
|
|
132
|
-
data=quality_data_obj, encoder=quality_data_encoder, tags=CryonirspTag.quality_data()
|
|
133
|
-
)
|
|
134
|
-
return 1
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def write_unused_frames_to_task(task: TransferCryoTrialData) -> int:
|
|
138
|
-
task.write(data="bad", encoder=str_encoder, tags=["FOO"])
|
|
139
|
-
task.write(
|
|
140
|
-
data="intermediate we don't care about",
|
|
141
|
-
encoder=str_encoder,
|
|
142
|
-
tags=[CryonirspTag.frame(), CryonirspTag.intermediate(), CryonirspTag.task("NOT_A_KEEPER")],
|
|
143
|
-
)
|
|
144
|
-
return 1
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
class AvailableOutputFiles(BaseModel):
|
|
148
|
-
"""Number of files of each type available for potential output"""
|
|
149
|
-
|
|
150
|
-
num_debug: int
|
|
151
|
-
num_intermediate: int
|
|
152
|
-
num_output: int
|
|
153
|
-
num_specific: int
|
|
154
|
-
num_dataset_inventory: int
|
|
155
|
-
num_asdf: int
|
|
156
|
-
num_quality_report: int
|
|
157
|
-
num_quality_data: int
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
@pytest.fixture
|
|
161
|
-
def transfer_task_with_files(recipe_run_id, recipe_run_configuration, tmp_path, mocker):
|
|
162
|
-
mocker.patch(
|
|
163
|
-
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
|
|
164
|
-
new=recipe_run_configuration,
|
|
165
|
-
)
|
|
166
|
-
proposal_id = "test_proposal_id"
|
|
167
|
-
with TransferCryoTrialData(
|
|
168
|
-
recipe_run_id=recipe_run_id,
|
|
169
|
-
workflow_name="workflow_name",
|
|
170
|
-
workflow_version="workflow_version",
|
|
171
|
-
) as task:
|
|
172
|
-
task.scratch = WorkflowFileSystem(
|
|
173
|
-
recipe_run_id=recipe_run_id,
|
|
174
|
-
scratch_base_path=tmp_path,
|
|
175
|
-
)
|
|
176
|
-
task.constants._update({"PROPOSAL_ID": proposal_id})
|
|
177
|
-
try:
|
|
178
|
-
write_unused_frames_to_task(task)
|
|
179
|
-
available_output_files = AvailableOutputFiles(
|
|
180
|
-
num_debug=write_debug_frames_to_task(task),
|
|
181
|
-
num_intermediate=write_intermediate_frames_to_task(task),
|
|
182
|
-
num_output=write_dummy_output_frames_to_task(task),
|
|
183
|
-
num_specific=write_specific_tags_to_task(task),
|
|
184
|
-
num_dataset_inventory=write_dataset_inventory_to_task(task),
|
|
185
|
-
num_asdf=write_asdf_to_task(task),
|
|
186
|
-
num_quality_report=write_quality_report_to_task(task),
|
|
187
|
-
num_quality_data=write_quality_data_to_task(task),
|
|
188
|
-
)
|
|
189
|
-
yield task, available_output_files
|
|
190
|
-
|
|
191
|
-
finally:
|
|
192
|
-
task._purge()
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
@pytest.mark.parametrize(
|
|
196
|
-
"debug_switch, intermediate_switch, output_switch, dataset_inventory_switch, asdf_switch, quality_report_switch, quality_data_switch, tag_lists",
|
|
197
|
-
[
|
|
198
|
-
pytest.param(0, 0, 0, 0, 0, 0, 0, [], id="none"),
|
|
199
|
-
pytest.param(1, 0, 0, 0, 0, 0, 0, [], id="debug_only"),
|
|
200
|
-
pytest.param(0, 1, 0, 0, 0, 0, 0, [], id="intermediate_only"),
|
|
201
|
-
pytest.param(0, 0, 1, 0, 0, 0, 0, [], id="output_only"),
|
|
202
|
-
pytest.param(0, 0, 0, 1, 0, 0, 0, [], id="dataset_inventory_only"),
|
|
203
|
-
pytest.param(0, 0, 0, 0, 1, 0, 0, [], id="asdf_only"),
|
|
204
|
-
pytest.param(0, 0, 0, 0, 0, 1, 0, [], id="quality_report_only"),
|
|
205
|
-
pytest.param(0, 0, 0, 0, 0, 0, 1, [], id="quality_data_only"),
|
|
206
|
-
pytest.param(0, 0, 0, 0, 0, 0, 0, tag_lists, id="specific_only"),
|
|
207
|
-
pytest.param(1, 1, 1, 1, 1, 1, 1, tag_lists, id="all"),
|
|
208
|
-
pytest.param(1, 1, 0, 0, 0, 0, 0, tag_lists, id="combo_debug_intermediate_specific"),
|
|
209
|
-
pytest.param(1, 1, 1, 0, 0, 0, 0, [], id="combo_debug_intermediate_output"),
|
|
210
|
-
pytest.param(0, 1, 1, 0, 0, 0, 0, tag_lists, id="combo_intermediate_output_specific"),
|
|
211
|
-
pytest.param(1, 0, 1, 0, 0, 0, 0, tag_lists, id="combo_debug_output_specific"),
|
|
212
|
-
],
|
|
213
|
-
)
|
|
214
|
-
def test_build_transfer_list(
|
|
215
|
-
transfer_task_with_files,
|
|
216
|
-
debug_switch,
|
|
217
|
-
intermediate_switch,
|
|
218
|
-
output_switch,
|
|
219
|
-
dataset_inventory_switch,
|
|
220
|
-
asdf_switch,
|
|
221
|
-
quality_report_switch,
|
|
222
|
-
quality_data_switch,
|
|
223
|
-
tag_lists,
|
|
224
|
-
):
|
|
225
|
-
"""
|
|
226
|
-
Given: A TransferCryoTrialData task with a recipe run configuration (RRC) and a collection of frames
|
|
227
|
-
When: Building the transfer list
|
|
228
|
-
Then: Only the files requested by the RRC switches are collected for transfer
|
|
229
|
-
"""
|
|
230
|
-
task, available_output_files = transfer_task_with_files
|
|
231
|
-
|
|
232
|
-
expected_num = 0
|
|
233
|
-
if debug_switch:
|
|
234
|
-
expected_num += available_output_files.num_debug
|
|
235
|
-
if intermediate_switch:
|
|
236
|
-
expected_num += available_output_files.num_intermediate
|
|
237
|
-
if output_switch:
|
|
238
|
-
expected_num += available_output_files.num_output
|
|
239
|
-
if dataset_inventory_switch:
|
|
240
|
-
expected_num += available_output_files.num_dataset_inventory
|
|
241
|
-
if asdf_switch:
|
|
242
|
-
expected_num += available_output_files.num_asdf
|
|
243
|
-
if quality_report_switch:
|
|
244
|
-
expected_num += available_output_files.num_quality_report
|
|
245
|
-
if quality_data_switch:
|
|
246
|
-
expected_num += available_output_files.num_quality_data
|
|
247
|
-
if tag_lists:
|
|
248
|
-
expected_num += available_output_files.num_specific
|
|
249
|
-
|
|
250
|
-
transfer_list = task.build_transfer_list()
|
|
251
|
-
assert len(transfer_list) == expected_num
|
|
File without changes
|