dkist-processing-test 1.21.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dkist-processing-test might be problematic. Click here for more details.
- dkist_processing_test/__init__.py +11 -0
- dkist_processing_test/config.py +10 -0
- dkist_processing_test/models/__init__.py +0 -0
- dkist_processing_test/models/constants.py +18 -0
- dkist_processing_test/models/parameters.py +35 -0
- dkist_processing_test/tasks/__init__.py +10 -0
- dkist_processing_test/tasks/exercise_numba.py +42 -0
- dkist_processing_test/tasks/fail.py +11 -0
- dkist_processing_test/tasks/fake_science.py +101 -0
- dkist_processing_test/tasks/high_memory.py +20 -0
- dkist_processing_test/tasks/manual.py +26 -0
- dkist_processing_test/tasks/movie.py +53 -0
- dkist_processing_test/tasks/noop.py +15 -0
- dkist_processing_test/tasks/parse.py +88 -0
- dkist_processing_test/tasks/quality.py +30 -0
- dkist_processing_test/tasks/trial_output_data.py +46 -0
- dkist_processing_test/tasks/write_l1.py +64 -0
- dkist_processing_test/tests/__init__.py +0 -0
- dkist_processing_test/tests/conftest.py +64 -0
- dkist_processing_test/tests/test_parameters.py +103 -0
- dkist_processing_test/tests/test_tasks.py +724 -0
- dkist_processing_test/tests/test_workflows.py +9 -0
- dkist_processing_test/workflows/__init__.py +1 -0
- dkist_processing_test/workflows/common_tasks.py +231 -0
- dkist_processing_test/workflows/end_to_end.py +58 -0
- dkist_processing_test/workflows/exercise_numba.py +13 -0
- dkist_processing_test/workflows/fail.py +16 -0
- dkist_processing_test/workflows/noop.py +28 -0
- dkist_processing_test/workflows/resource_queue.py +19 -0
- dkist_processing_test/workflows/trial_end_to_end.py +64 -0
- dkist_processing_test-1.21.3.dist-info/METADATA +79 -0
- dkist_processing_test-1.21.3.dist-info/RECORD +34 -0
- dkist_processing_test-1.21.3.dist-info/WHEEL +5 -0
- dkist_processing_test-1.21.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Test integrity of workflows."""
|
|
2
|
+
from dkist_processing_core.build_utils import validate_workflows
|
|
3
|
+
|
|
4
|
+
from dkist_processing_test import workflows
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_workflow_integrity():
|
|
8
|
+
"""Validate workflow to ensure acyclic-ness and export compilation"""
|
|
9
|
+
validate_workflows(workflows)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from dkist_processing_test.config import dkist_processing_test_configurations
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Workflow which exercises the common tasks in an end to end scenario
|
|
3
|
+
"""
|
|
4
|
+
from dkist_processing_common.tasks import CreateTrialAsdf
|
|
5
|
+
from dkist_processing_common.tasks import CreateTrialDatasetInventory
|
|
6
|
+
from dkist_processing_common.tasks import CreateTrialQualityReport
|
|
7
|
+
from dkist_processing_common.tasks import PublishCatalogAndQualityMessages
|
|
8
|
+
from dkist_processing_common.tasks import QualityL1Metrics
|
|
9
|
+
from dkist_processing_common.tasks import SubmitDatasetMetadata
|
|
10
|
+
from dkist_processing_common.tasks import Teardown
|
|
11
|
+
from dkist_processing_common.tasks import TransferL0Data
|
|
12
|
+
from dkist_processing_common.tasks import TransferL1Data
|
|
13
|
+
from dkist_processing_common.tasks import TrialTeardown
|
|
14
|
+
from dkist_processing_core import Workflow
|
|
15
|
+
|
|
16
|
+
from dkist_processing_test.tasks import AssembleTestMovie
|
|
17
|
+
from dkist_processing_test.tasks import GenerateCalibratedData
|
|
18
|
+
from dkist_processing_test.tasks import MakeTestMovieFrames
|
|
19
|
+
from dkist_processing_test.tasks import ParseL0TestInputData
|
|
20
|
+
from dkist_processing_test.tasks import TestAssembleQualityData
|
|
21
|
+
from dkist_processing_test.tasks import TestQualityL0Metrics
|
|
22
|
+
from dkist_processing_test.tasks import TransferTestTrialData
|
|
23
|
+
from dkist_processing_test.tasks import WriteL1Data
|
|
24
|
+
from dkist_processing_test.tasks.manual import ManualWithoutProvenance
|
|
25
|
+
from dkist_processing_test.tasks.manual import ManualWithProvenance
|
|
26
|
+
|
|
27
|
+
# TransferInputData Task
|
|
28
|
+
transfer_input_data = Workflow(
|
|
29
|
+
input_data="input",
|
|
30
|
+
output_data="output",
|
|
31
|
+
category="test",
|
|
32
|
+
detail="transfer-input-data",
|
|
33
|
+
workflow_package=__package__,
|
|
34
|
+
)
|
|
35
|
+
transfer_input_data.add_node(task=TransferL0Data, upstreams=None)
|
|
36
|
+
|
|
37
|
+
# ParseInputData Task
|
|
38
|
+
parse_input_data = Workflow(
|
|
39
|
+
input_data="input",
|
|
40
|
+
output_data="output",
|
|
41
|
+
category="test",
|
|
42
|
+
detail="parse-input-data",
|
|
43
|
+
workflow_package=__package__,
|
|
44
|
+
)
|
|
45
|
+
parse_input_data.add_node(task=ParseL0TestInputData, upstreams=None)
|
|
46
|
+
|
|
47
|
+
# L0Quality Task
|
|
48
|
+
quality_l0_metrics = Workflow(
|
|
49
|
+
input_data="input",
|
|
50
|
+
output_data="output",
|
|
51
|
+
category="test",
|
|
52
|
+
detail="quality-l0-metrics",
|
|
53
|
+
workflow_package=__package__,
|
|
54
|
+
)
|
|
55
|
+
quality_l0_metrics.add_node(task=TestQualityL0Metrics, upstreams=None)
|
|
56
|
+
|
|
57
|
+
# L1Quality Task
|
|
58
|
+
quality_l1_metrics = Workflow(
|
|
59
|
+
input_data="input",
|
|
60
|
+
output_data="output",
|
|
61
|
+
category="test",
|
|
62
|
+
detail="quality-l1-metrics",
|
|
63
|
+
workflow_package=__package__,
|
|
64
|
+
)
|
|
65
|
+
quality_l1_metrics.add_node(task=QualityL1Metrics, upstreams=None)
|
|
66
|
+
|
|
67
|
+
# TestAssembleQualityData Task
|
|
68
|
+
quality_assemble_data = Workflow(
|
|
69
|
+
input_data="input",
|
|
70
|
+
output_data="output",
|
|
71
|
+
category="test",
|
|
72
|
+
detail="quality-assemble-data",
|
|
73
|
+
workflow_package=__package__,
|
|
74
|
+
)
|
|
75
|
+
quality_assemble_data.add_node(task=TestAssembleQualityData, upstreams=None)
|
|
76
|
+
|
|
77
|
+
# GenerateL1CalibratedData Task
|
|
78
|
+
generate_calibrated_data = Workflow(
|
|
79
|
+
input_data="input",
|
|
80
|
+
output_data="output",
|
|
81
|
+
category="test",
|
|
82
|
+
detail="generate-calibrated-data",
|
|
83
|
+
workflow_package=__package__,
|
|
84
|
+
)
|
|
85
|
+
generate_calibrated_data.add_node(task=GenerateCalibratedData, upstreams=None)
|
|
86
|
+
|
|
87
|
+
# MakeTestMovieFrames task
|
|
88
|
+
make_test_movie_frames = Workflow(
|
|
89
|
+
input_data="input",
|
|
90
|
+
output_data="output",
|
|
91
|
+
category="test",
|
|
92
|
+
detail="make-test-movie-frames",
|
|
93
|
+
workflow_package=__package__,
|
|
94
|
+
)
|
|
95
|
+
make_test_movie_frames.add_node(task=MakeTestMovieFrames, upstreams=None)
|
|
96
|
+
|
|
97
|
+
# AssembleTestMovie Task
|
|
98
|
+
assemble_test_movie = Workflow(
|
|
99
|
+
input_data="input",
|
|
100
|
+
output_data="output",
|
|
101
|
+
category="test",
|
|
102
|
+
detail="assemble-test-movie",
|
|
103
|
+
workflow_package=__package__,
|
|
104
|
+
)
|
|
105
|
+
assemble_test_movie.add_node(task=AssembleTestMovie, upstreams=None)
|
|
106
|
+
|
|
107
|
+
# WriteL1 Task
|
|
108
|
+
write_l1 = Workflow(
|
|
109
|
+
input_data="input",
|
|
110
|
+
output_data="output",
|
|
111
|
+
category="test",
|
|
112
|
+
detail="write-l1",
|
|
113
|
+
workflow_package=__package__,
|
|
114
|
+
)
|
|
115
|
+
write_l1.add_node(task=WriteL1Data, upstreams=None)
|
|
116
|
+
|
|
117
|
+
# TransferOutputData Task
|
|
118
|
+
transfer_output_data = Workflow(
|
|
119
|
+
input_data="input",
|
|
120
|
+
output_data="output",
|
|
121
|
+
category="test",
|
|
122
|
+
detail="transfer-output-data",
|
|
123
|
+
workflow_package=__package__,
|
|
124
|
+
)
|
|
125
|
+
transfer_output_data.add_node(task=TransferL1Data, upstreams=None)
|
|
126
|
+
|
|
127
|
+
# TransferTrialData Task
|
|
128
|
+
transfer_trial_data = Workflow(
|
|
129
|
+
input_data="input",
|
|
130
|
+
output_data="output",
|
|
131
|
+
category="test",
|
|
132
|
+
detail="transfer-trial-data",
|
|
133
|
+
workflow_package=__package__,
|
|
134
|
+
)
|
|
135
|
+
transfer_trial_data.add_node(task=TransferTestTrialData, upstreams=None)
|
|
136
|
+
|
|
137
|
+
# SubmitDatasetMetadata Task
|
|
138
|
+
submit_dataset_metadata = Workflow(
|
|
139
|
+
input_data="input",
|
|
140
|
+
output_data="output",
|
|
141
|
+
category="test",
|
|
142
|
+
detail="submit-dataset-metadata",
|
|
143
|
+
workflow_package=__package__,
|
|
144
|
+
)
|
|
145
|
+
submit_dataset_metadata.add_node(task=SubmitDatasetMetadata, upstreams=None)
|
|
146
|
+
|
|
147
|
+
# PublishCatalogMessages Task
|
|
148
|
+
publish_catalog_messages = Workflow(
|
|
149
|
+
input_data="input",
|
|
150
|
+
output_data="output",
|
|
151
|
+
category="test",
|
|
152
|
+
detail="publish-messages",
|
|
153
|
+
workflow_package=__package__,
|
|
154
|
+
)
|
|
155
|
+
publish_catalog_messages.add_node(task=PublishCatalogAndQualityMessages, upstreams=None)
|
|
156
|
+
|
|
157
|
+
# Teardown Task
|
|
158
|
+
teardown = Workflow(
|
|
159
|
+
input_data="input",
|
|
160
|
+
output_data="output",
|
|
161
|
+
category="test",
|
|
162
|
+
detail="teardown",
|
|
163
|
+
workflow_package=__package__,
|
|
164
|
+
)
|
|
165
|
+
teardown.add_node(task=Teardown, upstreams=None)
|
|
166
|
+
|
|
167
|
+
# Trial Teardown Task
|
|
168
|
+
trial_teardown = Workflow(
|
|
169
|
+
input_data="input",
|
|
170
|
+
output_data="output",
|
|
171
|
+
category="test",
|
|
172
|
+
detail="trial-teardown",
|
|
173
|
+
workflow_package=__package__,
|
|
174
|
+
)
|
|
175
|
+
trial_teardown.add_node(task=TrialTeardown, upstreams=None)
|
|
176
|
+
|
|
177
|
+
# CreateTrialInventory Task
|
|
178
|
+
create_trial_inventory = Workflow(
|
|
179
|
+
input_data="input",
|
|
180
|
+
output_data="output",
|
|
181
|
+
category="test",
|
|
182
|
+
detail="create-trial-inventory",
|
|
183
|
+
workflow_package=__package__,
|
|
184
|
+
)
|
|
185
|
+
create_trial_inventory.add_node(
|
|
186
|
+
task=CreateTrialDatasetInventory, upstreams=None, pip_extras=["inventory"]
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
# CreateTrialAsdf Task
|
|
190
|
+
create_trial_asdf = Workflow(
|
|
191
|
+
input_data="input",
|
|
192
|
+
output_data="output",
|
|
193
|
+
category="test",
|
|
194
|
+
detail="create-trial-asdf",
|
|
195
|
+
workflow_package=__package__,
|
|
196
|
+
)
|
|
197
|
+
create_trial_asdf.add_node(task=CreateTrialAsdf, upstreams=None, pip_extras=["asdf"])
|
|
198
|
+
|
|
199
|
+
# CreateTrialQualityReport Task
|
|
200
|
+
create_trial_quality_report = Workflow(
|
|
201
|
+
input_data="input",
|
|
202
|
+
output_data="output",
|
|
203
|
+
category="test",
|
|
204
|
+
detail="create-trial-quality",
|
|
205
|
+
workflow_package=__package__,
|
|
206
|
+
)
|
|
207
|
+
create_trial_quality_report.add_node(
|
|
208
|
+
task=CreateTrialQualityReport, upstreams=None, pip_extras=["quality"]
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# ManualWithProvenance Task
|
|
213
|
+
manual_with_provenance = Workflow(
|
|
214
|
+
input_data="input",
|
|
215
|
+
output_data="output",
|
|
216
|
+
category="test",
|
|
217
|
+
detail="manual-with-provenance",
|
|
218
|
+
workflow_package=__package__,
|
|
219
|
+
)
|
|
220
|
+
manual_with_provenance.add_node(task=ManualWithProvenance, upstreams=None)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
# ManualWithoutProvenance Task
|
|
224
|
+
manual_without_provenance = Workflow(
|
|
225
|
+
input_data="input",
|
|
226
|
+
output_data="output",
|
|
227
|
+
category="test",
|
|
228
|
+
detail="manual-without-provenance",
|
|
229
|
+
workflow_package=__package__,
|
|
230
|
+
)
|
|
231
|
+
manual_without_provenance.add_node(task=ManualWithoutProvenance, upstreams=None)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Workflow which exercises the common tasks in an end to end scenario
|
|
3
|
+
"""
|
|
4
|
+
from dkist_processing_common.tasks import PublishCatalogAndQualityMessages
|
|
5
|
+
from dkist_processing_common.tasks import QualityL1Metrics
|
|
6
|
+
from dkist_processing_common.tasks import SubmitDatasetMetadata
|
|
7
|
+
from dkist_processing_common.tasks import Teardown
|
|
8
|
+
from dkist_processing_common.tasks import TransferL0Data
|
|
9
|
+
from dkist_processing_common.tasks import TransferL1Data
|
|
10
|
+
from dkist_processing_core import ResourceQueue
|
|
11
|
+
from dkist_processing_core import Workflow
|
|
12
|
+
|
|
13
|
+
from dkist_processing_test.tasks import AssembleTestMovie
|
|
14
|
+
from dkist_processing_test.tasks import GenerateCalibratedData
|
|
15
|
+
from dkist_processing_test.tasks import MakeTestMovieFrames
|
|
16
|
+
from dkist_processing_test.tasks import ParseL0TestInputData
|
|
17
|
+
from dkist_processing_test.tasks import TestAssembleQualityData
|
|
18
|
+
from dkist_processing_test.tasks import TestQualityL0Metrics
|
|
19
|
+
from dkist_processing_test.tasks import WriteL1Data
|
|
20
|
+
|
|
21
|
+
end_to_end = Workflow(
|
|
22
|
+
input_data="input",
|
|
23
|
+
output_data="output",
|
|
24
|
+
category="test",
|
|
25
|
+
detail="management-processes-e2e",
|
|
26
|
+
workflow_package=__package__,
|
|
27
|
+
)
|
|
28
|
+
end_to_end.add_node(task=TransferL0Data, upstreams=None)
|
|
29
|
+
|
|
30
|
+
# Science flow
|
|
31
|
+
end_to_end.add_node(task=ParseL0TestInputData, upstreams=TransferL0Data)
|
|
32
|
+
end_to_end.add_node(
|
|
33
|
+
task=GenerateCalibratedData,
|
|
34
|
+
resource_queue=ResourceQueue.HIGH_MEMORY,
|
|
35
|
+
upstreams=ParseL0TestInputData,
|
|
36
|
+
)
|
|
37
|
+
end_to_end.add_node(task=WriteL1Data, upstreams=GenerateCalibratedData)
|
|
38
|
+
|
|
39
|
+
# Movie flow
|
|
40
|
+
end_to_end.add_node(task=MakeTestMovieFrames, upstreams=GenerateCalibratedData)
|
|
41
|
+
end_to_end.add_node(task=AssembleTestMovie, upstreams=MakeTestMovieFrames)
|
|
42
|
+
|
|
43
|
+
# Quality flow
|
|
44
|
+
end_to_end.add_node(task=TestQualityL0Metrics, upstreams=ParseL0TestInputData)
|
|
45
|
+
end_to_end.add_node(task=QualityL1Metrics, upstreams=GenerateCalibratedData)
|
|
46
|
+
end_to_end.add_node(
|
|
47
|
+
task=TestAssembleQualityData, upstreams=[TestQualityL0Metrics, QualityL1Metrics]
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
# Output flow
|
|
51
|
+
end_to_end.add_node(task=SubmitDatasetMetadata, upstreams=[WriteL1Data, TestAssembleQualityData])
|
|
52
|
+
end_to_end.add_node(task=TransferL1Data, upstreams=[WriteL1Data, AssembleTestMovie])
|
|
53
|
+
end_to_end.add_node(
|
|
54
|
+
task=PublishCatalogAndQualityMessages, upstreams=[TransferL1Data, SubmitDatasetMetadata]
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# goodby
|
|
58
|
+
end_to_end.add_node(task=Teardown, upstreams=PublishCatalogAndQualityMessages)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# TransferInputData Task
|
|
2
|
+
from dkist_processing_core import Workflow
|
|
3
|
+
|
|
4
|
+
from dkist_processing_test.tasks import ExerciseNumba
|
|
5
|
+
|
|
6
|
+
exercise_numba = Workflow(
|
|
7
|
+
input_data="input",
|
|
8
|
+
output_data="output",
|
|
9
|
+
category="test",
|
|
10
|
+
detail="exercise_numba",
|
|
11
|
+
workflow_package=__package__,
|
|
12
|
+
)
|
|
13
|
+
exercise_numba.add_node(task=ExerciseNumba, upstreams=None)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Workflow which is designed to fail
|
|
3
|
+
"""
|
|
4
|
+
from dkist_processing_core import Workflow
|
|
5
|
+
|
|
6
|
+
from dkist_processing_test.tasks import FailTask
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
fail = Workflow(
|
|
10
|
+
input_data="input",
|
|
11
|
+
output_data="output",
|
|
12
|
+
category="test",
|
|
13
|
+
detail="fail",
|
|
14
|
+
workflow_package=__package__,
|
|
15
|
+
)
|
|
16
|
+
fail.add_node(task=FailTask, upstreams=None)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Workflow which exercises the api's but doesn't perform an action
|
|
3
|
+
"""
|
|
4
|
+
from dkist_processing_core import Workflow
|
|
5
|
+
|
|
6
|
+
from dkist_processing_test.tasks import NoOpTask
|
|
7
|
+
from dkist_processing_test.tasks import NoOpTask2
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
noop = Workflow(
|
|
11
|
+
input_data="input",
|
|
12
|
+
output_data="output",
|
|
13
|
+
category="test",
|
|
14
|
+
detail="noop",
|
|
15
|
+
workflow_package=__package__,
|
|
16
|
+
)
|
|
17
|
+
noop.add_node(task=NoOpTask, upstreams=None)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
noop_flow = Workflow(
|
|
21
|
+
input_data="input",
|
|
22
|
+
output_data="output",
|
|
23
|
+
category="test",
|
|
24
|
+
detail="noop-flow",
|
|
25
|
+
workflow_package=__package__,
|
|
26
|
+
)
|
|
27
|
+
noop_flow.add_node(task=NoOpTask, upstreams=None)
|
|
28
|
+
noop_flow.add_node(task=NoOpTask2, upstreams=NoOpTask)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Workflow which exercises the api's but doesn't perform an action
|
|
3
|
+
"""
|
|
4
|
+
from dkist_processing_core import ResourceQueue
|
|
5
|
+
from dkist_processing_core import Workflow
|
|
6
|
+
|
|
7
|
+
from dkist_processing_test.tasks import HighMemoryTask
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
high_memory_workflow = Workflow(
|
|
11
|
+
input_data="input",
|
|
12
|
+
output_data="output",
|
|
13
|
+
category="test",
|
|
14
|
+
detail="high_mem",
|
|
15
|
+
workflow_package=__package__,
|
|
16
|
+
)
|
|
17
|
+
high_memory_workflow.add_node(
|
|
18
|
+
task=HighMemoryTask, resource_queue=ResourceQueue.HIGH_MEMORY, upstreams=None
|
|
19
|
+
)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Workflow which exercises the common tasks end to end in a trial scenario
|
|
3
|
+
"""
|
|
4
|
+
from dkist_processing_common.tasks import CreateTrialAsdf
|
|
5
|
+
from dkist_processing_common.tasks import CreateTrialDatasetInventory
|
|
6
|
+
from dkist_processing_common.tasks import CreateTrialQualityReport
|
|
7
|
+
from dkist_processing_common.tasks import QualityL1Metrics
|
|
8
|
+
from dkist_processing_common.tasks import TransferL0Data
|
|
9
|
+
from dkist_processing_common.tasks import TrialTeardown
|
|
10
|
+
from dkist_processing_core import Workflow
|
|
11
|
+
|
|
12
|
+
from dkist_processing_test.tasks import AssembleTestMovie
|
|
13
|
+
from dkist_processing_test.tasks import GenerateCalibratedData
|
|
14
|
+
from dkist_processing_test.tasks import MakeTestMovieFrames
|
|
15
|
+
from dkist_processing_test.tasks import ParseL0TestInputData
|
|
16
|
+
from dkist_processing_test.tasks import TestAssembleQualityData
|
|
17
|
+
from dkist_processing_test.tasks import TestQualityL0Metrics
|
|
18
|
+
from dkist_processing_test.tasks import TransferTestTrialData
|
|
19
|
+
from dkist_processing_test.tasks import WriteL1Data
|
|
20
|
+
|
|
21
|
+
trial = Workflow(
|
|
22
|
+
input_data="input",
|
|
23
|
+
output_data="output",
|
|
24
|
+
category="test",
|
|
25
|
+
detail="trial-e2e",
|
|
26
|
+
workflow_package=__package__,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
trial.add_node(task=TransferL0Data, upstreams=None)
|
|
30
|
+
|
|
31
|
+
# Science flow
|
|
32
|
+
trial.add_node(task=ParseL0TestInputData, upstreams=TransferL0Data)
|
|
33
|
+
trial.add_node(task=GenerateCalibratedData, upstreams=ParseL0TestInputData)
|
|
34
|
+
trial.add_node(task=WriteL1Data, upstreams=GenerateCalibratedData)
|
|
35
|
+
|
|
36
|
+
# Movie flow
|
|
37
|
+
trial.add_node(task=MakeTestMovieFrames, upstreams=GenerateCalibratedData)
|
|
38
|
+
trial.add_node(task=AssembleTestMovie, upstreams=MakeTestMovieFrames)
|
|
39
|
+
|
|
40
|
+
# Quality flow
|
|
41
|
+
trial.add_node(task=TestQualityL0Metrics, upstreams=ParseL0TestInputData)
|
|
42
|
+
trial.add_node(task=QualityL1Metrics, upstreams=GenerateCalibratedData)
|
|
43
|
+
trial.add_node(task=TestAssembleQualityData, upstreams=[TestQualityL0Metrics, QualityL1Metrics])
|
|
44
|
+
|
|
45
|
+
# Trial data generation
|
|
46
|
+
trial.add_node(task=CreateTrialDatasetInventory, upstreams=WriteL1Data, pip_extras=["inventory"])
|
|
47
|
+
trial.add_node(task=CreateTrialAsdf, upstreams=WriteL1Data, pip_extras=["asdf"])
|
|
48
|
+
trial.add_node(
|
|
49
|
+
task=CreateTrialQualityReport, upstreams=TestAssembleQualityData, pip_extras=["quality"]
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
# Output flow
|
|
53
|
+
trial.add_node(
|
|
54
|
+
task=TransferTestTrialData,
|
|
55
|
+
upstreams=[
|
|
56
|
+
CreateTrialDatasetInventory,
|
|
57
|
+
CreateTrialAsdf,
|
|
58
|
+
CreateTrialQualityReport,
|
|
59
|
+
AssembleTestMovie,
|
|
60
|
+
],
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# goodby
|
|
64
|
+
trial.add_node(task=TrialTeardown, upstreams=TransferTestTrialData)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: dkist-processing-test
|
|
3
|
+
Version: 1.21.3
|
|
4
|
+
Summary: Example instrument code used by the DKIST science data processing pipelines to test processing infrastructure
|
|
5
|
+
Author-email: NSO / AURA <dkistdc@nso.edu>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://nso.edu/dkist/data-center/
|
|
8
|
+
Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-test/
|
|
9
|
+
Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
|
|
10
|
+
Classifier: Programming Language :: Python
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/x-rst
|
|
15
|
+
Requires-Dist: dkist-processing-common ==10.2.2
|
|
16
|
+
Requires-Dist: dkist-header-validator ==5.1.1
|
|
17
|
+
Requires-Dist: dkist-service-configuration ==2.2
|
|
18
|
+
Requires-Dist: dkist-fits-specifications ==4.7.0
|
|
19
|
+
Requires-Dist: numba ==0.59.1
|
|
20
|
+
Requires-Dist: astropy ==6.1.0
|
|
21
|
+
Requires-Dist: numpy ==1.26.4
|
|
22
|
+
Requires-Dist: dkist-spectral-lines ==3.0.0
|
|
23
|
+
Provides-Extra: asdf
|
|
24
|
+
Requires-Dist: dkist-processing-common[asdf] ; extra == 'asdf'
|
|
25
|
+
Requires-Dist: dkist-inventory[asdf] ==1.4.0 ; extra == 'asdf'
|
|
26
|
+
Provides-Extra: inventory
|
|
27
|
+
Requires-Dist: dkist-processing-common[inventory] ; extra == 'inventory'
|
|
28
|
+
Requires-Dist: dkist-inventory ==1.4.0 ; extra == 'inventory'
|
|
29
|
+
Provides-Extra: quality
|
|
30
|
+
Requires-Dist: dkist-quality ==1.1.1 ; extra == 'quality'
|
|
31
|
+
Provides-Extra: test
|
|
32
|
+
Requires-Dist: pytest ; extra == 'test'
|
|
33
|
+
Requires-Dist: pytest-xdist ; extra == 'test'
|
|
34
|
+
Requires-Dist: pytest-cov ; extra == 'test'
|
|
35
|
+
Requires-Dist: pytest-mock ; extra == 'test'
|
|
36
|
+
Requires-Dist: dkist-data-simulator >=5.2.0 ; extra == 'test'
|
|
37
|
+
Requires-Dist: dkist-processing-test[inventory] ; extra == 'test'
|
|
38
|
+
Requires-Dist: dkist-processing-test[asdf] ; extra == 'test'
|
|
39
|
+
Requires-Dist: dkist-processing-test[quality] ; extra == 'test'
|
|
40
|
+
|
|
41
|
+
dkist-processing-test
|
|
42
|
+
---------------------
|
|
43
|
+
|
|
44
|
+
Overview
|
|
45
|
+
--------
|
|
46
|
+
The dkist-processing-test library serves as an example implementation of a Tasks and Workflows using the
|
|
47
|
+
`dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ framework and
|
|
48
|
+
`dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_ Tasks.
|
|
49
|
+
|
|
50
|
+
The recommended project structure is to separate tasks and workflows into separate packages.
|
|
51
|
+
|
|
52
|
+
Build
|
|
53
|
+
-----
|
|
54
|
+
Artifacts are built through `bitbucket pipelines <bitbucket-pipelines.yml>`_
|
|
55
|
+
|
|
56
|
+
The pipeline can be used in other repos with a modification of the package and artifact locations
|
|
57
|
+
to use the names relevant to the target repo.
|
|
58
|
+
|
|
59
|
+
e.g. dkist-processing-test -> dkist-processing-vbi and dkist_processing_test -> dkist_processing_vbi
|
|
60
|
+
|
|
61
|
+
Deployment
|
|
62
|
+
----------
|
|
63
|
+
Deployment is done with `turtlebot <https://bitbucket.org/dkistdc/turtlebot/src/master/>`_ and follows
|
|
64
|
+
the process detailed in `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_
|
|
65
|
+
|
|
66
|
+
Environment Variables
|
|
67
|
+
---------------------
|
|
68
|
+
Only those specified by `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-common <https://pypi.org/project/dkist-processing-common/>`_
|
|
69
|
+
|
|
70
|
+
Development
|
|
71
|
+
-----------
|
|
72
|
+
|
|
73
|
+
.. code-block:: bash
|
|
74
|
+
|
|
75
|
+
git clone git@bitbucket.org:dkistdc/dkist-processing-test.git
|
|
76
|
+
cd dkist-processing-test
|
|
77
|
+
pre-commit install
|
|
78
|
+
pip install -e .[test]
|
|
79
|
+
pytest -v --cov dkist_processing_test
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
dkist_processing_test/__init__.py,sha256=t-Daj1wOIg5dgI1gklg6suSnI-xi6DKSb6DxHDWH96c,340
|
|
2
|
+
dkist_processing_test/config.py,sha256=LF80ReTk0ggJ3eZI4NZRZCgmR6E84adWU5iP6-JN-XY,383
|
|
3
|
+
dkist_processing_test/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
dkist_processing_test/models/constants.py,sha256=TxHYZHl3YQXN7-Voj7QgjgAajbxxB_qR8tRI9iTdERI,726
|
|
5
|
+
dkist_processing_test/models/parameters.py,sha256=5lUvGEmPh5JnGeLpJTpzrIao56bpRSyvfFZsRoONwYg,1256
|
|
6
|
+
dkist_processing_test/tasks/__init__.py,sha256=ZWoqlNgFrn2auTMGNX9YWmHW_3JwQzn04XE_FmCAU24,517
|
|
7
|
+
dkist_processing_test/tasks/exercise_numba.py,sha256=XfMVffPUgeiPoxSgo39tIjzFBJKKt3fnIYGaDFyBibc,1437
|
|
8
|
+
dkist_processing_test/tasks/fail.py,sha256=jiOiuqoX_JR5hx81REgvcSvb0GBlVKc9MIXKUO0Nhr4,245
|
|
9
|
+
dkist_processing_test/tasks/fake_science.py,sha256=Hon9h5TYc3fuHerm_1uLW2gNr4Y3W0oBN3nmIy4qw-M,4082
|
|
10
|
+
dkist_processing_test/tasks/high_memory.py,sha256=J3vBsivFkqs8eZZwBPpEORlHnFX1bGsbO3CjllJMRsc,540
|
|
11
|
+
dkist_processing_test/tasks/manual.py,sha256=gjol_EieMbclv0ZDkVqR2Xd7abutali6KNwoqaCJiAI,583
|
|
12
|
+
dkist_processing_test/tasks/movie.py,sha256=uFPWQlRFP5zzzYGT58IXebbG49su2I4AEpN2bIy7vGc,1831
|
|
13
|
+
dkist_processing_test/tasks/noop.py,sha256=txgpa0DYrjYPKBDfwfjHt7X9Dna4yaJeMCabu3p2Zmo,235
|
|
14
|
+
dkist_processing_test/tasks/parse.py,sha256=4bDKnPaWeZkQbUpYWCWk1O-Fjzu6WWRboFef3NFJDMs,2713
|
|
15
|
+
dkist_processing_test/tasks/quality.py,sha256=9QynI9R5mkYueb46wNBUhK62QSd2n791L0_w6nVxBjc,895
|
|
16
|
+
dkist_processing_test/tasks/trial_output_data.py,sha256=T34kGFxyR0IVaYnJhP3s9kv1_ExSNlnxx0n76cN32NU,1553
|
|
17
|
+
dkist_processing_test/tasks/write_l1.py,sha256=lpNyCI3kH_pnL0Wi1LHhNRjRHNxu6q76xccCNHm7uwA,2074
|
|
18
|
+
dkist_processing_test/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
dkist_processing_test/tests/conftest.py,sha256=UhtnrZUgbfX738Ej2MhxZMIjimHnUipfQbGxCLSaH8k,1939
|
|
20
|
+
dkist_processing_test/tests/test_parameters.py,sha256=nh8znKVxFhC46LYNi-DkhV24LFYTfFQI0duV0DwNoHs,3164
|
|
21
|
+
dkist_processing_test/tests/test_tasks.py,sha256=Yvjw8ImQO4QudC5F969LQZMoFT_959CYsldFnZh5r10,25388
|
|
22
|
+
dkist_processing_test/tests/test_workflows.py,sha256=NqRkstqcqwH80cKJ1uDw16G4MsDOuOiZRWdfH0STarQ,286
|
|
23
|
+
dkist_processing_test/workflows/__init__.py,sha256=Ryn_HXsVkZM_lLKyVteXC909Td2geylmz_1Z-8SE8c8,78
|
|
24
|
+
dkist_processing_test/workflows/common_tasks.py,sha256=4H1TaUNa5-HyE1NJQiAc3NAN62ZOTYb7MbR1yjKMg-k,6843
|
|
25
|
+
dkist_processing_test/workflows/end_to_end.py,sha256=XzSe7XtPF0e3fLXWL90LOZ0WKCoCKA4SL43N0CzgpCQ,2328
|
|
26
|
+
dkist_processing_test/workflows/exercise_numba.py,sha256=phpPYcIXEsmc0X1yls_YbXoKS6VXxhHe_8vNd_GyajU,347
|
|
27
|
+
dkist_processing_test/workflows/fail.py,sha256=KGhyAF7gKYYutP4aGa-1tDqPsC7Nr4xKYY0z-HByWi0,326
|
|
28
|
+
dkist_processing_test/workflows/noop.py,sha256=k2-6BpRkl1JDGaHPavxDzIlVx11KQtxKmsHmHTNQ9o0,666
|
|
29
|
+
dkist_processing_test/workflows/resource_queue.py,sha256=_jOT99O0MFN_ACne6i8T5ZIwe_lBQqGz2Ccb8JlqQMI,500
|
|
30
|
+
dkist_processing_test/workflows/trial_end_to_end.py,sha256=wkl_gk0zQ5TJ41CY0gEREPSIldZdTtLGwPDL_iwObe4,2425
|
|
31
|
+
dkist_processing_test-1.21.3.dist-info/METADATA,sha256=y2sE4f92uTABnvg8JemcYfquyNk2aLwdRSUxZCYrcAw,3292
|
|
32
|
+
dkist_processing_test-1.21.3.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
|
|
33
|
+
dkist_processing_test-1.21.3.dist-info/top_level.txt,sha256=Hs4oTIrG_r-svhk_RGFTEO4e3vqQoYlBzdv5mvJVF24,22
|
|
34
|
+
dkist_processing_test-1.21.3.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dkist_processing_test
|