dkist-processing-common 10.6.1rc3__py3-none-any.whl → 10.6.1rc5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- changelog/236.misc.1.rst +2 -1
- changelog/236.misc.rst +3 -1
- dkist_processing_common/models/graphql.py +21 -2
- dkist_processing_common/tasks/mixin/metadata_store.py +5 -6
- dkist_processing_common/tasks/output_data_base.py +1 -1
- dkist_processing_common/tasks/teardown.py +1 -1
- dkist_processing_common/tasks/transfer_input_data.py +4 -3
- dkist_processing_common/tasks/trial_output_data.py +12 -11
- dkist_processing_common/tasks/write_l1.py +3 -3
- dkist_processing_common/tests/conftest.py +94 -61
- dkist_processing_common/tests/test_input_dataset.py +2 -20
- dkist_processing_common/tests/test_teardown.py +16 -15
- dkist_processing_common/tests/test_transfer_input_data.py +74 -118
- dkist_processing_common/tests/test_trial_catalog.py +2 -2
- dkist_processing_common/tests/test_trial_output_data.py +9 -11
- dkist_processing_common/tests/test_write_l1.py +14 -2
- dkist_processing_common-10.6.1rc5.dist-info/METADATA +175 -0
- {dkist_processing_common-10.6.1rc3.dist-info → dkist_processing_common-10.6.1rc5.dist-info}/RECORD +20 -20
- {dkist_processing_common-10.6.1rc3.dist-info → dkist_processing_common-10.6.1rc5.dist-info}/WHEEL +1 -1
- dkist_processing_common-10.6.1rc3.dist-info/METADATA +0 -398
- {dkist_processing_common-10.6.1rc3.dist-info → dkist_processing_common-10.6.1rc5.dist-info}/top_level.txt +0 -0
|
@@ -1,164 +1,120 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import os
|
|
3
|
-
from itertools import chain
|
|
4
3
|
from pathlib import Path
|
|
5
|
-
from uuid import uuid4
|
|
6
4
|
|
|
7
5
|
import pytest
|
|
8
6
|
|
|
9
7
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
8
|
+
from dkist_processing_common.codecs.json import json_decoder
|
|
10
9
|
from dkist_processing_common.models.tags import Tag
|
|
11
10
|
from dkist_processing_common.tasks.transfer_input_data import TransferL0Data
|
|
12
11
|
from dkist_processing_common.tests.conftest import create_parameter_files
|
|
12
|
+
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TransferL0DataTask(TransferL0Data):
|
|
16
|
+
def run(self) -> None:
|
|
17
|
+
...
|
|
13
18
|
|
|
14
19
|
|
|
15
20
|
@pytest.fixture
|
|
16
|
-
def
|
|
17
|
-
|
|
21
|
+
def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
|
|
22
|
+
mocker.patch(
|
|
23
|
+
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient", new=FakeGQLClient
|
|
24
|
+
)
|
|
25
|
+
with TransferL0DataTask(
|
|
18
26
|
recipe_run_id=recipe_run_id,
|
|
19
27
|
workflow_name="workflow_name",
|
|
20
28
|
workflow_version="workflow_version",
|
|
29
|
+
) as task:
|
|
30
|
+
task.scratch = WorkflowFileSystem(
|
|
31
|
+
recipe_run_id=recipe_run_id,
|
|
32
|
+
scratch_base_path=tmp_path,
|
|
33
|
+
)
|
|
34
|
+
yield task
|
|
35
|
+
task._purge()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_download_dataset(transfer_l0_data_task):
|
|
39
|
+
"""
|
|
40
|
+
:Given: a TransferL0Data task with a valid input dataset
|
|
41
|
+
:When: downloading the dataset documents from the metadata store
|
|
42
|
+
:Then: the correct documents are written to disk
|
|
43
|
+
"""
|
|
44
|
+
# Given
|
|
45
|
+
task = transfer_l0_data_task
|
|
46
|
+
# When
|
|
47
|
+
task.download_input_dataset()
|
|
48
|
+
# Then
|
|
49
|
+
expected_observe_doc = FakeGQLClient.observe_frames_doc_object
|
|
50
|
+
observe_doc_from_file = next(
|
|
51
|
+
task.read(tags=Tag.input_dataset_observe_frames(), decoder=json_decoder)
|
|
21
52
|
)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
53
|
+
assert observe_doc_from_file == expected_observe_doc
|
|
54
|
+
expected_calibration_doc = FakeGQLClient.calibration_frames_doc_object
|
|
55
|
+
calibration_doc_from_file = next(
|
|
56
|
+
task.read(tags=Tag.input_dataset_calibration_frames(), decoder=json_decoder)
|
|
25
57
|
)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
"parameterValueId": 1,
|
|
33
|
-
"parameterValue": json.dumps([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
|
|
34
|
-
"parameterValueStartDate": "2000-01-01",
|
|
35
|
-
}
|
|
36
|
-
],
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
"parameterName": "param_name_2",
|
|
40
|
-
"parameterValues": [
|
|
41
|
-
{
|
|
42
|
-
"parameterValueId": 2,
|
|
43
|
-
"parameterValue": json.dumps(
|
|
44
|
-
{
|
|
45
|
-
"__file__": {
|
|
46
|
-
"bucket": "data",
|
|
47
|
-
"objectKey": f"parameters/param_name/{uuid4().hex}.dat",
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
),
|
|
51
|
-
"parameterValueStartDate": "2000-01-01",
|
|
52
|
-
},
|
|
53
|
-
{
|
|
54
|
-
"parameterValueId": 3,
|
|
55
|
-
"parameterValue": json.dumps(
|
|
56
|
-
{
|
|
57
|
-
"__file__": {
|
|
58
|
-
"bucket": "data",
|
|
59
|
-
"objectKey": f"parameters/param_name/{uuid4().hex}.dat",
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
),
|
|
63
|
-
"parameterValueStartDate": "2000-01-02",
|
|
64
|
-
},
|
|
65
|
-
],
|
|
66
|
-
},
|
|
67
|
-
{
|
|
68
|
-
"parameterName": "param_name_4",
|
|
69
|
-
"parameterValues": [
|
|
70
|
-
{
|
|
71
|
-
"parameterValueId": 4,
|
|
72
|
-
"parameterValue": json.dumps(
|
|
73
|
-
{"a": 1, "b": 3.14159, "c": "foo", "d": [1, 2, 3]}
|
|
74
|
-
),
|
|
75
|
-
"parameterValueStartDate": "2000-01-01",
|
|
76
|
-
}
|
|
77
|
-
],
|
|
78
|
-
},
|
|
79
|
-
]
|
|
80
|
-
input_dataset_observe_frames_part = [
|
|
81
|
-
{
|
|
82
|
-
"bucket": uuid4().hex[:6],
|
|
83
|
-
"object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
|
|
84
|
-
}
|
|
85
|
-
]
|
|
86
|
-
input_dataset_calibration_frames_part = [
|
|
87
|
-
{
|
|
88
|
-
"bucket": uuid4().hex[:6],
|
|
89
|
-
"object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
|
|
90
|
-
},
|
|
91
|
-
{
|
|
92
|
-
"bucket": uuid4().hex[:6],
|
|
93
|
-
"object_keys": [Path(uuid4().hex[:6]).as_posix() for _ in range(3)],
|
|
94
|
-
},
|
|
95
|
-
]
|
|
96
|
-
# load parameters file
|
|
97
|
-
file_path = task.scratch.workflow_base_path / Path(f"{uuid4().hex[:6]}.ext")
|
|
98
|
-
file_path.write_text(data=json.dumps(input_dataset_parameters_part))
|
|
99
|
-
task.tag(path=file_path, tags=Tag.input_dataset_parameters())
|
|
100
|
-
# create parameter files
|
|
101
|
-
expected_parameters = dict()
|
|
102
|
-
for item in input_dataset_parameters_part:
|
|
103
|
-
expected_parameters[item["parameterName"]] = item["parameterValues"]
|
|
104
|
-
create_parameter_files(task, expected_parameters)
|
|
105
|
-
# load observe frames file
|
|
106
|
-
file_path = task.scratch.workflow_base_path / Path(f"{uuid4().hex[:6]}.ext")
|
|
107
|
-
file_path.write_text(data=json.dumps(input_dataset_observe_frames_part))
|
|
108
|
-
task.tag(path=file_path, tags=Tag.input_dataset_observe_frames())
|
|
109
|
-
# load calibration frames file
|
|
110
|
-
file_path = task.scratch.workflow_base_path / Path(f"{uuid4().hex[:6]}.ext")
|
|
111
|
-
file_path.write_text(data=json.dumps(input_dataset_calibration_frames_part))
|
|
112
|
-
task.tag(path=file_path, tags=Tag.input_dataset_calibration_frames())
|
|
113
|
-
|
|
114
|
-
yield {
|
|
115
|
-
"task": task,
|
|
116
|
-
"parameters": input_dataset_parameters_part,
|
|
117
|
-
"observe": input_dataset_observe_frames_part,
|
|
118
|
-
"calibration": input_dataset_calibration_frames_part,
|
|
119
|
-
}
|
|
120
|
-
task._purge()
|
|
58
|
+
assert calibration_doc_from_file == expected_calibration_doc
|
|
59
|
+
expected_parameters_doc = FakeGQLClient.parameters_doc_object
|
|
60
|
+
parameters_doc_from_file = next(
|
|
61
|
+
task.read(tags=Tag.input_dataset_parameters(), decoder=json_decoder)
|
|
62
|
+
)
|
|
63
|
+
assert parameters_doc_from_file == expected_parameters_doc
|
|
121
64
|
|
|
122
65
|
|
|
123
|
-
def test_format_frame_transfer_items(
|
|
66
|
+
def test_format_frame_transfer_items(transfer_l0_data_task):
|
|
124
67
|
"""
|
|
125
|
-
:Given: a TransferL0Data task with a
|
|
126
|
-
:When: formatting
|
|
68
|
+
:Given: a TransferL0Data task with a downloaded input dataset
|
|
69
|
+
:When: formatting frames in the input dataset for transfer
|
|
127
70
|
:Then: the items are correctly loaded into GlobusTransferItem objects
|
|
128
71
|
"""
|
|
129
|
-
|
|
72
|
+
# Given
|
|
73
|
+
task = transfer_l0_data_task
|
|
74
|
+
task.download_input_dataset()
|
|
75
|
+
# When
|
|
76
|
+
transfer_items = task.format_frame_transfer_items()
|
|
77
|
+
# Then
|
|
130
78
|
source_filenames = []
|
|
131
79
|
destination_filenames = []
|
|
132
|
-
|
|
80
|
+
all_frames = (
|
|
81
|
+
FakeGQLClient.observe_frames_doc_object + FakeGQLClient.calibration_frames_doc_object
|
|
82
|
+
)
|
|
83
|
+
for frame_set in all_frames:
|
|
133
84
|
for key in frame_set["object_keys"]:
|
|
134
85
|
source_filenames.append(os.path.join("/", frame_set["bucket"], key))
|
|
135
86
|
destination_filenames.append(Path(key).name)
|
|
136
|
-
assert len(
|
|
137
|
-
for item in
|
|
87
|
+
assert len(transfer_items) == len(source_filenames)
|
|
88
|
+
for item in transfer_items:
|
|
138
89
|
assert item.source_path.as_posix() in source_filenames
|
|
139
90
|
assert item.destination_path.name in destination_filenames
|
|
140
91
|
assert not item.recursive
|
|
141
92
|
|
|
142
93
|
|
|
143
|
-
def test_format_parameter_file_transfer_items(
|
|
94
|
+
def test_format_parameter_file_transfer_items(transfer_l0_data_task):
|
|
144
95
|
"""
|
|
145
|
-
:Given: a TransferL0Data task with a
|
|
146
|
-
:When: formatting
|
|
96
|
+
:Given: a TransferL0Data task with a downloaded input dataset
|
|
97
|
+
:When: formatting parameter files in the input dataset for transfer
|
|
147
98
|
:Then: the items are correctly loaded into GlobusTransferItem objects
|
|
148
99
|
"""
|
|
149
|
-
|
|
100
|
+
# Given
|
|
101
|
+
task = transfer_l0_data_task
|
|
102
|
+
task.download_input_dataset()
|
|
103
|
+
create_parameter_files(task)
|
|
104
|
+
# When
|
|
105
|
+
transfer_items = task.format_parameter_transfer_items()
|
|
106
|
+
# Then
|
|
150
107
|
source_filenames = []
|
|
151
108
|
destination_filenames = []
|
|
152
|
-
|
|
109
|
+
parameters = FakeGQLClient.parameters_doc_object
|
|
110
|
+
for param in parameters:
|
|
153
111
|
for value in param["parameterValues"]:
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
value_dict = json.loads(param_value)
|
|
112
|
+
if "__file__" in value["parameterValue"]:
|
|
113
|
+
value_dict = json.loads(value["parameterValue"])
|
|
157
114
|
bucket = value_dict["__file__"]["bucket"]
|
|
158
115
|
object_key = value_dict["__file__"]["objectKey"]
|
|
159
116
|
source_filenames.append(os.path.join("/", bucket, object_key))
|
|
160
117
|
destination_filenames.append(Path(object_key).name)
|
|
161
|
-
transfer_items = task.format_parameter_transfer_items()
|
|
162
118
|
assert len(transfer_items) == len(source_filenames)
|
|
163
119
|
for transfer_item in transfer_items:
|
|
164
120
|
assert transfer_item.source_path.as_posix() in source_filenames
|
|
@@ -19,7 +19,7 @@ from dkist_processing_common.models.tags import Tag
|
|
|
19
19
|
from dkist_processing_common.tasks import CreateTrialAsdf
|
|
20
20
|
from dkist_processing_common.tasks import CreateTrialDatasetInventory
|
|
21
21
|
from dkist_processing_common.tasks import CreateTrialQualityReport
|
|
22
|
-
from dkist_processing_common.tests.conftest import
|
|
22
|
+
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
@pytest.fixture()
|
|
@@ -51,7 +51,7 @@ def create_trial_dataset_inventory_task(
|
|
|
51
51
|
"""An instance of CreateTrialDatasetInventory with L1 frames tagged in scratch."""
|
|
52
52
|
mocker.patch(
|
|
53
53
|
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
|
|
54
|
-
new=
|
|
54
|
+
new=FakeGQLClient,
|
|
55
55
|
)
|
|
56
56
|
task = CreateTrialDatasetInventory(
|
|
57
57
|
recipe_run_id=recipe_run_id,
|
|
@@ -29,11 +29,9 @@ def recipe_run_configuration(
|
|
|
29
29
|
response = super().execute_gql_query(**kwargs)
|
|
30
30
|
if isinstance(response, list):
|
|
31
31
|
if isinstance(response[0], RecipeRunResponse):
|
|
32
|
-
response[0].configuration =
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
"destination_bucket": destination_bucket,
|
|
36
|
-
}
|
|
32
|
+
response[0].configuration.trial_root_directory_name = custom_root_name
|
|
33
|
+
response[0].configuration.trial_directory_name = custom_dir_name
|
|
34
|
+
response[0].configuration.destination_bucket = destination_bucket
|
|
37
35
|
return response
|
|
38
36
|
|
|
39
37
|
return GQLClientWithConfiguration
|
|
@@ -48,12 +46,12 @@ def recipe_run_configuration_with_tag_lists(
|
|
|
48
46
|
response = super().execute_gql_query(**kwargs)
|
|
49
47
|
if isinstance(response, list):
|
|
50
48
|
if isinstance(response[0], RecipeRunResponse):
|
|
51
|
-
response[0].configuration =
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
49
|
+
response[0].configuration.trial_root_directory_name = custom_root_name
|
|
50
|
+
response[0].configuration.trial_directory_name = custom_dir_name
|
|
51
|
+
response[0].configuration.destination_bucket = destination_bucket
|
|
52
|
+
response[
|
|
53
|
+
0
|
|
54
|
+
].configuration.trial_exclusive_transfer_tag_lists = exclusive_tag_lists
|
|
57
55
|
return response
|
|
58
56
|
|
|
59
57
|
return GQLClientWithConfiguration
|
|
@@ -23,10 +23,21 @@ from dkist_processing_common.models.tags import Tag
|
|
|
23
23
|
from dkist_processing_common.models.wavelength import WavelengthRange
|
|
24
24
|
from dkist_processing_common.tasks.write_l1 import WriteL1Frame
|
|
25
25
|
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
26
|
-
from dkist_processing_common.tests.conftest import FakeGQLClientNoRecipeConfiguration
|
|
27
26
|
from dkist_processing_common.tests.conftest import TILE_SIZE
|
|
28
27
|
|
|
29
28
|
|
|
29
|
+
class FakeGQLClientDefaultRecipeConfiguration(FakeGQLClient):
|
|
30
|
+
def execute_gql_query(self, **kwargs):
|
|
31
|
+
response = super().execute_gql_query(**kwargs)
|
|
32
|
+
if type(response[0]) == RecipeRunResponse:
|
|
33
|
+
# Test converting returned None to a default configuration
|
|
34
|
+
new_response = response[0].model_validate(
|
|
35
|
+
{**response[0].model_dump(), "configuration": None}
|
|
36
|
+
)
|
|
37
|
+
response[0] = new_response
|
|
38
|
+
return response
|
|
39
|
+
|
|
40
|
+
|
|
30
41
|
class CompleteWriteL1Frame(WriteL1Frame):
|
|
31
42
|
def add_dataset_headers(
|
|
32
43
|
self, header: fits.Header, stokes: Literal["I", "Q", "U", "V"]
|
|
@@ -427,10 +438,11 @@ def test_rice_compression_with_default_tile_size(write_l1_task, mocker):
|
|
|
427
438
|
"""
|
|
428
439
|
mocker.patch(
|
|
429
440
|
"dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
|
|
430
|
-
new=
|
|
441
|
+
new=FakeGQLClientDefaultRecipeConfiguration,
|
|
431
442
|
)
|
|
432
443
|
task, _, _ = write_l1_task
|
|
433
444
|
task()
|
|
445
|
+
assert task.tile_size_param == None
|
|
434
446
|
files = list(task.read(tags=[Tag.frame(), Tag.output()]))
|
|
435
447
|
for file in files:
|
|
436
448
|
hdul = fits.open(file)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: dkist-processing-common
|
|
3
|
+
Version: 10.6.1rc5
|
|
4
|
+
Summary: Common task classes used by the DKIST science data processing pipelines
|
|
5
|
+
Author-email: NSO / AURA <dkistdc@nso.edu>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://nso.edu/dkist/data-center/
|
|
8
|
+
Project-URL: Repository, https://bitbucket.org/dkistdc/dkist-processing-common/
|
|
9
|
+
Project-URL: Documentation, https://docs.dkist.nso.edu/projects/common
|
|
10
|
+
Project-URL: Help, https://nso.atlassian.net/servicedesk/customer/portal/5
|
|
11
|
+
Classifier: Programming Language :: Python
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Requires-Python: >=3.11
|
|
15
|
+
Description-Content-Type: text/x-rst
|
|
16
|
+
Requires-Dist: asdf<4.0.0,>=3.5.0
|
|
17
|
+
Requires-Dist: astropy<7.0.0,>=6.0.0
|
|
18
|
+
Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
|
|
19
|
+
Requires-Dist: dkist-header-validator<6.0,>=5.0.0
|
|
20
|
+
Requires-Dist: dkist-processing-core==5.1.0
|
|
21
|
+
Requires-Dist: dkist-processing-pac<4.0,>=3.1
|
|
22
|
+
Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
|
|
23
|
+
Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
|
|
24
|
+
Requires-Dist: globus-sdk>=3.12.0
|
|
25
|
+
Requires-Dist: gqlclient[pydantic]==1.2.3
|
|
26
|
+
Requires-Dist: sqids==0.5.1
|
|
27
|
+
Requires-Dist: matplotlib>=3.4
|
|
28
|
+
Requires-Dist: moviepy>=2.0.0
|
|
29
|
+
Requires-Dist: numpy>=1.20.2
|
|
30
|
+
Requires-Dist: object-clerk==0.1.1
|
|
31
|
+
Requires-Dist: pandas>=1.4.2
|
|
32
|
+
Requires-Dist: pillow>=10.2.0
|
|
33
|
+
Requires-Dist: pydantic>=2.0
|
|
34
|
+
Requires-Dist: redis==4.6.0
|
|
35
|
+
Requires-Dist: requests>=2.23
|
|
36
|
+
Requires-Dist: scipy>=1.15.1
|
|
37
|
+
Requires-Dist: sunpy>=3.0.0
|
|
38
|
+
Requires-Dist: talus==1.1.0
|
|
39
|
+
Provides-Extra: test
|
|
40
|
+
Requires-Dist: pytest; extra == "test"
|
|
41
|
+
Requires-Dist: pytest-xdist; extra == "test"
|
|
42
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
43
|
+
Requires-Dist: pytest-mock; extra == "test"
|
|
44
|
+
Requires-Dist: hypothesis; extra == "test"
|
|
45
|
+
Requires-Dist: towncrier; extra == "test"
|
|
46
|
+
Requires-Dist: dkist-data-simulator>=5.0.0; extra == "test"
|
|
47
|
+
Requires-Dist: dkist-processing-common[inventory]; extra == "test"
|
|
48
|
+
Requires-Dist: dkist-processing-common[asdf]; extra == "test"
|
|
49
|
+
Requires-Dist: dkist-processing-common[quality]; extra == "test"
|
|
50
|
+
Provides-Extra: docs
|
|
51
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
52
|
+
Requires-Dist: sphinx-astropy; extra == "docs"
|
|
53
|
+
Requires-Dist: sphinx-changelog; extra == "docs"
|
|
54
|
+
Requires-Dist: sphinx-autoapi!=3.1.0; extra == "docs"
|
|
55
|
+
Requires-Dist: pytest; extra == "docs"
|
|
56
|
+
Requires-Dist: towncrier<22.12.0; extra == "docs"
|
|
57
|
+
Requires-Dist: dkist-sphinx-theme; extra == "docs"
|
|
58
|
+
Provides-Extra: inventory
|
|
59
|
+
Requires-Dist: dkist-inventory<2.0,>=1.6.0; extra == "inventory"
|
|
60
|
+
Provides-Extra: asdf
|
|
61
|
+
Requires-Dist: dkist-inventory[asdf]<2.0,>=1.6.0; extra == "asdf"
|
|
62
|
+
Provides-Extra: quality
|
|
63
|
+
Requires-Dist: dkist-quality<2.0,>=1.2.1; extra == "quality"
|
|
64
|
+
|
|
65
|
+
dkist-processing-common
|
|
66
|
+
=======================
|
|
67
|
+
|
|
68
|
+
This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-*instrument*` to
|
|
69
|
+
form the DKIST calibration processing stack.
|
|
70
|
+
|
|
71
|
+
Usage
|
|
72
|
+
-----
|
|
73
|
+
|
|
74
|
+
The classes in this repository should be used as the base of any DKIST processing pipeline tasks. Science tasks should subclass `ScienceTaskL0ToL1Base`.
|
|
75
|
+
|
|
76
|
+
Each class is built on an abstract base class with the `run` method left for a developer to fill out with the required steps that the task should take.
|
|
77
|
+
This class is then used as the callable object for the workflow and scheduling engine.
|
|
78
|
+
|
|
79
|
+
Example
|
|
80
|
+
-------
|
|
81
|
+
|
|
82
|
+
.. code-block:: python
|
|
83
|
+
|
|
84
|
+
from dkist_processing_common.tasks.base import ScienceTaskL0ToL1Base
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class RemoveArtifacts(ScienceTaskL0ToL1Base):
|
|
88
|
+
def run(self):
|
|
89
|
+
# task code here
|
|
90
|
+
total = 2 + 5
|
|
91
|
+
|
|
92
|
+
Deployment
|
|
93
|
+
----------
|
|
94
|
+
|
|
95
|
+
dkist-processing-common is deployed to `PyPI <https://pypi.org/project/dkist-processing-common/>`_
|
|
96
|
+
|
|
97
|
+
Development
|
|
98
|
+
-----------
|
|
99
|
+
|
|
100
|
+
There are two prerequisites for test execution on a local machine:
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
* Redis. A running instance of redis on the local machine is required. The tests will use the default host ip of localhost and port of 6379 to connect to the database.
|
|
104
|
+
|
|
105
|
+
* RabbitMQ. A running instance of rabbitmq on the local machine is required. The tests will use the default host of localhost and a port of 5672 to connect to the interservice bus.
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
To run the tests locally, clone the repository and install the package in editable mode with the test extras.
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
.. code-block:: bash
|
|
112
|
+
|
|
113
|
+
git clone git@bitbucket.org:dkistdc/dkist-processing-common.git
|
|
114
|
+
cd dkist-processing-common
|
|
115
|
+
pre-commit install
|
|
116
|
+
pip install -e .[test]
|
|
117
|
+
# Redis must be running
|
|
118
|
+
pytest -v --cov dkist_processing_common
|
|
119
|
+
|
|
120
|
+
Changelog
|
|
121
|
+
#########
|
|
122
|
+
|
|
123
|
+
When you make **any** change to this repository it **MUST** be accompanied by a changelog file.
|
|
124
|
+
The changelog for this repository uses the `towncrier <https://github.com/twisted/towncrier>`__ package.
|
|
125
|
+
Entries in the changelog for the next release are added as individual files (one per change) to the ``changelog/`` directory.
|
|
126
|
+
|
|
127
|
+
Writing a Changelog Entry
|
|
128
|
+
^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
129
|
+
|
|
130
|
+
A changelog entry accompanying a change should be added to the ``changelog/`` directory.
|
|
131
|
+
The name of a file in this directory follows a specific template::
|
|
132
|
+
|
|
133
|
+
<PULL REQUEST NUMBER>.<TYPE>[.<COUNTER>].rst
|
|
134
|
+
|
|
135
|
+
The fields have the following meanings:
|
|
136
|
+
|
|
137
|
+
* ``<PULL REQUEST NUMBER>``: This is the number of the pull request, so people can jump from the changelog entry to the diff on BitBucket.
|
|
138
|
+
* ``<TYPE>``: This is the type of the change and must be one of the values described below.
|
|
139
|
+
* ``<COUNTER>``: This is an optional field, if you make more than one change of the same type you can append a counter to the subsequent changes, i.e. ``100.bugfix.rst`` and ``100.bugfix.1.rst`` for two bugfix changes in the same PR.
|
|
140
|
+
|
|
141
|
+
The list of possible types is defined the the towncrier section of ``pyproject.toml``, the types are:
|
|
142
|
+
|
|
143
|
+
* ``feature``: This change is a new code feature.
|
|
144
|
+
* ``bugfix``: This is a change which fixes a bug.
|
|
145
|
+
* ``doc``: A documentation change.
|
|
146
|
+
* ``removal``: A deprecation or removal of public API.
|
|
147
|
+
* ``misc``: Any small change which doesn't fit anywhere else, such as a change to the package infrastructure.
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
Rendering the Changelog at Release Time
|
|
151
|
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
152
|
+
|
|
153
|
+
When you are about to tag a release first you must run ``towncrier`` to render the changelog.
|
|
154
|
+
The steps for this are as follows:
|
|
155
|
+
|
|
156
|
+
* Run `towncrier build --version vx.y.z` using the version number you want to tag.
|
|
157
|
+
* Agree to have towncrier remove the fragments.
|
|
158
|
+
* Add and commit your changes.
|
|
159
|
+
* Tag the release.
|
|
160
|
+
|
|
161
|
+
**NOTE:** If you forget to add a Changelog entry to a tagged release (either manually or automatically with ``towncrier``)
|
|
162
|
+
then the Bitbucket pipeline will fail. To be able to use the same tag you must delete it locally and on the remote branch:
|
|
163
|
+
|
|
164
|
+
.. code-block:: bash
|
|
165
|
+
|
|
166
|
+
# First, actually update the CHANGELOG and commit the update
|
|
167
|
+
git commit
|
|
168
|
+
|
|
169
|
+
# Delete tags
|
|
170
|
+
git tag -d vWHATEVER.THE.VERSION
|
|
171
|
+
git push --delete origin vWHATEVER.THE.VERSION
|
|
172
|
+
|
|
173
|
+
# Re-tag with the same version
|
|
174
|
+
git tag vWHATEVER.THE.VERSION
|
|
175
|
+
git push --tags origin main
|
{dkist_processing_common-10.6.1rc3.dist-info → dkist_processing_common-10.6.1rc5.dist-info}/RECORD
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
changelog/236.misc.1.rst,sha256=
|
|
3
|
-
changelog/236.misc.rst,sha256=
|
|
2
|
+
changelog/236.misc.1.rst,sha256=wN9uYenjgu25jpjAcLtCxLIAlLz658rqAV6sIvWBulI,158
|
|
3
|
+
changelog/236.misc.rst,sha256=V_aMXk7hePoQxIaiXCh7IKrvqe3Z0R_JYotQRMBAg9o,315
|
|
4
4
|
dkist_processing_common/__init__.py,sha256=490Fwm_GgqpwriQlsYfKcLUZNhZ6GkINtJqcYSIEKoU,319
|
|
5
5
|
dkist_processing_common/config.py,sha256=IcpaD_NvHZU-aLlUNOTdRC4V7ADIvVQwrZ2dHhIr4NY,4247
|
|
6
6
|
dkist_processing_common/manual.py,sha256=4sGj3cChdnEpwzq2GSNSbTpUf2oS3YUWiXna6ALfMF4,7037
|
|
@@ -24,7 +24,7 @@ dkist_processing_common/models/__init__.py,sha256=6LMqemdzVZ87fRrpAsbEnTtWZ02_Gu
|
|
|
24
24
|
dkist_processing_common/models/constants.py,sha256=1Eb8RDeuCr6brl237iGKxYLWCH49I6bOUEj_Tv-zFbQ,5441
|
|
25
25
|
dkist_processing_common/models/fits_access.py,sha256=Au9JROwhVla9zb_u0dN8mIWiSJd_Pca0oOr4N1hN0HY,4113
|
|
26
26
|
dkist_processing_common/models/flower_pot.py,sha256=59C5uGYKyMyncqQYxhzDZWl8k1DRZFB6s9RF-HFp9mY,5128
|
|
27
|
-
dkist_processing_common/models/graphql.py,sha256=
|
|
27
|
+
dkist_processing_common/models/graphql.py,sha256=t2PYEjEwSq08zq0HcVH2RG5FX4PVXgs4BdM2gunJ8fs,4747
|
|
28
28
|
dkist_processing_common/models/message.py,sha256=DRW7Qhl01dF5KagcqLta5U-uzdOMewrsHvMatDT6jnk,1684
|
|
29
29
|
dkist_processing_common/models/message_queue_binding.py,sha256=ROQ2ZQE3TCr4gVbz4WggvUSExAiWP8SD_GjjQl482M8,1012
|
|
30
30
|
dkist_processing_common/models/metric_code.py,sha256=McXAEF1Sa0_YlR1niXYLJWLFHhdLQhmYw9Xtpr5FGws,815
|
|
@@ -53,25 +53,25 @@ dkist_processing_common/tasks/__init__.py,sha256=uH8DTiQP-cx4vMK53S4LYGZGmbip5s0
|
|
|
53
53
|
dkist_processing_common/tasks/assemble_movie.py,sha256=9K4sgXyRKaX7UsFBIs138pG3AtClwLLopYw3ZQY3ok4,12771
|
|
54
54
|
dkist_processing_common/tasks/base.py,sha256=k_IJR5sVV6ennX0sbeb0C6dciqshdY7CKjtWHy_adm8,13143
|
|
55
55
|
dkist_processing_common/tasks/l1_output_data.py,sha256=IM-nvGaTM5r-z-9vHr2wovPVUpuNCah-cWIFMO2fcII,10576
|
|
56
|
-
dkist_processing_common/tasks/output_data_base.py,sha256=
|
|
56
|
+
dkist_processing_common/tasks/output_data_base.py,sha256=CC1TnCrChi8_iuMymr425CJqpY4jCggnVUMfqzFkpnw,3682
|
|
57
57
|
dkist_processing_common/tasks/parse_l0_input_data.py,sha256=iRMGdvhxBobNsTDQ0IEl0myDfB4P_xpxA00guuBWDj8,7986
|
|
58
58
|
dkist_processing_common/tasks/quality_metrics.py,sha256=g6MUq8s8jELDinkn6o45rfONyODw92JyVMrzb7Dd7OI,12458
|
|
59
|
-
dkist_processing_common/tasks/teardown.py,sha256=
|
|
60
|
-
dkist_processing_common/tasks/transfer_input_data.py,sha256=
|
|
59
|
+
dkist_processing_common/tasks/teardown.py,sha256=e4LKnphJDYDVDAez2tH7MxpZgCmxYsKrq9Zk0qAkzzM,2355
|
|
60
|
+
dkist_processing_common/tasks/transfer_input_data.py,sha256=bv2t0DN7nQ9opfl3VPdXG9m69zdzNIjordlCyZVNyBQ,5324
|
|
61
61
|
dkist_processing_common/tasks/trial_catalog.py,sha256=Y3DKstRfMS8nWWtJFMB0MUVPlZ1jWS_2jhJGMWwxy50,8748
|
|
62
|
-
dkist_processing_common/tasks/trial_output_data.py,sha256=
|
|
63
|
-
dkist_processing_common/tasks/write_l1.py,sha256=
|
|
62
|
+
dkist_processing_common/tasks/trial_output_data.py,sha256=aI_aRuu0qVO8zFGrr_9baxx9i3jUEHZSmsmbO6ytlkE,6960
|
|
63
|
+
dkist_processing_common/tasks/write_l1.py,sha256=C5IRUX1JO_Wa7suv_tgE4tH1E2eAUkro0rtj9EHduqw,22429
|
|
64
64
|
dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
|
|
65
65
|
dkist_processing_common/tasks/mixin/globus.py,sha256=QAV8VElxMAqxJ2KSB_bJaraceovYfjHXjOdocrTCkIA,6592
|
|
66
66
|
dkist_processing_common/tasks/mixin/input_dataset.py,sha256=dkW5vf_QPgWedHO_Lf9GjBxr1QrUCKs6gIXufUTi7GE,6813
|
|
67
67
|
dkist_processing_common/tasks/mixin/interservice_bus.py,sha256=I7BUh0o8AEX-FZv7gxCts6is0uq9lycWjtTB2KqwBrU,1080
|
|
68
|
-
dkist_processing_common/tasks/mixin/metadata_store.py,sha256=
|
|
68
|
+
dkist_processing_common/tasks/mixin/metadata_store.py,sha256=dMwjlhBI89xX0Pagub2Bk09z0fToipFN51DwyQEIHbk,11510
|
|
69
69
|
dkist_processing_common/tasks/mixin/object_store.py,sha256=Vn4l2XuCimii9Fc3gM-pQGIkTKMv_ldqljlxkLesZLU,3236
|
|
70
70
|
dkist_processing_common/tasks/mixin/quality/__init__.py,sha256=Bgu-DHW7yXLiehglldOCWluEkAP5qh0Hp1F30rh5NFw,383
|
|
71
71
|
dkist_processing_common/tasks/mixin/quality/_base.py,sha256=nZ9IC-O-hsLXa5-tk29B13CZyQIdhJCv0eO9cdkAhWc,8303
|
|
72
72
|
dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=WenTfa12guIUfm0GzkrK2gduKaOHs03e6RhE6j37Les,54304
|
|
73
73
|
dkist_processing_common/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
|
-
dkist_processing_common/tests/conftest.py,sha256=
|
|
74
|
+
dkist_processing_common/tests/conftest.py,sha256=B4BtMYCStjy3zvqefwJB6-ntSNOicIyQwfE0YeapCEE,30160
|
|
75
75
|
dkist_processing_common/tests/test_assemble_movie.py,sha256=XY_ruXSYP5k6s2gUAwlFdnhJ81eyWLSd2O9IkX4RXeo,4165
|
|
76
76
|
dkist_processing_common/tests/test_assemble_quality.py,sha256=fWSHK4UdVqgNjvxQuD40NBUnXrtmthUP7PUbISPV4MQ,16897
|
|
77
77
|
dkist_processing_common/tests/test_base.py,sha256=4ST3__jEHitEQaQs9-0OcqtyEJfIjZsk_6PRYZFV2-U,7124
|
|
@@ -80,7 +80,7 @@ dkist_processing_common/tests/test_constants.py,sha256=Kc9k5TdYy5QkRRlGav6kfI2dy
|
|
|
80
80
|
dkist_processing_common/tests/test_cs_step.py,sha256=RA0QD3D8eaL3YSOL_gIJ9wkngy14RQ2jbD-05KAziW4,2408
|
|
81
81
|
dkist_processing_common/tests/test_fits_access.py,sha256=e1B_N13vz6hFCBup6FlGItaH59dAmqXvox7jQuSyOB0,10946
|
|
82
82
|
dkist_processing_common/tests/test_flower_pot.py,sha256=X9_UI3maa3ZQncV3jYHgovWnawDsdEkEB5vw6EAB96o,3151
|
|
83
|
-
dkist_processing_common/tests/test_input_dataset.py,sha256=
|
|
83
|
+
dkist_processing_common/tests/test_input_dataset.py,sha256=AI5uqaDea4kOwpwAU5qQdzUbxMpBwD20YCAvB7nzD5o,18766
|
|
84
84
|
dkist_processing_common/tests/test_interservice_bus.py,sha256=M_iv2CLmx5TnCB1VUN4YjkQ2LEUjfCKk7-ZlkV62XEQ,3000
|
|
85
85
|
dkist_processing_common/tests/test_interservice_bus_mixin.py,sha256=8TTl0aypkq5gBPeyqSaQHbz_jmt5RmSD2oI8kT4Q1ZA,4195
|
|
86
86
|
dkist_processing_common/tests/test_output_data_base.py,sha256=Y9MFz5xw11tAnKjpHH7qrzsRYP1rZM_Trt4AylY0S6k,3120
|
|
@@ -95,13 +95,13 @@ dkist_processing_common/tests/test_submit_dataset_metadata.py,sha256=F1IKBFWhjjM
|
|
|
95
95
|
dkist_processing_common/tests/test_tags.py,sha256=UwlOJ45rkvbfbd5L5m5YltvOxQc8kGqJEn5V81H33U8,5023
|
|
96
96
|
dkist_processing_common/tests/test_task_name.py,sha256=kqFr59XX2K87xzfTlClzDV4-Je1dx72LvdaJ22UE8UU,1233
|
|
97
97
|
dkist_processing_common/tests/test_task_parsing.py,sha256=QXt1X6DTO3_liBD2c-t84DToLeEn7B3J-eteIyN4HEM,4027
|
|
98
|
-
dkist_processing_common/tests/test_teardown.py,sha256=
|
|
99
|
-
dkist_processing_common/tests/test_transfer_input_data.py,sha256=
|
|
98
|
+
dkist_processing_common/tests/test_teardown.py,sha256=w2sATQHkg2lMLvm6VFZF1mNGFYHwWj_SxvF9RQu-tuY,5362
|
|
99
|
+
dkist_processing_common/tests/test_transfer_input_data.py,sha256=MYPsZldzQ_j0AoGFqP6_ahn8Nr5mX8WAyG0wpcQReeI,4735
|
|
100
100
|
dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=27PifkyH3RZg0nsM-AjmrFJ-hbYuCk5Tt_0Zx8PJBfM,2109
|
|
101
|
-
dkist_processing_common/tests/test_trial_catalog.py,sha256=
|
|
102
|
-
dkist_processing_common/tests/test_trial_output_data.py,sha256=
|
|
101
|
+
dkist_processing_common/tests/test_trial_catalog.py,sha256=SZ-nyn0MXU9Lkg_94FbKER_cwiGoi06GYlzF_3AmvKg,6802
|
|
102
|
+
dkist_processing_common/tests/test_trial_output_data.py,sha256=cBCj0kXyF5NEMzKh6zPVksdoXyE8ju1opJgWgjdcJWA,12790
|
|
103
103
|
dkist_processing_common/tests/test_workflow_task_base.py,sha256=Z5aPW5LQtS0UWJiYho4X0r-2gPLfzpkmMwfmaoFLjMg,10517
|
|
104
|
-
dkist_processing_common/tests/test_write_l1.py,sha256=
|
|
104
|
+
dkist_processing_common/tests/test_write_l1.py,sha256=bIQd95Dhb8aHxCtOXV7Az4Fy_OeEIlspoSktdUR5lRA,20794
|
|
105
105
|
docs/Makefile,sha256=qnlVz6PuBqE39NfHWuUnHhNEA-EFgT2-WJNNNy9ttfk,4598
|
|
106
106
|
docs/changelog.rst,sha256=S2jPASsWlQxSlAPqdvNrYvhk9k3FcFWNXFNDYXBSjl4,120
|
|
107
107
|
docs/conf.py,sha256=FkX575cqTqZGCcLAjg2MlvE8Buj1Vt3CpHNgZxG256E,1890
|
|
@@ -110,7 +110,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
|
|
|
110
110
|
docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
|
|
111
111
|
docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
|
|
112
112
|
licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
|
|
113
|
-
dkist_processing_common-10.6.
|
|
114
|
-
dkist_processing_common-10.6.
|
|
115
|
-
dkist_processing_common-10.6.
|
|
116
|
-
dkist_processing_common-10.6.
|
|
113
|
+
dkist_processing_common-10.6.1rc5.dist-info/METADATA,sha256=zeyUZWd51nX30x1doSzvFHRNQraAhTthpfdoQCWgARM,6973
|
|
114
|
+
dkist_processing_common-10.6.1rc5.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
|
|
115
|
+
dkist_processing_common-10.6.1rc5.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
|
|
116
|
+
dkist_processing_common-10.6.1rc5.dist-info/RECORD,,
|