dkist-processing-test 1.9.2rc1__tar.gz → 1.9.3rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dkist-processing-test might be problematic. Click here for more details.
- {dkist-processing-test-1.9.2rc1/dkist_processing_test.egg-info → dkist-processing-test-1.9.3rc1}/PKG-INFO +1 -1
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/fake_science.py +7 -5
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/movie.py +9 -5
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tests/test_tasks.py +16 -8
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1/dkist_processing_test.egg-info}/PKG-INFO +1 -1
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test.egg-info/requires.txt +1 -1
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/setup.cfg +1 -1
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/.gitignore +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/.pre-commit-config.yaml +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/README.rst +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/bitbucket-pipelines.yml +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/__init__.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/models/__init__.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/models/parameters.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/__init__.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/exercise_numba.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/fail.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/high_memory.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/noop.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/parse.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/quality.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/trial_output_data.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tasks/write_l1.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tests/__init__.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tests/conftest.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tests/test_parameters.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/tests/test_workflows.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/__init__.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/common_tasks.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/end_to_end.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/exercise_numba.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/fail.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/noop.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/resource_queue.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/workflows/trial_end_to_end.py +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test.egg-info/SOURCES.txt +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test.egg-info/dependency_links.txt +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test.egg-info/top_level.txt +0 -0
- {dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dkist-processing-test
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.3rc1
|
|
4
4
|
Summary: Example Instrument code that is used by the DKIST Science Data Processing pipelines to test processing infrastructure.
|
|
5
5
|
Home-page: https://bitbucket.org/dkistdc/dkist-processing-test/src/main/
|
|
6
6
|
Author: NSO / AURA
|
|
@@ -4,10 +4,11 @@ Fake science task
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
from astropy.io import fits
|
|
6
6
|
from dkist_processing_common.codecs.fits import fits_array_encoder
|
|
7
|
+
from dkist_processing_common.codecs.fits import fits_hdu_decoder
|
|
8
|
+
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
7
9
|
from dkist_processing_common.codecs.json import json_encoder
|
|
8
10
|
from dkist_processing_common.models.tags import Tag
|
|
9
11
|
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
10
|
-
from dkist_processing_common.tasks.mixin.fits import FitsDataMixin
|
|
11
12
|
from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
|
|
12
13
|
|
|
13
14
|
from dkist_processing_test.models.parameters import TestParameters
|
|
@@ -15,7 +16,7 @@ from dkist_processing_test.models.parameters import TestParameters
|
|
|
15
16
|
__all__ = ["GenerateCalibratedData"]
|
|
16
17
|
|
|
17
18
|
|
|
18
|
-
class GenerateCalibratedData(WorkflowTaskBase,
|
|
19
|
+
class GenerateCalibratedData(WorkflowTaskBase, InputDatasetMixin):
|
|
19
20
|
|
|
20
21
|
record_provenance = True
|
|
21
22
|
|
|
@@ -58,7 +59,7 @@ class GenerateCalibratedData(WorkflowTaskBase, FitsDataMixin, InputDatasetMixin)
|
|
|
58
59
|
|
|
59
60
|
with self.apm_task_step("Loop over inputs"):
|
|
60
61
|
count = 1 # keep a running count to increment the dsps repeat number
|
|
61
|
-
for hdu in self.
|
|
62
|
+
for hdu in self.read(tags=Tag.input(), decoder=fits_hdu_decoder):
|
|
62
63
|
header = hdu.header
|
|
63
64
|
with self.apm_processing_step("Doing some calculations"):
|
|
64
65
|
header["DSPSNUM"] = count
|
|
@@ -83,13 +84,14 @@ class GenerateCalibratedData(WorkflowTaskBase, FitsDataMixin, InputDatasetMixin)
|
|
|
83
84
|
|
|
84
85
|
with self.apm_writing_step("Writing data"):
|
|
85
86
|
output_hdul = fits.HDUList([output_hdu])
|
|
86
|
-
self.
|
|
87
|
-
|
|
87
|
+
self.write(
|
|
88
|
+
data=output_hdul,
|
|
88
89
|
tags=[
|
|
89
90
|
Tag.calibrated(),
|
|
90
91
|
Tag.frame(),
|
|
91
92
|
Tag.stokes("I"),
|
|
92
93
|
Tag.dsps_repeat(count),
|
|
93
94
|
],
|
|
95
|
+
encoder=fits_hdulist_encoder,
|
|
94
96
|
)
|
|
95
97
|
count += 1
|
|
@@ -2,18 +2,19 @@
|
|
|
2
2
|
Fake MakeMovieFrames and AssembleTestMovie
|
|
3
3
|
"""
|
|
4
4
|
from astropy.io import fits
|
|
5
|
+
from dkist_processing_common.codecs.fits import fits_hdu_decoder
|
|
6
|
+
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
5
7
|
from dkist_processing_common.models.tags import Tag
|
|
6
8
|
from dkist_processing_common.parsers.l1_fits_access import L1FitsAccess
|
|
7
9
|
from dkist_processing_common.tasks import AssembleMovie
|
|
8
10
|
from dkist_processing_common.tasks import WorkflowTaskBase
|
|
9
|
-
from dkist_processing_common.tasks.mixin.fits import FitsDataMixin
|
|
10
11
|
from PIL import ImageDraw
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
__all__ = ["MakeTestMovieFrames", "AssembleTestMovie"]
|
|
14
15
|
|
|
15
16
|
|
|
16
|
-
class MakeTestMovieFrames(WorkflowTaskBase
|
|
17
|
+
class MakeTestMovieFrames(WorkflowTaskBase):
|
|
17
18
|
"""
|
|
18
19
|
Take each output frame, copy the header and data and write out
|
|
19
20
|
as a movie frame
|
|
@@ -22,16 +23,19 @@ class MakeTestMovieFrames(WorkflowTaskBase, FitsDataMixin):
|
|
|
22
23
|
def run(self):
|
|
23
24
|
for d in range(1, self.constants.num_dsps_repeats + 1):
|
|
24
25
|
with self.apm_task_step(f"Workign on dsps repeat {d}"):
|
|
25
|
-
for hdu in self.
|
|
26
|
+
for hdu in self.read(
|
|
27
|
+
tags=[Tag.output(), Tag.dsps_repeat(d)], decoder=fits_hdu_decoder
|
|
28
|
+
):
|
|
26
29
|
header = hdu.header
|
|
27
30
|
data = hdu.data
|
|
28
31
|
output_hdu = fits.PrimaryHDU(data=data, header=header)
|
|
29
32
|
output_hdul = fits.HDUList([output_hdu])
|
|
30
33
|
|
|
31
34
|
with self.apm_writing_step("Writing data"):
|
|
32
|
-
self.
|
|
33
|
-
|
|
35
|
+
self.write(
|
|
36
|
+
data=output_hdul,
|
|
34
37
|
tags=[Tag.movie_frame(), Tag.dsps_repeat(d)],
|
|
38
|
+
encoder=fits_hdulist_encoder,
|
|
35
39
|
)
|
|
36
40
|
|
|
37
41
|
|
|
@@ -15,6 +15,7 @@ from dkist_data_simulator.spec122 import Spec122Dataset
|
|
|
15
15
|
from dkist_header_validator import spec122_validator
|
|
16
16
|
from dkist_processing_common._util.scratch import WorkflowFileSystem
|
|
17
17
|
from dkist_processing_common.codecs.fits import fits_hdu_decoder
|
|
18
|
+
from dkist_processing_common.codecs.fits import fits_hdulist_encoder
|
|
18
19
|
from dkist_processing_common.models.tags import Tag
|
|
19
20
|
from dkist_processing_common.tests.conftest import FakeGQLClient
|
|
20
21
|
from dkist_service_configuration import logger
|
|
@@ -109,11 +110,15 @@ def generate_calibrated_data_task(
|
|
|
109
110
|
hdu.header["DSPSNUM"] = 1
|
|
110
111
|
hdul = fits.HDUList([hdu])
|
|
111
112
|
file_name = f"input_{idx}.fits"
|
|
112
|
-
task.
|
|
113
|
+
task.write(
|
|
114
|
+
data=hdul, tags=Tag.input(), relative_path=file_name, encoder=fits_hdulist_encoder
|
|
115
|
+
)
|
|
113
116
|
|
|
114
117
|
# Write parameter file
|
|
115
118
|
hdul = random_parameter_hdulist[0]
|
|
116
|
-
task.
|
|
119
|
+
task.write(
|
|
120
|
+
data=hdul, tags=Tag.parameter(parameter_file_object_key), encoder=fits_hdulist_encoder
|
|
121
|
+
)
|
|
117
122
|
|
|
118
123
|
# This needs to be after we've tagged the parameter file
|
|
119
124
|
assign_input_dataset_doc_to_task(task)
|
|
@@ -293,9 +298,10 @@ def write_l1_task(complete_common_header, request):
|
|
|
293
298
|
logger.info(f"{num_of_stokes_params=}")
|
|
294
299
|
hdul = fits.HDUList([hdu])
|
|
295
300
|
for i in range(num_of_stokes_params):
|
|
296
|
-
task.
|
|
297
|
-
|
|
301
|
+
task.write(
|
|
302
|
+
data=hdul,
|
|
298
303
|
tags=[Tag.calibrated(), Tag.frame(), Tag.stokes(stokes_params[i])],
|
|
304
|
+
encoder=fits_hdulist_encoder,
|
|
299
305
|
)
|
|
300
306
|
task.constants._update(
|
|
301
307
|
asdict(
|
|
@@ -354,12 +360,13 @@ def make_movie_frames_task(tmp_path, recipe_run_id):
|
|
|
354
360
|
data = np.ones((10, 10))
|
|
355
361
|
data[: d * 10, :] = 0.0
|
|
356
362
|
hdl = generate_214_l0_fits_frame(data=data, s122_header=header)
|
|
357
|
-
task.
|
|
358
|
-
|
|
363
|
+
task.write(
|
|
364
|
+
data=hdl,
|
|
359
365
|
tags=[
|
|
360
366
|
Tag.output(),
|
|
361
367
|
Tag.dsps_repeat(d + 1),
|
|
362
368
|
],
|
|
369
|
+
encoder=fits_hdulist_encoder,
|
|
363
370
|
)
|
|
364
371
|
yield task
|
|
365
372
|
task._purge()
|
|
@@ -406,12 +413,13 @@ def assemble_test_movie_task(tmp_path, recipe_run_id):
|
|
|
406
413
|
data = np.ones((10, 10))
|
|
407
414
|
data[: d * 10, :] = 0.0
|
|
408
415
|
hdl = generate_214_l0_fits_frame(data=data, s122_header=header)
|
|
409
|
-
task.
|
|
410
|
-
|
|
416
|
+
task.write(
|
|
417
|
+
data=hdl,
|
|
411
418
|
tags=[
|
|
412
419
|
Tag.movie_frame(),
|
|
413
420
|
Tag.dsps_repeat(d + 1),
|
|
414
421
|
],
|
|
422
|
+
encoder=fits_hdulist_encoder,
|
|
415
423
|
)
|
|
416
424
|
yield task
|
|
417
425
|
task._purge()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dkist-processing-test
|
|
3
|
-
Version: 1.9.
|
|
3
|
+
Version: 1.9.3rc1
|
|
4
4
|
Summary: Example Instrument code that is used by the DKIST Science Data Processing pipelines to test processing infrastructure.
|
|
5
5
|
Home-page: https://bitbucket.org/dkistdc/dkist-processing-test/src/main/
|
|
6
6
|
Author: NSO / AURA
|
|
@@ -17,7 +17,7 @@ setup_requires = setuptools_scm
|
|
|
17
17
|
packages = find:
|
|
18
18
|
include_package_data = True
|
|
19
19
|
install_requires =
|
|
20
|
-
dkist-processing-common ==
|
|
20
|
+
dkist-processing-common == 6.0.0rc1
|
|
21
21
|
dkist-header-validator == 5.1.0
|
|
22
22
|
dkist-service-configuration == 1.1.0
|
|
23
23
|
dkist-fits-specifications == 4.1.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dkist-processing-test-1.9.2rc1 → dkist-processing-test-1.9.3rc1}/dkist_processing_test/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|