dkist-processing-common 10.6.4__py3-none-any.whl → 10.7.1__py3-none-any.whl

dkist_processing_common/tasks/write_l1.py

@@ -18,7 +18,7 @@ from dkist_fits_specifications import __version__ as spec_version
 from dkist_fits_specifications.utils.formatter import reformat_spec214_header
 from dkist_header_validator import spec214_validator
 from dkist_header_validator.translator import remove_extra_axis_keys
-from dkist_header_validator.translator import sanitize_to_spec214_level1
+from dkist_header_validator.translator import remove_spec_122_keys_and_spec_214_l0_keys
 from dkist_spectral_lines.search import get_closest_spectral_line
 from dkist_spectral_lines.search import get_spectral_lines
 from scipy.stats import kurtosis
@@ -102,7 +102,9 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
 
                 # Check that the written file passes spec 214 validation if requested
                 if self.validate_l1_on_write:
-                    spec214_validator.validate(self.scratch.absolute_path(relative_path))
+                    spec214_validator.validate(
+                        self.scratch.absolute_path(relative_path), extra=False
+                    )
 
     @cached_property
     def tile_size_param(self) -> int | None:
@@ -170,19 +172,21 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
     def add_stats_headers(header: fits.Header, data: np.ndarray) -> fits.Header:
         """Fill out the spec 214 statistics header table."""
         data = data.flatten()
-        percentiles = np.nanpercentile(data, [1, 10, 25, 75, 90, 95, 98, 99])
+        percentiles = np.nanpercentile(data, [1, 2, 5, 10, 25, 75, 90, 95, 98, 99])
         header["DATAMIN"] = np.nanmin(data)
         header["DATAMAX"] = np.nanmax(data)
         header["DATAMEAN"] = np.nanmean(data)
         header["DATAMEDN"] = np.nanmedian(data)
-        header["DATA01"] = percentiles[0]
-        header["DATA10"] = percentiles[1]
-        header["DATA25"] = percentiles[2]
-        header["DATA75"] = percentiles[3]
-        header["DATA90"] = percentiles[4]
-        header["DATA95"] = percentiles[5]
-        header["DATA98"] = percentiles[6]
-        header["DATA99"] = percentiles[7]
+        header["DATAP01"] = percentiles[0]
+        header["DATAP02"] = percentiles[1]
+        header["DATAP05"] = percentiles[2]
+        header["DATAP10"] = percentiles[3]
+        header["DATAP25"] = percentiles[4]
+        header["DATAP75"] = percentiles[5]
+        header["DATAP90"] = percentiles[6]
+        header["DATAP95"] = percentiles[7]
+        header["DATAP98"] = percentiles[8]
+        header["DATAP99"] = percentiles[9]
         header["DATARMS"] = np.sqrt(np.nanmean(data**2))
         header["DATAKURT"] = kurtosis(data, nan_policy="omit")
         header["DATASKEW"] = skew(data, nan_policy="omit")
@@ -378,7 +382,7 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         # Add the dataset headers (abstract - implement in instrument task)
         header = self.add_dataset_headers(header=header, stokes=stokes_param)
         # Remove any headers not contained in spec 214
-        header = sanitize_to_spec214_level1(input_headers=header)
+        header = remove_spec_122_keys_and_spec_214_l0_keys(input_headers=header)
         # Remove any keys referring to axes that don't exist
         header = remove_extra_axis_keys(input_headers=header)
         return header

dkist_processing_common/tests/conftest.py

@@ -113,7 +113,7 @@ class CommonDataset(Spec122Dataset):
         self.add_constant_key("TELEVATN", 6.28)
         self.add_constant_key("TAZIMUTH", 3.14)
         self.add_constant_key("TTBLANGL", 1.23)
-        self.add_constant_key("INST_FOO", "bar")
+        self.add_constant_key("VISP_012", "bar")
         self.add_constant_key("DKIST004", "observe")
         self.add_constant_key("ID___005", "ip id")
         self.add_constant_key("PAC__004", "Sapphire Polarizer")

dkist_processing_common/tests/test_fits_access.py

@@ -22,19 +22,21 @@ def hdu_with_complete_common_header(complete_common_header):
 def hdu_with_special_keys(hdu_with_complete_common_header, request):
     """
     An HDU with data and a header that includes variations on the special keys that make headers
-    different from dict's.
+    different from dicts.
+
+    The base complete_common_header already has history and comment cards so this fixture adds extras.
     """
     add_special = request.param
     hdu = hdu_with_complete_common_header
     if add_special == "history":
         hdu.header.add_history("test history")
-        return hdu
+        return hdu, request.param
     if add_special == "comment":
-        hdu.header.add_history("test comment")
-        return hdu
+        hdu.header.add_comment("test comment")
+        return hdu, request.param
     if add_special == "continue":
         hdu.header["LONG_VAL"] = " " * 100
-        return hdu
+        return hdu, request.param
 
 
 @pytest.fixture()
@@ -220,7 +222,7 @@ def test_as_subclass(hdu_with_complete_common_header):
     class InstFitsAccess(L0FitsAccess):
         def __init__(self, hdu, name):
             super().__init__(hdu, name)
-            self.foo: str = self.header["INST_FOO"]
+            self.foo: str = self.header["VISP_012"]
 
     fits_obj = InstFitsAccess(hdu_with_complete_common_header, name="foo")
     assert fits_obj.foo == "bar"
@@ -282,11 +284,16 @@ def test_header_dict(hdu_with_special_keys):
     """
     Given: A FitsAccess object with data and a header including special header keys
     When: Accessing the header_dict method
-    Then: The object's header is successfully exported as a dict of the same length as the header
+    Then: The object's header is successfully exported as a dict of the same length as the header when accounting for multiple HISTORY or COMMENT cards
     """
-    fits_obj = FitsAccessBase(hdu_with_special_keys)
+    hdu, added_header_key = hdu_with_special_keys
+    fits_obj = FitsAccessBase(hdu)
     assert isinstance(fits_obj.header_dict, dict)
-    assert len(fits_obj.header_dict) == len(fits_obj.header)
+    if added_header_key == "continue":
+        assert len(fits_obj.header_dict) == len(fits_obj.header)
+    if added_header_key in ["history", "comment"]:
+        # HISTORY and COMMENT keys get "squeezed"
+        assert len(fits_obj.header_dict) == len(fits_obj.header) - 1
 
 
 @pytest.mark.parametrize(

dkist_processing_common/tests/test_write_l1.py

@@ -135,6 +135,7 @@ def write_l1_task(request, recipe_run_id, tmp_path):
         stokes_params = ["I", "Q", "U", "V"]
         used_stokes_params = []
         hdu = fits.PrimaryHDU(data=np.random.random(size=(1, 128, 128)) * 10, header=header)
+        hdu.header["IPTASK"] = "level0_only key to be removed"
         hdul = fits.HDUList([hdu])
         for i in range(num_of_stokes_params):
             task.write(
@@ -279,6 +280,9 @@ def test_write_l1_frame(
             # Test that FRAMEVOL still has its comment
             assert hdu.header.comments["FRAMEVOL"]
 
+            # Test that 'level0_only' keys are being removed
+            assert "IPTASK" not in hdu.header.keys()
+
 
 def test_replace_header_values(write_l1_task):
     """

{dkist_processing_common-10.6.4.dist-info → dkist_processing_common-10.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.6.4
+Version: 10.7.1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-10.6.4.dist-info → dkist_processing_common-10.7.1.dist-info}/RECORD

@@ -58,7 +58,7 @@ dkist_processing_common/tasks/teardown.py,sha256=e4LKnphJDYDVDAez2tH7MxpZgCmxYsK
 dkist_processing_common/tasks/transfer_input_data.py,sha256=afEW0glpCFMZRj90nFtQo_4XOQ4CuoOh86jahP6a-a0,5548
 dkist_processing_common/tasks/trial_catalog.py,sha256=Y3DKstRfMS8nWWtJFMB0MUVPlZ1jWS_2jhJGMWwxy50,8748
 dkist_processing_common/tasks/trial_output_data.py,sha256=aI_aRuu0qVO8zFGrr_9baxx9i3jUEHZSmsmbO6ytlkE,6960
-dkist_processing_common/tasks/write_l1.py,sha256=XhWLOw4hmql3n01SU26B0McPfYD_7oRKHna0Tswn8Lk,22494
+dkist_processing_common/tasks/write_l1.py,sha256=Jw8zQjlYjBQru3m1SB_ZDmicMkDRiUihy9VdwehvZc0,22683
 dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
 dkist_processing_common/tasks/mixin/globus.py,sha256=QAV8VElxMAqxJ2KSB_bJaraceovYfjHXjOdocrTCkIA,6592
 dkist_processing_common/tasks/mixin/input_dataset.py,sha256=dkW5vf_QPgWedHO_Lf9GjBxr1QrUCKs6gIXufUTi7GE,6813
@@ -69,14 +69,14 @@ dkist_processing_common/tasks/mixin/quality/__init__.py,sha256=Bgu-DHW7yXLiehgll
 dkist_processing_common/tasks/mixin/quality/_base.py,sha256=nZ9IC-O-hsLXa5-tk29B13CZyQIdhJCv0eO9cdkAhWc,8303
 dkist_processing_common/tasks/mixin/quality/_metrics.py,sha256=WenTfa12guIUfm0GzkrK2gduKaOHs03e6RhE6j37Les,54304
 dkist_processing_common/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dkist_processing_common/tests/conftest.py,sha256=B4BtMYCStjy3zvqefwJB6-ntSNOicIyQwfE0YeapCEE,30160
+dkist_processing_common/tests/conftest.py,sha256=h_ObhpXb1S0-db0Je8XoHNLkrXxXW_0B0TMhbnbDjMo,30160
 dkist_processing_common/tests/test_assemble_movie.py,sha256=XY_ruXSYP5k6s2gUAwlFdnhJ81eyWLSd2O9IkX4RXeo,4165
 dkist_processing_common/tests/test_assemble_quality.py,sha256=fWSHK4UdVqgNjvxQuD40NBUnXrtmthUP7PUbISPV4MQ,16897
 dkist_processing_common/tests/test_base.py,sha256=4ST3__jEHitEQaQs9-0OcqtyEJfIjZsk_6PRYZFV2-U,7124
 dkist_processing_common/tests/test_codecs.py,sha256=9Ln8FJs319rbHpCukO9lKLk3aDrdyDREjA4nCHsxDCA,20796
 dkist_processing_common/tests/test_constants.py,sha256=Kc9k5TdYy5QkRRlGav6kfI2dy5HHKqtpf9qOuaAfDZU,5903
 dkist_processing_common/tests/test_cs_step.py,sha256=RA0QD3D8eaL3YSOL_gIJ9wkngy14RQ2jbD-05KAziW4,2408
-dkist_processing_common/tests/test_fits_access.py,sha256=e1B_N13vz6hFCBup6FlGItaH59dAmqXvox7jQuSyOB0,10946
+dkist_processing_common/tests/test_fits_access.py,sha256=tyUPNbUqoTPhQgzua_doWP9l9ee4ir_LLV-I9rHktcs,11393
 dkist_processing_common/tests/test_flower_pot.py,sha256=X9_UI3maa3ZQncV3jYHgovWnawDsdEkEB5vw6EAB96o,3151
 dkist_processing_common/tests/test_input_dataset.py,sha256=AI5uqaDea4kOwpwAU5qQdzUbxMpBwD20YCAvB7nzD5o,18766
 dkist_processing_common/tests/test_interservice_bus.py,sha256=M_iv2CLmx5TnCB1VUN4YjkQ2LEUjfCKk7-ZlkV62XEQ,3000
@@ -100,7 +100,7 @@ dkist_processing_common/tests/test_transfer_l1_output_data.py,sha256=27PifkyH3RZ
 dkist_processing_common/tests/test_trial_catalog.py,sha256=SZ-nyn0MXU9Lkg_94FbKER_cwiGoi06GYlzF_3AmvKg,6802
 dkist_processing_common/tests/test_trial_output_data.py,sha256=cBCj0kXyF5NEMzKh6zPVksdoXyE8ju1opJgWgjdcJWA,12790
 dkist_processing_common/tests/test_workflow_task_base.py,sha256=Z5aPW5LQtS0UWJiYho4X0r-2gPLfzpkmMwfmaoFLjMg,10517
-dkist_processing_common/tests/test_write_l1.py,sha256=P_ep0_T2ZDZXPzFQmN7GgNs-RK6c5G6_DssPUW_sW7Q,21773
+dkist_processing_common/tests/test_write_l1.py,sha256=R_ljJdSC4hCHS_mLV6rKE7PVfpDWCaMXoN6n45QJNWM,21951
 docs/Makefile,sha256=qnlVz6PuBqE39NfHWuUnHhNEA-EFgT2-WJNNNy9ttfk,4598
 docs/changelog.rst,sha256=S2jPASsWlQxSlAPqdvNrYvhk9k3FcFWNXFNDYXBSjl4,120
 docs/conf.py,sha256=FkX575cqTqZGCcLAjg2MlvE8Buj1Vt3CpHNgZxG256E,1890
@@ -109,7 +109,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_common-10.6.4.dist-info/METADATA,sha256=xFPyZqrXkRO-vWgas9iUWZiduYlNn4qhPTOUCcfXigA,7154
-dkist_processing_common-10.6.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dkist_processing_common-10.6.4.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
-dkist_processing_common-10.6.4.dist-info/RECORD,,
+dkist_processing_common-10.7.1.dist-info/METADATA,sha256=7xA4uAWQZv0vUY7XMOiH-3g_G76q2YluFmK4967BK8s,7154
+dkist_processing_common-10.7.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dkist_processing_common-10.7.1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
+dkist_processing_common-10.7.1.dist-info/RECORD,,