dkist-processing-common 10.6.3__tar.gz → 10.7.0__tar.gz

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/CHANGELOG.rst

@@ -1,3 +1,35 @@
+v10.7.0 (2025-04-14)
+====================
+
+Features
+--------
+
+- New data percentiles are added to the headers to make the range symmetric. (`#242 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/242>`__)
+- L1 headers are now created by removing only the keys listed in SPEC-122 as opposed to removing all keys not present in SPEC-214. In addition, any SPEC-214 keys marked as `level0_only` are removed. This will allow spurious header keys to be discovered more easily. (`#246 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/246>`__)
+
+
+Bugfixes
+--------
+
+- Header keys that write the percentiles of the data are now correctly named (from DATA<pp> to DATAP<pp>). (`#242 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/242>`__)
+
+
+Misc
+----
+
+- Prevent header keys from being included if they are not a part of the Level 1 SPEC-0214 specification. (`#242 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/242>`__)
+- Add missing build dependency specifications. (`#247 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/247>`__)
+
+
+v10.6.4 (2025-03-31)
+====================
+
+Bugfixes
+--------
+
+- Allow input dataset IDs to be conditionally written into the L1 data headers. (`#243 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/243>`__)
+
+
 v10.6.3 (2025-03-27)
 ====================
 

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.6.3
+Version: 10.7.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/bitbucket-pipelines.yml

@@ -45,7 +45,7 @@ definitions:
           - pip
         name: Test
         script:
-          - pip install -U pip
+          - pip install -U pip wheel build setuptools_scm setuptools
           - pip install .[test]
           - pytest -v -n auto -m "not development" --cov dkist_processing_common --cov-report=xml --cov-report=term-missing
           - ./ci_scripts/latest/execute_script.sh upload_coverage.sh

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/dkist_processing_common/tasks/write_l1.py

@@ -18,7 +18,7 @@ from dkist_fits_specifications import __version__ as spec_version
 from dkist_fits_specifications.utils.formatter import reformat_spec214_header
 from dkist_header_validator import spec214_validator
 from dkist_header_validator.translator import remove_extra_axis_keys
-from dkist_header_validator.translator import sanitize_to_spec214_level1
+from dkist_header_validator.translator import remove_spec_122_keys_and_spec_214_l0_keys
 from dkist_spectral_lines.search import get_closest_spectral_line
 from dkist_spectral_lines.search import get_spectral_lines
 from scipy.stats import kurtosis
@@ -102,7 +102,9 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
 
                 # Check that the written file passes spec 214 validation if requested
                 if self.validate_l1_on_write:
-                    spec214_validator.validate(self.scratch.absolute_path(relative_path))
+                    spec214_validator.validate(
+                        self.scratch.absolute_path(relative_path), extra=False
+                    )
 
     @cached_property
     def tile_size_param(self) -> int | None:
@@ -170,19 +172,21 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
     def add_stats_headers(header: fits.Header, data: np.ndarray) -> fits.Header:
         """Fill out the spec 214 statistics header table."""
         data = data.flatten()
-        percentiles = np.nanpercentile(data, [1, 10, 25, 75, 90, 95, 98, 99])
+        percentiles = np.nanpercentile(data, [1, 2, 5, 10, 25, 75, 90, 95, 98, 99])
         header["DATAMIN"] = np.nanmin(data)
         header["DATAMAX"] = np.nanmax(data)
         header["DATAMEAN"] = np.nanmean(data)
         header["DATAMEDN"] = np.nanmedian(data)
-        header["DATA01"] = percentiles[0]
-        header["DATA10"] = percentiles[1]
-        header["DATA25"] = percentiles[2]
-        header["DATA75"] = percentiles[3]
-        header["DATA90"] = percentiles[4]
-        header["DATA95"] = percentiles[5]
-        header["DATA98"] = percentiles[6]
-        header["DATA99"] = percentiles[7]
+        header["DATAP01"] = percentiles[0]
+        header["DATAP02"] = percentiles[1]
+        header["DATAP05"] = percentiles[2]
+        header["DATAP10"] = percentiles[3]
+        header["DATAP25"] = percentiles[4]
+        header["DATAP75"] = percentiles[5]
+        header["DATAP90"] = percentiles[6]
+        header["DATAP95"] = percentiles[7]
+        header["DATAP98"] = percentiles[8]
+        header["DATAP99"] = percentiles[9]
         header["DATARMS"] = np.sqrt(np.nanmean(data**2))
         header["DATAKURT"] = kurtosis(data, nan_policy="omit")
         header["DATASKEW"] = skew(data, nan_policy="omit")
@@ -214,11 +218,12 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header["CADMAX"] = self.constants.maximum_cadence
         header["CADVAR"] = self.constants.variance_cadence
         # Keywords to support reprocessing
-        if ids_par_id := self.metadata_store_input_dataset_parameters.inputDatasetPartId:
-            header["IDSPARID"] = ids_par_id
-        header["IDSOBSID"] = self.metadata_store_input_dataset_observe_frames.inputDatasetPartId
-        if ids_cal_id := self.metadata_store_input_dataset_calibration_frames.inputDatasetPartId:
-            header["IDSCALID"] = ids_cal_id
+        if parameters := self.metadata_store_input_dataset_parameters:
+            header["IDSPARID"] = parameters.inputDatasetPartId
+        if observe_frames := self.metadata_store_input_dataset_observe_frames:
+            header["IDSOBSID"] = observe_frames.inputDatasetPartId
+        if calibration_frames := self.metadata_store_input_dataset_calibration_frames:
+            header["IDSCALID"] = calibration_frames.inputDatasetPartId
         header["WKFLNAME"] = self.workflow_name
         header["WKFLVERS"] = self.workflow_version
         header = self.add_contributing_id_headers(header=header)
@@ -377,7 +382,7 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         # Add the dataset headers (abstract - implement in instrument task)
         header = self.add_dataset_headers(header=header, stokes=stokes_param)
         # Remove any headers not contained in spec 214
-        header = sanitize_to_spec214_level1(input_headers=header)
+        header = remove_spec_122_keys_and_spec_214_l0_keys(input_headers=header)
         # Remove any keys referring to axes that don't exist
         header = remove_extra_axis_keys(input_headers=header)
         return header

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/dkist_processing_common/tests/conftest.py

@@ -113,7 +113,7 @@ class CommonDataset(Spec122Dataset):
         self.add_constant_key("TELEVATN", 6.28)
         self.add_constant_key("TAZIMUTH", 3.14)
         self.add_constant_key("TTBLANGL", 1.23)
-        self.add_constant_key("INST_FOO", "bar")
+        self.add_constant_key("VISP_012", "bar")
         self.add_constant_key("DKIST004", "observe")
         self.add_constant_key("ID___005", "ip id")
         self.add_constant_key("PAC__004", "Sapphire Polarizer")

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/dkist_processing_common/tests/test_fits_access.py

@@ -220,7 +220,7 @@ def test_as_subclass(hdu_with_complete_common_header):
     class InstFitsAccess(L0FitsAccess):
         def __init__(self, hdu, name):
             super().__init__(hdu, name)
-            self.foo: str = self.header["INST_FOO"]
+            self.foo: str = self.header["VISP_012"]
 
     fits_obj = InstFitsAccess(hdu_with_complete_common_header, name="foo")
     assert fits_obj.foo == "bar"

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/dkist_processing_common/tests/test_transfer_input_data.py

@@ -6,6 +6,7 @@ import pytest
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.codecs.json import json_decoder
+from dkist_processing_common.models.graphql import InputDatasetRecipeRunResponse
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks.transfer_input_data import TransferL0Data
 from dkist_processing_common.tests.conftest import create_parameter_files
@@ -17,13 +18,14 @@ class TransferL0DataTask(TransferL0Data):
         ...
 
 
-class FakeGQLClientMissingPart(FakeGQLClient):
+class FakeGQLClientMissingInputDatasetPart(FakeGQLClient):
     """Same metadata mocker with calibration input dataset part missing."""
 
     def execute_gql_query(self, **kwargs):
         original_response = super().execute_gql_query(**kwargs)
-        # Remove calibration frames part
-        del original_response[0].recipeInstance.inputDataset.inputDatasetInputDatasetParts[2]
+        # Remove calibration frames part if getting InputDatasetRecipeRunResponse:
+        if kwargs.get("query_response_cls") == InputDatasetRecipeRunResponse:
+            del original_response[0].recipeInstance.inputDataset.inputDatasetInputDatasetParts[2]
         return original_response
 
 
@@ -53,7 +55,7 @@ def transfer_l0_data_task(recipe_run_id, tmp_path, mocker):
 @pytest.fixture
 def transfer_l0_data_task_missing_part(recipe_run_id, tmp_path, mocker):
     yield from _transfer_l0_data_task_with_client(
-        recipe_run_id, tmp_path, mocker, FakeGQLClientMissingPart
+        recipe_run_id, tmp_path, mocker, FakeGQLClientMissingInputDatasetPart
     )
 
 

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/dkist_processing_common/tests/test_write_l1.py

@@ -24,6 +24,9 @@ from dkist_processing_common.models.wavelength import WavelengthRange
 from dkist_processing_common.tasks.write_l1 import WriteL1Frame
 from dkist_processing_common.tests.conftest import FakeGQLClient
 from dkist_processing_common.tests.conftest import TILE_SIZE
+from dkist_processing_common.tests.test_transfer_input_data import (
+    FakeGQLClientMissingInputDatasetPart,
+)
 
 
 class FakeGQLClientDefaultRecipeConfiguration(FakeGQLClient):
@@ -132,6 +135,7 @@ def write_l1_task(request, recipe_run_id, tmp_path):
         stokes_params = ["I", "Q", "U", "V"]
         used_stokes_params = []
         hdu = fits.PrimaryHDU(data=np.random.random(size=(1, 128, 128)) * 10, header=header)
+        hdu.header["IPTASK"] = "level0_only key to be removed"
         hdul = fits.HDUList([hdu])
         for i in range(num_of_stokes_params):
             task.write(
@@ -276,6 +280,9 @@ def test_write_l1_frame(
             # Test that FRAMEVOL still has its comment
             assert hdu.header.comments["FRAMEVOL"]
 
+            # Test that 'level0_only' keys are being removed
+            assert "IPTASK" not in hdu.header.keys()
+
 
 def test_replace_header_values(write_l1_task):
     """
@@ -484,6 +491,29 @@ def test_reprocessing_keys(write_l1_task, mocker):
         assert header["PRODUCT"] == task.compute_product_id(header["IDSOBSID"], header["PROCTYPE"])
 
 
+def test_missing_input_dataset_part(write_l1_task, mocker):
+    """
+    :Given: a Write_L1 task with a missing calibration frames part
+    :When: running the task
+    :Then: the input dataset part keys are correctly written without throwing an exception
+    """
+    mocker.patch(
+        "dkist_processing_common.tasks.mixin.metadata_store.GraphQLClient",
+        new=FakeGQLClientMissingInputDatasetPart,
+    )
+    task, _, _ = write_l1_task
+    task()
+    files = list(task.read(tags=[Tag.frame(), Tag.output()]))
+    for file in files:
+        header = fits.open(file)[1].header
+        assert header["IDSPARID"] == task.metadata_store_input_dataset_parameters.inputDatasetPartId
+        assert (
+            header["IDSOBSID"]
+            == task.metadata_store_input_dataset_observe_frames.inputDatasetPartId
+        )
+        assert "IDSCALID" not in header
+
+
 @pytest.mark.parametrize(
     "ids_obs_id, proc_type",
     [

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 10.6.3
+Version: 10.7.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-10.6.3 → dkist_processing_common-10.7.0}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=64", "setuptools_scm>=8"]
+requires = ["setuptools>=64", "setuptools_scm>=8", "wheel", "build"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages]