dkist-processing-common 12.4.0__tar.gz → 12.4.2__tar.gz

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/CHANGELOG.rst

@@ -1,3 +1,27 @@
+v12.4.2 (2026-02-11)
+====================
+
+Features
+--------
+
+- Update `dkist-processing-common` to use `dkist-inventory` >= v1.11.4rc1 in order to add calibration and observation frames relevant to a particular processing pipeline run to the metadata ASDF file generated in trial workflows. (`#275 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/275>`__)
+
+
+Misc
+----
+
+- Remove towncrier dependency from the project. (`#299 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/299>`__)
+
+
+v12.4.1 (2026-02-04)
+====================
+
+Bugfixes
+--------
+
+- Use a different method for calculating DATA_RMS due to overflow with the current method. (`#296 <https://bitbucket.org/dkistdc/dkist-processing-common/pull-requests/296>`__)
+
+
 v12.4.0 (2026-02-04)
 ====================
 

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.4.0
+Version: 12.4.2
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -43,23 +43,22 @@ Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: hypothesis; extra == "test"
-Requires-Dist: towncrier; extra == "test"
 Requires-Dist: dkist-data-simulator>=5.2.6; extra == "test"
 Requires-Dist: dkist-processing-common[inventory]; extra == "test"
 Requires-Dist: dkist-processing-common[asdf]; extra == "test"
 Requires-Dist: dkist-processing-common[quality]; extra == "test"
 Provides-Extra: docs
+Requires-Dist: setuptools; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-astropy; extra == "docs"
 Requires-Dist: sphinx-changelog; extra == "docs"
 Requires-Dist: sphinx-autoapi!=3.1.0; extra == "docs"
 Requires-Dist: pytest; extra == "docs"
-Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.11.2; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.4; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.2; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.4; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/dkist_processing_common/tasks/trial_catalog.py

@@ -143,7 +143,17 @@ class CreateTrialAsdf(OutputDataBase):
     def run(self) -> None:
         """Generate an ASDF file simulating the ASDF file that would be produced when cataloging the dataset."""
         with self.telemetry_span("Collate input dataset parameters"):
-            parameters = self.parse_input_dataset_parameters()
+            parameters = self._parse_input_dataset_part_documents(Tag.input_dataset_parameters())
+
+        with self.telemetry_span("Collate input dataset observation frames"):
+            observation_frames = self._parse_input_dataset_part_documents(
+                Tag.input_dataset_observe_frames()
+            )
+
+        with self.telemetry_span("Collate input dataset calibration frames"):
+            calibration_frames = self._parse_input_dataset_part_documents(
+                Tag.input_dataset_calibration_frames()
+            )
 
         with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
@@ -151,6 +161,8 @@ class CreateTrialAsdf(OutputDataBase):
                 hdu=1,  # compressed
                 relative_to=self.scratch.workflow_base_path,
                 parameters=parameters,
+                observation_frames=observation_frames,
+                calibration_frames=calibration_frames,
             )
 
         trial_history = [
@@ -177,14 +189,19 @@ class CreateTrialAsdf(OutputDataBase):
                     ),
                 )
 
-    def parse_input_dataset_parameters(self) -> list[dict[str, Any]]:
+    def _parse_input_dataset_part_documents(self, tag: Tag) -> list[dict[str, Any]]:
         """
-        Return the parameters associated with the dataset.
+        Parse InputDatasetPartDocumentList records for a given tag.
+
+        Parameters
+        ----------
+        tag
+            The Tag used to locate the input dataset part documents.
 
         Returns
         -------
         list[dict[str, Any]]
-            A list of dictionaries, each containing a parameter name and its values.
+            A list of dictionaries representing the ``doc_list`` for the given tag.
 
         Raises
         ------
@@ -192,21 +209,20 @@ class CreateTrialAsdf(OutputDataBase):
             If there is not exactly one ``InputDatasetPartDocumentList`` found.
         """
         part_docs_iter = self.read(
-            tags=Tag.input_dataset_parameters(),
+            tags=tag,
             decoder=basemodel_decoder,
             model=InputDatasetPartDocumentList,
         )
         docs = list(part_docs_iter)
 
         if not docs:
-            logger.warning("No parameter list decoded from files")
+            logger.warning(f"No input dataset part document list decoded from files for tag {tag}")
             return []
 
         if len(docs) > 1:
-            raise ValueError(f"Expected 1 parameter list, found {len(docs)}")
+            raise ValueError(f"Expected 1 document list for tag {tag}, found {len(docs)}")
 
-        parameters = docs[0].model_dump(by_alias=True).get("doc_list", [])
-        return parameters
+        return docs[0].model_dump(by_alias=True).get("doc_list", [])
 
 
 class CreateTrialQualityReport(OutputDataBase):

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/dkist_processing_common/tasks/write_l1.py

@@ -152,7 +152,7 @@ class WriteL1Frame(WriteL1Base, ABC):
         header["DATAP95"] = percentiles[8]
         header["DATAP98"] = percentiles[9]
         header["DATAP99"] = percentiles[10]
-        header["DATARMS"] = np.sqrt(np.dot(valid_data, valid_data) / valid_data.size)
+        header["DATARMS"] = np.sqrt(np.mean(valid_data**2))
         header["DATAKURT"] = kurtosis(valid_data)
         header["DATASKEW"] = skew(valid_data)
         return header

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/dkist_processing_common/tests/test_trial_catalog.py

@@ -5,6 +5,7 @@ from string import ascii_uppercase
 from uuid import uuid4
 
 import astropy.units as u
+import numpy as np
 import pytest
 from astropy.io import fits
 from dkist_data_simulator.dataset_extras import DatasetExtraBase
@@ -19,12 +20,14 @@ from dkist_processing_common.codecs.bytes import bytes_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
 from dkist_processing_common.codecs.json import json_decoder
 from dkist_processing_common.codecs.quality import quality_data_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetFrames
 from dkist_processing_common.models.input_dataset import InputDatasetParameter
 from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.tasks import CreateTrialAsdf
 from dkist_processing_common.tasks import CreateTrialDatasetInventory
 from dkist_processing_common.tasks import CreateTrialQualityReport
+from dkist_processing_common.tests.mock_metadata_store import default_observe_frames_doc
 from dkist_processing_common.tests.mock_metadata_store import input_dataset_parameters_part_factory
 
 
@@ -80,6 +83,34 @@ def scratch_with_l1_frames_and_parameters(
     return scratch
 
 
+@pytest.fixture()
+def scratch_with_l1_frames_and_other_frames(
+    scratch_with_l1_frames,
+) -> WorkflowFileSystem:
+    """
+    Scratch instance for a recipe run id with tagged L1 frames and
+    input observation/calibration frames.
+    """
+    scratch = scratch_with_l1_frames
+
+    frames_doc = InputDatasetPartDocumentList(
+        doc_list=[InputDatasetFrames(**frame) for frame in default_observe_frames_doc]
+    )
+
+    file_obj = basemodel_encoder(frames_doc)
+    scratch.write(
+        file_obj,
+        tags=Tag.input_dataset_observe_frames(),
+        relative_path=f"{uuid4().hex}_obs_frames.json",
+    )
+    scratch.write(
+        file_obj,
+        tags=Tag.input_dataset_calibration_frames(),
+        relative_path=f"{uuid4().hex}_cal_frames.json",
+    )
+    return scratch
+
+
 @pytest.fixture()
 def create_trial_dataset_inventory_task(
     recipe_run_id,
@@ -137,6 +168,22 @@ def create_trial_asdf_task_with_params(
     task._purge()
 
 
+@pytest.fixture()
+def create_trial_asdf_task_with_frames(
+    recipe_run_id, tmp_path, scratch_with_l1_frames_and_other_frames, fake_constants_db
+) -> CreateTrialAsdf:
+    """An instance of CreateTrialAsdf with L1 frames and input frame docs tagged in scratch."""
+    task = CreateTrialAsdf(
+        recipe_run_id=recipe_run_id,
+        workflow_name="trial_asdf",
+        workflow_version="trial_asdf_version",
+    )
+    task.scratch = scratch_with_l1_frames_and_other_frames
+    task.constants._update(fake_constants_db)
+    yield task
+    task._purge()
+
+
 @pytest.fixture()
 def create_trial_quality_report_task(
     recipe_run_id, tmp_path, fake_constants_db
@@ -242,6 +289,61 @@ def test_create_trial_asdf(with_params, request, recipe_run_id, mock_input_datas
         assert ds.meta["parameters"] == []
 
 
+def test_create_trial_asdf_with_frames(create_trial_asdf_task_with_frames):
+    """
+    :Given: CreateTrialAsdf with L1 frames and input frame docs in scratch
+    :When: CreateTrialAsdf is run
+    :Then: The ASDF tree contains observation and calibration input frame metadata
+    """
+    task = create_trial_asdf_task_with_frames
+    # When
+    task()
+
+    # Then
+    asdf_tags = [Tag.output(), Tag.asdf()]
+    results = list(task.read(tags=asdf_tags, decoder=asdf_decoder))
+    assert len(results) == 1
+
+    tree = results[0]
+    assert isinstance(tree, dict)
+
+    ds = tree["dataset"]
+    assert "observation_input_frames" in ds.meta
+    assert "calibration_input_frames" in ds.meta
+
+    obs_frames = ds.meta["observation_input_frames"]
+    cal_frames = ds.meta["calibration_input_frames"]
+
+    assert isinstance(obs_frames, list)
+    assert isinstance(cal_frames, list)
+    assert len(obs_frames) == len(default_observe_frames_doc)
+    assert len(cal_frames) == len(default_observe_frames_doc)
+
+    obs_meta = obs_frames[0]
+    cal_meta = cal_frames[0]
+
+    assert isinstance(obs_meta["bucket"], str)
+    assert isinstance(cal_meta["bucket"], str)
+
+    obs_keys = np.asarray(obs_meta["object_keys"])
+    cal_keys = np.asarray(cal_meta["object_keys"])
+
+    if obs_keys.dtype.kind != "S":
+        obs_keys = obs_keys.astype("S")
+    if cal_keys.dtype.kind != "S":
+        cal_keys = cal_keys.astype("S")
+
+    assert obs_keys.dtype.kind == "S"
+    assert cal_keys.dtype.kind == "S"
+
+    obs_keys_str = [k.decode("ascii") for k in obs_keys.tolist()]
+    cal_keys_str = [k.decode("ascii") for k in cal_keys.tolist()]
+
+    expected_keys = set(default_observe_frames_doc[0]["object_keys"])
+    assert set(obs_keys_str) == expected_keys
+    assert set(cal_keys_str) == expected_keys
+
+
 def test_create_trial_quality_report(create_trial_quality_report_task):
     """
     :Given: An instance of CreateTrialQualityReport with tagged quality data

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 12.4.0
+Version: 12.4.2
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -43,23 +43,22 @@ Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: hypothesis; extra == "test"
-Requires-Dist: towncrier; extra == "test"
 Requires-Dist: dkist-data-simulator>=5.2.6; extra == "test"
 Requires-Dist: dkist-processing-common[inventory]; extra == "test"
 Requires-Dist: dkist-processing-common[asdf]; extra == "test"
 Requires-Dist: dkist-processing-common[quality]; extra == "test"
 Provides-Extra: docs
+Requires-Dist: setuptools; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-astropy; extra == "docs"
 Requires-Dist: sphinx-changelog; extra == "docs"
 Requires-Dist: sphinx-autoapi!=3.1.0; extra == "docs"
 Requires-Dist: pytest; extra == "docs"
-Requires-Dist: towncrier<22.12.0; extra == "docs"
 Requires-Dist: dkist-sphinx-theme; extra == "docs"
 Provides-Extra: inventory
-Requires-Dist: dkist-inventory<2.0,>=1.11.2; extra == "inventory"
+Requires-Dist: dkist-inventory<2.0,>=1.11.4; extra == "inventory"
 Provides-Extra: asdf
-Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.2; extra == "asdf"
+Requires-Dist: dkist-inventory[asdf]<2.0,>=1.11.4; extra == "asdf"
 Provides-Extra: quality
 Requires-Dist: dkist-quality<3.0,>=2.0.0; extra == "quality"
 

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/dkist_processing_common.egg-info/requires.txt

@@ -24,19 +24,19 @@ sunpy>=3.0.0
 talus==1.3.4
 
 [asdf]
-dkist-inventory[asdf]<2.0,>=1.11.2
+dkist-inventory[asdf]<2.0,>=1.11.4
 
 [docs]
+setuptools
 sphinx
 sphinx-astropy
 sphinx-changelog
 sphinx-autoapi!=3.1.0
 pytest
-towncrier<22.12.0
 dkist-sphinx-theme
 
 [inventory]
-dkist-inventory<2.0,>=1.11.2
+dkist-inventory<2.0,>=1.11.4
 
 [quality]
 dkist-quality<3.0,>=2.0.0
@@ -47,7 +47,6 @@ pytest-xdist
 pytest-cov
 pytest-mock
 hypothesis
-towncrier
 dkist-data-simulator>=5.2.6
 dkist-processing-common[inventory]
 dkist-processing-common[asdf]

{dkist_processing_common-12.4.0 → dkist_processing_common-12.4.2}/pyproject.toml

@@ -60,26 +60,25 @@ test = [
   "pytest-cov",
   "pytest-mock",
   "hypothesis",
-  "towncrier",
   "dkist-data-simulator >= 5.2.6",
   "dkist-processing-common[inventory]",
   "dkist-processing-common[asdf]",
   "dkist-processing-common[quality]",
 ]
 docs = [
+  "setuptools",
   "sphinx",
   "sphinx-astropy",
   "sphinx-changelog",
   "sphinx-autoapi != 3.1.0",
   "pytest",
-  "towncrier < 22.12.0",
   "dkist-sphinx-theme",
 ]
 inventory = [
-  "dkist-inventory >= 1.11.2, <2.0",
+  "dkist-inventory >= 1.11.4, <2.0",
 ]
 asdf = [
-  "dkist-inventory[asdf] >= 1.11.2, <2.0",
+  "dkist-inventory[asdf] >= 1.11.4, <2.0",
 ]
 quality = [
   "dkist-quality >= 2.0.0, <3.0",