dkist-processing-cryonirsp 1.7.6__tar.gz → 1.7.7rc1__tar.gz

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-cryonirsp
-Version: 1.7.6
+Version: 1.7.7rc1
 Summary: Science processing code for the Cryo-NIRSP instrument on DKIST
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,7 +17,7 @@ Requires-Dist: Pillow==10.4.0
 Requires-Dist: astropy==7.0.2
 Requires-Dist: dkist-fits-specifications==4.17.0
 Requires-Dist: dkist-header-validator==5.2.1
-Requires-Dist: dkist-processing-common==10.8.3
+Requires-Dist: dkist-processing-common==10.8.4rc1
 Requires-Dist: dkist-processing-math==2.2.0
 Requires-Dist: dkist-processing-pac==3.1.1
 Requires-Dist: dkist-spectral-lines==3.0.0
@@ -88,7 +88,7 @@ Requires-Dist: WTForms==3.2.1; extra == "frozen"
 Requires-Dist: Werkzeug==2.2.3; extra == "frozen"
 Requires-Dist: aioftp==0.25.1; extra == "frozen"
 Requires-Dist: aiohappyeyeballs==2.6.1; extra == "frozen"
-Requires-Dist: aiohttp==3.12.4; extra == "frozen"
+Requires-Dist: aiohttp==3.12.6; extra == "frozen"
 Requires-Dist: aiosignal==1.3.2; extra == "frozen"
 Requires-Dist: alembic==1.16.1; extra == "frozen"
 Requires-Dist: amqp==5.3.1; extra == "frozen"
@@ -114,16 +114,16 @@ Requires-Dist: asdf_transform_schemas==0.6.0; extra == "frozen"
 Requires-Dist: asgiref==3.8.1; extra == "frozen"
 Requires-Dist: asteval==1.0.6; extra == "frozen"
 Requires-Dist: astropy==7.0.2; extra == "frozen"
-Requires-Dist: astropy-iers-data==0.2025.5.26.0.37.21; extra == "frozen"
+Requires-Dist: astropy-iers-data==0.2025.6.2.0.38.23; extra == "frozen"
 Requires-Dist: asyncpg==0.30.0; extra == "frozen"
 Requires-Dist: attrs==25.3.0; extra == "frozen"
 Requires-Dist: babel==2.17.0; extra == "frozen"
 Requires-Dist: billiard==4.2.1; extra == "frozen"
 Requires-Dist: blinker==1.9.0; extra == "frozen"
-Requires-Dist: boto3==1.38.26; extra == "frozen"
-Requires-Dist: botocore==1.38.26; extra == "frozen"
+Requires-Dist: boto3==1.38.27; extra == "frozen"
+Requires-Dist: botocore==1.38.27; extra == "frozen"
 Requires-Dist: cachelib==0.13.0; extra == "frozen"
-Requires-Dist: celery==5.5.2; extra == "frozen"
+Requires-Dist: celery==5.5.3; extra == "frozen"
 Requires-Dist: certifi==2025.4.26; extra == "frozen"
 Requires-Dist: cffi==1.17.1; extra == "frozen"
 Requires-Dist: charset-normalizer==3.4.2; extra == "frozen"
@@ -144,9 +144,9 @@ Requires-Dist: dacite==1.9.2; extra == "frozen"
 Requires-Dist: decorator==5.2.1; extra == "frozen"
 Requires-Dist: dill==0.4.0; extra == "frozen"
 Requires-Dist: dkist-header-validator==5.2.1; extra == "frozen"
-Requires-Dist: dkist-processing-common==10.8.3; extra == "frozen"
+Requires-Dist: dkist-processing-common==10.8.4rc1; extra == "frozen"
 Requires-Dist: dkist-processing-core==5.1.1; extra == "frozen"
-Requires-Dist: dkist-processing-cryonirsp==1.7.6; extra == "frozen"
+Requires-Dist: dkist-processing-cryonirsp==1.7.7rc1; extra == "frozen"
 Requires-Dist: dkist-processing-math==2.2.0; extra == "frozen"
 Requires-Dist: dkist-processing-pac==3.1.1; extra == "frozen"
 Requires-Dist: dkist-service-configuration==2.2; extra == "frozen"
@@ -165,7 +165,8 @@ Requires-Dist: globus-sdk==3.56.1; extra == "frozen"
 Requires-Dist: google-re2==1.1.20240702; extra == "frozen"
 Requires-Dist: googleapis-common-protos==1.70.0; extra == "frozen"
 Requires-Dist: gqlclient==1.2.3; extra == "frozen"
-Requires-Dist: grpcio==1.71.0; extra == "frozen"
+Requires-Dist: greenlet==3.2.2; extra == "frozen"
+Requires-Dist: grpcio==1.72.1; extra == "frozen"
 Requires-Dist: gunicorn==23.0.0; extra == "frozen"
 Requires-Dist: h11==0.16.0; extra == "frozen"
 Requires-Dist: httpcore==1.0.9; extra == "frozen"
@@ -182,7 +183,7 @@ Requires-Dist: jsonschema==4.24.0; extra == "frozen"
 Requires-Dist: jsonschema-specifications==2025.4.1; extra == "frozen"
 Requires-Dist: jupyter_core==5.8.1; extra == "frozen"
 Requires-Dist: kiwisolver==1.4.8; extra == "frozen"
-Requires-Dist: kombu==5.5.3; extra == "frozen"
+Requires-Dist: kombu==5.5.4; extra == "frozen"
 Requires-Dist: largestinteriorrectangle==0.2.1; extra == "frozen"
 Requires-Dist: lazy-object-proxy==1.11.0; extra == "frozen"
 Requires-Dist: lazy_loader==0.4; extra == "frozen"
@@ -229,7 +230,7 @@ Requires-Dist: platformdirs==4.3.8; extra == "frozen"
 Requires-Dist: pluggy==1.6.0; extra == "frozen"
 Requires-Dist: prison==0.2.1; extra == "frozen"
 Requires-Dist: proglog==0.1.12; extra == "frozen"
-Requires-Dist: prometheus_client==0.22.0; extra == "frozen"
+Requires-Dist: prometheus_client==0.22.1; extra == "frozen"
 Requires-Dist: prompt_toolkit==3.0.51; extra == "frozen"
 Requires-Dist: propcache==0.3.1; extra == "frozen"
 Requires-Dist: protobuf==5.29.5; extra == "frozen"
@@ -273,12 +274,12 @@ Requires-Dist: talus==1.1.0; extra == "frozen"
 Requires-Dist: tenacity==9.1.2; extra == "frozen"
 Requires-Dist: termcolor==3.1.0; extra == "frozen"
 Requires-Dist: text-unidecode==1.3; extra == "frozen"
-Requires-Dist: tifffile==2025.5.26; extra == "frozen"
+Requires-Dist: tifffile==2025.6.1; extra == "frozen"
 Requires-Dist: tornado==6.5.1; extra == "frozen"
 Requires-Dist: tqdm==4.67.1; extra == "frozen"
 Requires-Dist: traitlets==5.14.3; extra == "frozen"
 Requires-Dist: typing-inspection==0.4.1; extra == "frozen"
-Requires-Dist: typing_extensions==4.13.2; extra == "frozen"
+Requires-Dist: typing_extensions==4.14.0; extra == "frozen"
 Requires-Dist: tzdata==2025.2; extra == "frozen"
 Requires-Dist: uc-micro-py==1.0.3; extra == "frozen"
 Requires-Dist: uncertainties==3.2.3; extra == "frozen"

dkist_processing_cryonirsp-1.7.7rc1/changelog/190.misc.rst

@@ -0,0 +1 @@
+Remove use of input dataset mixin imported from dkist-processing-common.

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/models/parameters.py

@@ -136,8 +136,8 @@ class CryonirspParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmI
     @cached_property
     def linearization_thresholds(self) -> np.ndarray:
         """Name of parameter associated with the linearization thresholds."""
-        param_dict = self._find_parameter_for_arm("cryonirsp_linearization_thresholds")
-        value = self._load_param_value_from_numpy_save(param_dict)
+        param_obj = self._find_parameter_for_arm("cryonirsp_linearization_thresholds")
+        value = self._load_param_value_from_numpy_save(param_obj=param_obj)
         # float64 data can blow up the memory required for linearization - convert to float32
         if np.issubdtype(value.dtype, np.float64):
             value = value.astype(np.float32, casting="same_kind")
@@ -275,8 +275,8 @@ class CryonirspParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmI
         - wavelength in nanometers
         - transmission at given wavelength
         """
-        param_dict = self._find_most_recent_past_value("cryonirsp_solar_atlas")
-        return self._load_param_value_from_numpy_save(param_dict)
+        param_obj = self._find_most_recent_past_value("cryonirsp_solar_atlas")
+        return self._load_param_value_from_numpy_save(param_obj=param_obj)
 
     @cached_property
     def telluric_atlas(self) -> np.ndarray:
@@ -286,5 +286,5 @@ class CryonirspParameters(ParameterBase, ParameterWavelengthMixin, ParameterArmI
         - wavelength in nanometers
         - transmission at given wavelength
         """
-        param_dict = self._find_most_recent_past_value("cryonirsp_telluric_atlas")
-        return self._load_param_value_from_numpy_save(param_dict)
+        param_obj = self._find_most_recent_past_value("cryonirsp_telluric_atlas")
+        return self._load_param_value_from_numpy_save(param_obj=param_obj)

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tasks/cryonirsp_base.py

@@ -2,7 +2,6 @@
 from abc import ABC
 
 from dkist_processing_common.tasks import WorkflowTaskBase
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
 from dkist_processing_common.tasks.mixin.quality import QualityMixin
 
 from dkist_processing_cryonirsp.models.constants import CryonirspConstants
@@ -12,7 +11,6 @@ from dkist_processing_cryonirsp.tasks.mixin.corrections import CorrectionsMixin
 
 class CryonirspTaskBase(
     WorkflowTaskBase,
-    InputDatasetMixin,
     CorrectionsMixin,
     QualityMixin,
     ABC,
@@ -50,7 +48,7 @@ class CryonirspTaskBase(
             workflow_version=workflow_version,
         )
         self.parameters = CryonirspParameters(
-            self.input_dataset_parameters,
+            scratch=self.scratch,
             obs_ip_start_time=self.constants.obs_ip_start_time,
             wavelength=self.constants.wavelength,
             arm_id=self.constants.arm_id,

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tasks/parse.py

@@ -20,7 +20,6 @@ from dkist_processing_common.parsers.unique_bud import UniqueBud
 from dkist_processing_common.tasks import default_constant_bud_factory
 from dkist_processing_common.tasks import default_tag_flower_factory
 from dkist_processing_common.tasks import ParseDataBase
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
 
 from dkist_processing_cryonirsp.models.constants import CryonirspBudName
 from dkist_processing_cryonirsp.models.parameters import CryonirspParsingParameters
@@ -142,7 +141,7 @@ class ParseL0CryonirspRampData(ParseDataBase):
         return [Tag.input(), Tag.frame()]
 
 
-class ParseL0CryonirspLinearizedData(ParseDataBase, InputDatasetMixin):
+class ParseL0CryonirspLinearizedData(ParseDataBase):
     """
     Parse linearity corrected CryoNIRSP input data to add common and Cryonirsp specific constants.
 
@@ -168,7 +167,7 @@ class ParseL0CryonirspLinearizedData(ParseDataBase, InputDatasetMixin):
             workflow_name=workflow_name,
             workflow_version=workflow_version,
         )
-        self.parameters = CryonirspParsingParameters(self.input_dataset_parameters)
+        self.parameters = CryonirspParsingParameters(scratch=self.scratch)
 
     @property
     def fits_parsing_class(self):

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tasks/sp_dispersion_axis_correction.py

@@ -10,7 +10,6 @@ from astropy.io import fits
 from astropy.time import Time
 from astropy.wcs import WCS
 from dkist_processing_common.codecs.asdf import asdf_encoder
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
 from dkist_service_configuration.logging import logger
 from scipy.ndimage import gaussian_filter1d
 from scipy.optimize import differential_evolution
@@ -26,7 +25,7 @@ from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
 __all__ = ["SPDispersionAxisCorrection"]
 
 
-class SPDispersionAxisCorrection(CryonirspTaskBase, InputDatasetMixin):
+class SPDispersionAxisCorrection(CryonirspTaskBase):
     """Task class for correcting the dispersion axis wavelength values.
 
     Parameters

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tasks/write_l1.py

@@ -14,7 +14,6 @@ from astropy.time.core import Time
 from dkist_processing_common.codecs.asdf import asdf_decoder
 from dkist_processing_common.models.wavelength import WavelengthRange
 from dkist_processing_common.tasks import WriteL1Frame
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
 from sunpy.coordinates import GeocentricEarthEquatorial
 from sunpy.coordinates import Helioprojective
 
@@ -25,7 +24,7 @@ from dkist_processing_cryonirsp.models.tags import CryonirspTag
 __all__ = ["CIWriteL1Frame", "SPWriteL1Frame"]
 
 
-class CryonirspWriteL1Frame(WriteL1Frame, ABC, InputDatasetMixin):
+class CryonirspWriteL1Frame(WriteL1Frame, ABC):
     """
     Task class for writing out calibrated l1 CryoNIRSP frames.
 
@@ -51,7 +50,7 @@ class CryonirspWriteL1Frame(WriteL1Frame, ABC, InputDatasetMixin):
             workflow_version=workflow_version,
         )
         self.parameters = CryonirspParameters(
-            self.input_dataset_parameters,
+            scratch=self.scratch,
             obs_ip_start_time=self.constants.obs_ip_start_time,
             wavelength=self.constants.wavelength,
             arm_id=self.constants.arm_id,

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/conftest.py

@@ -1,11 +1,9 @@
 import json
-import os
 from collections.abc import Iterable
 from dataclasses import asdict
 from dataclasses import dataclass
 from dataclasses import field
 from dataclasses import is_dataclass
-from pathlib import Path
 from random import randint
 from typing import Any
 from typing import Callable
@@ -21,8 +19,15 @@ from dkist_data_simulator.spec122 import Spec122Dataset
 from dkist_header_validator import spec122_validator
 from dkist_header_validator.translator import sanitize_to_spec214_level1
 from dkist_header_validator.translator import translate_spec122_to_spec214_l0
+from dkist_processing_common.codecs.array import array_encoder
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
 from dkist_processing_common.codecs.fits import fits_array_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetFilePointer
+from dkist_processing_common.models.input_dataset import InputDatasetObject
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.tasks import WorkflowTaskBase
+from pydantic import Field
+from pydantic import model_validator
 
 from dkist_processing_cryonirsp.models.constants import CryonirspConstants
 from dkist_processing_cryonirsp.models.exposure_conditions import AllowableOpticalDensityFilterNames
@@ -250,58 +255,70 @@ class WavelengthParameter:
         return hash((self.values, self.wavelength))
 
 
-@dataclass
-class FileParameter:
-    """For parameters that are files on disk."""
-
-    param_path: str
-    is_file: bool = True
-    objectKey: str = "not_used_because_its_already_converted"
-    bucket: str = "not_used_because_we_dont_transfer"
-    # Note: we have these as already-parsed file parameter (i.e., no "__file__") mainly because it allows us to have
-    #       parameter files that are outside of the workflow basepath (where they would not be able to be tagged with
-    #       PARAMETER_FILE). This is a pattern that we see in grogu testing.
-    #       A downside of this approach is that we are slightly more fragile to changes in the underlying __file__ parsing
-    #       in `*-common`. Apologies to any future devs who run into this problem. To fix it you'll need to make
-    #       downstream fixtures aware of the actual files so they can be tagged prior to the instantiation of the
-    #       Parameter object on some Task.
+class FileObject(InputDatasetObject):
+    """For files on disk, with attributes overridden to have defaults."""
+
+    bucket: str | None = None
+    object_key: str | None = None
+
+    def __hash__(self):
+        return hash((self.bucket, self.object_key, self.tag))
+
+
+class FileParameter(InputDatasetFilePointer):
+    """For parameters that are files, with additional attributes to make FileObjects."""
+
+    object_key: str | None = Field(
+        default="dummy_default_value", exclude=True
+    )  # Not necessary, but useful for GROGU
+    file_pointer: FileObject = Field(default_factory=lambda: FileObject(), alias="__file__")
+
+    @model_validator(mode="after")
+    def _populate_file_object(self):
+        self.file_pointer = FileObject(
+            bucket="not_used_because_we_dont_transfer",
+            object_key=self.object_key,
+            tag=CryonirspTag.parameter((self.object_key)),
+        )
+        return self
 
 
 # These constants are used to prevent name errors in _create_parameter_files
-# and in CryonirspTestingParameters
+# and in CryonirspTestingParameters.
 LINEARIZATION_THRESHOLDS_CI = "cryonirsp_linearization_thresholds_ci.npy"
 LINEARIZATION_THRESHOLDS_SP = "cryonirsp_linearization_thresholds_sp.npy"
 SOLAR_ATLAS = "cryonirsp_solar_atlas.npy"
 TELLURIC_ATLAS = "cryonirsp_telluric_atlas.npy"
 
 
-def _create_parameter_files(param_path: Path) -> None:
+def _create_parameter_files(task: WorkflowTaskBase) -> None:
     # linearization thresholds
     thresh = np.ones((10, 10), dtype=np.float64) * 100.0
-    np.save(os.path.join(param_path, LINEARIZATION_THRESHOLDS_CI), thresh)
-    np.save(os.path.join(param_path, LINEARIZATION_THRESHOLDS_SP), thresh)
+    task.write(
+        data=thresh, tags=CryonirspTag.parameter(LINEARIZATION_THRESHOLDS_CI), encoder=array_encoder
+    )
+    task.write(
+        data=thresh, tags=CryonirspTag.parameter(LINEARIZATION_THRESHOLDS_SP), encoder=array_encoder
+    )
 
     # solar and telluric atlases
     atlas_wavelengths = range(300, 16600)
     atlas_transmission = np.random.rand(16300)
     atlas = [atlas_wavelengths, atlas_transmission]
-    np.save(os.path.join(param_path, SOLAR_ATLAS), atlas)
-    np.save(os.path.join(param_path, TELLURIC_ATLAS), atlas)
+    task.write(data=atlas, tags=CryonirspTag.parameter(SOLAR_ATLAS), encoder=array_encoder)
+    task.write(data=atlas, tags=CryonirspTag.parameter(TELLURIC_ATLAS), encoder=array_encoder)
 
 
 TestingParameters = TypeVar("TestingParameters", bound="CryonirspTestingParameters")
 
 
 def cryonirsp_testing_parameters_factory(
-    param_path: Path | str = "", create_files: bool = True
+    task: WorkflowTaskBase, create_files: bool = True
 ) -> TestingParameters:
     """Create the InputDatasetParameterValue objects and write the parameter files."""
-    if isinstance(param_path, str):
-        param_path = Path(param_path)
-    absolute_path = param_path.absolute()
 
     if create_files:
-        _create_parameter_files(absolute_path)
+        _create_parameter_files(task=task)
 
     @dataclass
     class CryonirspTestingParameters:
@@ -339,7 +356,7 @@ def cryonirsp_testing_parameters_factory(
         cryonirsp_fringe_correction_lowpass_cutoff_period: float = 40.0
         cryonirsp_linearization_thresholds_ci: FileParameter = field(
             default_factory=lambda: FileParameter(
-                param_path=str(absolute_path / LINEARIZATION_THRESHOLDS_CI)
+                object_key=LINEARIZATION_THRESHOLDS_CI,
             )
         )
         cryonirsp_linearization_polyfit_coeffs_ci: list[float] = field(
@@ -347,7 +364,7 @@ def cryonirsp_testing_parameters_factory(
         )
         cryonirsp_linearization_thresholds_sp: FileParameter = field(
             default_factory=lambda: FileParameter(
-                param_path=str(absolute_path / LINEARIZATION_THRESHOLDS_SP)
+                object_key=LINEARIZATION_THRESHOLDS_SP,
             )
         )
         cryonirsp_linearization_polyfit_coeffs_sp: list[float] = field(
@@ -364,10 +381,14 @@ def cryonirsp_testing_parameters_factory(
             WavelengthParameter(values=(-4.26, -4.26, -4.26, -4.26))
         )
         cryonirsp_solar_atlas: FileParameter = field(
-            default_factory=lambda: FileParameter(param_path=str(absolute_path / SOLAR_ATLAS))
+            default_factory=lambda: FileParameter(
+                object_key=SOLAR_ATLAS,
+            )
         )
         cryonirsp_telluric_atlas: FileParameter = field(
-            default_factory=lambda: FileParameter(param_path=str(absolute_path / TELLURIC_ATLAS))
+            default_factory=lambda: FileParameter(
+                object_key=TELLURIC_ATLAS,
+            )
         )
         cryonirsp_camera_mirror_focal_length_mm: float = 932.0
         cryonirsp_pixel_pitch_micron: float = 18.0
@@ -387,11 +408,15 @@ def testing_obs_ip_start_time() -> str:
 
 @pytest.fixture(scope="session")
 def input_dataset_document_simple_parameters_part():
-    def get_input_dataset_parameters_part(parameters):
+    """Convert a dataclass of parameterValues into an actual input dataset parameters part."""
+
+    def make_input_dataset_parameters_part(parameters):
         parameters_list = []
 
         value_id = randint(1000, 2000)
         for pn, pv in asdict(parameters).items():
+            if isinstance(pv, FileParameter):
+                pv = pv.model_dump()
             values = [
                 {
                     "parameterValueId": value_id,
@@ -403,7 +428,7 @@ def input_dataset_document_simple_parameters_part():
             parameters_list.append(parameter)
         return parameters_list
 
-    return get_input_dataset_parameters_part
+    return make_input_dataset_parameters_part
 
 
 @pytest.fixture(scope="session")
@@ -412,6 +437,8 @@ def assign_input_dataset_doc_to_task(
     testing_wavelength,
     testing_obs_ip_start_time,
 ):
+    """Load the parameters the way that tasks load the parameters in task init."""
+
     def update_task(
         task,
         parameters,
@@ -419,12 +446,15 @@ def assign_input_dataset_doc_to_task(
         arm_id: str = "SP",
         obs_ip_start_time=testing_obs_ip_start_time,
     ):
-        doc_path = task.scratch.workflow_base_path / "dataset_doc.json"
-        with open(doc_path, "w") as f:
-            f.write(json.dumps(input_dataset_document_simple_parameters_part(parameters)))
-        task.tag(doc_path, CryonirspTag.input_dataset_parameters())
+        task.write(
+            data=InputDatasetPartDocumentList(
+                doc_list=input_dataset_document_simple_parameters_part(parameters)
+            ),
+            tags=CryonirspTag.input_dataset_parameters(),
+            encoder=basemodel_encoder,
+        )
         task.parameters = parameter_class(
-            task.input_dataset_parameters,
+            scratch=task.scratch,
             wavelength=testing_wavelength,
             arm_id=arm_id,
             obs_ip_start_time=obs_ip_start_time,

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/local_trial_workflows/local_trial_helpers.py

@@ -8,6 +8,8 @@ from random import randint
 import asdf
 from astropy.io import fits
 from dkist_header_validator import spec122_validator
+from dkist_processing_common.codecs.basemodel import basemodel_encoder
+from dkist_processing_common.models.input_dataset import InputDatasetPartDocumentList
 from dkist_processing_common.models.task_name import TaskName
 from dkist_processing_common.tasks import TransferTrialData
 from dkist_processing_common.tasks import WorkflowTaskBase
@@ -18,7 +20,7 @@ from dkist_processing_cryonirsp.models.tags import CryonirspTag
 from dkist_processing_cryonirsp.models.task_name import CryonirspTaskName
 from dkist_processing_cryonirsp.tasks.cryonirsp_base import CryonirspTaskBase
 from dkist_processing_cryonirsp.tests.conftest import cryonirsp_testing_parameters_factory
-
+from dkist_processing_cryonirsp.tests.conftest import FileParameter
 
 # These are the workflow versions of the polyfit coefficient parameters
 WORKFLOW_LINEARIZATION_POLYFIT_COEFFS_CI = [
@@ -424,13 +426,13 @@ class LoadInstPolCal(LoadTaskTags):
 def translate_122_to_214_task(suffix: str):
     class Translate122To214L0(WorkflowTaskBase):
         def run(self) -> None:
-            raw_dir = Path(self.scratch.scratch_base_path) / f"CRYO-NIRSP{self.recipe_run_id:03n}"
+            raw_dir = Path(self.scratch.scratch_base_path) / f"CRYONIRSP{self.recipe_run_id:03n}"
             if not os.path.exists(self.scratch.workflow_base_path):
                 os.makedirs(self.scratch.workflow_base_path)
 
             if not raw_dir.exists():
                 raise FileNotFoundError(
-                    f"Expected to find a raw CRYO-NIRSP{self.recipe_run_id:03n} folder in {self.scratch.scratch_base_path}"
+                    f"Expected to find a raw CRYONIRSP{self.recipe_run_id:03n} folder in {self.scratch.scratch_base_path}"
                 )
 
             for file in raw_dir.glob(f"*.{suffix}"):
@@ -465,26 +467,38 @@ def translate_122_to_214_task(suffix: str):
 def create_input_dataset_parameter_document(param_path: Path):
     class CreateInputDatasetParameterDocument(WorkflowTaskBase):
         def run(self) -> None:
-            doc_path = self.scratch.workflow_base_path / "input_dataset_parameters.json"
-            with open(doc_path, "w") as f:
-                f.write(json.dumps(self.input_dataset_document_simple_parameters_part))
-            self.tag(doc_path, CryonirspTag.input_dataset_parameters())
-            logger.info(f"Wrote input dataset doc to {doc_path}")
+            relative_path = "input_dataset_parameters.json"
+            self.write(
+                data=InputDatasetPartDocumentList(
+                    doc_list=self.input_dataset_document_simple_parameters_part
+                ),
+                relative_path=relative_path,
+                tags=CryonirspTag.input_dataset_parameters(),
+                encoder=basemodel_encoder,
+            )
+            logger.info(f"Wrote input dataset parameter doc to {relative_path}")
+            self.copy_and_tag_parameter_files(param_path=param_path)
+            logger.info(f"Copied input dataset parameter files from {param_path}")
 
         @property
-        def input_dataset_document_simple_parameters_part(self):
-            parameters_list = []
-            value_id = randint(1000, 2000)
-            param_class = cryonirsp_testing_parameters_factory(
-                param_path=param_path, create_files=False
-            )
+        def parameter_dict(self):
+            param_class = cryonirsp_testing_parameters_factory(task=self, create_files=False)
             params = asdict(
                 param_class(
                     cryonirsp_linearization_polyfit_coeffs_ci=WORKFLOW_LINEARIZATION_POLYFIT_COEFFS_CI,
                     cryonirsp_linearization_polyfit_coeffs_sp=WORKFLOW_LINEARIZATION_POLYFIT_COEFFS_SP,
                 )
             )
+            return params
+
+        @property
+        def input_dataset_document_simple_parameters_part(self):
+            parameters_list = []
+            value_id = randint(1000, 2000)
+            params = self.parameter_dict
             for pn, pv in params.items():
+                if isinstance(pv, FileParameter):
+                    pv = pv.model_dump()
                 values = [
                     {
                         "parameterValueId": value_id,
@@ -497,6 +511,22 @@ def create_input_dataset_parameter_document(param_path: Path):
 
             return parameters_list
 
+        def copy_and_tag_parameter_files(self, param_path=param_path):
+            # Copy parameter files from param_path to a place where they can be tagged
+            params = self.parameter_dict
+            destination_dir = Path(self.scratch.workflow_base_path) / "parameters"
+            destination_dir.mkdir(parents=True, exist_ok=True)
+            for pn, pv in params.items():
+                if isinstance(pv, FileParameter):
+                    file_path = next(param_path.rglob(pv.file_pointer.object_key))
+                    if file_path.parent != destination_dir:
+                        shutil.copy(file_path, destination_dir)
+                        logger.info(
+                            f"Copied parameter file for '{pn}' to {destination_dir / file_path}"
+                        )
+                        file_path = next(destination_dir.rglob(pv.file_pointer.object_key))
+                    self.tag(path=file_path, tags=pv.file_pointer.tag)
+
     return CreateInputDatasetParameterDocument
 
 

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/test_bad_pixel_maps.py

@@ -43,7 +43,7 @@ def compute_bad_pixel_map_task(
             task.scratch = WorkflowFileSystem(
                 scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
             )
-            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            param_class = cryonirsp_testing_parameters_factory(task)
             assign_input_dataset_doc_to_task(task, param_class())
             start_time = datetime.now()
             ds = CryonirspHeadersValidCISolarGainFrames(

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/test_ci_beam_boundaries.py

@@ -38,7 +38,7 @@ def compute_beam_boundaries_task(
             task.scratch = WorkflowFileSystem(
                 scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
             )
-            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            param_class = cryonirsp_testing_parameters_factory(task)
             assign_input_dataset_doc_to_task(task, param_class())
             # Create fake bad pixel map
             task.write(

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/test_ci_science.py

@@ -70,7 +70,7 @@ def ci_science_calibration_task(
                 scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
             )
 
-            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            param_class = cryonirsp_testing_parameters_factory(task)
             assign_input_dataset_doc_to_task(task, param_class())
 
             # Need a beam boundary file

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/test_corrections.py

@@ -4,7 +4,6 @@ import numpy as np
 import pytest
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common.tasks import WorkflowTaskBase
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
 from dkist_processing_common.tests.conftest import FakeGQLClient
 
 from dkist_processing_cryonirsp.models.constants import CryonirspConstants
@@ -21,7 +20,7 @@ column_error_bad_pixel_map = base_bad_pixel_map.copy()
 column_error_bad_pixel_map[:, 6] = 1
 
 
-class BadPixelMapTask(WorkflowTaskBase, CorrectionsMixin, InputDatasetMixin):
+class BadPixelMapTask(WorkflowTaskBase, CorrectionsMixin):
     constants: CryonirspConstants
 
     @property
@@ -58,7 +57,7 @@ def bad_pixel_mask_task(
             task.scratch = WorkflowFileSystem(
                 scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
             )
-            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            param_class = cryonirsp_testing_parameters_factory(task)
             assign_input_dataset_doc_to_task(task, param_class())
             yield task
         finally:

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/test_cryo_base.py

@@ -42,7 +42,7 @@ def cryo_science_task(
             task.scratch = WorkflowFileSystem(
                 scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
             )
-            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            param_class = cryonirsp_testing_parameters_factory(task)
             assign_input_dataset_doc_to_task(task, param_class())
             yield task
         finally:

{dkist_processing_cryonirsp-1.7.6 → dkist_processing_cryonirsp-1.7.7rc1}/dkist_processing_cryonirsp/tests/test_dark.py

@@ -52,7 +52,7 @@ def sp_dark_calibration_task(
             task.scratch = WorkflowFileSystem(
                 scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
             )
-            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            param_class = cryonirsp_testing_parameters_factory(task)
             assign_input_dataset_doc_to_task(task, param_class())
 
             # Create fake beam border intermediate arrays
@@ -137,7 +137,7 @@ def ci_dark_calibration_task(
             task.scratch = WorkflowFileSystem(
                 scratch_base_path=tmp_path, recipe_run_id=recipe_run_id
             )
-            param_class = cryonirsp_testing_parameters_factory(param_path=tmp_path)
+            param_class = cryonirsp_testing_parameters_factory(task)
             assign_input_dataset_doc_to_task(task, param_class())
 
             # Need a beam boundary file