dkist-processing-common 10.5.4__py3-none-any.whl → 12.1.0rc1__py3-none-any.whl

dkist_processing_common/tasks/write_l1.py

@@ -1,4 +1,5 @@
 """Task(s) for writing level 1 data as 214 compliant fits files."""
+
 import importlib
 import logging
 import uuid
@@ -6,27 +7,30 @@ from abc import ABC
 from abc import abstractmethod
 from functools import cached_property
 from pathlib import Path
+from string import ascii_uppercase
 from typing import Literal
 
 import astropy.units as u
 import numpy as np
-from astropy.coordinates import EarthLocation
 from astropy.io import fits
 from astropy.time import Time
 from dkist_fits_specifications import __version__ as spec_version
 from dkist_fits_specifications.utils.formatter import reformat_spec214_header
 from dkist_header_validator import spec214_validator
 from dkist_header_validator.translator import remove_extra_axis_keys
-from dkist_header_validator.translator import sanitize_to_spec214_level1
+from dkist_header_validator.translator import remove_spec_122_keys_and_spec_214_l0_keys
 from dkist_spectral_lines.search import get_closest_spectral_line
 from dkist_spectral_lines.search import get_spectral_lines
 from scipy.stats import kurtosis
 from scipy.stats import skew
+from sqids import Sqids
 from sunpy.coordinates import HeliocentricInertial
 from sunpy.coordinates import Helioprojective
 
 from dkist_processing_common.codecs.fits import fits_access_decoder
 from dkist_processing_common.codecs.fits import fits_hdulist_encoder
+from dkist_processing_common.models.dkist_location import location_of_dkist
+from dkist_processing_common.models.fried_parameter import r0_valid
 from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.models.wavelength import WavelengthRange
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
@@ -49,7 +53,7 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
     def run(self) -> None:
         """Run method for this task."""
         for stokes_param in self.constants.stokes_params:
-            with self.apm_task_step(f"Get calibrated frames for stokes param {stokes_param}"):
+            with self.telemetry_span(f"Get calibrated frames for stokes param {stokes_param}"):
                 tags = [Tag.frame(), Tag.calibrated(), Tag.stokes(stokes_param)]
                 calibrated_fits_objects = self.read(
                     tags=tags,
@@ -100,22 +104,24 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
 
                 # Check that the written file passes spec 214 validation if requested
                 if self.validate_l1_on_write:
-                    spec214_validator.validate(self.scratch.absolute_path(relative_path))
+                    spec214_validator.validate(
+                        self.scratch.absolute_path(relative_path), extra=False
+                    )
 
     @cached_property
-    def tile_size_param(self) -> int:
+    def tile_size_param(self) -> int | None:
         """Get the tile size parameter for compression."""
-        return self.metadata_store_recipe_run_configuration().get("tile_size", None)
+        return self.metadata_store_recipe_run.configuration.tile_size
 
     @cached_property
     def validate_l1_on_write(self) -> bool:
         """Check for validate on write."""
-        return self.metadata_store_recipe_run_configuration().get("validate_l1_on_write", True)
+        return self.metadata_store_recipe_run.configuration.validate_l1_on_write
 
     @cached_property
     def workflow_had_manual_intervention(self):
         """Indicate determining if any provenance capturing steps had manual intervention."""
-        for provenance_record in self.metadata_store_recipe_run_provenance:
+        for provenance_record in self.metadata_store_recipe_run.recipeRunProvenances:
             if provenance_record.isTaskManual:
                 return True
         return False
@@ -157,23 +163,32 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header["DATE-END"] = self.calculate_date_end(header=header)
         return header
 
+    @staticmethod
+    def compute_product_id(ids_obs_id: int, proc_type: str) -> str:
+        """Compute the productId from IDSOBSID and PROCTYPE."""
+        sqid_factory = Sqids(alphabet=ascii_uppercase, min_length=5)
+        sqid = sqid_factory.encode([ids_obs_id])
+        return f"{proc_type}-{sqid}"
+
     @staticmethod
     def add_stats_headers(header: fits.Header, data: np.ndarray) -> fits.Header:
         """Fill out the spec 214 statistics header table."""
         data = data.flatten()
-        percentiles = np.nanpercentile(data, [1, 10, 25, 75, 90, 95, 98, 99])
+        percentiles = np.nanpercentile(data, [1, 2, 5, 10, 25, 75, 90, 95, 98, 99])
         header["DATAMIN"] = np.nanmin(data)
         header["DATAMAX"] = np.nanmax(data)
         header["DATAMEAN"] = np.nanmean(data)
         header["DATAMEDN"] = np.nanmedian(data)
-        header["DATA01"] = percentiles[0]
-        header["DATA10"] = percentiles[1]
-        header["DATA25"] = percentiles[2]
-        header["DATA75"] = percentiles[3]
-        header["DATA90"] = percentiles[4]
-        header["DATA95"] = percentiles[5]
-        header["DATA98"] = percentiles[6]
-        header["DATA99"] = percentiles[7]
+        header["DATAP01"] = percentiles[0]
+        header["DATAP02"] = percentiles[1]
+        header["DATAP05"] = percentiles[2]
+        header["DATAP10"] = percentiles[3]
+        header["DATAP25"] = percentiles[4]
+        header["DATAP75"] = percentiles[5]
+        header["DATAP90"] = percentiles[6]
+        header["DATAP95"] = percentiles[7]
+        header["DATAP98"] = percentiles[8]
+        header["DATAP99"] = percentiles[9]
         header["DATARMS"] = np.sqrt(np.nanmean(data**2))
         header["DATAKURT"] = kurtosis(data, nan_policy="omit")
         header["DATASKEW"] = skew(data, nan_policy="omit")
@@ -192,30 +207,45 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header["FRAMEVOL"] = -1.0
         header["PROCTYPE"] = "L1"
         header["RRUNID"] = self.recipe_run_id
-        header["RECIPEID"] = self.metadata_store_recipe_id
-        header["RINSTID"] = self.metadata_store_recipe_instance_id
+        header["RECIPEID"] = self.metadata_store_recipe_run.recipeInstance.recipeId
+        header["RINSTID"] = self.metadata_store_recipe_run.recipeInstanceId
         header["EXTNAME"] = "observation"
         header["SOLARNET"] = 1
         header["OBS_HDU"] = 1
         header["FILENAME"] = self.l1_filename(header=header, stokes=stokes)
         header["STOKES"] = stokes
-        # Cadence keywords
-        header["CADENCE"] = self.constants.average_cadence
-        header["CADMIN"] = self.constants.minimum_cadence
-        header["CADMAX"] = self.constants.maximum_cadence
-        header["CADVAR"] = self.constants.variance_cadence
         # Keywords to support reprocessing
-        if ids_par_id := self.metadata_store_input_dataset_parameters_part_id:
-            header["IDSPARID"] = ids_par_id
-        if ids_obs_id := self.metadata_store_input_dataset_observe_frames_part_id:
-            header["IDSOBSID"] = ids_obs_id
-        if ids_cal_id := self.metadata_store_input_dataset_calibration_frames_part_id:
-            header["IDSCALID"] = ids_cal_id
+        if parameters := self.metadata_store_input_dataset_parameters:
+            header["IDSPARID"] = parameters.inputDatasetPartId
+        if observe_frames := self.metadata_store_input_dataset_observe_frames:
+            header["IDSOBSID"] = observe_frames.inputDatasetPartId
+        if calibration_frames := self.metadata_store_input_dataset_calibration_frames:
+            header["IDSCALID"] = calibration_frames.inputDatasetPartId
         header["WKFLNAME"] = self.workflow_name
         header["WKFLVERS"] = self.workflow_version
         header = self.add_contributing_id_headers(header=header)
         header["MANPROCD"] = self.workflow_had_manual_intervention
-        # Spectral line keywords
+        header["PRODUCT"] = self.compute_product_id(header["IDSOBSID"], header["PROCTYPE"])
+        return header
+
+    def add_timing_headers(self, header: fits.Header) -> fits.Header:
+        """
+        Add timing headers to the FITS header.
+
+        This method adds or updates headers related to frame timings.
+        """
+        # Cadence keywords
+        header["CADENCE"] = self.constants.average_cadence
+        header["CADMIN"] = self.constants.minimum_cadence
+        header["CADMAX"] = self.constants.maximum_cadence
+        header["CADVAR"] = self.constants.variance_cadence
+        return header
+
+    def add_spectral_line_headers(
+        self,
+        header: fits.Header,
+    ) -> fits.Header:
+        """Add datacenter table keys relating to spectral lines."""
         wavelength_range = self.get_wavelength_range(header=header)
         spectral_lines = get_spectral_lines(
             wavelength_min=wavelength_range.min,
@@ -236,55 +266,31 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         For spectrographs, this is the wavelengths covered by the spectral axis of the data.
         """
 
-    @property
-    def location_of_dkist(self) -> EarthLocation:
-        """Return hard-coded EarthLocation of the DKIST.
-
-        Cartesian geocentric coordinates of DKIST on Earth as retrieved from
-        https://github.com/astropy/astropy-data/blob/gh-pages/coordinates/sites.json#L838
-        """
-        _dkist_site_info = {
-            "aliases": ["DKIST", "ATST"],
-            "name": "Daniel K. Inouye Solar Telescope",
-            "elevation": 3067,
-            "elevation_unit": "meter",
-            "latitude": 20.7067,
-            "latitude_unit": "degree",
-            "longitude": 203.7436,
-            "longitude_unit": "degree",
-            "timezone": "US/Hawaii",
-            "source": "DKIST website: https://www.nso.edu/telescopes/dki-solar-telescope/",
-        }
-        location_of_dkist = EarthLocation.from_geodetic(
-            _dkist_site_info["longitude"] * u.Unit(_dkist_site_info["longitude_unit"]),
-            _dkist_site_info["latitude"] * u.Unit(_dkist_site_info["latitude_unit"]),
-            _dkist_site_info["elevation"] * u.Unit(_dkist_site_info["elevation_unit"]),
-        )
-
-        return location_of_dkist
-
     def add_solarnet_headers(self, header: fits.Header) -> fits.Header:
         """Add headers recommended by solarnet that haven't already been added."""
         header["DATE-AVG"] = self.calculate_date_avg(header=header)
         header["TELAPSE"] = self.calculate_telapse(header=header)
         header["DATEREF"] = header["DATE-BEG"]
-        dkist_loc = self.location_of_dkist
-        header["OBSGEO-X"] = dkist_loc.x.to_value(unit=u.m)
-        header["OBSGEO-Y"] = dkist_loc.y.to_value(unit=u.m)
-        header["OBSGEO-Z"] = dkist_loc.z.to_value(unit=u.m)
+        header["OBSGEO-X"] = location_of_dkist.x.to_value(unit=u.m)
+        header["OBSGEO-Y"] = location_of_dkist.y.to_value(unit=u.m)
+        header["OBSGEO-Z"] = location_of_dkist.z.to_value(unit=u.m)
         obstime = Time(header["DATE-AVG"])
         header["OBS_VR"] = (
-            dkist_loc.get_gcrs(obstime=obstime)
+            location_of_dkist.get_gcrs(obstime=obstime)
             .transform_to(HeliocentricInertial(obstime=obstime))
             .d_distance.to_value(unit=u.m / u.s)
         )  # relative velocity of observer with respect to the sun in m/s
         header["SOLARRAD"] = self.calculate_solar_angular_radius(obstime=obstime)
         header["SPECSYS"] = "TOPOCENT"  # no wavelength correction made due to doppler velocity
         header["VELOSYS"] = 0.0  # no wavelength correction made due to doppler velocity
-        header["WAVEBAND"] = get_closest_spectral_line(wavelength=header["LINEWAV"] * u.nm).name
         wavelength_range = self.get_wavelength_range(header=header)
         header["WAVEMIN"] = wavelength_range.min.to_value(u.nm)
         header["WAVEMAX"] = wavelength_range.max.to_value(u.nm)
+        waveband: str | None = self.get_waveband(
+            wavelength=header["LINEWAV"] * u.nm, wavelength_range=wavelength_range
+        )
+        if waveband:
+            header["WAVEBAND"] = waveband
         return header
 
     def l1_filename(self, header: fits.Header, stokes: Literal["I", "Q", "U", "V"]):
@@ -355,8 +361,14 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         """
         # Replace header values in place
         header = self.replace_header_values(header=header, data=data)
-        # Remove r0 value if AO not locked
-        header = self.remove_invalid_r0_values(header=header)
+        # Remove r0 value if r0 conditions are not met
+        r0_is_valid = r0_valid(
+            r0=header["ATMOS_R0"],
+            ao_lock=header.get("AO_LOCK", None),
+            num_out_of_bounds_ao_values=header.get("OOBSHIFT", None),
+        )
+        if not r0_is_valid:
+            header.pop("ATMOS_R0", None)
         # Add the stats table
         header = self.add_stats_headers(header=header, data=data)
         # Add the datacenter table
@@ -367,8 +379,12 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         header = self.add_doc_headers(header=header)
         # Add the dataset headers (abstract - implement in instrument task)
         header = self.add_dataset_headers(header=header, stokes=stokes_param)
+        # Add the timing headers
+        header = self.add_timing_headers(header=header)
+        # Add the spectral line headers
+        header = self.add_spectral_line_headers(header=header)
         # Remove any headers not contained in spec 214
-        header = sanitize_to_spec214_level1(input_headers=header)
+        header = remove_spec_122_keys_and_spec_214_l0_keys(input_headers=header)
         # Remove any keys referring to axes that don't exist
         header = remove_extra_axis_keys(input_headers=header)
         return header
@@ -409,9 +425,9 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         inst_name = self.constants.instrument.lower()
         calvers = self.version_from_module_name()
         header["CALVERS"] = calvers
-        header[
-            "CAL_URL"
-        ] = f"{self.docs_base_url}/projects/{inst_name}/en/v{calvers}/{self.workflow_name}.html"
+        header["CAL_URL"] = (
+            f"{self.docs_base_url}/projects/{inst_name}/en/v{calvers}/{self.workflow_name}.html"
+        )
         return header
 
     def version_from_module_name(self) -> str:
@@ -489,7 +505,7 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
         as seen by an observer located at the DKIST site at the given time of observation.
         """
         dummy_theta_coord = 0 * u.arcsec
-        dkist_at_obstime = self.location_of_dkist.get_itrs(obstime=obstime)
+        dkist_at_obstime = location_of_dkist.get_itrs(obstime=obstime)
         sun_coordinate = Helioprojective(
             Tx=dummy_theta_coord, Ty=dummy_theta_coord, observer=dkist_at_obstime
         )
@@ -498,6 +514,20 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
     @staticmethod
     def remove_invalid_r0_values(header: fits.Header) -> fits.Header:
         """Remove the Fried parameter r0 from the header if the AO is not locked."""
-        if not header["AO_LOCK"]:
+        if header.get("AO_LOCK") is not True:
             header.pop("ATMOS_R0", None)
         return header
+
+    @staticmethod
+    def get_waveband(wavelength: u.Quantity, wavelength_range: WavelengthRange) -> str | None:
+        """
+        Get the spectral line information of the closest spectral line to the wavelength argument.
+
+        If the spectral line rest wavelength in air does not fall in the wavelength range of the data,
+        do not populate the keyword.
+        """
+        closest_line = get_closest_spectral_line(wavelength=wavelength)
+        rest_wavelength = closest_line.rest_wavelength_in_air
+        if rest_wavelength < wavelength_range.min or rest_wavelength > wavelength_range.max:
+            return None
+        return closest_line.name

dkist_processing_common/tests/conftest.py

@@ -1,6 +1,7 @@
 """
 Global test fixtures
 """
+
 import json
 from collections import defaultdict
 from copy import deepcopy
@@ -32,22 +33,9 @@ from dkist_processing_pac.optics.telescope import Telescope
 from dkist_processing_common._util.constants import ConstantsDb
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common._util.tags import TagDB
-from dkist_processing_common.models.graphql import InputDatasetInputDatasetPartResponse
-from dkist_processing_common.models.graphql import InputDatasetPartResponse
-from dkist_processing_common.models.graphql import InputDatasetPartTypeResponse
-from dkist_processing_common.models.graphql import InputDatasetRecipeInstanceResponse
-from dkist_processing_common.models.graphql import InputDatasetRecipeRunResponse
-from dkist_processing_common.models.graphql import InputDatasetResponse
-from dkist_processing_common.models.graphql import RecipeInstanceResponse
-from dkist_processing_common.models.graphql import RecipeRunProvenanceResponse
-from dkist_processing_common.models.graphql import RecipeRunResponse
-from dkist_processing_common.models.graphql import RecipeRunStatusResponse
-from dkist_processing_common.models.tags import Tag
 from dkist_processing_common.parsers.l0_fits_access import L0FitsAccess
 from dkist_processing_common.tasks import WorkflowTaskBase
-from dkist_processing_common.tasks.mixin.input_dataset import InputDatasetMixin
-
-TILE_SIZE = 64
+from dkist_processing_common.tests.mock_metadata_store import fake_gql_client
 
 
 @pytest.fixture()
@@ -100,6 +88,21 @@ def constants_db(recipe_run_id) -> ConstantsDb:
     constants.close()
 
 
+@pytest.fixture()
+def fake_constants_db() -> dict:
+    """
+    A fake constants DB to prevent key errors.
+
+    Usage on a task: task.constants._update(fake_constants_db)
+    """
+    db = {
+        "PROPOSAL_ID": "PROPID",
+        "INSTRUMENT": "INSTRUMENT",
+        "OBS_IP_START_TIME": "20240416T160000",
+    }
+    return db
+
+
 class CommonDataset(Spec122Dataset):
     def __init__(self, polarimetric: bool = True):
         super().__init__(
@@ -113,7 +116,7 @@ class CommonDataset(Spec122Dataset):
         self.add_constant_key("TELEVATN", 6.28)
         self.add_constant_key("TAZIMUTH", 3.14)
         self.add_constant_key("TTBLANGL", 1.23)
-        self.add_constant_key("INST_FOO", "bar")
+        self.add_constant_key("VISP_012", "bar")
         self.add_constant_key("DKIST004", "observe")
         self.add_constant_key("ID___005", "ip id")
         self.add_constant_key("PAC__004", "Sapphire Polarizer")
@@ -286,18 +289,18 @@ def cs_step_angle_round_ndigits() -> int:
 
 
 @pytest.fixture(scope="session")
-def angle_random_max_perturabtion(cs_step_angle_round_ndigits) -> float:
+def angle_random_max_perturbation(cs_step_angle_round_ndigits) -> float:
     # Ensures that we always round down to zero.
     # E.g., if ndigits = 1 then this value will be 0.049.
     return 10**-cs_step_angle_round_ndigits / 2 - 10 ** -(cs_step_angle_round_ndigits + 2)
 
 
 @pytest.fixture(scope="session")
-def grouped_cal_sequence_headers(angle_random_max_perturabtion) -> dict[int, list[L0FitsAccess]]:
+def grouped_cal_sequence_headers(angle_random_max_perturbation) -> dict[int, list[L0FitsAccess]]:
     ds = CalibrationSequenceDataset(
         array_shape=(1, 2, 2),
         time_delta=2.0,
-        angle_max_random_perturbation=angle_random_max_perturabtion,
+        angle_max_random_perturbation=angle_random_max_perturbation,
     )
     header_list = [
         spec122_validator.validate_and_translate_to_214_l0(d.header(), return_type=fits.HDUList)[
@@ -332,99 +335,9 @@ def max_cs_step_time_sec() -> float:
     return 20.0
 
 
-class FakeGQLClient:
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def execute_gql_query(self, **kwargs):
-        query_base = kwargs["query_base"]
-        if query_base == "recipeRunStatuses":
-            return [RecipeRunStatusResponse(recipeRunStatusId=1)]
-        if query_base == "recipeRuns":
-            if kwargs.get("query_response_cls") == InputDatasetRecipeRunResponse:
-                return [
-                    InputDatasetRecipeRunResponse(
-                        recipeInstance=InputDatasetRecipeInstanceResponse(
-                            inputDataset=InputDatasetResponse(
-                                inputDatasetId=1,
-                                isActive=True,
-                                inputDatasetInputDatasetParts=[
-                                    InputDatasetInputDatasetPartResponse(
-                                        inputDatasetPart=InputDatasetPartResponse(
-                                            inputDatasetPartId=1,
-                                            inputDatasetPartDocument='[{"parameterName": "", "parameterValues": [{"parameterValueId": 1, "parameterValue": "[[1,2,3],[4,5,6],[7,8,9]]", "parameterValueStartDate": "1/1/2000"}]}]',
-                                            inputDatasetPartType=InputDatasetPartTypeResponse(
-                                                inputDatasetPartTypeName="parameters"
-                                            ),
-                                        )
-                                    ),
-                                    InputDatasetInputDatasetPartResponse(
-                                        inputDatasetPart=InputDatasetPartResponse(
-                                            inputDatasetPartId=2,
-                                            inputDatasetPartDocument="""[
-                                            {
-                                                "bucket": "bucket_name",
-                                                "object_keys": [
-                                                    "key1",
-                                                    "key2"
-                                                ]
-                                            },
-                                        ]""",
-                                            inputDatasetPartType=InputDatasetPartTypeResponse(
-                                                inputDatasetPartTypeName="observe_frames"
-                                            ),
-                                        )
-                                    ),
-                                    InputDatasetInputDatasetPartResponse(
-                                        inputDatasetPart=InputDatasetPartResponse(
-                                            inputDatasetPartId=3,
-                                            inputDatasetPartDocument="""[
-                                            {
-                                                "bucket": "bucket_name",
-                                                "object_keys": [
-                                                    "key3",
-                                                    "key4"
-                                                ]
-                                            },
-                                        ]""",
-                                            inputDatasetPartType=InputDatasetPartTypeResponse(
-                                                inputDatasetPartTypeName="calibration_frames"
-                                            ),
-                                        )
-                                    ),
-                                ],
-                            ),
-                        ),
-                    ),
-                ]
-
-            return [
-                RecipeRunResponse(
-                    recipeInstanceId=1,
-                    recipeInstance=RecipeInstanceResponse(
-                        recipeId=1,
-                        inputDatasetId=1,
-                    ),
-                    configuration=f'{{"tile_size": {TILE_SIZE}}}',
-                    recipeRunProvenances=[
-                        RecipeRunProvenanceResponse(recipeRunProvenanceId=1, isTaskManual=False),
-                    ],
-                ),
-            ]
-
-    @staticmethod
-    def execute_gql_mutation(**kwargs):
-        ...
-
-
-class FakeGQLClientNoRecipeConfiguration(FakeGQLClient):
-    def execute_gql_query(self, **kwargs):
-        response = super().execute_gql_query(**kwargs)
-        response[0].configuration = None
-        return response
-
-
-# All the following stuff is copied from dkist-processing-pac
+####################################
+# Copied from dkist-processing-pac #
+####################################
 def compute_telgeom(time_hst: Time):
     dkist_lon = (156 + 15 / 60.0 + 21.7 / 3600.0) * (-1)
     dkist_lat = 20 + 42 / 60.0 + 27.0 / 3600.0
@@ -488,7 +401,7 @@ class CalibrationSequenceStepDataset(Spec122Dataset):
         return "none" if self.pol_status == "clear" else str(self.pol_theta)
 
     @key_function("PAC__006")
-    def retarter_status(self, key: str) -> str:
+    def retarder_status(self, key: str) -> str:
         return self.ret_status
 
     @key_function("PAC__007")
@@ -740,7 +653,12 @@ def post_fit_polcal_fitter(
     return fitter
 
 
-class InputDatasetTask(WorkflowTaskBase, InputDatasetMixin):
+#################
+# Input Dataset #
+#################
+
+
+class InputDatasetTask(WorkflowTaskBase):
     def run(self):
         pass
 
@@ -768,65 +686,6 @@ def task_with_input_dataset(
         task.scratch.workflow_base_path = tmp_path / str(recipe_run_id)
         for part, tag in input_dataset_parts:
             file_path = task.scratch.workflow_base_path / Path(f"{uuid4().hex[:6]}.ext")
-            file_path.write_text(data=json.dumps(part))
+            file_path.write_text(data=json.dumps({"doc_list": part}))
             task.tag(path=file_path, tags=tag)
         yield task
-
-
-def create_parameter_files(task: WorkflowTaskBase, expected_parameters: dict):
-    """
-    Create the parameter files required by the task.
-
-    Parameters
-    ----------
-    task
-        The task associated with these parameters
-
-    expected_parameters
-        A dict of parameters with the format shown below
-
-    Returns
-    -------
-    None
-
-    expected_parameters is a dict with the parameter names as the keys
-    and the values are a list of value dicts for each parameter:
-    expected_parameters =
-        { 'parameter_name_1': [param_dict_1, param_dict_2, ...],
-          'parameter_name_2': [param_dict_1, param_dict_2, ...],
-           ...
-        }
-    where the param_dicts have the following format:
-    sample_param_dict =
-        { "parameterValueId": <param_id>,
-          "parameterValue": <param_value>,
-          "parameterValueStartDate": <start_date>
-        }
-    """
-    # Loop over all the parameter values. Each value is a list of parameterValue dicts
-    for expected_parameter_values in expected_parameters.values():
-        for value_dict in expected_parameter_values:
-            if "__file__" not in value_dict["parameterValue"]:
-                continue
-            value = json.loads(value_dict["parameterValue"])
-            param_path = value["__file__"]["objectKey"]
-            file_path = task.scratch.workflow_base_path / Path(param_path)
-            if not file_path.parent.exists():
-                file_path.parent.mkdir(parents=True, exist_ok=True)
-            file_path.write_text(data="")
-            task.tag(path=file_path, tags=Tag.parameter(param_path))
-
-
-@pytest.fixture()
-def fake_constants_db() -> dict:
-    """
-    A fake constants DB to prevent key errors.
-
-    Usage on a task: task.constants._update(fake_constants_db)
-    """
-    db = {
-        "PROPOSAL_ID": "PROPID",
-        "INSTRUMENT": "INSTRUMENT",
-        "OBS_IP_START_TIME": "20240416T160000",
-    }
-    return db