imap-processing 0.19.0__py3-none-any.whl → 0.19.3__py3-none-any.whl

imap_processing/cli.py

@@ -24,8 +24,7 @@ import imap_data_access
 import numpy as np
 import spiceypy
 import xarray as xr
-from imap_data_access import ScienceFilePath
-from imap_data_access.io import download
+from imap_data_access.io import IMAPDataAccessError, download
 from imap_data_access.processing_input import (
     ProcessingInputCollection,
     ProcessingInputType,
@@ -407,32 +406,23 @@ class ProcessInstrument(ABC):
             A list of file paths to upload to the SDC.
         """
         if self.upload_to_sdc:
-            # Validate that the files don't already exist
-            for filename in products:
-                file_path = ScienceFilePath(filename)
-                existing_file = imap_data_access.query(
-                    instrument=file_path.instrument,
-                    data_level=file_path.data_level,
-                    descriptor=file_path.descriptor,
-                    start_date=file_path.start_date,
-                    end_date=file_path.start_date,
-                    repointing=file_path.repointing,
-                    version=file_path.version,
-                    extension="cdf",
-                    table="science",
-                )
-                if existing_file:
-                    raise ProcessInstrument.ImapFileExistsError(
-                        f"File {filename} already exists in the IMAP SDC. "
-                        "No files were uploaded."
-                        f"Generated files: {products}."
-                    )
-
-            if len(products) == 0:
+            if not products:
                 logger.info("No files to upload.")
+                return
+
             for filename in products:
-                logger.info(f"Uploading file: {filename}")
-                imap_data_access.upload(filename)
+                try:
+                    logger.info(f"Uploading file: {filename}")
+                    imap_data_access.upload(filename)
+                except IMAPDataAccessError as e:
+                    msg = str(e)
+                    if "FileAlreadyExists" in msg and "409" in msg:
+                        logger.warning("Skipping upload of existing file, %s", filename)
+                        continue
+                    else:
+                        logger.error(f"Upload failed with error: {msg}")
+                except Exception as e:
+                    logger.error(f"Upload failed unknown error: {e}")
 
     @final
     def process(self) -> None:
@@ -676,7 +666,7 @@ class Glows(ProcessInstrument):
         datasets: list[xr.Dataset] = []
 
         if self.data_level == "l1a":
-            science_files = dependencies.get_file_paths(source="glows")
+            science_files = dependencies.get_file_paths(source="glows", data_type="l0")
             if len(science_files) != 1:
                 raise ValueError(
                     f"GLOWS L1A requires exactly one input science file, received: "
@@ -685,33 +675,30 @@ class Glows(ProcessInstrument):
             datasets = glows_l1a(science_files[0])
 
         if self.data_level == "l1b":
-            science_files = dependencies.get_file_paths(source="glows")
+            science_files = dependencies.get_file_paths(source="glows", data_type="l1a")
             if len(science_files) != 1:
                 raise ValueError(
                     f"GLOWS L1B requires exactly one input science file, received: "
                     f"{science_files}."
                 )
             input_dataset = load_cdf(science_files[0])
-            # TODO: Replace this by reading from AWS/ProcessingInputs
-
-            glows_ancillary_dir = Path(__file__).parent / "glows" / "ancillary"
 
             # Create file lists for each ancillary type
-            excluded_regions_files = [
-                glows_ancillary_dir
-                / "imap_glows_map-of-excluded-regions_20250923_v002.dat"
-            ]
-            uv_sources_files = [
-                glows_ancillary_dir / "imap_glows_map-of-uv-sources_20250923_v002.dat"
-            ]
-            suspected_transients_files = [
-                glows_ancillary_dir
-                / "imap_glows_suspected-transients_20250923_v002.dat"
-            ]
-            exclusions_by_instr_team_files = [
-                glows_ancillary_dir
-                / "imap_glows_exclusions-by-instr-team_20250923_v002.dat"
-            ]
+            excluded_regions_files = dependencies.get_processing_inputs(
+                descriptor="map-of-excluded-regions"
+            )[0]
+            uv_sources_files = dependencies.get_processing_inputs(
+                descriptor="map-of-uv-sources"
+            )[0]
+            suspected_transients_files = dependencies.get_processing_inputs(
+                descriptor="suspected-transients"
+            )[0]
+            exclusions_by_instr_team_files = dependencies.get_processing_inputs(
+                descriptor="exclusions-by-instr-team"
+            )[0]
+            pipeline_settings = dependencies.get_processing_inputs(
+                descriptor="pipeline-settings"
+            )[0]
 
             # Use end date buffer for ancillary data
             current_day = np.datetime64(
@@ -730,6 +717,9 @@ class Glows(ProcessInstrument):
             exclusions_by_instr_team_combiner = GlowsAncillaryCombiner(
                 exclusions_by_instr_team_files, day_buffer
             )
+            pipeline_settings_combiner = GlowsAncillaryCombiner(
+                pipeline_settings, day_buffer
+            )
 
             datasets = [
                 glows_l1b(
@@ -738,6 +728,7 @@ class Glows(ProcessInstrument):
                     uv_sources_combiner.combined_dataset,
                     suspected_transients_combiner.combined_dataset,
                     exclusions_by_instr_team_combiner.combined_dataset,
+                    pipeline_settings_combiner.combined_dataset,
                 )
             ]
 
@@ -1020,16 +1011,19 @@ class Lo(ProcessInstrument):
         elif self.data_level == "l1b":
             data_dict = {}
             science_files = dependencies.get_file_paths(source="lo", data_type="l1a")
+            ancillary_files = dependencies.get_file_paths(
+                source="lo", data_type="ancillary"
+            )
             logger.info(f"Science files for L1B: {science_files}")
             for file in science_files:
                 dataset = load_cdf(file)
                 data_dict[dataset.attrs["Logical_source"]] = dataset
-            datasets = lo_l1b.lo_l1b(data_dict)
+            datasets = lo_l1b.lo_l1b(data_dict, ancillary_files)
 
         elif self.data_level == "l1c":
             data_dict = {}
             anc_dependencies: list = dependencies.get_file_paths(
-                source="lo", descriptor="goodtimes"
+                source="lo", data_type="ancillary"
             )
             science_files = dependencies.get_file_paths(source="lo", descriptor="de")
             for file in science_files:
@@ -1039,13 +1033,11 @@ class Lo(ProcessInstrument):
 
         elif self.data_level == "l2":
             data_dict = {}
-            # TODO: Add ancillary descriptors when maps using them are
-            #  implemented.
-            anc_dependencies = []
             science_files = dependencies.get_file_paths(source="lo", descriptor="pset")
-            psets = []
-            for file in science_files:
-                psets.append(load_cdf(file))
+            anc_dependencies = dependencies.get_file_paths(data_type="ancillary")
+
+            # Load all pset files into datasets
+            psets = [load_cdf(file) for file in science_files]
             data_dict[psets[0].attrs["Logical_source"]] = psets
             datasets = lo_l2.lo_l2(data_dict, anc_dependencies, self.descriptor)
         return datasets
@@ -1228,8 +1220,8 @@ class Spacecraft(ProcessInstrument):
             The list of processed products.
         """
         print(f"Processing Spacecraft {self.data_level}")
-
-        if self.data_level == "l1a":
+        processed_dataset = []
+        if self.descriptor == "quaternions":
             # File path is expected output file path
             input_files = dependencies.get_file_paths(source="spacecraft")
             if len(input_files) > 1:
@@ -1238,26 +1230,21 @@ class Spacecraft(ProcessInstrument):
                     f"{input_files}. Expected only one dependency."
                 )
             datasets = list(quaternions.process_quaternions(input_files[0]))
-            return datasets
-        elif self.data_level == "spice":
+            processed_dataset.extend(datasets)
+        elif self.descriptor == "pointing-attitude":
             spice_inputs = dependencies.get_file_paths(
                 data_type=SPICESource.SPICE.value
             )
             ah_paths = [path for path in spice_inputs if ".ah" in path.suffixes]
-            if len(ah_paths) != 1:
-                raise ValueError(
-                    f"Unexpected spice dependencies found for Spacecraft "
-                    f"pointing_kernel: {ah_paths}. Expected exactly one "
-                    f"attitude history file."
-                )
             pointing_kernel_paths = pointing_frame.generate_pointing_attitude_kernel(
-                ah_paths[0]
+                ah_paths[-1]
             )
-            return pointing_kernel_paths
+            processed_dataset.extend(pointing_kernel_paths)
         else:
             raise NotImplementedError(
                 f"Spacecraft processing not implemented for level {self.data_level}"
             )
+        return processed_dataset
 
 
 class Swapi(ProcessInstrument):
@@ -1461,7 +1448,10 @@ class Ultra(ProcessInstrument):
             }
             science_files = dependencies.get_file_paths(source="ultra", data_type="l1b")
             l1b_dict = {
-                dataset.attrs["Logical_source"]: dataset
+                # TODO remove
+                dataset.attrs["Logical_source"].replace(
+                    "cullingmask", "goodtimes"
+                ): dataset
                 for dataset in [load_cdf(sci_file) for sci_file in science_files]
             }
             combined = {**l1a_dict, **l1b_dict}
@@ -1470,11 +1460,12 @@ class Ultra(ProcessInstrument):
             for path in anc_paths:
                 ancillary_files[path.stem.split("_")[2]] = path
             spice_paths = dependencies.get_file_paths(data_type="spice")
-            if spice_paths:
-                has_spice = True
+            # Only the helio pset needs IMAP frames
+            if any("imap_frames" in path.as_posix() for path in spice_paths):
+                imap_frames = True
             else:
-                has_spice = False
-            datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, has_spice)
+                imap_frames = False
+            datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, imap_frames)
         elif self.data_level == "l2":
             all_pset_filepaths = dependencies.get_file_paths(
                 source="ultra", descriptor="pset"

imap_processing/codice/codice_l0.py

@@ -39,7 +39,8 @@ def decom_packets(packet_file: Path) -> dict[int, xr.Dataset]:
     # TODO: Currently need to use the 'old' packet definition for housekeeping
     #       because the simulated housekeeping data being used has various
     #       mis-matches from the telemetry definition. This may be updated
-    #       once new simulated housekeeping data are acquired.
+    #       once new simulated housekeeping data are acquired. See GitHub issue
+    #       #2135.
     if "hskp" in str(packet_file):
         xtce_filename = "P_COD_NHK.xml"
     else:

imap_processing/codice/codice_l1a.py

@@ -125,7 +125,7 @@ class CoDICEL1aPipeline:
         # orientation and the azimuth determine which spin sector the data
         # gets stored in.
         # TODO: All these nested for-loops are bad. Try to find a better
-        #       solution.
+        #       solution. See GitHub issue #2136.
         for i, epoch_data in enumerate(self.data):
             for energy_index in range(num_energies):
                 pixel_orientation = constants.PIXEL_ORIENTATIONS[energy_index]
@@ -345,7 +345,7 @@ class CoDICEL1aPipeline:
             # energy dimension
             # TODO: This bit of code may no longer be needed once I can figure
             #       out how to run hi-sectored product through the
-            #       create_binned_dataset function
+            #       create_binned_dataset function. See GitHub issue #2137.
             if self.config["dataset_name"] == "imap_codice_l1a_hi-sectored":
                 dims = [
                     f"energy_{variable_name}" if item == "esa_step" else item
@@ -367,7 +367,7 @@ class CoDICEL1aPipeline:
         # longer need the "esa_step" coordinate
         # TODO: This bit of code may no longer be needed once I can figure
         #       out how to run hi-sectored product through the
-        #       create_binned_dataset function
+        #       create_binned_dataset function. See GitHub issue #2137.
         if self.config["dataset_name"] == "imap_codice_l1a_hi-sectored":
             for species in self.config["energy_table"]:
                 dataset = self.define_energy_bins(dataset, species)
@@ -822,9 +822,6 @@ def group_ialirt_data(
 
     # Workaround to get this function working for both I-ALiRT spacecraft
     # data and CoDICE-specific I-ALiRT test data from Joey
-    # TODO: Once CoDICE I-ALiRT processing is more established, we can probably
-    #       do away with processing the test data from Joey and just use the
-    #       I-ALiRT data that is constructed closer to what we expect in-flight.
     if hasattr(packets, "acquisition_time"):
         time_key = "acquisition_time"
         counter_key = "counter"
@@ -880,7 +877,7 @@ def create_binned_dataset(
         Xarray dataset containing the final processed dataset.
     """
     # TODO: hi-sectored data product should be processed similar to hi-omni,
-    #       so I should be able to use this method.
+    #       so I should be able to use this method. See GitHub issue #2137.
 
     # Get the four "main" parameters for processing
     table_id, plan_id, plan_step, view_id = get_params(dataset)
@@ -901,7 +898,7 @@ def create_binned_dataset(
         attrs=pipeline.cdf_attrs.get_variable_attributes("epoch", check_schema=False),
     )
     # TODO: Figure out how to calculate epoch centers and deltas and store them
-    #       in variables here
+    #       in variables here. See GitHub issue #1501.
     dataset = xr.Dataset(
         coords={"epoch": coord},
         attrs=pipeline.cdf_attrs.get_global_attributes(pipeline.config["dataset_name"]),
@@ -941,7 +938,7 @@ def create_binned_dataset(
     return dataset
 
 
-def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
+def create_direct_event_dataset(apid: int, unpacked_dataset: xr.Dataset) -> xr.Dataset:
     """
     Create dataset for direct event data.
 
@@ -955,7 +952,7 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     dictionary. Padding is added to any fields that have less than 10000 events.
 
     In order to process these data, we must take the decommed raw data, group
-    the packets appropriately based on their `seq_flgs`, decompress the data,
+    the unpacked_dataset appropriately based on their `seq_flgs`, decompress the data,
     then arrange the data into CDF data variables for each priority and bit
     field. For example, P2_SpinAngle represents the spin angles for the 2nd
     priority data.
@@ -964,8 +961,8 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     ----------
     apid : int
         The APID of the packet.
-    packets : xarray.Dataset
-        The packets to process.
+    unpacked_dataset : xarray.Dataset
+        The unpacked dataset to process.
 
     Returns
     -------
@@ -973,13 +970,13 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
         Xarray dataset containing the direct event data.
     """
     # Group and decompress the data
-    grouped_data = group_data(packets)
+    grouped_data = group_data(unpacked_dataset)
     decompressed_data = [
         decompress(group, CoDICECompression.LOSSLESS) for group in grouped_data
     ]
 
     # Reshape the packet data into CDF-ready variables
-    data = reshape_de_data(packets, decompressed_data, apid)
+    reshaped_de_data = reshape_de_data(unpacked_dataset, decompressed_data, apid)
 
     # Gather the CDF attributes
     cdf_attrs = ImapCdfAttributes()
@@ -989,11 +986,11 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     # Determine the epochs to use in the dataset, which are the epochs whenever
     # there is a start of a segment and the priority is 0
     epoch_indices = np.where(
-        ((packets.seq_flgs.data == 3) | (packets.seq_flgs.data == 1))
-        & (packets.priority.data == 0)
+        ((unpacked_dataset.seq_flgs.data == 3) | (unpacked_dataset.seq_flgs.data == 1))
+        & (unpacked_dataset.priority.data == 0)
     )[0]
-    acq_start_seconds = packets.acq_start_seconds[epoch_indices]
-    acq_start_subseconds = packets.acq_start_subseconds[epoch_indices]
+    acq_start_seconds = unpacked_dataset.acq_start_seconds[epoch_indices]
+    acq_start_subseconds = unpacked_dataset.acq_start_subseconds[epoch_indices]
 
     # Calculate epoch variables
     epochs, epochs_delta_minus, epochs_delta_plus = calculate_epoch_values(
@@ -1051,20 +1048,19 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     )
 
     # Create the CDF data variables for each Priority and Field
-    for i in range(constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["num_priorities"]):
-        for field in constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["cdf_fields"]:
-            variable_name = f"p{i}_{field}"
-            attrs = cdf_attrs.get_variable_attributes(variable_name)
-            if field in ["num_events", "data_quality"]:
-                dims = ["epoch"]
-            else:
-                dims = ["epoch", "event_num"]
-            dataset[variable_name] = xr.DataArray(
-                np.array(data[variable_name]),
-                name=variable_name,
-                dims=dims,
-                attrs=attrs,
-            )
+    for field in constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["cdf_fields"]:
+        if field in ["num_events", "data_quality"]:
+            attrs = cdf_attrs.get_variable_attributes("de_2d_attrs")
+            dims = ["epoch", "priority"]
+        else:
+            attrs = cdf_attrs.get_variable_attributes("de_3d_attrs")
+            dims = ["epoch", "priority", "event_num"]
+        dataset[field] = xr.DataArray(
+            np.array(reshaped_de_data[field]),
+            name=field,
+            dims=dims,
+            attrs=attrs,
+        )
 
     return dataset
 
@@ -1490,7 +1486,7 @@ def reshape_de_data(
         CDF variable names, and the values represent the data.
     """
     # Dictionary to hold all the (soon to be restructured) direct event data
-    data: dict[str, np.ndarray] = {}
+    de_data: dict[str, np.ndarray] = {}
 
     # Extract some useful variables
     num_priorities = constants.DE_DATA_PRODUCT_CONFIGURATIONS[apid]["num_priorities"]
@@ -1510,18 +1506,20 @@ def reshape_de_data(
 
     # Initialize data arrays for each priority and field to store the data
     # We also need arrays to hold number of events and data quality
-    for priority_num in range(num_priorities):
-        for field in bit_structure:
-            if field not in ["Priority", "Spare"]:
-                data[f"p{priority_num}_{field}"] = np.full(
-                    (num_epochs, 10000),
-                    bit_structure[field]["fillval"],
-                    dtype=bit_structure[field]["dtype"],
-                )
-        data[f"p{priority_num}_num_events"] = np.full(
-            num_epochs, 65535, dtype=np.uint16
-        )
-        data[f"p{priority_num}_data_quality"] = np.full(num_epochs, 255, dtype=np.uint8)
+    for field in bit_structure:
+        # if these two, no need to store
+        if field not in ["Priority", "Spare"]:
+            de_data[f"{field}"] = np.full(
+                (num_epochs, num_priorities, 10000),
+                bit_structure[field]["fillval"],
+                dtype=bit_structure[field]["dtype"],
+            )
+    # Add other additional fields of l1a
+    de_data["num_events"] = np.full(
+        (num_epochs, num_priorities), 65535, dtype=np.uint16
+    )
+
+    de_data["data_quality"] = np.full((num_epochs, num_priorities), 255, dtype=np.uint8)
 
     # decompressed_data is one large list of values of length
     # (<number of epochs> * <number of priorities>)
@@ -1545,8 +1543,8 @@ def reshape_de_data(
 
             # Number of events and data quality can be determined at this stage
             num_events = num_events_arr[epoch_start:epoch_end][i]
-            data[f"p{priority_num}_num_events"][epoch_index] = num_events
-            data[f"p{priority_num}_data_quality"][epoch_index] = data_quality[i]
+            de_data["num_events"][epoch_index, priority_num] = num_events
+            de_data["data_quality"][epoch_index, priority_num] = data_quality[i]
 
             # Iterate over each event
             for event_index in range(num_events):
@@ -1577,12 +1575,12 @@ def reshape_de_data(
                     )
 
                     # Set the value into the data array
-                    data[f"p{priority_num}_{field_name}"][epoch_index, event_index] = (
+                    de_data[f"{field_name}"][epoch_index, priority_num, event_index] = (
                         value
                     )
                     bit_position += field_components["bit_length"]
 
-    return data
+    return de_data
 
 
 def process_codice_l1a(file_path: Path) -> list[xr.Dataset]:

imap_processing/codice/codice_l1b.py

@@ -9,18 +9,18 @@ from imap_processing.codice.codice_l1b import process_codice_l1b
 dataset = process_codice_l1b(l1a_filenanme)
 """
 
-# TODO: Figure out how to convert hi-priority data product. Need an updated
-#       algorithm document that describes this.
-
 import logging
 from pathlib import Path
 
 import numpy as np
 import xarray as xr
 
+from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 from imap_processing.cdf.utils import load_cdf
 from imap_processing.codice import constants
+from imap_processing.codice.utils import CODICEAPID
+from imap_processing.utils import packet_file_to_datasets
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -49,9 +49,6 @@ def convert_to_rates(
     rates_data : np.ndarray
         The converted data array.
     """
-    # TODO: Temporary workaround to create CDFs for SIT-4. Revisit after SIT-4.
-    acq_times = 1
-
     if descriptor in [
         "lo-counters-aggregated",
         "lo-counters-singles",
@@ -65,6 +62,13 @@ def convert_to_rates(
     ]:
         # Applying rate calculation described in section 10.2 of the algorithm
         # document
+        # In order to divide by acquisition times, we must reshape the acq
+        # time data array to match the data variable shape
+        dims = [1] * dataset[variable_name].data.ndim
+        dims[1] = 128
+        acq_times = dataset.acquisition_time_per_step.data.reshape(dims)
+
+        # Now perform the calculation
         rates_data = dataset[variable_name].data / (
             acq_times
             * 1e-6  # Converting from microseconds to seconds
@@ -83,10 +87,8 @@ def convert_to_rates(
         rates_data = dataset[variable_name].data / (
             constants.L1B_DATA_PRODUCT_CONFIGURATIONS[descriptor]["num_spin_sectors"]
             * constants.L1B_DATA_PRODUCT_CONFIGURATIONS[descriptor]["num_spins"]
-            * acq_times
+            * constants.HI_ACQUISITION_TIME
         )
-    elif descriptor == "hskp":
-        rates_data = dataset[variable_name].data / acq_times
 
     return rates_data
 
@@ -131,35 +133,43 @@ def process_codice_l1b(file_path: Path) -> xr.Dataset:
     # Update the global attributes
     l1b_dataset.attrs = cdf_attrs.get_global_attributes(dataset_name)
 
-    # Determine which variables need to be converted from counts to rates
-    # TODO: Figure out exactly which hskp variables need to be converted
-    # Housekeeping and binned datasets are treated a bit differently since
-    # not all variables need to be converted
+    # TODO: This was thrown together quickly and should be double-checked
     if descriptor == "hskp":
-        # TODO: Check with Joey if any housekeeping data needs to be converted
-        variables_to_convert = []
-    elif descriptor == "hi-sectored":
-        variables_to_convert = ["h", "he3he4", "cno", "fe"]
-    elif descriptor == "hi-omni":
-        variables_to_convert = ["h", "he3", "he4", "c", "o", "ne_mg_si", "fe", "uh"]
-    elif descriptor == "hi-ialirt":
-        variables_to_convert = ["h"]
+        xtce_filename = "codice_packet_definition.xml"
+        xtce_packet_definition = Path(
+            f"{imap_module_directory}/codice/packet_definitions/{xtce_filename}"
+        )
+        packet_file = (
+            imap_module_directory
+            / "tests"
+            / "codice"
+            / "data"
+            / "imap_codice_l0_raw_20241110_v001.pkts"
+        )
+        datasets: dict[int, xr.Dataset] = packet_file_to_datasets(
+            packet_file, xtce_packet_definition, use_derived_value=True
+        )
+        l1b_dataset = datasets[CODICEAPID.COD_NHK]
+
+        # TODO: Drop the same variables as we do in L1a? (see line 1103 in
+        #       codice_l1a.py
+
     else:
         variables_to_convert = getattr(
             constants, f"{descriptor.upper().replace('-', '_')}_VARIABLE_NAMES"
         )
 
-    # Apply the conversion to rates
-    for variable_name in variables_to_convert:
-        l1b_dataset[variable_name].data = convert_to_rates(
-            l1b_dataset, descriptor, variable_name
-        )
-
-        # Set the variable attributes
-        cdf_attrs_key = f"{descriptor}-{variable_name}"
-        l1b_dataset[variable_name].attrs = cdf_attrs.get_variable_attributes(
-            cdf_attrs_key, check_schema=False
-        )
+        # Apply the conversion to rates
+        for variable_name in variables_to_convert:
+            l1b_dataset[variable_name].data = convert_to_rates(
+                l1b_dataset, descriptor, variable_name
+            )
+
+            # Set the variable attributes
+            cdf_attrs_key = f"{descriptor}-{variable_name}"
+            l1b_dataset[variable_name].attrs = cdf_attrs.get_variable_attributes(
+                cdf_attrs_key, check_schema=False
+            )
 
     logger.info(f"\nFinal data product:\n{l1b_dataset}\n")