imap-processing 0.16.2__py3-none-any.whl → 0.18.0__py3-none-any.whl

imap_processing/cli.py

@@ -68,6 +68,7 @@ from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
 from imap_processing.mag.l1b.mag_l1b import mag_l1b
 from imap_processing.mag.l1c.mag_l1c import mag_l1c
+from imap_processing.mag.l1d.mag_l1d import mag_l1d
 from imap_processing.mag.l2.mag_l2 import mag_l2
 from imap_processing.spacecraft import quaternions
 from imap_processing.spice import pointing_frame, repoint, spin
@@ -516,7 +517,7 @@ class ProcessInstrument(ABC):
         self,
         processed_data: list[xr.Dataset | Path],
         dependencies: ProcessingInputCollection,
-    ) -> None:
+    ) -> list[Path]:
         """
         Complete post-processing.
 
@@ -544,10 +545,17 @@ class ProcessInstrument(ABC):
             method.
         dependencies : ProcessingInputCollection
             Object containing dependencies to process.
+
+        Returns
+        -------
+        list[Path]
+            List of paths to CDF files produced.
         """
+        products: list[Path] = []
+
         if len(processed_data) == 0:
             logger.info("No products to write to CDF file.")
-            return
+            return products
 
         logger.info("Writing products to local storage")
 
@@ -567,7 +575,6 @@ class ProcessInstrument(ABC):
         # start_date.
         # If it is start_date, skip repointing in the output filename.
 
-        products = []
         for ds in processed_data:
             if isinstance(ds, xr.Dataset):
                 ds.attrs["Data_version"] = self.version[1:]  # Strip 'v' from version
@@ -581,6 +588,7 @@ class ProcessInstrument(ABC):
                 products.append(ds)
 
         self.upload_products(products)
+        return products
 
     @final
     def cleanup(self) -> None:
@@ -896,29 +904,28 @@ class Idex(ProcessInstrument):
             dependency = load_cdf(science_files[0])
             datasets = [idex_l2a(dependency)]
         elif self.data_level == "l2b":
-            if len(dependency_list) != 3:
+            if len(dependency_list) < 3 or len(dependency_list) > 4:
                 raise ValueError(
                     f"Unexpected dependencies found for IDEX L2B:"
-                    f"{dependency_list}. Expected only three dependencies."
+                    f"{dependency_list}. Expected three or four dependencies."
                 )
             sci_files = dependencies.get_file_paths(
                 source="idex", descriptor="sci-1week"
             )
-            dependency = load_cdf(sci_files[0])
+            sci_dependencies = [load_cdf(f) for f in sci_files]
             hk_files = dependencies.get_file_paths(source="idex", descriptor="evt")
-            hk_dependencies = [load_cdf(dep) for dep in hk_files]
-            datasets = [idex_l2b(dependency, hk_dependencies)]
+            # Remove duplicate housekeeping files
+            hk_dependencies = [load_cdf(dep) for dep in list(set(hk_files))]
+            datasets = [idex_l2b(sci_dependencies, hk_dependencies)]
         elif self.data_level == "l2c":
             if len(dependency_list) != 1:
                 raise ValueError(
                     f"Unexpected dependencies found for IDEX L2C:"
                     f"{dependency_list}. Expected only one dependency."
                 )
-            sci_files = dependencies.get_file_paths(
-                source="idex", descriptor="sci-1week"
-            )
-            dependency = load_cdf(sci_files[0])
-            datasets = idex_l2c(dependency)
+            sci_files = dependencies.get_file_paths(source="idex", descriptor="sci-1mo")
+            dependencies = [load_cdf(f) for f in sci_files]
+            datasets = [idex_l2c(dependencies)]
         return datasets
 
 
@@ -1074,6 +1081,21 @@ class Mag(ProcessInstrument):
                     f"Invalid dependencies found for MAG L1C:"
                     f"{dependencies}. Expected one or two dependencies."
                 )
+        if self.data_level == "l1d":
+            science_files = dependencies.get_file_paths(source="mag", data_type="l1c")
+            science_files.extend(
+                dependencies.get_file_paths(source="mag", data_type="l1b")
+            )
+            input_data = [load_cdf(dep) for dep in science_files]
+            calibration = dependencies.get_processing_inputs(
+                descriptor="l1d-calibration"
+            )
+            combined_calibration = MagAncillaryCombiner(calibration[0], day_buffer)
+            datasets = mag_l1d(
+                input_data,
+                combined_calibration.combined_dataset,
+                current_day,
+            )
 
         if self.data_level == "l2":
             science_files = dependencies.get_file_paths(source="mag", data_type="l1b")
@@ -1122,6 +1144,13 @@ class Mag(ProcessInstrument):
                 mode=DataMode(descriptor_no_frame.upper()),
             )
 
+        for ds in datasets:
+            if "raw" not in ds.attrs["Logical_source"] and not np.all(
+                ds["epoch"].values[1:] > ds["epoch"].values[:-1]
+            ):
+                raise ValueError(
+                    "Timestamps for output file are not monotonically increasing."
+                )
         return datasets
 
 

imap_processing/codice/codice_l1a.py

@@ -54,8 +54,6 @@ class CoDICEL1aPipeline:
 
     Methods
     -------
-    calculate_epoch_values()
-        Calculate and return the values to be used for `epoch`.
     decompress_data(science_values)
         Perform decompression on the data.
     define_coordinates()
@@ -89,28 +87,6 @@ class CoDICEL1aPipeline:
         self.plan_step = plan_step
         self.view_id = view_id
 
-    def calculate_epoch_values(self) -> NDArray[int]:
-        """
-        Calculate and return the values to be used for `epoch`.
-
-        On CoDICE, the epoch values are derived from the `acq_start_seconds` and
-        `acq_start_subseconds` fields in the packet.
-
-        Note that the `acq_start_subseconds` field needs to be converted from
-        microseconds to seconds.
-
-        Returns
-        -------
-        epoch : NDArray[int]
-            List of epoch values.
-        """
-        epoch = met_to_ttj2000ns(
-            self.dataset["acq_start_seconds"]
-            + self.dataset["acq_start_subseconds"] / 1e6
-        )
-
-        return epoch
-
     def decompress_data(self, science_values: list[NDArray[str]] | list[str]) -> None:
         """
         Perform decompression on the data.
@@ -167,17 +143,30 @@ class CoDICEL1aPipeline:
         self.coords = {}
 
         coord_names = [
-            "epoch",
-            *self.config["output_dims"].keys(),
-            *[key + "_label" for key in self.config["output_dims"].keys()],
+            *self.config["dims"].keys(),
+            *[key + "_label" for key in self.config["dims"].keys()],
         ]
 
+        # Define epoch coordinates
+        epochs, epoch_delta_minus, epoch_delta_plus = calculate_epoch_values(
+            self.dataset.acq_start_seconds, self.dataset.acq_start_subseconds
+        )
+        for name, var in [
+            ("epoch", epochs),
+            ("epoch_delta_minus", epoch_delta_minus),
+            ("epoch_delta_plus", epoch_delta_plus),
+        ]:
+            coord = xr.DataArray(
+                var,
+                name=name,
+                dims=[name],
+                attrs=self.cdf_attrs.get_variable_attributes(name, check_schema=False),
+            )
+            self.coords[name] = coord
+
         # Define the values for the coordinates
         for name in coord_names:
-            if name == "epoch":
-                values = self.calculate_epoch_values()
-                dims = [name]
-            elif name in [
+            if name in [
                 "esa_step",
                 "inst_az",
                 "spin_sector",
@@ -185,7 +174,7 @@ class CoDICEL1aPipeline:
                 "spin_sector_index",
                 "ssd_index",
             ]:
-                values = np.arange(self.config["output_dims"][name])
+                values = np.arange(self.config["dims"][name])
                 dims = [name]
             elif name == "spin_sector_pairs_label":
                 values = np.array(
@@ -199,15 +188,22 @@ class CoDICEL1aPipeline:
                     ]
                 )
                 dims = [name]
+            elif name == "inst_az_label":
+                if self.config["dataset_name"] == "imap_codice_l1a_lo-nsw-angular":
+                    values = [str(x) for x in range(4, 23)]
+                elif self.config["dataset_name"] == "imap_codice_l1a_lo-sw-angular":
+                    values = ["1", "2", "3", "23", "24"]
+                else:
+                    values = np.arange(self.config["dims"]["inst_az"]).astype(str)
+                dims = ["inst_az"]
             elif name in [
                 "spin_sector_label",
                 "esa_step_label",
-                "inst_az_label",
                 "spin_sector_index_label",
                 "ssd_index_label",
             ]:
                 key = name.removesuffix("_label")
-                values = np.arange(self.config["output_dims"][key]).astype(str)
+                values = np.arange(self.config["dims"][key]).astype(str)
                 dims = [key]
 
             coord = xr.DataArray(
@@ -241,16 +237,16 @@ class CoDICEL1aPipeline:
         # Stack the data so that it is easier to reshape and iterate over
         all_data = np.stack(self.data)
 
-        # The dimension of all_data is something like (epoch, num_counters,
-        # num_energy_steps, num_positions, num_spin_sectors) (or may be slightly
+        # The dimension of all_data is something like (epoch, num_energy_steps,
+        # num_positions, num_spin_sectors, num_counters) (or may be slightly
         # different depending on the data product). In any case, iterate over
         # the num_counters dimension to isolate the data for each counter so
         # each counter's data can be placed in a separate CDF data variable.
         for counter, variable_name in zip(
-            range(all_data.shape[1]), self.config["variable_names"]
+            range(all_data.shape[-1]), self.config["variable_names"]
         ):
             # Extract the counter data
-            counter_data = all_data[:, counter, ...]
+            counter_data = all_data[..., counter]
 
             # Get the CDF attributes
             descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1]
@@ -260,7 +256,7 @@ class CoDICEL1aPipeline:
             # For most products, the final CDF dimensions always has "epoch" as
             # the first dimension followed by the dimensions for the specific
             # data product
-            dims = ["epoch", *list(self.config["output_dims"].keys())]
+            dims = ["epoch", *list(self.config["dims"].keys())]
 
             # However, CoDICE-Hi products use specific energy bins for the
             # energy dimension
@@ -409,6 +405,12 @@ class CoDICEL1aPipeline:
                 dims = ["epoch"]
                 attrs = self.cdf_attrs.get_variable_attributes("spin_period")
 
+            # The k-factor is a constant that maps voltages to energies
+            elif variable_name == "k_factor":
+                variable_data = np.array([constants.K_FACTOR], dtype=np.float32)
+                dims = [""]
+                attrs = self.cdf_attrs.get_variable_attributes("k_factor")
+
             # Add variable to the dataset
             dataset[variable_name] = xr.DataArray(
                 variable_data,
@@ -600,52 +602,27 @@ class CoDICEL1aPipeline:
 
         These data need to be divided up by species or priorities (or
         what I am calling "counters" as a general term), and re-arranged into
-        4D arrays representing dimensions such as time, spin sectors, positions,
-        and energies (depending on the data product).
+        multidimensional arrays representing dimensions such as time,
+        spin sectors, positions, and energies (depending on the data product).
 
         However, the existence and order of these dimensions can vary depending
-        on the specific data product, so we define this in the "input_dims"
-        and "output_dims" values configuration dictionary; the "input_dims"
-        defines how the dimensions are written into the packet data, while
-        "output_dims" defines how the dimensions should be written to the final
-        CDF product.
+        on the specific data product, so we define this in the "dims" key of the
+        configuration dictionary.
         """
         # This will contain the reshaped data for all counters
         self.data = []
 
-        # First reshape the data based on how it is written to the data array of
-        # the packet data. The number of counters is the first dimension / axis,
-        # with the exception of lo-counters-aggregated which is treated slightly
-        # differently
-        if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
-            reshape_dims = (
-                self.config["num_counters"],
-                *self.config["input_dims"].values(),
-            )
-        else:
-            reshape_dims = (
-                *self.config["input_dims"].values(),
-                self.config["num_counters"],
-            )
-
-        # Then, transpose the data based on how the dimensions should be written
-        # to the CDF file. Since this is specific to each data product, we need
-        # to determine this dynamically based on the "output_dims" config.
-        # Again, lo-counters-aggregated is treated slightly differently
-        input_keys = ["num_counters", *self.config["input_dims"].keys()]
-        output_keys = ["num_counters", *self.config["output_dims"].keys()]
-        if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
-            transpose_axes = [input_keys.index(dim) for dim in output_keys]
-        else:
-            transpose_axes = [1, 2, 0]  # [esa_step, spin_sector_pairs, num_counters]
-
+        # Reshape the data based on how it is written to the data array of
+        # the packet data. The number of counters is the last dimension / axis.
+        reshape_dims = (
+            *self.config["dims"].values(),
+            self.config["num_counters"],
+        )
         for packet_data in self.raw_data:
             reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape(
                 reshape_dims
             )
-            reshaped_cdf_data = np.transpose(reshaped_packet_data, axes=transpose_axes)
-
-            self.data.append(reshaped_cdf_data)
+            self.data.append(reshaped_packet_data)
 
         # No longer need to keep the raw data around
         del self.raw_data
@@ -674,6 +651,54 @@ class CoDICEL1aPipeline:
         self.cdf_attrs.add_instrument_variable_attrs("codice", "l1a")
 
 
+def calculate_epoch_values(
+    acq_start_seconds: xr.DataArray, acq_start_subseconds: xr.DataArray
+) -> tuple[NDArray[int], NDArray[int], NDArray[int]]:
+    """
+    Calculate and return the values to be used for `epoch`.
+
+    On CoDICE, the epoch values are derived from the `acq_start_seconds` and
+    `acq_start_subseconds` fields in the packet.
+
+    Note that the `acq_start_subseconds` field needs to be converted from
+    microseconds to seconds.
+
+    Parameters
+    ----------
+    acq_start_seconds : xarray.DataArray
+        The acquisition times to calculate the epoch values from.
+    acq_start_subseconds : xarray.DataArray
+        The subseconds portion of the acquisition times.
+
+    Returns
+    -------
+    epoch : NDArray[int]
+        List of centered epoch values.
+    epoch_delta_minus: NDArray[int]
+        List of values that represent the length of time from acquisition
+        start to the center of the acquisition time bin.
+    epoch_delta_plus: NDArray[int]
+        List of values that represent the length of time from the center of
+        the acquisition time bin to the end of acquisition.
+    """
+    # First calculate an epoch value based on the acquisition start
+    acq_start = met_to_ttj2000ns(acq_start_seconds + acq_start_subseconds / 1e6)
+
+    # Apply correction to center the epoch bin
+    epoch = (acq_start[:-1] + acq_start[1:]) // 2
+    epoch_delta_minus = epoch - acq_start[:-1]
+    epoch_delta_plus = acq_start[1:] - epoch
+
+    # Since the centers and deltas are determined by averaging sequential bins,
+    # the last elements must be calculated differently. For this, we just use
+    # the last acquisition start and the previous deltas
+    epoch = np.concatenate([epoch, [acq_start[-1]]])
+    epoch_delta_minus = np.concatenate([epoch_delta_minus, [epoch_delta_minus[-1]]])
+    epoch_delta_plus = np.concatenate([epoch_delta_plus, [epoch_delta_plus[-1]]])
+
+    return epoch, epoch_delta_minus, epoch_delta_plus
+
+
 def group_ialirt_data(
     packets: xr.Dataset, data_field_range: range, prefix: str
 ) -> list[bytearray]:
@@ -777,6 +802,8 @@ def create_binned_dataset(
         dims=["epoch"],
         attrs=pipeline.cdf_attrs.get_variable_attributes("epoch", check_schema=False),
     )
+    # TODO: Figure out how to calculate epoch centers and deltas and store them
+    #       in variables here
     dataset = xr.Dataset(
         coords={"epoch": coord},
         attrs=pipeline.cdf_attrs.get_global_attributes(pipeline.config["dataset_name"]),
@@ -869,7 +896,11 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     )[0]
     acq_start_seconds = packets.acq_start_seconds[epoch_indices]
     acq_start_subseconds = packets.acq_start_subseconds[epoch_indices]
-    epochs = met_to_ttj2000ns(acq_start_seconds + acq_start_subseconds / 1e6)
+
+    # Calculate epoch variables
+    epochs, epochs_delta_minus, epochs_delta_plus = calculate_epoch_values(
+        acq_start_seconds, acq_start_subseconds
+    )
 
     # Define coordinates
     epoch = xr.DataArray(
@@ -878,6 +909,20 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
         dims=["epoch"],
         attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False),
     )
+    epoch_delta_minus = xr.DataArray(
+        epochs_delta_minus,
+        name="epoch_delta_minus",
+        dims=["epoch_delta_minus"],
+        attrs=cdf_attrs.get_variable_attributes(
+            "epoch_delta_minus", check_schema=False
+        ),
+    )
+    epoch_delta_plus = xr.DataArray(
+        epochs_delta_plus,
+        name="epoch_delta_plus",
+        dims=["epoch_delta_plus"],
+        attrs=cdf_attrs.get_variable_attributes("epoch_delta_plus", check_schema=False),
+    )
     event_num = xr.DataArray(
         np.arange(10000),
         name="event_num",
@@ -893,12 +938,14 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
 
     # Create the dataset to hold the data variables
     if apid == CODICEAPID.COD_LO_PHA:
-        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_lo-pha")
+        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_lo-direct-events")
     elif apid == CODICEAPID.COD_HI_PHA:
-        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_hi-pha")
+        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_hi-direct-events")
     dataset = xr.Dataset(
         coords={
             "epoch": epoch,
+            "epoch_delta_minus": epoch_delta_minus,
+            "epoch_delta_plus": epoch_delta_plus,
             "event_num": event_num,
             "event_num_label": event_num_label,
         },

imap_processing/codice/codice_l1b.py

@@ -116,7 +116,7 @@ def process_codice_l1b(file_path: Path) -> xr.Dataset:
     descriptor = dataset_name.removeprefix("imap_codice_l1b_")
 
     # Direct event data products do not have a level L1B
-    if descriptor in ["lo-pha", "hi-pha"]:
+    if descriptor in ["lo-direct-events", "hi-direct-events"]:
         logger.warning("Encountered direct event data product. Skipping L1b processing")
         return None
 

imap_processing/codice/codice_l2.py

@@ -45,17 +45,8 @@ def process_codice_l2(file_path: Path) -> xr.Dataset:
     # TODO: Could clean this up by using imap-data-access methods?
     dataset_name = l1_dataset.attrs["Logical_source"]
     data_level = dataset_name.removeprefix("imap_codice_").split("_")[0]
-    descriptor = dataset_name.removeprefix(f"imap_codice_{data_level}_")
     dataset_name = dataset_name.replace(data_level, "l2")
 
-    # TODO: Temporary work-around to replace "PHA" naming convention with
-    #       "direct events" This will eventually be changed at the L1 level and
-    #       thus this will eventually be removed.
-    if descriptor == "lo-pha":
-        dataset_name = dataset_name.replace("lo-pha", "lo-direct-events")
-    elif descriptor == "hi-pha":
-        dataset_name = dataset_name.replace("hi-pha", "hi-direct-events")
-
     # Use the L1 data product as a starting point for L2
     l2_dataset = l1_dataset.copy()