imap-processing 0.19.3__py3-none-any.whl → 1.0.0__py3-none-any.whl

imap_processing/cdf/config/imap_enamaps_l2-healpix_variable_attrs.yaml

@@ -52,9 +52,8 @@ ena_intensity:
   DEPEND_2: pixel_index
   LABL_PTR_1: energy_label
   LABL_PTR_2: pixel_index_label
-  DISPLAY_TYPE: image
 
-ena_intensity_stat_unc:
+ena_intensity_stat_uncert:
   DEPEND_1: energy
   DEPEND_2: pixel_index
   LABL_PTR_1: energy_label

imap_processing/cdf/config/imap_enamaps_l2-rectangular_variable_attrs.yaml

@@ -19,9 +19,10 @@ longitude_label:
   DEPEND_1: longitude
 
 longitude_delta:
-  VAR_TYPE: metadata
+  VAR_TYPE: support_data
   dtype: float32
   CATDESC: Half-width of longitude pixel
+  DEPEND_1: longitude
   FORMAT: F12.6
   UNITS: degrees
   FIELDNAM: longitude delta
@@ -35,9 +36,10 @@ latitude_label:
   DEPEND_1: latitude
 
 latitude_delta:
-  VAR_TYPE: metadata
+  VAR_TYPE: support_data
   dtype: float32
   CATDESC: Half-width of latitude pixel
+  DEPEND_1: latitude
   FORMAT: F12.6
   UNITS: degrees
   FIELDNAM: latitude delta
@@ -65,7 +67,7 @@ ena_intensity:
   LABL_PTR_2: longitude_label
   LABL_PTR_3: latitude_label
 
-ena_intensity_stat_unc:
+ena_intensity_stat_uncert:
   DEPEND_1: energy
   DEPEND_2: longitude
   DEPEND_3: latitude

imap_processing/cdf/config/imap_hi_global_cdf_attrs.yaml

@@ -54,8 +54,7 @@ imap_hi_l1c_pset_attrs:
     Logical_source: imap_hi_l1c_{sensor}-pset
     Logical_source_description: IMAP-Hi Instrument Level-1C Pointing Set Data.
 
-# TODO: Finalize these global attributes
-imap_hi_l2_enamap-sf:
+imap_hi_l2_enamap:
     Data_type: L2_{descriptor}>Level-2 ENA Intensity Map for Hi{sensor}
     Logical_source: imap_hi_l2_{descriptor}
     Logical_source_description: IMAP-Hi Instrument Level-2 ENA Intensity Map Data for Hi{sensor} on rectangular tiling.

imap_processing/cdf/config/imap_hi_variable_attrs.yaml

@@ -252,6 +252,7 @@ hi_de_ccsds_index:
 # LABL_PTR_i expects VAR_TYPE of metadata with char data type
 hi_hist_angle_label:
   CATDESC: Angle bin centers for histogram data.
+  DEPEND_1: angle
   FIELDNAM: ANGLE
   FORMAT: A5
   VAR_TYPE: metadata

imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml

@@ -65,15 +65,15 @@ dead_time_ratio:
 
 sensitivity:
   <<: *energy_dependent
-  CATDESC: Calibration/sensitivity factor.
+  CATDESC: Averaged instrument sensitive area.
   FIELDNAM: sensitivity
   LABLAXIS: sensitivity
   # TODO: come back to format
-  UNITS: counts/second
+  UNITS: cm^2
 
 geometric_function:
   <<: *energy_dependent
-  CATDESC: The effective sensitive area as a function of theta and phi in instrument frame (energy independent).
+  CATDESC: Averaged aperture area as seen by the backstop.
   FIELDNAM: geometric_function
   LABLAXIS: Geometric Factor
   UNITS: cm^2
@@ -81,7 +81,7 @@ geometric_function:
 
 efficiency:
   <<: *energy_dependent
-  CATDESC: Estimated event efficiency for particles path through the instrument.
+  CATDESC: Efficiency of the instrument in converting ENAs to valid events.
   FIELDNAM: efficiency
   LABLAXIS: Efficiency
   UNITS: " "
@@ -90,6 +90,7 @@ efficiency:
 
 exposure_factor:
   <<: *energy_dependent
+  DEPEND_0: epoch
   CATDESC: Exposure time with the deadtime correction applied for a pointing.
   FIELDNAM: exposure_factor
   LABLAXIS: exposure factor
@@ -101,19 +102,20 @@ helio_exposure_factor:
   CATDESC: Exposure time with the deadtime correction applied for a pointing.
   FIELDNAM: exposure_factor
   LABLAXIS: exposure factor
+  DEPEND_0: epoch
   # TODO: come back to format
   UNITS: seconds
 
 scatter_theta:
   <<: *energy_dependent
-  CATDESC: Scattering theta values for a pointing.
+  CATDESC: Scattering theta values for a pointing (reported as gaussian uncertainty).
   FIELDNAM: scatter_theta
   LABLAXIS: scatter theta
   UNITS: degrees
 
 scatter_phi:
   <<: *energy_dependent
-  CATDESC: Scattering phi values for a pointing.
+  CATDESC: Scattering phi values for a pointing (reported as gaussian uncertainty).
   FIELDNAM: scatter_phi
   LABLAXIS: scatter phi
   UNITS: degrees

imap_processing/cdf/utils.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import datetime
 import logging
 import re
 import warnings
@@ -148,6 +149,10 @@ def write_cdf(
     dataset.attrs["Logical_file_id"] = file_path.stem
     # Add the processing version to the dataset attributes
     dataset.attrs["ground_software_version"] = imap_processing._version.__version__
+    dataset.attrs["Generation_date"] = datetime.datetime.now(
+        datetime.timezone.utc
+    ).strftime("%Y%m%d")
+    dataset.attrs["Generated_by"] = "IMAP Science Data Center"
 
     # Convert the xarray object to a CDF
     if "l1" in data_level:

imap_processing/cli.py

@@ -51,7 +51,7 @@ from imap_processing.cdf.utils import load_cdf, write_cdf
 #   call cdf.utils.write_cdf
 from imap_processing.codice import codice_l1a, codice_l1b, codice_l2
 from imap_processing.glows.l1a.glows_l1a import glows_l1a
-from imap_processing.glows.l1b.glows_l1b import glows_l1b
+from imap_processing.glows.l1b.glows_l1b import glows_l1b, glows_l1b_de
 from imap_processing.glows.l2.glows_l2 import glows_l2
 from imap_processing.hi import hi_l1a, hi_l1b, hi_l1c, hi_l2
 from imap_processing.hit.l1a.hit_l1a import hit_l1a
@@ -682,55 +682,60 @@ class Glows(ProcessInstrument):
                     f"{science_files}."
                 )
             input_dataset = load_cdf(science_files[0])
+            if "hist" in self.descriptor:
+                # Create file lists for each ancillary type
+                excluded_regions_files = dependencies.get_processing_inputs(
+                    descriptor="map-of-excluded-regions"
+                )[0]
+                uv_sources_files = dependencies.get_processing_inputs(
+                    descriptor="map-of-uv-sources"
+                )[0]
+                suspected_transients_files = dependencies.get_processing_inputs(
+                    descriptor="suspected-transients"
+                )[0]
+                exclusions_by_instr_team_files = dependencies.get_processing_inputs(
+                    descriptor="exclusions-by-instr-team"
+                )[0]
+                pipeline_settings = dependencies.get_processing_inputs(
+                    descriptor="pipeline-settings"
+                )[0]
 
-            # Create file lists for each ancillary type
-            excluded_regions_files = dependencies.get_processing_inputs(
-                descriptor="map-of-excluded-regions"
-            )[0]
-            uv_sources_files = dependencies.get_processing_inputs(
-                descriptor="map-of-uv-sources"
-            )[0]
-            suspected_transients_files = dependencies.get_processing_inputs(
-                descriptor="suspected-transients"
-            )[0]
-            exclusions_by_instr_team_files = dependencies.get_processing_inputs(
-                descriptor="exclusions-by-instr-team"
-            )[0]
-            pipeline_settings = dependencies.get_processing_inputs(
-                descriptor="pipeline-settings"
-            )[0]
-
-            # Use end date buffer for ancillary data
-            current_day = np.datetime64(
-                f"{self.start_date[:4]}-{self.start_date[4:6]}-{self.start_date[6:]}"
-            )
-            day_buffer = current_day + np.timedelta64(3, "D")
-
-            # Create combiners for each ancillary dataset
-            excluded_regions_combiner = GlowsAncillaryCombiner(
-                excluded_regions_files, day_buffer
-            )
-            uv_sources_combiner = GlowsAncillaryCombiner(uv_sources_files, day_buffer)
-            suspected_transients_combiner = GlowsAncillaryCombiner(
-                suspected_transients_files, day_buffer
-            )
-            exclusions_by_instr_team_combiner = GlowsAncillaryCombiner(
-                exclusions_by_instr_team_files, day_buffer
-            )
-            pipeline_settings_combiner = GlowsAncillaryCombiner(
-                pipeline_settings, day_buffer
-            )
+                # Use end date buffer for ancillary data
+                current_day = np.datetime64(
+                    f"{self.start_date[:4]}-{self.start_date[4:6]}-{self.start_date[6:]}"
+                )
+                day_buffer = current_day + np.timedelta64(3, "D")
 
-            datasets = [
-                glows_l1b(
-                    input_dataset,
-                    excluded_regions_combiner.combined_dataset,
-                    uv_sources_combiner.combined_dataset,
-                    suspected_transients_combiner.combined_dataset,
-                    exclusions_by_instr_team_combiner.combined_dataset,
-                    pipeline_settings_combiner.combined_dataset,
+                # Create combiners for each ancillary dataset
+                excluded_regions_combiner = GlowsAncillaryCombiner(
+                    excluded_regions_files, day_buffer
+                )
+                uv_sources_combiner = GlowsAncillaryCombiner(
+                    uv_sources_files, day_buffer
+                )
+                suspected_transients_combiner = GlowsAncillaryCombiner(
+                    suspected_transients_files, day_buffer
+                )
+                exclusions_by_instr_team_combiner = GlowsAncillaryCombiner(
+                    exclusions_by_instr_team_files, day_buffer
+                )
+                pipeline_settings_combiner = GlowsAncillaryCombiner(
+                    pipeline_settings, day_buffer
                 )
-            ]
+
+                datasets = [
+                    glows_l1b(
+                        input_dataset,
+                        excluded_regions_combiner.combined_dataset,
+                        uv_sources_combiner.combined_dataset,
+                        suspected_transients_combiner.combined_dataset,
+                        exclusions_by_instr_team_combiner.combined_dataset,
+                        pipeline_settings_combiner.combined_dataset,
+                    )
+                ]
+            else:
+                # Direct events
+                datasets = [glows_l1b_de(input_dataset)]
 
         if self.data_level == "l2":
             science_files = dependencies.get_file_paths(source="glows")
@@ -804,13 +809,26 @@ class Hi(ProcessInstrument):
             datasets = hi_l1c.hi_l1c(load_cdf(science_paths[0]), anc_paths[0])
         elif self.data_level == "l2":
             science_paths = dependencies.get_file_paths(source="hi", data_type="l1c")
-            # TODO get ancillary paths
-            geometric_factors_path = ""
-            esa_energies_path = ""
+            anc_dependencies = dependencies.get_processing_inputs(data_type="ancillary")
+            if len(anc_dependencies) != 3:
+                raise ValueError(
+                    f"Expected three ancillary dependencies for L2 processing including"
+                    f"cal-prod, esa-energies, and esa-eta-fit-factors."
+                    f"Got {[anc_dep.descriptor for anc_dep in anc_dependencies]}"
+                    "."
+                )
+            # Get individual L2 ancillary dependencies
+            # Strip the "45sensor" or "90sensor" off the ancillary descriptor and
+            # create a mapping from descriptor to path
+            l2_ancillary_path_dict = {
+                "-".join(dep.descriptor.split("-")[1:]): dep.imap_file_paths[
+                    0
+                ].construct_path()
+                for dep in anc_dependencies
+            }
             datasets = hi_l2.hi_l2(
                 science_paths,
-                geometric_factors_path,
-                esa_energies_path,
+                l2_ancillary_path_dict,
                 self.descriptor,
             )
         else:
@@ -1119,9 +1137,9 @@ class Mag(ProcessInstrument):
             input_data = [load_cdf(dep) for dep in science_files]
             # Input datasets can be in any order, and are validated within mag_l1c
             if len(input_data) == 1:
-                datasets = [mag_l1c(input_data[0])]
+                datasets = [mag_l1c(input_data[0], current_day)]
             elif len(input_data) == 2:
-                datasets = [mag_l1c(input_data[0], input_data[1])]
+                datasets = [mag_l1c(input_data[0], current_day, input_data[1])]
             else:
                 raise ValueError(
                     f"Invalid dependencies found for MAG L1C:"

imap_processing/codice/codice_l1a.py

@@ -89,7 +89,7 @@ class CoDICEL1aPipeline:
         self.plan_step = plan_step
         self.view_id = view_id
 
-    def apply_despinning(self) -> None:
+    def apply_despinning(self) -> None:  # noqa: PLR0912 (too many branches)
         """
         Apply the despinning algorithm to lo- angular and priority products.
 
@@ -108,10 +108,10 @@ class CoDICEL1aPipeline:
         # The dimensions are dependent on the specific data product
         if "angular" in self.config["dataset_name"]:
             despun_dims: tuple[int, ...] = (
+                num_counters,
                 num_energies,
                 num_positions,
                 num_spins,
-                num_counters,
             )
         elif "priority" in self.config["dataset_name"]:
             despun_dims = (num_energies, num_spins, num_counters)
@@ -130,23 +130,33 @@ class CoDICEL1aPipeline:
             for energy_index in range(num_energies):
                 pixel_orientation = constants.PIXEL_ORIENTATIONS[energy_index]
                 for spin_sector_index in range(num_spin_sectors):
-                    for azimuth_index in range(num_spins):
-                        if pixel_orientation == "A" and azimuth_index < 12:
+                    for azimuth_index in range(num_positions):
+                        if "-sw-" in self.config["dataset_name"]:
+                            # do something
+                            position_index = constants.SW_INDEX_TO_POSITION[
+                                azimuth_index
+                            ]
+                        elif "-nsw-" in self.config["dataset_name"]:
+                            position_index = constants.NSW_INDEX_TO_POSITION[
+                                azimuth_index
+                            ]
+
+                        if pixel_orientation == "A" and position_index < 12:
                             despun_spin_sector = spin_sector_index
-                        elif pixel_orientation == "A" and azimuth_index >= 12:
+                        elif pixel_orientation == "A" and position_index >= 12:
                             despun_spin_sector = spin_sector_index + 12
-                        elif pixel_orientation == "B" and azimuth_index < 12:
+                        elif pixel_orientation == "B" and position_index < 12:
                             despun_spin_sector = spin_sector_index + 12
-                        elif pixel_orientation == "B" and azimuth_index >= 12:
+                        elif pixel_orientation == "B" and position_index >= 12:
                             despun_spin_sector = spin_sector_index
 
                         if "angular" in self.config["dataset_name"]:
                             spin_data = epoch_data[
-                                energy_index, :, spin_sector_index, :
-                            ]  # (5, 4)
-                            despun_data[i][energy_index, :, despun_spin_sector, :] = (
-                                spin_data
-                            )
+                                :, energy_index, azimuth_index, spin_sector_index
+                            ]
+                            despun_data[i][
+                                :, energy_index, azimuth_index, despun_spin_sector
+                            ] = spin_data
                         elif "priority" in self.config["dataset_name"]:
                             spin_data = epoch_data[energy_index, spin_sector_index, :]
                             despun_data[i][energy_index, despun_spin_sector, :] = (
@@ -325,11 +335,25 @@ class CoDICEL1aPipeline:
         # different depending on the data product). In any case, iterate over
         # the num_counters dimension to isolate the data for each counter so
         # each counter's data can be placed in a separate CDF data variable.
+        # For Lo SW species, all_data has shape (9, 16, 128, 1) -> (epochs,
+        # num_counters, num_energy_steps, num_spin_sectors)
+        if self._is_different_dimension():
+            # For Lo species datasets, counters are the second dimension (index 1)
+            num_counters = all_data.shape[1]
+        else:
+            # For all other datasets, counters are the last dimension
+            num_counters = all_data.shape[-1]
+
         for counter, variable_name in zip(
-            range(all_data.shape[-1]), self.config["variable_names"], strict=False
+            range(num_counters), self.config["variable_names"], strict=False
         ):
             # Extract the counter data
-            counter_data = all_data[..., counter]
+            if self._is_different_dimension():
+                counter_data = all_data[:, counter, :, :]
+            elif "sectored" in self.config["dataset_name"]:
+                counter_data = all_data[:, counter, :, :, :]
+            else:
+                counter_data = all_data[..., counter]
 
             # Get the CDF attributes
             descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1]
@@ -708,10 +732,25 @@ class CoDICEL1aPipeline:
 
         # Reshape the data based on how it is written to the data array of
         # the packet data. The number of counters is the last dimension / axis.
-        reshape_dims = (
-            *self.config["dims"].values(),
-            self.config["num_counters"],
-        )
+        if self._is_different_dimension():
+            # For Lo species datasets, counters are the first dimension
+            reshape_dims = (
+                self.config["num_counters"],
+                *self.config["dims"].values(),
+            )
+        elif "sectored" in self.config["dataset_name"]:
+            # For sectored datasets, counters are the second dimension
+            reshape_dims = (
+                self.config["num_counters"],
+                *self.config["dims"].values(),
+            )
+        else:
+            # For all other datasets, counters are the last dimension
+            reshape_dims = (
+                *self.config["dims"].values(),
+                self.config["num_counters"],
+            )
+
         for packet_data in self.raw_data:
             reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape(
                 reshape_dims
@@ -725,6 +764,26 @@ class CoDICEL1aPipeline:
         # No longer need to keep the raw data around
         del self.raw_data
 
+    def _is_different_dimension(self) -> bool:
+        """
+        Check if the current dataset is a Lo species dataset.
+
+        Lo species datasets have a different data structure where counters are the
+        second dimension (index 1) instead of the last dimension.
+
+        Returns
+        -------
+        bool
+            True if the dataset is a Lo species dataset
+            (lo-sw-species or lo-nsw-species), False otherwise.
+        """
+        return self.config["dataset_name"] in [
+            "imap_codice_l1a_lo-sw-species",
+            "imap_codice_l1a_lo-nsw-species",
+            "imap_codice_l1a_lo-sw-angular",
+            "imap_codice_l1a_lo-nsw-angular",
+        ]
+
     def set_data_product_config(self, apid: int, dataset: xr.Dataset) -> None:
         """
         Set the various settings for defining the data products.
@@ -1615,23 +1674,23 @@ def process_codice_l1a(file_path: Path) -> list[xr.Dataset]:
         # Housekeeping data
         if apid == CODICEAPID.COD_NHK:
             processed_dataset = create_hskp_dataset(dataset)
-            logger.info(f"\nFinal data product:\n{processed_dataset}\n")
+            logger.info(f"\nProcessed {CODICEAPID(apid).name} packet\n")
 
         # Event data
         elif apid in [CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA]:
             processed_dataset = create_direct_event_dataset(apid, dataset)
-            logger.info(f"\nFinal data product:\n{processed_dataset}\n")
+            logger.info(f"\nProcessed {CODICEAPID(apid).name} packet\n")
 
         # I-ALiRT data
         elif apid in [CODICEAPID.COD_LO_IAL, CODICEAPID.COD_HI_IAL]:
             processed_dataset = create_ialirt_dataset(apid, dataset)
-            logger.info(f"\nFinal data product:\n{processed_dataset}\n")
+            logger.info(f"\nProcessed {CODICEAPID(apid).name} packet\n")
 
         # hi-omni data
         elif apid == CODICEAPID.COD_HI_OMNI_SPECIES_COUNTS:
             science_values = [packet.data for packet in dataset.data]
             processed_dataset = create_binned_dataset(apid, dataset, science_values)
-            logger.info(f"\nFinal data product:\n{processed_dataset}\n")
+            logger.info(f"\nProcessed {CODICEAPID(apid).name} packet\n")
 
         # Everything else
         elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING:
@@ -1649,7 +1708,7 @@ def process_codice_l1a(file_path: Path) -> list[xr.Dataset]:
             pipeline.define_coordinates()
             processed_dataset = pipeline.define_data_variables()
 
-            logger.info(f"\nFinal data product:\n{processed_dataset}\n")
+            logger.info(f"\nProcessed {CODICEAPID(apid).name} packet\n")
 
         # For APIDs that don't require processing
         else:

imap_processing/codice/codice_l1b.py

@@ -56,8 +56,6 @@ def convert_to_rates(
         "lo-sw-angular",
         "lo-nsw-priority",
         "lo-sw-priority",
-        "lo-nsw-species",
-        "lo-sw-species",
         "lo-ialirt",
     ]:
         # Applying rate calculation described in section 10.2 of the algorithm
@@ -66,14 +64,35 @@ def convert_to_rates(
         # time data array to match the data variable shape
         dims = [1] * dataset[variable_name].data.ndim
         dims[1] = 128
-        acq_times = dataset.acquisition_time_per_step.data.reshape(dims)
-
+        acq_times = dataset.acquisition_time_per_step.data.reshape(dims)  # (128)
         # Now perform the calculation
         rates_data = dataset[variable_name].data / (
             acq_times
-            * 1e-6  # Converting from microseconds to seconds
+            * 1e-3  # Converting from milliseconds to seconds
             * constants.L1B_DATA_PRODUCT_CONFIGURATIONS[descriptor]["num_spin_sectors"]
         )
+    elif descriptor in [
+        "lo-nsw-species",
+        "lo-sw-species",
+    ]:
+        # Applying rate calculation described in section 10.2 of the algorithm
+        # document
+        # In order to divide by acquisition times, we must reshape the acq
+        # time data array to match the data variable shape (epoch, esa_step, sector)
+        dims = [1] * dataset[variable_name].data.ndim
+        dims[1] = 128
+        acq_times = dataset.acquisition_time_per_step.data.reshape(dims)  # (128)
+        # acquisition time have an array of shape (128,). We match n_sector to that.
+        # Per CoDICE, fill first 127 with default value of 12. Then fill last with 11.
+        n_sector = np.full(128, 12, dtype=int)
+        n_sector[-1] = 11
+
+        # Now perform the calculation
+        rates_data = dataset[variable_name].data / (
+            acq_times
+            * 1e-3  # Converting from milliseconds to seconds
+            * n_sector[:, np.newaxis]  # Spin sectors
+        )
     elif descriptor in [
         "hi-counters-aggregated",
         "hi-counters-singles",
@@ -164,13 +183,23 @@ def process_codice_l1b(file_path: Path) -> xr.Dataset:
             l1b_dataset[variable_name].data = convert_to_rates(
                 l1b_dataset, descriptor, variable_name
             )
-
             # Set the variable attributes
             cdf_attrs_key = f"{descriptor}-{variable_name}"
             l1b_dataset[variable_name].attrs = cdf_attrs.get_variable_attributes(
                 cdf_attrs_key, check_schema=False
             )
 
+        if descriptor in ["lo-sw-species", "lo-nsw-species"]:
+            # Do not carry these variable attributes from L1a to L1b
+            drop_variables = [
+                "k_factor",
+                "nso_half_spin",
+                "sw_bias_gain_mode",
+                "st_bias_gain_mode",
+                "spin_period",
+            ]
+            l1b_dataset = l1b_dataset.drop_vars(drop_variables)
+
     logger.info(f"\nFinal data product:\n{l1b_dataset}\n")
 
     return l1b_dataset