imap-processing 0.17.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

imap_processing/cdf/config/imap_mag_global_cdf_attrs.yaml

@@ -97,4 +97,18 @@ imap_mag_l2_burst-dsrf:
   Data_type: L2_burst-dsrf>Level 2 burst rate data in DSRF
   Logical_source: imap_mag_l2_burst-dsrf
   Logical_source_description: IMAP Mission Burst Rate Instrument Level-2 Data in
-    Despun Spacecraft Reference Frame.
+    Despun Spacecraft Reference Frame.
+
+imap_mag_l2_norm-srf:
+    <<: *default
+    Data_type: L2_norm-srf>Level 2 normal rate data in SRF
+    Logical_source: imap_mag_l2_norm-srf
+    Logical_source_description: IMAP Mission Normal Rate Instrument Level-2 Data in
+        Spacecraft Reference Frame.
+
+imap_mag_l2_burst-srf:
+    <<: *default
+    Data_type: L2_burst-srf>Level 2 burst rate data in SRF
+    Logical_source: imap_mag_l2_burst-srf
+    Logical_source_description: IMAP Mission Burst Rate Instrument Level-2 Data in
+        Spacecraft Reference Frame.

imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml

@@ -32,6 +32,66 @@ imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-ang:
     Logical_source: imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-ang
     Logical_source_description: IMAP-Ultra Instrument Level-1A PHxTOF Hi Angular Data.
 
+imap_ultra_l1a_45sensor-histogram-ena-phxtof-hi-nrg:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A TOF
+    Logical_source: imap_ultra_l1a_45sensor-histogram-ena-phxtof-hi-nrg
+    Logical_source_description: IMAP-Ultra Instrument Level-1A PHxTOF Hi Energy Data.
+
+imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-nrg:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A TOF
+    Logical_source: imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-nrg
+    Logical_source_description: IMAP-Ultra Instrument Level-1A PHxTOF Hi Energy Data.
+
+imap_ultra_l1a_45sensor-histogram-ena-phxtof-hi-time:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A TOF
+    Logical_source: imap_ultra_l1a_45sensor-histogram-ena-phxtof-hi-time
+    Logical_source_description: IMAP-Ultra Instrument Level-1A PHxTOF Hi Time Data.
+
+imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-time:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A TOF
+    Logical_source: imap_ultra_l1a_90sensor-histogram-ena-phxtof-hi-time
+    Logical_source_description: IMAP-Ultra Instrument Level-1A PHxTOF Hi Time Data.
+
+imap_ultra_l1a_45sensor-histogram-ena-extof-hi-ang:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A TOF
+    Logical_source: imap_ultra_l1a_45sensor-histogram-ena-extof-hi-ang
+    Logical_source_description: IMAP-Ultra Instrument Level-1A ExTOF Hi Angular Data.
+
+imap_ultra_l1a_90sensor-histogram-ena-extof-hi-ang:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A TOF
+    Logical_source: imap_ultra_l1a_90sensor-histogram-ena-extof-hi-ang
+    Logical_source_description: IMAP-Ultra Instrument Level-1A ExTOF Hi Angular Data.
+
+imap_ultra_l1a_45sensor-histogram-ena-extof-hi-time:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A ExTOF
+    Logical_source: imap_ultra_l1a_45sensor-histogram-ena-extof-hi-time
+    Logical_source_description: IMAP-Ultra Instrument Level-1A ExTOF Hi Time Data.
+
+imap_ultra_l1a_90sensor-histogram-ena-extof-hi-time:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A ExTOF
+    Logical_source: imap_ultra_l1a_90sensor-histogram-ena-extof-hi-time
+    Logical_source_description: IMAP-Ultra Instrument Level-1A ExTOF Hi Time Data.
+
+imap_ultra_l1a_45sensor-histogram-ena-extof-hi-nrg:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A ExTOF
+    Logical_source: imap_ultra_l1a_45sensor-histogram-ena-extof-hi-nrg
+    Logical_source_description: IMAP-Ultra Instrument Level-1A ExTOF Hi Energy Data.
+
+imap_ultra_l1a_90sensor-histogram-ena-extof-hi-nrg:
+    <<: *instrument_base
+    Data_type: L1A_Histogram>Level-1A ExTOF
+    Logical_source: imap_ultra_l1a_90sensor-histogram-ena-extof-hi-nrg
+    Logical_source_description: IMAP-Ultra Instrument Level-1A ExTOF Hi Energy Data.
+
 imap_ultra_l1a_45sensor-rates:
     <<: *instrument_base
     Data_type: L1A_Rates>Level-1A Rates

imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml

@@ -256,6 +256,20 @@ theta:
   LABLAXIS: theta
   UNITS: degrees
 
+geometric_factor_blades:
+  <<: *default_float32
+  CATDESC: Ultra sensitive area within the field of view.
+  FIELDNAM: geometric_function_blades
+  LABLAXIS: geometric function blades
+  UNITS: " "
+
+geometric_factor_noblades:
+  <<: *default_float32
+  CATDESC: Ultra sensitive area within the field of regard.
+  FIELDNAM: geometric_function_noblades
+  LABLAXIS: geometric function noblades
+  UNITS: " "
+
 phi_fwhm:
   <<: *default_float32
   CATDESC: FWHM of the phi distribution.
@@ -334,7 +348,8 @@ spin_start_time:
   LABLAXIS: spin start time
   # TODO: come back to format
   UNITS: s
-  DEPEND_0: spin_number
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
 
 spin_period:
   <<: *default
@@ -342,7 +357,8 @@ spin_period:
   FIELDNAM: spin_period
   LABLAXIS: spin_period
   UNITS: s
-  DEPEND_0: spin_number
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
 
 spin_rate:
   <<: *default
@@ -350,7 +366,8 @@ spin_rate:
   FIELDNAM: spin_rate
   LABLAXIS: spin_rate
   UNITS: rpm
-  DEPEND_0: spin_number
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
 
 rate_start_pulses:
   <<: *default
@@ -397,9 +414,46 @@ ena_rates:
   <<: *default_float32
   CATDESC: Rates calculated from de packet.
   FIELDNAM: ena_rates
-  LABLAXIS: ena_rates
-  DEPEND_0: spin_number
-  DEPEND_1: energy_bin_geometric_mean
+  LABLAXIS: ena rates
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  DEPEND_2: energy_bin_geometric_mean
+  UNITS: " "
+
+start_pulses_per_spin:
+  <<: *default_float32
+  CATDESC: Total start pulses per spin.
+  FIELDNAM: start_pulses_per_spin
+  LABLAXIS: start pulses per spin
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  UNITS: " "
+
+stop_pulses_per_spin:
+  <<: *default_float32
+  CATDESC: Total start pulses per spin.
+  FIELDNAM: stop_pulses_per_spin
+  LABLAXIS: stop pulses per spin
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  UNITS: " "
+
+coin_pulses_per_spin:
+  <<: *default_float32
+  CATDESC: Total coincidence pulses per spin.
+  FIELDNAM: coin_pulses_per_spin
+  LABLAXIS: coin_pulses_per_spin
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  UNITS: " "
+
+rejected_events_per_spin:
+  <<: *default_uint16
+  CATDESC: Rejected events per spin.
+  FIELDNAM: rejected_events_per_spin
+  LABLAXIS: rejected events per spin
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
   UNITS: " "
 
 ena_rates_threshold:
@@ -407,8 +461,9 @@ ena_rates_threshold:
   CATDESC: Rates threshold used for flagging data.
   FIELDNAM: ena_rates_threshold
   LABLAXIS: ena_rates_threshold
-  DEPEND_0: spin_number
-  DEPEND_1: energy_bin_geometric_mean
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  DEPEND_2: energy_bin_geometric_mean
   UNITS: " "
 
 quality_ena_rates:
@@ -416,8 +471,9 @@ quality_ena_rates:
   CATDESC: Spin filter derived from Ultra ena rates. Bitwise flagging used to filter the spin.
   FIELDNAM: quality_ena_rates
   LABLAXIS: quality_ena_rates
-  DEPEND_0: spin_number
-  DEPEND_1: energy_bin_geometric_mean
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  DEPEND_2: energy_bin_geometric_mean
   UNITS: " "
 
 quality_attitude:
@@ -427,7 +483,9 @@ quality_attitude:
   LABLAXIS: quality attitude
   # TODO: come back to format
   UNITS: " "
-  DEPEND_0: spin_number
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  DEPEND_2: energy_bin_geometric_mean
 
 quality_instruments:
   <<: *default_uint16
@@ -436,3 +494,25 @@ quality_instruments:
   LABLAXIS: quality instruments
   # TODO: come back to format
   UNITS: " "
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  DEPEND_2: energy_bin_geometric_mean
+
+quality_hk:
+  <<: *default_uint16
+  CATDESC: Spin filter derived from housekeeping data. Bitwise flagging used to filter the spin.
+  FIELDNAM: quality_hk
+  LABLAXIS: quality hk
+  # TODO: come back to format
+  UNITS: " "
+  DEPEND_0: epoch
+  DEPEND_1: spin_number
+  DEPEND_2: energy_bin_geometric_mean
+
+quality_fov:
+  <<: *default_uint16
+  CATDESC: Quality flag for direct event for events outside of the FOV.
+  FIELDNAM: quality_fov
+  LABLAXIS: quality fov
+  # TODO: come back to format
+  UNITS: " "

imap_processing/cli.py

@@ -68,6 +68,7 @@ from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
 from imap_processing.mag.l1b.mag_l1b import mag_l1b
 from imap_processing.mag.l1c.mag_l1c import mag_l1c
+from imap_processing.mag.l1d.mag_l1d import mag_l1d
 from imap_processing.mag.l2.mag_l2 import mag_l2
 from imap_processing.spacecraft import quaternions
 from imap_processing.spice import pointing_frame, repoint, spin
@@ -516,7 +517,7 @@ class ProcessInstrument(ABC):
         self,
         processed_data: list[xr.Dataset | Path],
         dependencies: ProcessingInputCollection,
-    ) -> None:
+    ) -> list[Path]:
         """
         Complete post-processing.
 
@@ -544,10 +545,17 @@ class ProcessInstrument(ABC):
             method.
         dependencies : ProcessingInputCollection
             Object containing dependencies to process.
+
+        Returns
+        -------
+        list[Path]
+            List of paths to CDF files produced.
         """
+        products: list[Path] = []
+
         if len(processed_data) == 0:
             logger.info("No products to write to CDF file.")
-            return
+            return products
 
         logger.info("Writing products to local storage")
 
@@ -567,7 +575,6 @@ class ProcessInstrument(ABC):
         # start_date.
         # If it is start_date, skip repointing in the output filename.
 
-        products = []
         for ds in processed_data:
             if isinstance(ds, xr.Dataset):
                 ds.attrs["Data_version"] = self.version[1:]  # Strip 'v' from version
@@ -581,6 +588,7 @@ class ProcessInstrument(ABC):
                 products.append(ds)
 
         self.upload_products(products)
+        return products
 
     @final
     def cleanup(self) -> None:
@@ -916,8 +924,8 @@ class Idex(ProcessInstrument):
                     f"{dependency_list}. Expected only one dependency."
                 )
             sci_files = dependencies.get_file_paths(source="idex", descriptor="sci-1mo")
-            dependency = load_cdf(sci_files[0])
-            datasets = idex_l2c(dependency)
+            dependencies = [load_cdf(f) for f in sci_files]
+            datasets = [idex_l2c(dependencies)]
         return datasets
 
 
@@ -1073,6 +1081,21 @@ class Mag(ProcessInstrument):
                     f"Invalid dependencies found for MAG L1C:"
                     f"{dependencies}. Expected one or two dependencies."
                 )
+        if self.data_level == "l1d":
+            science_files = dependencies.get_file_paths(source="mag", data_type="l1c")
+            science_files.extend(
+                dependencies.get_file_paths(source="mag", data_type="l1b")
+            )
+            input_data = [load_cdf(dep) for dep in science_files]
+            calibration = dependencies.get_processing_inputs(
+                descriptor="l1d-calibration"
+            )
+            combined_calibration = MagAncillaryCombiner(calibration[0], day_buffer)
+            datasets = mag_l1d(
+                input_data,
+                combined_calibration.combined_dataset,
+                current_day,
+            )
 
         if self.data_level == "l2":
             science_files = dependencies.get_file_paths(source="mag", data_type="l1b")

imap_processing/codice/codice_l1a.py

@@ -143,8 +143,8 @@ class CoDICEL1aPipeline:
         self.coords = {}
 
         coord_names = [
-            *self.config["output_dims"].keys(),
-            *[key + "_label" for key in self.config["output_dims"].keys()],
+            *self.config["dims"].keys(),
+            *[key + "_label" for key in self.config["dims"].keys()],
         ]
 
         # Define epoch coordinates
@@ -174,7 +174,7 @@ class CoDICEL1aPipeline:
                 "spin_sector_index",
                 "ssd_index",
             ]:
-                values = np.arange(self.config["output_dims"][name])
+                values = np.arange(self.config["dims"][name])
                 dims = [name]
             elif name == "spin_sector_pairs_label":
                 values = np.array(
@@ -188,15 +188,22 @@ class CoDICEL1aPipeline:
                     ]
                 )
                 dims = [name]
+            elif name == "inst_az_label":
+                if self.config["dataset_name"] == "imap_codice_l1a_lo-nsw-angular":
+                    values = [str(x) for x in range(4, 23)]
+                elif self.config["dataset_name"] == "imap_codice_l1a_lo-sw-angular":
+                    values = ["1", "2", "3", "23", "24"]
+                else:
+                    values = np.arange(self.config["dims"]["inst_az"]).astype(str)
+                dims = ["inst_az"]
             elif name in [
                 "spin_sector_label",
                 "esa_step_label",
-                "inst_az_label",
                 "spin_sector_index_label",
                 "ssd_index_label",
             ]:
                 key = name.removesuffix("_label")
-                values = np.arange(self.config["output_dims"][key]).astype(str)
+                values = np.arange(self.config["dims"][key]).astype(str)
                 dims = [key]
 
             coord = xr.DataArray(
@@ -230,16 +237,16 @@ class CoDICEL1aPipeline:
         # Stack the data so that it is easier to reshape and iterate over
         all_data = np.stack(self.data)
 
-        # The dimension of all_data is something like (epoch, num_counters,
-        # num_energy_steps, num_positions, num_spin_sectors) (or may be slightly
+        # The dimension of all_data is something like (epoch, num_energy_steps,
+        # num_positions, num_spin_sectors, num_counters) (or may be slightly
         # different depending on the data product). In any case, iterate over
         # the num_counters dimension to isolate the data for each counter so
         # each counter's data can be placed in a separate CDF data variable.
         for counter, variable_name in zip(
-            range(all_data.shape[1]), self.config["variable_names"]
+            range(all_data.shape[-1]), self.config["variable_names"]
         ):
             # Extract the counter data
-            counter_data = all_data[:, counter, ...]
+            counter_data = all_data[..., counter]
 
             # Get the CDF attributes
             descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1]
@@ -249,7 +256,7 @@ class CoDICEL1aPipeline:
             # For most products, the final CDF dimensions always has "epoch" as
             # the first dimension followed by the dimensions for the specific
             # data product
-            dims = ["epoch", *list(self.config["output_dims"].keys())]
+            dims = ["epoch", *list(self.config["dims"].keys())]
 
             # However, CoDICE-Hi products use specific energy bins for the
             # energy dimension
@@ -398,6 +405,12 @@ class CoDICEL1aPipeline:
                 dims = ["epoch"]
                 attrs = self.cdf_attrs.get_variable_attributes("spin_period")
 
+            # The k-factor is a constant that maps voltages to energies
+            elif variable_name == "k_factor":
+                variable_data = np.array([constants.K_FACTOR], dtype=np.float32)
+                dims = [""]
+                attrs = self.cdf_attrs.get_variable_attributes("k_factor")
+
             # Add variable to the dataset
             dataset[variable_name] = xr.DataArray(
                 variable_data,
@@ -589,52 +602,27 @@ class CoDICEL1aPipeline:
 
         These data need to be divided up by species or priorities (or
         what I am calling "counters" as a general term), and re-arranged into
-        4D arrays representing dimensions such as time, spin sectors, positions,
-        and energies (depending on the data product).
+        multidimensional arrays representing dimensions such as time,
+        spin sectors, positions, and energies (depending on the data product).
 
         However, the existence and order of these dimensions can vary depending
-        on the specific data product, so we define this in the "input_dims"
-        and "output_dims" values configuration dictionary; the "input_dims"
-        defines how the dimensions are written into the packet data, while
-        "output_dims" defines how the dimensions should be written to the final
-        CDF product.
+        on the specific data product, so we define this in the "dims" key of the
+        configuration dictionary.
         """
         # This will contain the reshaped data for all counters
         self.data = []
 
-        # First reshape the data based on how it is written to the data array of
-        # the packet data. The number of counters is the first dimension / axis,
-        # with the exception of lo-counters-aggregated which is treated slightly
-        # differently
-        if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
-            reshape_dims = (
-                self.config["num_counters"],
-                *self.config["input_dims"].values(),
-            )
-        else:
-            reshape_dims = (
-                *self.config["input_dims"].values(),
-                self.config["num_counters"],
-            )
-
-        # Then, transpose the data based on how the dimensions should be written
-        # to the CDF file. Since this is specific to each data product, we need
-        # to determine this dynamically based on the "output_dims" config.
-        # Again, lo-counters-aggregated is treated slightly differently
-        input_keys = ["num_counters", *self.config["input_dims"].keys()]
-        output_keys = ["num_counters", *self.config["output_dims"].keys()]
-        if self.config["dataset_name"] != "imap_codice_l1a_lo-counters-aggregated":
-            transpose_axes = [input_keys.index(dim) for dim in output_keys]
-        else:
-            transpose_axes = [1, 2, 0]  # [esa_step, spin_sector_pairs, num_counters]
-
+        # Reshape the data based on how it is written to the data array of
+        # the packet data. The number of counters is the last dimension / axis.
+        reshape_dims = (
+            *self.config["dims"].values(),
+            self.config["num_counters"],
+        )
         for packet_data in self.raw_data:
             reshaped_packet_data = np.array(packet_data, dtype=np.uint32).reshape(
                 reshape_dims
             )
-            reshaped_cdf_data = np.transpose(reshaped_packet_data, axes=transpose_axes)
-
-            self.data.append(reshaped_cdf_data)
+            self.data.append(reshaped_packet_data)
 
         # No longer need to keep the raw data around
         del self.raw_data
@@ -950,9 +938,9 @@ def create_direct_event_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
 
     # Create the dataset to hold the data variables
     if apid == CODICEAPID.COD_LO_PHA:
-        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_lo-pha")
+        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_lo-direct-events")
     elif apid == CODICEAPID.COD_HI_PHA:
-        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_hi-pha")
+        attrs = cdf_attrs.get_global_attributes("imap_codice_l1a_hi-direct-events")
     dataset = xr.Dataset(
         coords={
             "epoch": epoch,

imap_processing/codice/codice_l1b.py

@@ -116,7 +116,7 @@ def process_codice_l1b(file_path: Path) -> xr.Dataset:
     descriptor = dataset_name.removeprefix("imap_codice_l1b_")
 
     # Direct event data products do not have a level L1B
-    if descriptor in ["lo-pha", "hi-pha"]:
+    if descriptor in ["lo-direct-events", "hi-direct-events"]:
         logger.warning("Encountered direct event data product. Skipping L1b processing")
         return None
 

imap_processing/codice/codice_l2.py

@@ -45,17 +45,8 @@ def process_codice_l2(file_path: Path) -> xr.Dataset:
     # TODO: Could clean this up by using imap-data-access methods?
     dataset_name = l1_dataset.attrs["Logical_source"]
     data_level = dataset_name.removeprefix("imap_codice_").split("_")[0]
-    descriptor = dataset_name.removeprefix(f"imap_codice_{data_level}_")
     dataset_name = dataset_name.replace(data_level, "l2")
 
-    # TODO: Temporary work-around to replace "PHA" naming convention with
-    #       "direct events" This will eventually be changed at the L1 level and
-    #       thus this will eventually be removed.
-    if descriptor == "lo-pha":
-        dataset_name = dataset_name.replace("lo-pha", "lo-direct-events")
-    elif descriptor == "hi-pha":
-        dataset_name = dataset_name.replace("hi-pha", "hi-direct-events")
-
     # Use the L1 data product as a starting point for L2
     l2_dataset = l1_dataset.copy()