imap-processing 0.15.0__tar.gz → 0.16.0__tar.gz

{imap_processing-0.15.0 → imap_processing-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imap-processing
-Version: 0.15.0
+Version: 0.16.0
 Summary: IMAP Science Operations Center Processing
 License: MIT
 Keywords: IMAP,SDC,SOC,Science Operations

{imap_processing-0.15.0 → imap_processing-0.16.0}/imap_processing/_version.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced later during substitution.
-__version__ = "0.15.0"
-__version_tuple__ = (0, 15, 0)
+__version__ = "0.16.0"
+__version_tuple__ = (0, 16, 0)

{imap_processing-0.15.0 → imap_processing-0.16.0}/imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml

@@ -15,6 +15,47 @@ component:
   LABLAXIS: component
   FORMAT: A3
   VAR_TYPE: metadata
+  UNITS: " "
+
+esa_step:
+  CATDESC: ESA step number
+  FIELDNAM: esa_step
+  LABLAXIS: esa_step
+  FORMAT: I2
+  VALIDMIN: 0
+  VALIDMAX: 7
+  VAR_TYPE: metadata
+  UNITS: " "
+
+energy_ranges:
+    CATDESC: CoDICE-Hi energy ranges
+    FIELDNAM: energy_ranges
+    LABLAXIS: energy_ranges
+    FORMAT: I3
+    VALIDMIN: 0
+    VALIDMAX: 14
+    VAR_TYPE: metadata
+    UNITS: " "
+
+azimuth:
+    CATDESC: CoDICE-Hi azimuth
+    FIELDNAM: azimuth
+    LABLAXIS: azimuth
+    FORMAT: I3
+    VALIDMIN: 0
+    VALIDMAX: 4
+    VAR_TYPE: metadata
+    UNITS: " "
+
+spin_angle_bin:
+    CATDESC: CoDICE-Hi spin_angle_bin
+    FIELDNAM: spin_angle_bin
+    LABLAXIS: spin_angle_bin
+    FORMAT: I3
+    VALIDMIN: 0
+    VALIDMAX: 4
+    VAR_TYPE: metadata
+    UNITS: " "
 
 default_uint8_attrs: &default_uint8
   <<: *default
@@ -72,6 +113,72 @@ default_float64_attrs: &default_float64
   VALIDMAX: 1.7976931348623157e+308
   dtype: float64
 
+codicehi_h:
+  <<: *default_float32
+  CATDESC: H intensities in 15 energy ranges and binned into 4 azimuths and 4 spin angle bins
+  FIELDNAM: H intensities
+  LABLAXIS: Diff. Intensity
+  UNITS: "# / cm2-sr-s- MeV"
+  VALIDMIN: 0
+  VALIDMAX: 100000000.0
+  DEPEND_1: energy_ranges
+  DEPEND_2: azimuth
+  DEPEND_3: spin_angle_bin
+
+codicelo_c_over_o_abundance:
+  <<: *default_float32
+  CATDESC: C/O abundance ratio
+  FIELDNAM: C/O
+  LABLAXIS: ratio
+  UNITS: " "
+  VALIDMIN: 0
+  VALIDMAX: 100000000.0
+
+codicelo_mg_over_o_abundance:
+  <<: *default_float32
+  CATDESC: Mg/O abundance ratio
+  FIELDNAM: Mg/O
+  LABLAXIS: ratio
+  UNITS: " "
+  VALIDMIN: 0
+  VALIDMAX: 100000000.0
+
+codicelo_fe_over_o_abundance:
+  <<: *default_float32
+  CATDESC: Fe/O abundance ratio
+  FIELDNAM: Fe/O
+  LABLAXIS: ratio
+  UNITS: " "
+  VALIDMIN: 0
+  VALIDMAX: 100000000.0
+
+codicelo_c_plus_6_over_c_plus_5_ratio:
+  <<: *default_float32
+  CATDESC: C+6/C+5 charge state ratio
+  FIELDNAM: C+6/C+5
+  LABLAXIS: ratio
+  UNITS: " "
+  VALIDMIN: 0
+  VALIDMAX: 100000000.0
+
+codicelo_o_plus_7_over_o_plus_6_ratio:
+  <<: *default_float32
+  CATDESC: O+7/O+6 charge state ratio
+  FIELDNAM: O+7/O+6
+  LABLAXIS: ratio
+  UNITS: " "
+  VALIDMIN: 0
+  VALIDMAX: 100000000.0
+
+codicelo_fe_low_over_fe_high_ratio:
+  <<: *default_float32
+  CATDESC: Fe low/Fe high charge state ratio
+  FIELDNAM: Fe low/Fe high
+  LABLAXIS: ratio
+  UNITS: " "
+  VALIDMIN: 0
+  VALIDMAX: 100000000.0
+
 hit_e_a_side_low_en:
   <<: *default_uint32
   CATDESC: Low energy (~300 keV) electrons (A-side)
@@ -215,150 +322,26 @@ swapi_pseudo_proton_temperature:
   VALIDMAX: 50000000.0
 
 # SWE Normalized Counts, First Half Cycle
-swe_normalized_counts_half_1_esa_0:
+swe_normalized_counts_half_1:
   <<: *default_uint32
   CATDESC: Normalized electron counts (Half Cycle 1, ESA step 0)
-  FIELDNAM: swe_normalized_counts_half_1_esa_0
+  FIELDNAM: swe_normalized_counts_half_1
   LABLAXIS: swe_norm_cts_q1_esa0
   UNITS: Normalized counts
   VALIDMIN: 0
   VALIDMAX: 500000
-
-swe_normalized_counts_half_1_esa_1:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 1, ESA step 1)
-  FIELDNAM: swe_normalized_counts_half_1_esa_1
-  LABLAXIS: swe_norm_cts_q1_esa1
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_1_esa_2:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 1, ESA step 2)
-  FIELDNAM: swe_normalized_counts_half_1_esa_2
-  LABLAXIS: swe_norm_cts_q1_esa2
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_1_esa_3:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 1, ESA step 3)
-  FIELDNAM: swe_normalized_counts_half_1_esa_3
-  LABLAXIS: swe_norm_cts_q1_esa3
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_1_esa_4:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 1, ESA step 4)
-  FIELDNAM: swe_normalized_counts_half_1_esa_4
-  LABLAXIS: swe_norm_cts_q1_esa4
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_1_esa_5:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 1, ESA step 5)
-  FIELDNAM: swe_normalized_counts_half_1_esa_5
-  LABLAXIS: swe_norm_cts_q1_esa5
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_1_esa_6:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 1, ESA step 6)
-  FIELDNAM: swe_normalized_counts_half_1_esa_6
-  LABLAXIS: swe_norm_cts_q1_esa6
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_1_esa_7:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 1, ESA step 7)
-  FIELDNAM: swe_normalized_counts_half_1_esa_7
-  LABLAXIS: swe_norm_cts_q1_esa7
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
+  DEPEND_1: esa_step
 
 # SWE Normalized Counts, Second Half Cycle
-swe_normalized_counts_half_2_esa_0:
+swe_normalized_counts_half_2:
   <<: *default_uint32
   CATDESC: Normalized electron counts (Half Cycle 2, ESA step 0)
-  FIELDNAM: swe_normalized_counts_half_2_esa_0
+  FIELDNAM: swe_normalized_counts_half_2
   LABLAXIS: swe_norm_cts_q2_esa0
   UNITS: Normalized counts
   VALIDMIN: 0
   VALIDMAX: 500000
-
-swe_normalized_counts_half_2_esa_1:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 2, ESA step 1)
-  FIELDNAM: swe_normalized_counts_half_2_esa_1
-  LABLAXIS: swe_norm_cts_q2_esa1
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_2_esa_2:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 2, ESA step 2)
-  FIELDNAM: swe_normalized_counts_half_2_esa_2
-  LABLAXIS: swe_norm_cts_q2_esa2
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_2_esa_3:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 2, ESA step 3)
-  FIELDNAM: swe_normalized_counts_half_2_esa_3
-  LABLAXIS: swe_norm_cts_q2_esa3
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_2_esa_4:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 2, ESA step 4)
-  FIELDNAM: swe_normalized_counts_half_2_esa_4
-  LABLAXIS: swe_norm_cts_q2_esa4
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_2_esa_5:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 2, ESA step 5)
-  FIELDNAM: swe_normalized_counts_half_2_esa_5
-  LABLAXIS: swe_norm_cts_q2_esa5
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_2_esa_6:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 2, ESA step 6)
-  FIELDNAM: swe_normalized_counts_half_2_esa_6
-  LABLAXIS: swe_norm_cts_q2_esa6
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
-
-swe_normalized_counts_half_2_esa_7:
-  <<: *default_uint32
-  CATDESC: Normalized electron counts (Half Cycle 2, ESA step 7)
-  FIELDNAM: swe_normalized_counts_half_2_esa_7
-  LABLAXIS: swe_norm_cts_q2_esa7
-  UNITS: Normalized counts
-  VALIDMIN: 0
-  VALIDMAX: 500000
+  DEPEND_1: esa_step
 
 swe_counterstreaming_electrons:
     <<: *default_uint8

{imap_processing-0.15.0 → imap_processing-0.16.0}/imap_processing/cli.py

@@ -51,10 +51,7 @@ from imap_processing.codice import codice_l1a, codice_l1b, codice_l2
 from imap_processing.glows.l1a.glows_l1a import glows_l1a
 from imap_processing.glows.l1b.glows_l1b import glows_l1b
 from imap_processing.glows.l2.glows_l2 import glows_l2
-from imap_processing.hi.l1a import hi_l1a
-from imap_processing.hi.l1b import hi_l1b
-from imap_processing.hi.l1c import hi_l1c
-from imap_processing.hi.l2 import hi_l2
+from imap_processing.hi import hi_l1a, hi_l1b, hi_l1c, hi_l2
 from imap_processing.hit.l1a.hit_l1a import hit_l1a
 from imap_processing.hit.l1b.hit_l1b import hit_l1b
 from imap_processing.hit.l2.hit_l2 import hit_l2

{imap_processing-0.15.0 → imap_processing-0.16.0}/imap_processing/codice/codice_l1a.py

@@ -674,7 +674,9 @@ class CoDICEL1aPipeline:
         self.cdf_attrs.add_instrument_variable_attrs("codice", "l1a")
 
 
-def group_ialirt_data(packets: xr.Dataset, data_field_range: range) -> list[bytearray]:
+def group_ialirt_data(
+    packets: xr.Dataset, data_field_range: range, prefix: str
+) -> list[bytearray]:
     """
     Group together the individual I-ALiRT data fields.
 
@@ -684,6 +686,8 @@ def group_ialirt_data(packets: xr.Dataset, data_field_range: range) -> list[byte
         The dataset containing the I-ALiRT data packets.
     data_field_range : range
         The range of the individual data fields (15 or lo, 6 for hi).
+    prefix : str
+        The prefix used to index the data (i.e. ``cod_lo`` or ``cod_hi``).
 
     Returns
     -------
@@ -693,14 +697,28 @@ def group_ialirt_data(packets: xr.Dataset, data_field_range: range) -> list[byte
     current_data_stream = bytearray()
     grouped_data = []
 
+    # Workaround to get this function working for both I-ALiRT spacecraft
+    # data and CoDICE-specific I-ALiRT test data from Joey
+    # TODO: Once CoDICE I-ALiRT processing is more established, we can probably
+    #       do away with processing the test data from Joey and just use the
+    #       I-ALiRT data that is constructed closer to what we expect in-flight.
+    if hasattr(packets, "acquisition_time"):
+        time_key = "acquisition_time"
+        counter_key = "counter"
+        data_key = "data"
+    else:
+        time_key = f"{prefix}_acq"
+        counter_key = f"{prefix}_counter"
+        data_key = f"{prefix}_data"
+
     # When a counter value of 255 is encountered, this signifies the
     # end of the data stream
-    for packet_num in range(0, len(packets.acquisition_time.data)):
-        counter = packets.counter.data[packet_num]
+    for packet_num in range(0, len(packets[time_key].data)):
+        counter = packets[counter_key].data[packet_num]
         if counter != 255:
             for field in data_field_range:
                 current_data_stream.extend(
-                    bytearray([packets[f"data_{field:02}"].data[packet_num]])
+                    bytearray([packets[f"{data_key}_{field:02}"].data[packet_num]])
                 )
         else:
             # At this point, if there are data, the data stream is ready
@@ -1003,72 +1021,79 @@ def create_ialirt_dataset(apid: int, packets: xr.Dataset) -> xr.Dataset:
     # See sections 10.4.1 and 10.4.2 in the algorithm document
     if apid == CODICEAPID.COD_LO_IAL:
         data_field_range = range(0, 15)
+        prefix = "cod_lo"
     elif apid == CODICEAPID.COD_HI_IAL:
         data_field_range = range(0, 5)
+        prefix = "cod_hi"
 
     # Group together packets of I-ALiRT data to form complete data sets
-    grouped_data = group_ialirt_data(packets, data_field_range)
-
-    # Process each group to get the science data and corresponding metadata
-    science_values, metadata_values = process_ialirt_data_streams(grouped_data)
-
-    # How data are processed is different for lo-iarlirt and hi-ialirt
-    if apid == CODICEAPID.COD_HI_IAL:
-        # Set some necessary values and process as a binned dataset similar to
-        # a hi-omni data product
-        metadata_for_processing = [
-            "table_id",
-            "plan_id",
-            "plan_step",
-            "view_id",
-            "spin_period",
-            "suspect",
-        ]
-        for var in metadata_for_processing:
-            packets[var] = metadata_values[var.upper()]
-        dataset = create_binned_dataset(apid, packets, science_values)
-
-    elif apid == CODICEAPID.COD_LO_IAL:
-        # Create a nominal instance of the pipeline and process similar to a
-        # lo-sw-species data product
-        pipeline = CoDICEL1aPipeline(
-            metadata_values["TABLE_ID"][0],
-            metadata_values["PLAN_ID"][0],
-            metadata_values["PLAN_STEP"][0],
-            metadata_values["VIEW_ID"][0],
-        )
-        pipeline.set_data_product_config(apid, packets)
-        pipeline.decompress_data(science_values)
-        pipeline.reshape_data()
-
-        # The calculate_epoch_values method needs acq_start_seconds and
-        # acq_start_subseconds attributes on the dataset
-        pipeline.dataset["acq_start_seconds"] = (
-            "_",
-            metadata_values["ACQ_START_SECONDS"],
-        )
-        pipeline.dataset["acq_start_subseconds"] = (
-            "_",
-            metadata_values["ACQ_START_SUBSECONDS"],
-        )
+    grouped_data = group_ialirt_data(packets, data_field_range, prefix)
+
+    if grouped_data:
+        # Process each group to get the science data and corresponding metadata
+        science_values, metadata_values = process_ialirt_data_streams(grouped_data)
+
+        # How data are processed is different for lo-iarlirt and hi-ialirt
+        if apid == CODICEAPID.COD_HI_IAL:
+            # Set some necessary values and process as a binned dataset similar to
+            # a hi-omni data product
+            metadata_for_processing = [
+                "table_id",
+                "plan_id",
+                "plan_step",
+                "view_id",
+                "spin_period",
+                "suspect",
+            ]
+            for var in metadata_for_processing:
+                packets[var] = metadata_values[var.upper()]
+            dataset = create_binned_dataset(apid, packets, science_values)
+
+        elif apid == CODICEAPID.COD_LO_IAL:
+            # Create a nominal instance of the pipeline and process similar to a
+            # lo-sw-species data product
+            pipeline = CoDICEL1aPipeline(
+                metadata_values["TABLE_ID"][0],
+                metadata_values["PLAN_ID"][0],
+                metadata_values["PLAN_STEP"][0],
+                metadata_values["VIEW_ID"][0],
+            )
+            pipeline.set_data_product_config(apid, packets)
+            pipeline.decompress_data(science_values)
+            pipeline.reshape_data()
 
-        pipeline.define_coordinates()
+            # The calculate_epoch_values method needs acq_start_seconds and
+            # acq_start_subseconds attributes on the dataset
+            pipeline.dataset["acq_start_seconds"] = (
+                "_",
+                metadata_values["ACQ_START_SECONDS"],
+            )
+            pipeline.dataset["acq_start_subseconds"] = (
+                "_",
+                metadata_values["ACQ_START_SUBSECONDS"],
+            )
 
-        # The dataset also needs the metadata that will be carried through
-        # to the final data product
-        for field in [
-            "spin_period",
-            "suspect",
-            "st_bias_gain_mode",
-            "sw_bias_gain_mode",
-            "rgfo_half_spin",
-            "nso_half_spin",
-        ]:
-            pipeline.dataset[field] = ("_", metadata_values[field.upper()])
+            pipeline.define_coordinates()
 
-        dataset = pipeline.define_data_variables()
+            # The dataset also needs the metadata that will be carried through
+            # to the final data product
+            for field in [
+                "spin_period",
+                "suspect",
+                "st_bias_gain_mode",
+                "sw_bias_gain_mode",
+                "rgfo_half_spin",
+                "nso_half_spin",
+            ]:
+                pipeline.dataset[field] = ("_", metadata_values[field.upper()])
 
-    return dataset
+            dataset = pipeline.define_data_variables()
+
+        return dataset
+
+    else:
+        logger.warning("No I-ALiRT data found")
+        return None
 
 
 def get_de_metadata(packets: xr.Dataset, segment: int) -> bytes:

{imap_processing-0.15.0 → imap_processing-0.16.0}/imap_processing/codice/codice_l2.py

@@ -12,7 +12,6 @@ dataset = process_codice_l2(l1_filename)
 import logging
 from pathlib import Path
 
-import numpy as np
 import xarray as xr
 
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -77,13 +76,6 @@ def process_codice_l2(file_path: Path) -> xr.Dataset:
     # TODO: Add L2-specific algorithms/functionality here. For SIT-4, we can
     #       just keep the data as-is.
 
-    # Workaround to fix monitomically increasing epoch issue
-    epoch_attrs = cdf_attrs.get_variable_attributes("epoch", check_schema=False)
-    new_epoch_values = np.array([843543730220584064 + i for i in range(113)])
-    l2_dataset = l2_dataset.assign_coords(
-        epoch=("epoch", new_epoch_values, epoch_attrs)
-    )
-
     logger.info(f"\nFinal data product:\n{l2_dataset}\n")
 
     return l2_dataset

{imap_processing-0.15.0 → imap_processing-0.16.0}/imap_processing/codice/constants.py

@@ -158,6 +158,17 @@ LO_PHA_VARIABLE_NAMES = [
     f"P{n}_{field}" for n in range(8) for field in LO_PHA_CDF_FIELDS
 ]
 
+# Final I-ALiRT data product fields
+CODICE_LO_IAL_DATA_FIELDS = [
+    "c_over_o_abundance",
+    "mg_over_o_abundance",
+    "fe_over_o_abundance",
+    "c_plus_6_over_c_plus_5_ratio",
+    "o_plus_7_over_o_plus_6_ratio",
+    "fe_low_over_fe_high_ratio",
+]
+CODICE_HI_IAL_DATA_FIELDS = ["h"]
+
 # lo- and hi-counters-aggregated data product variables are dynamically
 # determined based on the number of active counters
 # TODO: Try to convince Joey to move to lower case variable names with
@@ -207,13 +218,13 @@ HI_COUNTERS_AGGREGATED_ACTIVE_VARIABLES = {
     "Reserved3": False,
     "Reserved4": False,
     "Reserved5": False,
-    "LowTOFCutoff": True,
-    "Reserved6": True,
-    "Reserved7": True,
+    "LowTOFCutoff": False,
+    "Reserved6": False,
+    "Reserved7": False,
     "ASIC1FlagInvalid": True,
     "ASIC2FlagInvalid": True,
-    "ASIC1ChannelInvalid": True,
-    "ASIC2ChannelInvalid": True,
+    "ASIC1ChannelInvalid": False,
+    "ASIC2ChannelInvalid": False,
 }
 HI_COUNTERS_AGGREGATED_VARIABLE_NAMES = [
     name