imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml

@@ -3,210 +3,268 @@ default_attrs: &default
   DEPEND_0: epoch
   DISPLAY_TYPE: time_series
   FILLVAL: -9223372036854775808
-  FORMAT: I19
-  VALIDMIN: 0
+  FORMAT: I12
+  VALIDMIN: -9223372036854775808
   VALIDMAX: 9223372036854775807
   VAR_TYPE: data
+  UNITS: " "
 
-x_front:
+default_uint8_attrs: &default_uint8
+  <<: *default
+  FILLVAL: 255
+  FORMAT: I3
+  VALIDMIN: 0
+  VALIDMAX: 255
+  dtype: uint8
+
+default_uint16_attrs: &default_uint16
+  <<: *default
+  FILLVAL: 65535
+  FORMAT: I5
+  VALIDMIN: 0
+  VALIDMAX: 65535
+  dtype: uint16
+
+default_uint32_attrs: &default_uint32
+  <<: *default
+  FILLVAL: 4294967295
+  FORMAT: I10
+  VALIDMIN: 0
+  VALIDMAX: 4294967295
+  dtype: uint32
+
+default_int64_attrs: &default_int64
+  <<: *default
+  FILLVAL: -9223372036854775808
+  FORMAT: I20
+  VALIDMIN: -9223372036854775808
+  VALIDMAX: 9223372036854775807
+  dtype: int64
+
+default_int32_attrs: &default_int32
+  <<: *default
+  FILLVAL: -2147483648
+  FORMAT: I10
+  VALIDMIN: -2147483648
+  VALIDMAX: 2147483647
+  dtype: int32
+
+default_float32_attrs: &default_float32
   <<: *default
+  FILLVAL: .NAN
+  FORMAT: F12.6
+  VALIDMIN: -3.4028235e+38
+  VALIDMAX: 3.4028235e+38
+  dtype: float32
+
+x_front:
+  <<: *default_float32
   CATDESC: x front position
   FIELDNAM: Event x-position on front foil
   LABLAXIS: x front position
-  # TODO: come back to format
-  UNITS: mm
+  UNITS: mm / 100
 
 y_front:
-  <<: *default
+  <<: *default_float32
   CATDESC: y front position
   FIELDNAM: Event y-position on front foil
   LABLAXIS: y front position
-  # TODO: come back to format
-  UNITS: mm
+  UNITS: mm / 100
 
 x_back:
-  <<: *default
+  <<: *default_float32
   CATDESC: x back position
   FIELDNAM: x_back
   LABLAXIS: x back position
-  # TODO: come back to format
-  UNITS: mm
+  UNITS: mm / 100
 
 y_back:
-  <<: *default
+  <<: *default_float32
   CATDESC: y back position
   FIELDNAM: y_back
   LABLAXIS: y back position
-  # TODO: come back to format
-  UNITS: mm
+  UNITS: mm / 100
 
 x_coin:
-  <<: *default
+  <<: *default_float32
   CATDESC: x coincidence position
   FIELDNAM: x_coin
   LABLAXIS: x coincidence position
-  # TODO: come back to format
-  UNITS: mm
+  UNITS: mm / 100
 
 tof_start_stop:
-  <<: *default
+  <<: *default_float32
   CATDESC: Particle time of flight from start to stop
   FIELDNAM: tof_start_stop
   LABLAXIS: tof start stop
-  # TODO: come back to format
-  UNITS: ns
+  UNITS: nanosecond / 10
 
 tof_stop_coin:
-  <<: *default
+  <<: *default_float32
   CATDESC: Particle time of flight from stop to coincidence
   FIELDNAM: tof_stop_coin
   LABLAXIS: tof stop coin
-  # TODO: come back to format
-  UNITS: ns
+  UNITS: nanosecond / 10
 
 tof_corrected:
-  <<: *default
+  <<: *default_float32
   CATDESC: Corrected time of flight
   FIELDNAM: tof_corrected
   LABLAXIS: tof corrected
-  # TODO: come back to format
-  UNITS: ns
+  UNITS: nanosecond / 10
 
-eventtype:
-  <<: *default
+event_type:
+  <<: *default_uint8
   CATDESC: Eventtype (1-2 top and bottom stop types; 8-15 ssd stop types)
-  FIELDNAM: eventtype
-  LABLAXIS: eventtype
-  # TODO: come back to format
+  FIELDNAM: Event type
+  LABLAXIS: event type
+  UNITS: " "
+
+coincidence_type:
+  <<: *default_uint8
+  CATDESC: Coincidence type for the direct event (1-2 top and bottom)
+  FIELDNAM: Coincidence type
+  LABLAXIS: coincidence type
+  UNITS: " "
+
+start_type:
+  <<: *default_uint8
+  CATDESC: Start type for the direct event (1-2 left and right)
+  FIELDNAM: Start type
+  LABLAXIS: start type
   UNITS: " "
 
 vx_ultra:
-  <<: *default
+  <<: *default_float32
   CATDESC: Normalized component of the velocity vector in x direction in the instrument frame.
   FIELDNAM: vx_ultra
   LABLAXIS: vx ultra
-  # TODO: come back to format
   UNITS: " "
 
 vy_ultra:
-  <<: *default
+  <<: *default_float32
   CATDESC: Normalized component of the velocity vector in y direction in the instrument frame.
   FIELDNAM: vy_ultra
   LABLAXIS: vy ultra
-  # TODO: come back to format
   UNITS: " "
 
 vz_ultra:
-  <<: *default
+  <<: *default_float32
   CATDESC: Normalized component of the velocity vector in z direction in the instrument frame.
   FIELDNAM: vz_ultra
   LABLAXIS: vz ultra
-  # TODO: come back to format
   UNITS: " "
 
 energy:
-  <<: *default
+  <<: *default_float32
   CATDESC: Energy measured using the pulse height from the stop anode (DN) or Energy measured using the SSD (keV) depending on eventtype.
   FIELDNAM: energy
   LABLAXIS: energy
-  # TODO: come back to format
   UNITS: keV
 
 species:
   <<: *default
-  CATDESC: Species bin.
-  FIELDNAM: species
-  LABLAXIS: species
-  # TODO: come back to format
+  DISPLAY_TYPE: no_plot
+  CATDESC: Label species type.
+  FIELDNAM: Label species type.
+  FILLVAL: "UNKNOWN"
+  FORMAT: A8
+  dtype: str
   UNITS: " "
 
+front_back_distance:
+  <<: *default_float32
+  CATDESC: Distance front to back.
+  FIELDNAM: front_back_distance
+  LABLAXIS: front_back_distance
+  UNITS: mm / 100
+
+path_length:
+  <<: *default_float32
+  CATDESC: Path length.
+  FIELDNAM: path_length
+  LABLAXIS: path_length
+  UNITS: mm / 100
+
 event_efficiency:
-  <<: *default
+  <<: *default_float32
   CATDESC: Estimated event efficiency for this path through the instrument.
   FIELDNAM: event_efficiency
   LABLAXIS: event efficiency
-  # TODO: come back to format
   UNITS: " "
 
 vx_sc:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in x direction in the spacecraft frame.
+  <<: *default_float32
+  CATDESC: x-component of the velocity vector of the ena in the spacecraft frame.
   FIELDNAM: vx_sc
   LABLAXIS: vx sc
-  # TODO: come back to format
   UNITS: " "
 
 vy_sc:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in y direction in the spacecraft frame.
+  <<: *default_float32
+  CATDESC: y-component of the velocity vector of the ena in the spacecraft frame.
   FIELDNAM: vy_sc
   LABLAXIS: vy sc
-  # TODO: come back to format
   UNITS: " "
 
 vz_sc:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in z direction in the spacecraft frame.
+  <<: *default_float32
+  CATDESC: z-component of the velocity vector of the ena in the spacecraft frame.
   FIELDNAM: vz_sc
   LABLAXIS: vz sc
-  # TODO: come back to format
   UNITS: " "
 
 vx_dps_sc:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in x direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted)
+  <<: *default_float32
+  CATDESC: x-component of the velocity vector of the ena in the DPS frame at rest WRT spacecraft.
   FIELDNAM: vx_dps_sc
   LABLAXIS: vx dps sc
-  # TODO: come back to format
   UNITS: " "
 
 vy_dps_sc:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in y direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted)
+  <<: *default_float32
+  CATDESC: y-component of the velocity vector of the ena in the DPS frame at rest WRT spacecraft.
   FIELDNAM: vy_dps_sc
   LABLAXIS: vy dps sc
-  # TODO: come back to format
   UNITS: " "
 
 vz_dps_sc:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in z direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted)
+  <<: *default_float32
+  CATDESC: z-component of the velocity vector of the ena in the DPS frame at rest WRT spacecraft.
   FIELDNAM: vz_dps_sc
   LABLAXIS: vz dps sc
-  # TODO: come back to format
   UNITS: " "
 
 vx_dps_helio:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in x direction in the DPS frame at rest WRT heliosphere.
+  <<: *default_float32
+  CATDESC: x-component of the velocity vector of the ena in the DPS frame at rest WRT heliosphere.
   FIELDNAM: vx_dps_helio
   LABLAXIS: vx dps helio
-  # TODO: come back to format
   UNITS: " "
 
 vy_dps_helio:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in y direction in the DPS frame at rest WRT heliosphere.
+  <<: *default_float32
+  CATDESC: y-component of the velocity vector of the ena in the DPS frame at rest WRT heliosphere.
   FIELDNAM: vy_dps_helio
   LABLAXIS: vy dps helio
-  # TODO: come back to format
   UNITS: " "
 
 vz_dps_helio:
-  <<: *default
-  CATDESC: Normalized component of the velocity vector in z direction in the DPS frame at rest WRT heliosphere.
+  <<: *default_float32
+  CATDESC: z-component of the velocity vector of the ena in the DPS frame at rest WRT heliosphere.
   FIELDNAM: vz_dps_helio
   LABLAXIS: vz dps helio
-  # TODO: come back to format
   UNITS: " "
 
-eventtimes:
-  <<: *default
-  CATDESC: Event times calculated from event and universal spin table.
-  FIELDNAM: eventtimes
-  LABLAXIS: event times
-  # TODO: come back to format
-  UNITS: " "
+de_event_met:
+  <<: *default_int64
+  CATDESC: Mission Elapsed Time (MET) of Direct Event
+  DISPLAY_TYPE: no_plot
+  FIELDNAM: Mission Elapsed Time
+  LABLAXIS: DE MET
+  UNITS: ns
+  VAR_TYPE: support_data
+  SCALE_TYP: linear
 
 spin_number:
   <<: *default

imap_processing/cdf/config/imap_variable_schema.yaml

@@ -0,0 +1,13 @@
+# This file acts as an overwrite option for the variable schema described by SAMMI.
+# SAMMI generally has good defaults and its schema should be used, but this is a place
+# to override those defaults if necessary.
+
+# This is only required while issue #7 is incomplete.
+attribute_key:
+  LABLAXIS:
+      description: >
+        should be a short character string (approximately 10 characters, but preferably 6 characters - more only if absolutely required for clarity) which can be used to label a y-axis for a plot or to provide a heading for a data listing.
+        Only one of LABLAXIS or LABL_PTR_i should be present.
+      required: false # not required if using LABL_PTR_i
+      valid_values: null
+      alternate: LABL_PTR_1

imap_processing/cdf/imap_cdf_manager.py

@@ -5,38 +5,31 @@ This class should be used in all cases, and should contain any IMAP specific ass
 or code.
 """
 
-from pathlib import Path
-from typing import Optional
+from __future__ import annotations
 
-from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager
+from pathlib import Path
 
+from sammi.cdf_attribute_manager import CdfAttributeManager
 
-class ImapCdfAttributes(CdfAttributeManager):
-    """
-    Contains IMAP specific tools and settings for CDF management.
 
-    Parameters
-    ----------
-    source_dir : pathlib.Path or None
-        Source directory.
-    """
+class ImapCdfAttributes(CdfAttributeManager):  # type: ignore [misc]
+    """Contains IMAP specific tools and settings for CDF management."""
 
-    def __init__(self, source_dir: Optional[Path] = None):
-        """
-        Set the path to the config directory.
+    def __init__(self) -> None:
+        """Set the path to the config directory."""
+        self.source_dir = Path(__file__).parent / "config"
 
-        Parameters
-        ----------
-        source_dir : pathlib.Path or None
-            Source directory.
-        """
-        if source_dir is None:
-            super().__init__(Path(__file__).parent / "config")
-        else:
-            super().__init__(source_dir)
+        # TODO: Overwriting schema is not working.
+        super().__init__(
+            variable_schema_layers=[self.source_dir / "imap_variable_schema.yaml"],
+            use_defaults=True,
+        )
 
         # Load constants attrs that everyone uses
-        self.load_variable_attributes("imap_constant_attrs.yaml")
+        self.load_variable_attributes(self.source_dir / "imap_constant_attrs.yaml")
+        self.load_global_attributes(
+            self.source_dir / "imap_default_global_cdf_attrs.yaml"
+        )
 
     def add_instrument_global_attrs(self, instrument: str) -> None:
         """
@@ -48,9 +41,13 @@ class ImapCdfAttributes(CdfAttributeManager):
             Instrument name.
         """
         # Looks for file named "imap_{instrument}_global_cdf_attrs.yaml"
-        self.load_global_attributes(f"imap_{instrument}_global_cdf_attrs.yaml")
+        self.load_global_attributes(
+            self.source_dir / f"imap_{instrument}_global_cdf_attrs.yaml"
+        )
 
-    def add_instrument_variable_attrs(self, instrument: str, level: str) -> None:
+    def add_instrument_variable_attrs(
+        self, instrument: str, level: str | None = None
+    ) -> None:
         """
         Add instrument specific variable attributes.
 
@@ -61,4 +58,11 @@ class ImapCdfAttributes(CdfAttributeManager):
         level : str
             Data level.
         """
-        self.load_variable_attributes(f"imap_{instrument}_{level}_variable_attrs.yaml")
+        if level is None:
+            self.load_variable_attributes(
+                self.source_dir / f"imap_{instrument}_variable_attrs.yaml"
+            )
+        else:
+            self.load_variable_attributes(
+                self.source_dir / f"imap_{instrument}_{level}_variable_attrs.yaml"
+            )

imap_processing/cdf/utils.py

@@ -13,15 +13,11 @@ from cdflib.xarray.cdf_to_xarray import ISTP_TO_XARRAY_ATTRS
 
 import imap_processing
 from imap_processing._version import __version__, __version_tuple__  # noqa: F401
+from imap_processing.spice.time import J2000_EPOCH
 
 logger = logging.getLogger(__name__)
 
 
-# Reference start time (launch time or epoch)
-# DEFAULT_EPOCH = np.datetime64("2010-01-01T00:01:06.184", "ns")
-J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns")
-
-
 def load_cdf(
     file_path: Path, remove_xarray_attrs: bool = True, **kwargs: dict
 ) -> xr.Dataset:
@@ -96,6 +92,8 @@ def write_cdf(
     # Logical_source looks like "imap_swe_l2_counts-1min"
     instrument, data_level, descriptor = dataset.attrs["Logical_source"].split("_")[1:]
     # Convert J2000 epoch referenced data to datetime64
+    # TODO: This implementation of epoch to time string results in an error of
+    #       5 seconds due to 5 leap-second occurrences since the J2000 epoch.
     dt64 = J2000_EPOCH + dataset["epoch"].values[0].astype("timedelta64[ns]")
     start_time = np.datetime_as_string(dt64, unit="D").replace("-", "")
 

imap_processing/cli.py

@@ -44,6 +44,7 @@ from imap_processing.hi.l1c import hi_l1c
 from imap_processing.hit.l1a.hit_l1a import hit_l1a
 from imap_processing.hit.l1b.hit_l1b import hit_l1b
 from imap_processing.idex.idex_l1a import PacketParser
+from imap_processing.idex.idex_l1b import idex_l1b
 from imap_processing.lo.l1a import lo_l1a
 from imap_processing.lo.l1b import lo_l1b
 from imap_processing.lo.l1c import lo_l1c
@@ -551,19 +552,21 @@ class Hit(ProcessInstrument):
                     f"Unexpected dependencies found for HIT L1A:"
                     f"{dependencies}. Expected only one dependency."
                 )
-            # process data and write all processed data to CDF files
+            # process data to L1A products
             datasets = hit_l1a(dependencies[0], self.version)
 
         elif self.data_level == "l1b":
-            if len(dependencies) > 1:
-                raise ValueError(
-                    f"Unexpected dependencies found for HIT L1B:"
-                    f"{dependencies}. Expected only one dependency."
-                )
-            # process data and write all processed data to CDF files
-            l1a_dataset = load_cdf(dependencies[0])
-
-            datasets = hit_l1b(l1a_dataset, self.version)
+            data_dict = {}
+            for i, dependency in enumerate(dependencies):
+                if self.dependencies[i]["data_level"] == "l0":
+                    # Add path to CCSDS file to process housekeeping
+                    data_dict["imap_hit_l0_raw"] = dependency
+                else:
+                    # Add L1A datasets to process science data
+                    dataset = load_cdf(dependency)
+                    data_dict[dataset.attrs["Logical_source"]] = dataset
+            # process data to L1B products
+            datasets = hit_l1b(data_dict, self.version)
 
         return datasets
 
@@ -588,15 +591,23 @@ class Idex(ProcessInstrument):
         print(f"Processing IDEX {self.data_level}")
         datasets: list[xr.Dataset] = []
 
-        if self.data_level == "l1":
+        if self.data_level == "l1a":
             if len(dependencies) > 1:
                 raise ValueError(
-                    f"Unexpected dependencies found for IDEX L1:"
+                    f"Unexpected dependencies found for IDEX L1a:"
                     f"{dependencies}. Expected only one dependency."
                 )
             # read CDF file
-
-            datasets = PacketParser(dependencies[0], self.version).data
+            datasets = [PacketParser(dependencies[0], self.version).data]
+        elif self.data_level == "l1b":
+            if len(dependencies) > 1:
+                raise ValueError(
+                    f"Unexpected dependencies found for IDEX L1b:"
+                    f"{dependencies}. Expected only one dependency."
+                )
+            # process data
+            dependency = load_cdf(dependencies[0])
+            datasets = [idex_l1b(dependency, self.version)]
         return datasets