imap-processing 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

imap_processing/ultra/utils/ultra_l1_utils.py

@@ -6,7 +6,9 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
 
 
 def create_dataset(
-    data_dict: dict, name: str, level: str, data_version: str
+    data_dict: dict,
+    name: str,
+    level: str,
 ) -> xr.Dataset:
     """
     Create xarray for L1b data.
@@ -19,8 +21,6 @@ def create_dataset(
         Name of the dataset.
     level : str
         Level of the dataset.
-    data_version : str
-        Version of the data.
 
     Returns
     -------
@@ -30,15 +30,25 @@ def create_dataset(
     cdf_manager = ImapCdfAttributes()
     cdf_manager.add_instrument_global_attrs("ultra")
     cdf_manager.add_instrument_variable_attrs("ultra", level)
-    cdf_manager.add_global_attribute("Data_version", data_version)
 
+    # L1b extended spin, badtimes, and cullingmask data products
     if "spin_number" in data_dict.keys():
         coords = {
             "spin_number": data_dict["spin_number"],
             "energy_bin_geometric_mean": data_dict["energy_bin_geometric_mean"],
+            # Start time aligns with the universal spin table
+            "epoch": data_dict["epoch"],
         }
         default_dimension = "spin_number"
-
+    # L1c pset data products
+    elif "healpix" in data_dict:
+        coords = {
+            "healpix": data_dict["healpix"],
+            "energy_bin_geometric_mean": data_dict["energy_bin_geometric_mean"],
+            "epoch": data_dict["epoch"],
+        }
+        default_dimension = "healpix"
+    # L1b de data product
     else:
         epoch_time = xr.DataArray(
             data_dict["epoch"],
@@ -68,6 +78,8 @@ def create_dataset(
         "velocity_sc",
         "velocity_dps_sc",
         "velocity_dps_helio",
+        "direct_event_unit_velocity",
+        "direct_event_unit_position",
     }
     rates_keys = {
         "ena_rates",
@@ -75,30 +87,37 @@ def create_dataset(
         "quality_ena_rates",
     }
 
-    for key in data_dict.keys():
-        if key in ["epoch", "spin_number", "energy_bin_geometric_mean"]:
+    for key, data in data_dict.items():
+        # Skip keys that are coordinates.
+        if key in ["epoch", "spin_number", "energy_bin_geometric_mean", "healpix"]:
             continue
         elif key in velocity_keys:
             dataset[key] = xr.DataArray(
-                data_dict[key],
+                data,
                 dims=["epoch", "component"],
                 attrs=cdf_manager.get_variable_attributes(key),
             )
-        elif key == "ena_rates_threshold":
+        elif key in ("ena_rates_threshold", "energy_bin_delta"):
             dataset[key] = xr.DataArray(
-                data_dict[key],
+                data,
                 dims=["energy_bin_geometric_mean"],
                 attrs=cdf_manager.get_variable_attributes(key),
             )
         elif key in rates_keys:
             dataset[key] = xr.DataArray(
-                data_dict[key],
+                data,
                 dims=["energy_bin_geometric_mean", "spin_number"],
                 attrs=cdf_manager.get_variable_attributes(key),
             )
+        elif key == "counts":
+            dataset[key] = xr.DataArray(
+                data,
+                dims=["energy_bin_geometric_mean", "healpix"],
+                attrs=cdf_manager.get_variable_attributes(key),
+            )
         else:
             dataset[key] = xr.DataArray(
-                data_dict[key],
+                data,
                 dims=[default_dimension],
                 attrs=cdf_manager.get_variable_attributes(key),
             )

imap_processing/utils.py

@@ -79,6 +79,19 @@ def convert_raw_to_eu(
         informational columns: ('packetName', 'mnemonic', 'convertAs') and
         conversion columns named 'c0', 'c1', 'c2', etc. Conversion columns
         specify the array of polynomial coefficients used for the conversion.
+        If the column 'convertAs' is 'SEGMENTED_POLY' then there must be columns
+        'dn_range_start' and 'dn_range_stop' that specifies the raw DN range and the
+        coefficients that should be used for the conversion.
+
+        E.g.:
+
+        mnemonic       convertAs …       dn_range_start   dn_range_stop  c0    c1…
+        -------------------------------------------------------------------------
+        temperature  |  SEGMENTED_POLY | 0              | 2063         | 0.1  | 0.2
+        temperature  |  SEGMENTED_POLY | 2064           | 3853         | 0    | 0.1
+        temperature  |  SEGMENTED_POLY | 3854           | 4094         | 0.6  | 0.3
+        sensor_v     |  UNSEGMENTED_POLY |              |              | 0.04 | .110
+
         Comment lines are allowed in the csv file specified by starting with
         the '#' character.
     packet_name : str
@@ -86,7 +99,8 @@ def convert_raw_to_eu(
     **read_csv_kwargs : dict
         In order to allow for some flexibility in the format of the csv
         conversion table, any additional keywords passed to this function are
-        passed in the call to `pandas.read_csv()`. See pandas documentation
+        passed in the call to `pandas.read_csv()`.
+        See pandas documentation
         for a list of keywords and their functionality:
         https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html.
 
@@ -95,47 +109,73 @@ def convert_raw_to_eu(
     dataset : xr.Dataset
         Raw data converted to engineering unit as needed.
     """
-    # Make sure there is column called "index" with unique
-    # value such as 0, 1, 2, 3, ...
     eu_conversion_df = pd.read_csv(
         conversion_table_path,
         **read_csv_kwargs,
     )
 
-    # Look up all metadata fields for the packet name
-    packet_df = eu_conversion_df.loc[eu_conversion_df["packetName"] == packet_name]
-
-    # for each metadata field, convert raw value to engineering unit
-    for _, row in packet_df.iterrows():
-        if row["convertAs"] == "UNSEGMENTED_POLY":
+    # Iterate through every variable in the dataset and check if there is an entry for
+    # That variable in the conversion table.
+    for var in dataset.variables:
+        packet_df = eu_conversion_df.loc[
+            (eu_conversion_df["packetName"] == packet_name)
+            &
+            # Filter for mnemonic case-insensitive
+            (eu_conversion_df["mnemonic"].str.lower() == var.lower())
+        ].reset_index(drop=True)
+
+        if packet_df.empty:
+            continue
+
+        if np.all(packet_df["convertAs"] == "UNSEGMENTED_POLY"):
+            if len(packet_df.index) > 1:
+                raise ValueError(
+                    "For unsegmented polynomial conversions, there should "
+                    "only be one row per mnemonic and packet name."
+                )
+            row = packet_df.iloc[0]
             # On this line, we are getting the coefficients from the
             # table and then reverse them because the np.polyval is
             # expecting coefficient in descending order
             # coeff columns must have names 'c0', 'c1', 'c2', ...
             coeff_values = row.filter(regex=r"c\d").values[::-1]
-            row_key = row["mnemonic"]
-            # TODO: remove this check once everyone has lowercase
-            # all of CDF data variable names to match SPDF requirement.
-            # Right now, check if dataset mnemonics is lowercase of row mnemonics.
-            # If so, make them match
-            mnemonics = row_key.lower() if row_key.lower() in dataset else row_key
-            try:
-                # Convert the raw value to engineering unit
-                dataset[mnemonics].data = np.polyval(
-                    coeff_values, dataset[mnemonics].data
+            # Convert the raw value to engineering unit
+            dataset[var].data = np.polyval(coeff_values, dataset[var].data)
+
+        elif np.all(packet_df["convertAs"] == "SEGMENTED_POLY"):
+            data = dataset[var].data
+            # Check if any of the raw DN values fall outside the ranges
+            bad_mask = np.logical_or(
+                data < packet_df["dn_range_start"].min(),
+                data > packet_df["dn_range_stop"].max(),
+            )
+            if np.any(data[bad_mask]):
+                raise ValueError(
+                    "Raw DN values found outside of the expected range"
+                    f"for mnemonic: {var}"
                 )
-                # Modify units attribute
-                if "unit" in row:
-                    dataset[mnemonics].attrs.update({"units": row["unit"]})
-            except KeyError:
-                # TODO: Don't catch this error once packet definitions stabilize
-                logger.warning(f"Input dataset does not contain key: {row_key}")
+            # Create conditions and corresponding functions for np.piecewise
+            conditions = [
+                (data >= row["dn_range_start"]) & (data <= row["dn_range_stop"])
+                for _, row in packet_df.iterrows()
+            ]
+            functions = [
+                lambda x, r=row: np.polyval(r.filter(regex=r"c\d").values[::-1], x)
+                for _, row in packet_df.iterrows()
+            ]
+            # Convert the raw value to engineering unit
+            dataset[var].data = np.piecewise(data, conditions, functions)
+
         else:
             raise ValueError(
-                f"Unexpected conversion type: {row['convertAs']} encountered in"
-                f" engineering unit conversion table: {conversion_table_path}"
+                "Column 'convertAs' must all be UNSEGMENTED_POLY or "
+                "SEGMENTED_POLY for a packet name and mnemonic"
             )
 
+        # Modify units attribute
+        if "unit" in packet_df:
+            dataset[var].attrs.update({"UNITS": packet_df.iloc[0]["unit"]})
+
     return dataset
 
 
@@ -312,7 +352,7 @@ def packet_file_to_datasets(
         # to the packet_name being a part of the variable name in the XTCE definition
         ds = ds.rename(
             {
-                # partition splits the string into 3 parts: before ".", ".", after "."
+                # partition splits the string into 3 parts: before ".", after "."
                 # if there was no ".", the second part is an empty string, so we use
                 # the original key in that case
                 key: key.partition(".")[2] or key
@@ -339,5 +379,5 @@ def convert_to_binary_string(data: bytes) -> str:
     binary_data : str
         The binary data as a string.
     """
-    binary_str_data = f"{int.from_bytes(data, byteorder='big'):0{len(data)*8}b}"
+    binary_str_data = f"{int.from_bytes(data, byteorder='big'):0{len(data) * 8}b}"
     return binary_str_data

{imap_processing-0.11.0.dist-info → imap_processing-0.13.0.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: imap-processing
-Version: 0.11.0
+Version: 0.13.0
 Summary: IMAP Science Operations Center Processing
 License: MIT
 Keywords: IMAP,SDC,SOC,Science Operations
 Author: IMAP SDC Developers
 Author-email: imap-sdc@lists.lasp.colorado.edu
-Requires-Python: >=3.9,<4
+Requires-Python: >=3.10,<4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -16,7 +16,6 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: POSIX
 Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -25,11 +24,15 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Provides-Extra: dev
 Provides-Extra: doc
+Provides-Extra: map-visualization
 Provides-Extra: test
 Provides-Extra: tools
+Requires-Dist: astropy-healpix (>=1.0)
 Requires-Dist: cdflib (>=1.3.1,<2.0.0)
-Requires-Dist: imap-data-access (>=0.10.1)
+Requires-Dist: healpy (>=1.18.0,<2.0.0) ; extra == "map-visualization"
+Requires-Dist: imap-data-access (>=0.21.0)
 Requires-Dist: mypy (==1.10.1) ; extra == "dev"
+Requires-Dist: netcdf4 (>=1.7.2,<2.0.0) ; extra == "test"
 Requires-Dist: numpy (<=3)
 Requires-Dist: numpydoc (>=1.5.0,<2.0.0) ; extra == "doc"
 Requires-Dist: openpyxl (>=3.0.7) ; extra == "test" or extra == "tools"
@@ -38,15 +41,16 @@ Requires-Dist: pre-commit (>=3.3.3,<4.0.0) ; extra == "dev"
 Requires-Dist: pydata-sphinx-theme ; extra == "doc"
 Requires-Dist: pytest (>=6.2.5) ; extra == "test"
 Requires-Dist: pytest-cov (>=4.0.0,<5.0.0) ; extra == "test"
-Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: pytest-xdist (>=3.2,<4.0) ; extra == "test"
 Requires-Dist: requests (>=2.32.3,<3.0.0) ; extra == "test"
 Requires-Dist: ruff (==0.2.1) ; extra == "dev"
 Requires-Dist: sammi-cdf (>=1.0,<2.0)
+Requires-Dist: scipy (>=1.13,<2.0)
 Requires-Dist: space_packet_parser (>=5.0.1,<6.0.0)
 Requires-Dist: sphinx ; extra == "doc"
 Requires-Dist: sphinxcontrib-openapi (>=0.8.3,<0.9.0) ; extra == "doc"
 Requires-Dist: spiceypy (>=6.0.0)
-Requires-Dist: xarray (>=2023.0.0)
+Requires-Dist: xarray (>=2024.10.0)
 Description-Content-Type: text/markdown
 
 # IMAP (Interstellar Mapping and Acceleration Probe)