imap-processing 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

imap_processing/mag/l0/decom_mag.py

@@ -129,7 +129,9 @@ def generate_dataset(
         direction.astype(str),
         name="direction_label",
         dims=["direction_label"],
-        attrs=attribute_manager.get_variable_attributes("direction_label"),
+        attrs=attribute_manager.get_variable_attributes(
+            "direction_label", check_schema=False
+        ),
     )
 
     # TODO: Epoch here refers to the start of the sample. Confirm that this is
@@ -154,11 +156,10 @@ def generate_dataset(
         coords={
             "epoch": epoch_time,
             "direction": direction,
-            "direction_label": direction_label,
         },
         attrs=attribute_manager.get_global_attributes(logical_id),
     )
-
+    output["direction_label"] = direction_label
     output["raw_vectors"] = raw_vectors
 
     for key, value in support_data.items():

imap_processing/mag/l1a/mag_l1a.py

@@ -158,7 +158,6 @@ def process_packets(
                 "timedelta64[ns]"
             )
         ).astype("datetime64[D]")
-
         primary_packet_properties = MagL1aPacketProperties(
             mag_l0.SHCOARSE,
             primary_start_time,
@@ -284,9 +283,7 @@ def generate_dataset(
 
     # TODO: Just leave time in datetime64 type with vector as dtype object to avoid this
     # Get the timestamp from the end of the vector
-    time_data = single_file_l1a.vectors[:, 4].astype(
-        np.dtype("datetime64[ns]"), copy=False
-    )
+    time_data = single_file_l1a.vectors[:, 4]
 
     compression = xr.DataArray(
         np.arange(2),
@@ -326,17 +323,21 @@ def generate_dataset(
     )
 
     direction_label = xr.DataArray(
-        direction.astype(str),
+        direction.values.astype(str),
         name="direction_label",
         dims=["direction_label"],
-        attrs=attribute_manager.get_variable_attributes("direction_label"),
+        attrs=attribute_manager.get_variable_attributes(
+            "direction_label", check_schema=False
+        ),
     )
 
     compression_label = xr.DataArray(
-        compression.astype(str),
+        compression.values.astype(str),
         name="compression_label",
         dims=["compression_label"],
-        attrs=attribute_manager.get_variable_attributes("compression_label"),
+        attrs=attribute_manager.get_variable_attributes(
+            "compression_label", check_schema=False
+        ),
     )
 
     output = xr.Dataset(
@@ -344,12 +345,11 @@ def generate_dataset(
             "epoch": epoch_time,
             "direction": direction,
             "compression": compression,
-            "direction_label": direction_label,
-            "compression_label": compression_label,
         },
         attrs=attribute_manager.get_global_attributes(logical_file_id),
     )
-
+    output["direction_label"] = direction_label
+    output["compression_label"] = compression_label
     output["vectors"] = vectors
     output["compression_flags"] = compression_flags
 

imap_processing/mag/l1b/mag_l1b.py

@@ -64,9 +64,11 @@ def mag_l1b_processing(input_dataset: xr.Dataset) -> xr.Dataset:
     """
     # TODO: There is a time alignment step that will add a lot of complexity.
     # This needs to be done once we have some SPICE time data.
+    mag_attributes = ImapCdfAttributes()
+    mag_attributes.add_instrument_variable_attrs("mag", "l1")
 
-    dims = [["direction"]]
-    new_dims = [["direction"]]
+    dims = [["direction"], ["compression"]]
+    new_dims = [["direction"], ["compression"]]
     # TODO: This should definitely be loaded from AWS
     calibration_dataset = load_cdf(
         Path(__file__).parent / "imap_calibration_mag_20240229_v01.cdf"
@@ -78,8 +80,9 @@ def mag_l1b_processing(input_dataset: xr.Dataset) -> xr.Dataset:
         calibration_matrix = calibration_dataset["MFITOURFI"]
 
     l1b_fields = xr.apply_ufunc(
-        calibrate,
+        update_vector,
         input_dataset["vectors"],
+        input_dataset["compression_flags"],
         input_core_dims=dims,
         output_core_dims=new_dims,
         vectorize=True,
@@ -88,13 +91,93 @@ def mag_l1b_processing(input_dataset: xr.Dataset) -> xr.Dataset:
     )
 
     output_dataset = input_dataset.copy()
-    output_dataset["vectors"] = l1b_fields
+    output_dataset["vectors"].data = l1b_fields[0].data
+
+    output_dataset["epoch"].attrs = mag_attributes.get_variable_attributes("epoch")
+    output_dataset["direction"].attrs = mag_attributes.get_variable_attributes(
+        "direction_attrs"
+    )
+    output_dataset["compression"].attrs = mag_attributes.get_variable_attributes(
+        "compression_attrs"
+    )
+    output_dataset["direction_label"].attrs = mag_attributes.get_variable_attributes(
+        "direction_label", check_schema=False
+    )
+    output_dataset["compression_label"].attrs = mag_attributes.get_variable_attributes(
+        "compression_label", check_schema=False
+    )
 
-    # TODO add/update attributes
     return output_dataset
 
 
-def calibrate(
+def update_vector(
+    input_vector: np.ndarray,
+    input_compression: np.ndarray,
+    calibration_matrix: xr.DataArray,
+) -> tuple[np.ndarray, np.ndarray]:
+    """
+    Apply calibration and compression scaling to vector.
+
+    This calls, in sequence, calibrate_vector and rescale_vector to apply L1B processing
+    to the input vector.
+
+    Parameters
+    ----------
+    input_vector : numpy.ndarray
+        One input vector to update, looking like (x, y, z, range).
+    input_compression : numpy.ndarray
+        Compression flags corresponding to the vector, looking like (is_compressed,
+        compression_width).
+    calibration_matrix : xr.DataArray
+        DataArray containing the full set of calibration matrices, for each range.
+        Size is ((3, 3, 4)).
+
+    Returns
+    -------
+    tuple[numpy.ndarray, numpy.ndarray]
+        Updated vector and the same compression flags.
+    """
+    vector = calibrate_vector(input_vector, calibration_matrix)
+    return rescale_vector(vector, input_compression), input_compression
+
+
+def rescale_vector(
+    input_vector: np.ndarray, compression_flags: np.ndarray
+) -> np.ndarray:
+    """
+    Rescale vector based on compression flags.
+
+    If the first value of compression_flags is zero, this just returns the input_vector
+    unchanged. Otherwise, the vector is scaled using the compression width, which is
+    the second part of compression_flags.
+
+    The vector is scaled using the following equation:
+    M = 2 ^ (16-width)
+    output_vector = input_vector * M
+
+    Therefore, for a 16 bit width, the same vector is returned.
+
+    Parameters
+    ----------
+    input_vector : numpy.ndarray
+        One input vector to update, looking like (x, y, z, range).
+    compression_flags : numpy.ndarray
+        Compression flags corresponding to the vector, looking like (is_compressed,
+        compression_width).
+
+    Returns
+    -------
+    output_vector : numpy.ndarray
+        Updated vector.
+    """
+    if not compression_flags[0]:
+        return input_vector
+    else:
+        factor = np.float_power(2, (16 - compression_flags[1]))
+        return input_vector * factor  # type: ignore
+
+
+def calibrate_vector(
     input_vector: np.ndarray, calibration_matrix: xr.DataArray = None
 ) -> np.ndarray:
     """
@@ -121,5 +204,4 @@ def calibrate(
     updated_vector[:3] = np.matmul(
         input_vector[:3], calibration_matrix.values[:, :, int(input_vector[3])]
     )
-
     return updated_vector

imap_processing/spice/geometry.py

@@ -19,6 +19,7 @@ import numpy as np
 import numpy.typing as npt
 import pandas as pd
 import spiceypy as spice
+from numpy.typing import NDArray
 
 from imap_processing.spice.kernels import ensure_spice
 
@@ -303,8 +304,6 @@ def get_spacecraft_to_instrument_spin_phase_offset(instrument: SpiceFrame) -> fl
     return offset_lookup[instrument]
 
 
-@typing.no_type_check
-@ensure_spice
 def frame_transform(
     et: Union[float, npt.NDArray],
     position: npt.NDArray,
@@ -350,10 +349,11 @@ def frame_transform(
                 f"Invalid position shape: {position.shape}. "
                 f"Each input position vector must have 3 elements."
             )
-        if not len(position) == len(et):
+        if not len(position) == np.asarray(et).size:
             raise ValueError(
                 "Mismatch in number of position vectors and Ephemeris times provided."
-                f"Position has {len(position)} elements and et has {len(et)} elements."
+                f"Position has {len(position)} elements and et has "
+                f"{np.asarray(et).size} elements."
             )
 
     # rotate will have shape = (3, 3) or (n, 3, 3)
@@ -369,6 +369,8 @@ def frame_transform(
     return result
 
 
+@typing.no_type_check
+@ensure_spice
 def get_rotation_matrix(
     et: Union[float, npt.NDArray],
     from_frame: SpiceFrame,
@@ -443,3 +445,123 @@ def instrument_pointing(
     if isinstance(et, typing.Collection):
         return np.rad2deg([spice.reclat(vec)[1:] for vec in pointing])
     return np.rad2deg(spice.reclat(pointing)[1:])
+
+
+def basis_vectors(
+    et: Union[float, npt.NDArray],
+    from_frame: SpiceFrame,
+    to_frame: SpiceFrame,
+) -> npt.NDArray:
+    """
+    Get the basis vectors of the `from_frame` expressed in the `to_frame`.
+
+    The rotation matrix defining a frame transform are the basis vectors of
+    the `from_frame` expressed in the `to_frame`. This function just transposes
+    each rotation matrix retrieved from the `get_rotation_matrix` function so
+    that basis vectors can be indexed 0 for x, 1 for y, and 2 for z.
+
+    Parameters
+    ----------
+    et : float or np.ndarray
+        Ephemeris time(s) for which to get the rotation matrices.
+    from_frame : SpiceFrame
+        Reference frame to transform from.
+    to_frame : SpiceFrame
+        Reference frame to transform to.
+
+    Returns
+    -------
+    basis_vectors : np.ndarray
+        If `et` is a float, the returned basis vector matrix is of shape `(3, 3)`. If
+        `et` is a np.ndarray, the returned basis vector matrix is of shape `(n, 3, 3)`
+        where `n` matches the number of elements in et and basis vectors are the rows
+        of the 3 by 3 matrices.
+
+    Examples
+    --------
+    >>> from imap_processing.spice.geometry import basis_vectors
+    ... from imap_processing.spice.time import j2000ns_to_j2000s
+    ... et = j2000ns_to_j2000s(dataset.epoch.values)
+    ... basis_vectors = basis_vectors(
+    ...     et, SpiceFrame.IMAP_SPACECRAFT, SpiceFrame.ECLIPJ2000
+    ... )
+    ... spacecraft_x = basis_vectors[:, 0]
+    ... spacecraft_y = basis_vectors[:, 1]
+    ... spacecraft_z = basis_vectors[:, 2]
+    """
+    return np.moveaxis(get_rotation_matrix(et, from_frame, to_frame), -1, -2)
+
+
+def cartesian_to_spherical(
+    v: NDArray,
+) -> NDArray:
+    """
+    Convert cartesian coordinates to spherical coordinates.
+
+    Parameters
+    ----------
+    v : np.ndarray
+        A NumPy array with shape (n, 3) where each
+        row represents a vector
+        with x, y, z-components.
+
+    Returns
+    -------
+    spherical_coords : np.ndarray
+        A NumPy array with shape (n, 3), where each row contains
+        the spherical coordinates (r, azimuth, elevation):
+
+        - r : Distance of the point from the origin.
+        - azimuth : angle in the xy-plane in radians [0, 2*pi].
+        - elevation : angle from the z-axis in radians [-pi/2, pi/2].
+    """
+    # Magnitude of the velocity vector
+    magnitude_v = np.linalg.norm(v, axis=-1, keepdims=True)
+
+    vhat = v / magnitude_v
+
+    # Elevation angle (angle from the z-axis, range: [-pi/2, pi/2])
+    el = np.arcsin(vhat[..., 2])
+
+    # Azimuth angle (angle in the xy-plane, range: [0, 2*pi])
+    az = np.arctan2(vhat[..., 1], vhat[..., 0])
+
+    # Ensure azimuth is from 0 to 2PI
+    az = az % (2 * np.pi)
+    spherical_coords = np.stack(
+        (np.squeeze(magnitude_v), np.degrees(az), np.degrees(el)), axis=-1
+    )
+
+    return spherical_coords
+
+
+def spherical_to_cartesian(spherical_coords: NDArray) -> NDArray:
+    """
+    Convert spherical coordinates to Cartesian coordinates.
+
+    Parameters
+    ----------
+    spherical_coords : np.ndarray
+        A NumPy array with shape (n, 3), where each row contains
+        the spherical coordinates (r, azimuth, elevation):
+
+        - r : Distance of the point from the origin.
+        - azimuth : angle in the xy-plane in radians [0, 2*pi].
+        - elevation : angle from the z-axis in radians [-pi/2, pi/2].
+
+    Returns
+    -------
+    cartesian_coords : np.ndarray
+        Cartesian coordinates.
+    """
+    r = spherical_coords[..., 0]
+    azimuth = spherical_coords[..., 1]
+    elevation = spherical_coords[..., 2]
+
+    x = r * np.cos(elevation) * np.cos(azimuth)
+    y = r * np.cos(elevation) * np.sin(azimuth)
+    z = r * np.sin(elevation)
+
+    cartesian_coords = np.stack((x, y, z), axis=-1)
+
+    return cartesian_coords

imap_processing/swapi/l1/swapi_l1.py

@@ -480,7 +480,7 @@ def process_swapi_science(
     # ====================================================
     cdf_manager = ImapCdfAttributes()
     cdf_manager.add_instrument_global_attrs("swapi")
-    cdf_manager.load_variable_attributes("imap_swapi_variable_attrs.yaml")
+    cdf_manager.add_instrument_variable_attrs(instrument="swapi", level=None)
 
     # ===================================================================
     # Quality flags

imap_processing/swapi/l2/swapi_l2.py

@@ -43,7 +43,7 @@ def swapi_l2(l1_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
     # Load the CDF attributes
     cdf_manager = ImapCdfAttributes()
     cdf_manager.add_instrument_global_attrs("swapi")
-    cdf_manager.load_variable_attributes("imap_swapi_variable_attrs.yaml")
+    cdf_manager.add_instrument_variable_attrs(instrument="swapi", level=None)
 
     # Copy over only certain variables from L1 to L2 dataset
     l1_data_keys = [

imap_processing/swe/l1b/swe_l1b_science.py

@@ -95,7 +95,7 @@ def deadtime_correction(counts: np.ndarray, acq_duration: int) -> npt.NDArray:
     counts : numpy.ndarray
         Counts data before deadtime corrections.
     acq_duration : int
-        This is ACQ_DURATION from science packet.
+        This is ACQ_DURATION from science packet. acq_duration is in microseconds.
 
     Returns
     -------
@@ -104,10 +104,10 @@ def deadtime_correction(counts: np.ndarray, acq_duration: int) -> npt.NDArray:
     """
     # deadtime is 360 ns
     deadtime = 360e-9
-    correct = 1.0 - (deadtime * counts / (acq_duration / 1000.0))
+    correct = 1.0 - (deadtime * (counts / (acq_duration * 1e-6)))
     correct = np.maximum(0.1, correct)
     corrected_count = np.divide(counts, correct)
-    return corrected_count
+    return corrected_count.astype(np.float64)
 
 
 def convert_counts_to_rate(data: np.ndarray, acq_duration: int) -> npt.NDArray:
@@ -121,17 +121,17 @@ def convert_counts_to_rate(data: np.ndarray, acq_duration: int) -> npt.NDArray:
     data : numpy.ndarray
         Counts data.
     acq_duration : int
-        Acquisition duration. acq_duration is in millieseconds.
+        Acquisition duration. acq_duration is in microseconds.
 
     Returns
     -------
     numpy.ndarray
         Count rates array in seconds.
     """
-    # convert milliseconds to seconds
-    # Todo: check with SWE team about int or float types.
-    acq_duration = int(acq_duration / 1000.0)
-    return data / acq_duration
+    # convert microseconds to seconds
+    acq_duration_sec = acq_duration * 1e-6
+    count_rate = data / acq_duration_sec
+    return count_rate.astype(np.float64)
 
 
 def calculate_calibration_factor(time: int) -> None:

imap_processing/tests/ccsds/test_data/expected_output.xml

@@ -64,6 +64,7 @@
 				<xtce:EnumerationList>
 					<xtce:Enumeration value="0" label="OFF" />
 					<xtce:Enumeration value="1" label="ON" />
+					<xtce:Enumeration value="2" label="NONE" />
 				</xtce:EnumerationList>
 			</xtce:EnumeratedParameterType>
 			<xtce:IntegerParameterType name="TEST_PACKET2.SHCOARSE" signed="false">

imap_processing/tests/ccsds/test_excel_to_xtce.py

@@ -195,10 +195,10 @@ def xtce_excel_file(tmp_path):
     }
 
     states = {
-        "packetName": ["TEST_PACKET"] * 2,
-        "mnemonic": ["VAR_STATE"] * 2,
-        "value": [0, 1],
-        "state": ["OFF", "ON"],
+        "packetName": ["TEST_PACKET"] * 3,
+        "mnemonic": ["VAR_STATE"] * 3,
+        "value": [0, 1, "0x2"],
+        "state": ["OFF", "ON", "NONE"],
     }
 
     # Write the DataFrame to an excel file

imap_processing/tests/cdf/test_imap_cdf_manager.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-from unittest import mock
 
 # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
@@ -25,15 +24,6 @@ def test_add_instrument_global_attrs():
     assert instrument2_instrument["Project"] == "STP>Solar Terrestrial Probes"
 
 
-@mock.patch(
-    "imap_processing.cdf.cdf_attribute_manager.CdfAttributeManager.load_variable_attributes"
-)
-def testing_source_dir(mock_load_variable_attributes):
-    # Create an ImapCdfAttributes object
-    imap_cdf_manager = ImapCdfAttributes(Path(__file__).parent.parent / "cdf")
-    assert str(imap_cdf_manager.source_dir) == str(Path(__file__).parent.parent / "cdf")
-
-
 def test_add_instrument_variable_attrs():
     # Create an ImapCdfAttributes object
     imap_cdf_manager = ImapCdfAttributes()

imap_processing/tests/codice/conftest.py

@@ -2,23 +2,7 @@ from imap_processing import imap_module_directory
 
 TEST_DATA_PATH = imap_module_directory / "tests" / "codice" / "data"
 
-TEST_PACKETS = [
-    TEST_DATA_PATH / "imap_codice_l0_hskp_20100101_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-counters-singles_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-omni_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-sectored_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_hi-pha_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-counters-singles_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-sw-angular_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-nsw-angular_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-sw-priority_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-nsw-priority_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-sw-species_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-nsw-species_20240429_v001.pkts",
-    TEST_DATA_PATH / "imap_codice_l0_lo-pha_20240429_v001.pkts",
-]
+TEST_L0_FILE = TEST_DATA_PATH / "imap_codice_l0_raw_20241110_v001.pkts"
 
 TEST_L1A_FILES = [
     TEST_DATA_PATH / "imap_codice_l1a_hskp_20100101_v001.pkts",

imap_processing/tests/codice/test_codice_l0.py

@@ -37,7 +37,7 @@ def decom_test_data() -> xr.Dataset:
 
     packet_file = Path(
         f"{imap_module_directory}/tests/codice/data/"
-        f"imap_codice_l0_hskp_20100101_v001.pkts"
+        f"imap_codice_l0_raw_20241110_v001.pkts"
     )
 
     packet = codice_l0.decom_packets(packet_file)[1136]
@@ -69,6 +69,9 @@ def validation_data() -> pd.core.frame.DataFrame:
     return validation_data
 
 
+@pytest.mark.xfail(
+    reason="Need to update to validate against new validation CDFs. See issue #1154."
+)
 def test_eu_hskp_data(
     decom_test_data: xr.Dataset,
     validation_data: pd.core.frame.DataFrame,
@@ -106,6 +109,9 @@ def test_eu_hskp_data(
             assert round(eu_val, 5) == round(validation_val, 5)
 
 
+@pytest.mark.xfail(
+    reason="Need to update to validate against new validation CDFs. See issue #1154."
+)
 def test_raw_hskp_data(
     decom_test_data: xr.Dataset,
     validation_data: pd.core.frame.DataFrame,
@@ -140,5 +146,5 @@ def test_total_packets_in_data_file(decom_test_data: xr.Dataset):
         The decommutated housekeeping packet
     """
 
-    total_packets = 99
+    total_packets = 31778
     assert len(decom_test_data.epoch) == total_packets