imap-processing 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

imap_processing/cli.py

@@ -36,7 +36,10 @@ from imap_data_access.processing_input import (
 
 import imap_processing
 from imap_processing._version import __version__, __version_tuple__  # noqa: F401
-from imap_processing.ancillary.ancillary_dataset_combiner import MagAncillaryCombiner
+from imap_processing.ancillary.ancillary_dataset_combiner import (
+    GlowsAncillaryCombiner,
+    MagAncillaryCombiner,
+)
 from imap_processing.cdf.utils import load_cdf, write_cdf
 
 # TODO: change how we import things and also folder
@@ -59,7 +62,6 @@ from imap_processing.idex.idex_l1a import PacketParser
 from imap_processing.idex.idex_l1b import idex_l1b
 from imap_processing.idex.idex_l2a import idex_l2a
 from imap_processing.idex.idex_l2b import idex_l2b
-from imap_processing.idex.idex_l2c import idex_l2c
 from imap_processing.lo.l1a import lo_l1a
 from imap_processing.lo.l1b import lo_l1b
 from imap_processing.lo.l1c import lo_l1c
@@ -68,6 +70,7 @@ from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
 from imap_processing.mag.l1b.mag_l1b import mag_l1b
 from imap_processing.mag.l1c.mag_l1c import mag_l1c
+from imap_processing.mag.l1d.mag_l1d import mag_l1d
 from imap_processing.mag.l2.mag_l2 import mag_l2
 from imap_processing.spacecraft import quaternions
 from imap_processing.spice import pointing_frame, repoint, spin
@@ -378,7 +381,7 @@ class ProcessInstrument(ABC):
         data_level: str,
         data_descriptor: str,
         dependency_str: str,
-        start_date: str | None,
+        start_date: str,
         repointing: str | None,
         version: str,
         upload_to_sdc: bool,
@@ -516,7 +519,7 @@ class ProcessInstrument(ABC):
         self,
         processed_data: list[xr.Dataset | Path],
         dependencies: ProcessingInputCollection,
-    ) -> None:
+    ) -> list[Path]:
         """
         Complete post-processing.
 
@@ -544,10 +547,17 @@ class ProcessInstrument(ABC):
             method.
         dependencies : ProcessingInputCollection
             Object containing dependencies to process.
+
+        Returns
+        -------
+        list[Path]
+            List of paths to CDF files produced.
         """
+        products: list[Path] = []
+
         if len(processed_data) == 0:
             logger.info("No products to write to CDF file.")
-            return
+            return products
 
         logger.info("Writing products to local storage")
 
@@ -567,7 +577,6 @@ class ProcessInstrument(ABC):
         # start_date.
         # If it is start_date, skip repointing in the output filename.
 
-        products = []
         for ds in processed_data:
             if isinstance(ds, xr.Dataset):
                 ds.attrs["Data_version"] = self.version[1:]  # Strip 'v' from version
@@ -581,6 +590,7 @@ class ProcessInstrument(ABC):
                 products.append(ds)
 
         self.upload_products(products)
+        return products
 
     @final
     def cleanup(self) -> None:
@@ -678,11 +688,58 @@ class Glows(ProcessInstrument):
             science_files = dependencies.get_file_paths(source="glows")
             if len(science_files) != 1:
                 raise ValueError(
-                    f"GLOWS L1A requires exactly one input science file, received: "
+                    f"GLOWS L1B requires exactly one input science file, received: "
                     f"{science_files}."
                 )
             input_dataset = load_cdf(science_files[0])
-            datasets = [glows_l1b(input_dataset)]
+            # TODO: Replace this by reading from AWS/ProcessingInputs
+
+            glows_ancillary_dir = Path(__file__).parent / "glows" / "ancillary"
+
+            # Create file lists for each ancillary type
+            excluded_regions_files = [
+                glows_ancillary_dir
+                / "imap_glows_map-of-excluded-regions_20250923_v002.dat"
+            ]
+            uv_sources_files = [
+                glows_ancillary_dir / "imap_glows_map-of-uv-sources_20250923_v002.dat"
+            ]
+            suspected_transients_files = [
+                glows_ancillary_dir
+                / "imap_glows_suspected-transients_20250923_v002.dat"
+            ]
+            exclusions_by_instr_team_files = [
+                glows_ancillary_dir
+                / "imap_glows_exclusions-by-instr-team_20250923_v002.dat"
+            ]
+
+            # Use end date buffer for ancillary data
+            current_day = np.datetime64(
+                f"{self.start_date[:4]}-{self.start_date[4:6]}-{self.start_date[6:]}"
+            )
+            day_buffer = current_day + np.timedelta64(3, "D")
+
+            # Create combiners for each ancillary dataset
+            excluded_regions_combiner = GlowsAncillaryCombiner(
+                excluded_regions_files, day_buffer
+            )
+            uv_sources_combiner = GlowsAncillaryCombiner(uv_sources_files, day_buffer)
+            suspected_transients_combiner = GlowsAncillaryCombiner(
+                suspected_transients_files, day_buffer
+            )
+            exclusions_by_instr_team_combiner = GlowsAncillaryCombiner(
+                exclusions_by_instr_team_files, day_buffer
+            )
+
+            datasets = [
+                glows_l1b(
+                    input_dataset,
+                    excluded_regions_combiner.combined_dataset,
+                    uv_sources_combiner.combined_dataset,
+                    suspected_transients_combiner.combined_dataset,
+                    exclusions_by_instr_team_combiner.combined_dataset,
+                )
+            ]
 
         if self.data_level == "l2":
             science_files = dependencies.get_file_paths(source="glows")
@@ -730,10 +787,18 @@ class Hi(ProcessInstrument):
         elif self.data_level == "l1b":
             l0_files = dependencies.get_file_paths(source="hi", descriptor="raw")
             if l0_files:
-                datasets = hi_l1b.hi_l1b(l0_files[0])
+                datasets = hi_l1b.housekeeping(l0_files[0])
             else:
-                l1a_files = dependencies.get_file_paths(source="hi", data_type="l1a")
-                datasets = hi_l1b.hi_l1b(load_cdf(l1a_files[0]))
+                l1a_de_file = dependencies.get_file_paths(
+                    source="hi", data_type="l1a", descriptor="de"
+                )[0]
+                l1b_hk_file = dependencies.get_file_paths(
+                    source="hi", data_type="l1b", descriptor="hk"
+                )[0]
+                esa_energies_csv = dependencies.get_file_paths(data_type="ancillary")[0]
+                datasets = hi_l1b.annotate_direct_events(
+                    load_cdf(l1a_de_file), load_cdf(l1b_hk_file), esa_energies_csv
+                )
         elif self.data_level == "l1c":
             science_paths = dependencies.get_file_paths(source="hi", data_type="l1b")
             if len(science_paths) != 1:
@@ -788,37 +853,41 @@ class Hit(ProcessInstrument):
 
         dependency_list = dependencies.processing_input
         if self.data_level == "l1a":
-            # 1 science files and 2 spice files
-            if len(dependency_list) > 3:
+            # Two inputs - L0 and SPICE
+            if len(dependency_list) > 2:
                 raise ValueError(
                     f"Unexpected dependencies found for HIT L1A:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"{dependency_list}. Expected only 2 dependencies, "
+                    f"L0 and time kernels."
                 )
             # process data to L1A products
             science_files = dependencies.get_file_paths(source="hit", descriptor="raw")
-            datasets = hit_l1a(science_files[0])
+            datasets = hit_l1a(science_files[0], self.start_date)
 
         elif self.data_level == "l1b":
-            data_dict = {}
-            # TODO: Sean removed the file number error handling to work with the
-            #  new SPICE dependencies for SIT-4. Need to review and make changes
-            #  if needed.
             l0_files = dependencies.get_file_paths(source="hit", descriptor="raw")
             l1a_files = dependencies.get_file_paths(source="hit", data_type="l1a")
-            if len(l0_files) > 0:
-                # Add path to CCSDS file to process housekeeping
-                data_dict["imap_hit_l0_raw"] = l0_files[0]
+            if len(l0_files) == 1:
+                # Path to CCSDS file to process housekeeping
+                dependency = l0_files[0]
             else:
+                # 1 science file
+                if len(l1a_files) > 1:
+                    raise ValueError(
+                        f"Unexpected dependencies found for HIT L1B:"
+                        f"{l1a_files}. Expected only one dependency."
+                    )
                 # Add L1A dataset to process science data
-                l1a_dataset = load_cdf(l1a_files[0])
-                data_dict[l1a_dataset.attrs["Logical_source"]] = l1a_dataset
+                dependency = load_cdf(l1a_files[0])
             # process data to L1B products
-            datasets = hit_l1b(data_dict)
+            datasets = [hit_l1b(dependency, self.descriptor)]
+
         elif self.data_level == "l2":
+            # 1 science files and 4 ancillary files
             if len(dependency_list) != 5:
                 raise ValueError(
                     f"Unexpected dependencies found for HIT L2:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"{dependency_list}. Expected only five dependencies."
                 )
             # Add L1B dataset to process science data
             science_files = dependencies.get_file_paths(
@@ -838,7 +907,7 @@ class Hit(ProcessInstrument):
                 )
             l1b_dataset = load_cdf(science_files[0])
             # process data to L2 products
-            datasets = hit_l2(l1b_dataset, ancillary_files)
+            datasets = [hit_l2(l1b_dataset, ancillary_files)]
 
         return datasets
 
@@ -887,14 +956,18 @@ class Idex(ProcessInstrument):
             dependency = load_cdf(science_files[0])
             datasets = [idex_l1b(dependency)]
         elif self.data_level == "l2a":
-            if len(dependency_list) != 1:
+            if len(dependency_list) != 3:
                 raise ValueError(
                     f"Unexpected dependencies found for IDEX L2A:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"{dependency_list}. Expected three dependencies."
                 )
             science_files = dependencies.get_file_paths(source="idex")
             dependency = load_cdf(science_files[0])
-            datasets = [idex_l2a(dependency)]
+            anc_paths = dependencies.get_file_paths(data_type="ancillary")
+            ancillary_files = {}
+            for path in anc_paths:
+                ancillary_files[path.stem.split("_")[2]] = path
+            datasets = [idex_l2a(dependency, ancillary_files)]
         elif self.data_level == "l2b":
             if len(dependency_list) < 3 or len(dependency_list) > 4:
                 raise ValueError(
@@ -908,16 +981,7 @@ class Idex(ProcessInstrument):
             hk_files = dependencies.get_file_paths(source="idex", descriptor="evt")
             # Remove duplicate housekeeping files
             hk_dependencies = [load_cdf(dep) for dep in list(set(hk_files))]
-            datasets = [idex_l2b(sci_dependencies, hk_dependencies)]
-        elif self.data_level == "l2c":
-            if len(dependency_list) != 1:
-                raise ValueError(
-                    f"Unexpected dependencies found for IDEX L2C:"
-                    f"{dependency_list}. Expected only one dependency."
-                )
-            sci_files = dependencies.get_file_paths(source="idex", descriptor="sci-1mo")
-            dependency = load_cdf(sci_files[0])
-            datasets = idex_l2c(dependency)
+            datasets = idex_l2b(sci_dependencies, hk_dependencies)
         return datasets
 
 
@@ -977,15 +1041,13 @@ class Lo(ProcessInstrument):
             data_dict = {}
             # TODO: Add ancillary descriptors when maps using them are
             #  implemented.
-            anc_dependencies = dependencies.get_file_paths(
-                source="lo",
-            )
+            anc_dependencies = []
             science_files = dependencies.get_file_paths(source="lo", descriptor="pset")
             psets = []
             for file in science_files:
                 psets.append(load_cdf(file))
             data_dict[psets[0].attrs["Logical_source"]] = psets
-            datasets = lo_l2.lo_l2(data_dict, anc_dependencies)
+            datasets = lo_l2.lo_l2(data_dict, anc_dependencies, self.descriptor)
         return datasets
 
 
@@ -1073,6 +1135,21 @@ class Mag(ProcessInstrument):
                     f"Invalid dependencies found for MAG L1C:"
                     f"{dependencies}. Expected one or two dependencies."
                 )
+        if self.data_level == "l1d":
+            science_files = dependencies.get_file_paths(source="mag", data_type="l1c")
+            science_files.extend(
+                dependencies.get_file_paths(source="mag", data_type="l1b")
+            )
+            input_data = [load_cdf(dep) for dep in science_files]
+            calibration = dependencies.get_processing_inputs(
+                descriptor="l1d-calibration"
+            )
+            combined_calibration = MagAncillaryCombiner(calibration[0], day_buffer)
+            datasets = mag_l1d(
+                input_data,
+                combined_calibration.combined_dataset,
+                current_day,
+            )
 
         if self.data_level == "l2":
             science_files = dependencies.get_file_paths(source="mag", data_type="l1b")