imap-processing 0.18.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/cli.py

@@ -24,8 +24,7 @@ import imap_data_access
 import numpy as np
 import spiceypy
 import xarray as xr
-from imap_data_access import ScienceFilePath
-from imap_data_access.io import download
+from imap_data_access.io import IMAPDataAccessError, download
 from imap_data_access.processing_input import (
     ProcessingInputCollection,
     ProcessingInputType,
@@ -36,7 +35,10 @@ from imap_data_access.processing_input import (
 
 import imap_processing
 from imap_processing._version import __version__, __version_tuple__  # noqa: F401
-from imap_processing.ancillary.ancillary_dataset_combiner import MagAncillaryCombiner
+from imap_processing.ancillary.ancillary_dataset_combiner import (
+    GlowsAncillaryCombiner,
+    MagAncillaryCombiner,
+)
 from imap_processing.cdf.utils import load_cdf, write_cdf
 
 # TODO: change how we import things and also folder
@@ -59,7 +61,6 @@ from imap_processing.idex.idex_l1a import PacketParser
 from imap_processing.idex.idex_l1b import idex_l1b
 from imap_processing.idex.idex_l2a import idex_l2a
 from imap_processing.idex.idex_l2b import idex_l2b
-from imap_processing.idex.idex_l2c import idex_l2c
 from imap_processing.lo.l1a import lo_l1a
 from imap_processing.lo.l1b import lo_l1b
 from imap_processing.lo.l1c import lo_l1c
@@ -379,7 +380,7 @@ class ProcessInstrument(ABC):
         data_level: str,
         data_descriptor: str,
         dependency_str: str,
-        start_date: str | None,
+        start_date: str,
         repointing: str | None,
         version: str,
         upload_to_sdc: bool,
@@ -405,32 +406,23 @@ class ProcessInstrument(ABC):
             A list of file paths to upload to the SDC.
         """
         if self.upload_to_sdc:
-            # Validate that the files don't already exist
-            for filename in products:
-                file_path = ScienceFilePath(filename)
-                existing_file = imap_data_access.query(
-                    instrument=file_path.instrument,
-                    data_level=file_path.data_level,
-                    descriptor=file_path.descriptor,
-                    start_date=file_path.start_date,
-                    end_date=file_path.start_date,
-                    repointing=file_path.repointing,
-                    version=file_path.version,
-                    extension="cdf",
-                    table="science",
-                )
-                if existing_file:
-                    raise ProcessInstrument.ImapFileExistsError(
-                        f"File {filename} already exists in the IMAP SDC. "
-                        "No files were uploaded."
-                        f"Generated files: {products}."
-                    )
-
-            if len(products) == 0:
+            if not products:
                 logger.info("No files to upload.")
+                return
+
             for filename in products:
-                logger.info(f"Uploading file: {filename}")
-                imap_data_access.upload(filename)
+                try:
+                    logger.info(f"Uploading file: {filename}")
+                    imap_data_access.upload(filename)
+                except IMAPDataAccessError as e:
+                    msg = str(e)
+                    if "FileAlreadyExists" in msg and "409" in msg:
+                        logger.warning("Skipping upload of existing file, %s", filename)
+                        continue
+                    else:
+                        logger.error(f"Upload failed with error: {msg}")
+                except Exception as e:
+                    logger.error(f"Upload failed unknown error: {e}")
 
     @final
     def process(self) -> None:
@@ -674,7 +666,7 @@ class Glows(ProcessInstrument):
         datasets: list[xr.Dataset] = []
 
         if self.data_level == "l1a":
-            science_files = dependencies.get_file_paths(source="glows")
+            science_files = dependencies.get_file_paths(source="glows", data_type="l0")
             if len(science_files) != 1:
                 raise ValueError(
                     f"GLOWS L1A requires exactly one input science file, received: "
@@ -683,14 +675,62 @@ class Glows(ProcessInstrument):
             datasets = glows_l1a(science_files[0])
 
         if self.data_level == "l1b":
-            science_files = dependencies.get_file_paths(source="glows")
+            science_files = dependencies.get_file_paths(source="glows", data_type="l1a")
             if len(science_files) != 1:
                 raise ValueError(
-                    f"GLOWS L1A requires exactly one input science file, received: "
+                    f"GLOWS L1B requires exactly one input science file, received: "
                     f"{science_files}."
                 )
             input_dataset = load_cdf(science_files[0])
-            datasets = [glows_l1b(input_dataset)]
+
+            # Create file lists for each ancillary type
+            excluded_regions_files = dependencies.get_processing_inputs(
+                descriptor="map-of-excluded-regions"
+            )[0]
+            uv_sources_files = dependencies.get_processing_inputs(
+                descriptor="map-of-uv-sources"
+            )[0]
+            suspected_transients_files = dependencies.get_processing_inputs(
+                descriptor="suspected-transients"
+            )[0]
+            exclusions_by_instr_team_files = dependencies.get_processing_inputs(
+                descriptor="exclusions-by-instr-team"
+            )[0]
+            pipeline_settings = dependencies.get_processing_inputs(
+                descriptor="pipeline-settings"
+            )[0]
+
+            # Use end date buffer for ancillary data
+            current_day = np.datetime64(
+                f"{self.start_date[:4]}-{self.start_date[4:6]}-{self.start_date[6:]}"
+            )
+            day_buffer = current_day + np.timedelta64(3, "D")
+
+            # Create combiners for each ancillary dataset
+            excluded_regions_combiner = GlowsAncillaryCombiner(
+                excluded_regions_files, day_buffer
+            )
+            uv_sources_combiner = GlowsAncillaryCombiner(uv_sources_files, day_buffer)
+            suspected_transients_combiner = GlowsAncillaryCombiner(
+                suspected_transients_files, day_buffer
+            )
+            exclusions_by_instr_team_combiner = GlowsAncillaryCombiner(
+                exclusions_by_instr_team_files, day_buffer
+            )
+            pipeline_settings_combiner = GlowsAncillaryCombiner(
+                pipeline_settings, day_buffer
+            )
+
+            datasets = [
+                glows_l1b(
+                    input_dataset,
+                    excluded_regions_combiner.combined_dataset,
+                    uv_sources_combiner.combined_dataset,
+                    suspected_transients_combiner.combined_dataset,
+                    exclusions_by_instr_team_combiner.combined_dataset,
+                    pipeline_settings_combiner.combined_dataset,
+                )
+            ]
 
         if self.data_level == "l2":
             science_files = dependencies.get_file_paths(source="glows")
@@ -738,10 +778,18 @@ class Hi(ProcessInstrument):
         elif self.data_level == "l1b":
             l0_files = dependencies.get_file_paths(source="hi", descriptor="raw")
             if l0_files:
-                datasets = hi_l1b.hi_l1b(l0_files[0])
+                datasets = hi_l1b.housekeeping(l0_files[0])
             else:
-                l1a_files = dependencies.get_file_paths(source="hi", data_type="l1a")
-                datasets = hi_l1b.hi_l1b(load_cdf(l1a_files[0]))
+                l1a_de_file = dependencies.get_file_paths(
+                    source="hi", data_type="l1a", descriptor="de"
+                )[0]
+                l1b_hk_file = dependencies.get_file_paths(
+                    source="hi", data_type="l1b", descriptor="hk"
+                )[0]
+                esa_energies_csv = dependencies.get_file_paths(data_type="ancillary")[0]
+                datasets = hi_l1b.annotate_direct_events(
+                    load_cdf(l1a_de_file), load_cdf(l1b_hk_file), esa_energies_csv
+                )
         elif self.data_level == "l1c":
             science_paths = dependencies.get_file_paths(source="hi", data_type="l1b")
             if len(science_paths) != 1:
@@ -796,37 +844,41 @@ class Hit(ProcessInstrument):
 
         dependency_list = dependencies.processing_input
         if self.data_level == "l1a":
-            # 1 science files and 2 spice files
-            if len(dependency_list) > 3:
+            # Two inputs - L0 and SPICE
+            if len(dependency_list) > 2:
                 raise ValueError(
                     f"Unexpected dependencies found for HIT L1A:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"{dependency_list}. Expected only 2 dependencies, "
+                    f"L0 and time kernels."
                 )
             # process data to L1A products
             science_files = dependencies.get_file_paths(source="hit", descriptor="raw")
-            datasets = hit_l1a(science_files[0])
+            datasets = hit_l1a(science_files[0], self.start_date)
 
         elif self.data_level == "l1b":
-            data_dict = {}
-            # TODO: Sean removed the file number error handling to work with the
-            #  new SPICE dependencies for SIT-4. Need to review and make changes
-            #  if needed.
             l0_files = dependencies.get_file_paths(source="hit", descriptor="raw")
             l1a_files = dependencies.get_file_paths(source="hit", data_type="l1a")
-            if len(l0_files) > 0:
-                # Add path to CCSDS file to process housekeeping
-                data_dict["imap_hit_l0_raw"] = l0_files[0]
+            if len(l0_files) == 1:
+                # Path to CCSDS file to process housekeeping
+                dependency = l0_files[0]
             else:
+                # 1 science file
+                if len(l1a_files) > 1:
+                    raise ValueError(
+                        f"Unexpected dependencies found for HIT L1B:"
+                        f"{l1a_files}. Expected only one dependency."
+                    )
                 # Add L1A dataset to process science data
-                l1a_dataset = load_cdf(l1a_files[0])
-                data_dict[l1a_dataset.attrs["Logical_source"]] = l1a_dataset
+                dependency = load_cdf(l1a_files[0])
             # process data to L1B products
-            datasets = hit_l1b(data_dict)
+            datasets = [hit_l1b(dependency, self.descriptor)]
+
         elif self.data_level == "l2":
+            # 1 science files and 4 ancillary files
             if len(dependency_list) != 5:
                 raise ValueError(
                     f"Unexpected dependencies found for HIT L2:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"{dependency_list}. Expected only five dependencies."
                 )
             # Add L1B dataset to process science data
             science_files = dependencies.get_file_paths(
@@ -846,7 +898,7 @@ class Hit(ProcessInstrument):
                 )
             l1b_dataset = load_cdf(science_files[0])
             # process data to L2 products
-            datasets = hit_l2(l1b_dataset, ancillary_files)
+            datasets = [hit_l2(l1b_dataset, ancillary_files)]
 
         return datasets
 
@@ -895,14 +947,18 @@ class Idex(ProcessInstrument):
             dependency = load_cdf(science_files[0])
             datasets = [idex_l1b(dependency)]
         elif self.data_level == "l2a":
-            if len(dependency_list) != 1:
+            if len(dependency_list) != 3:
                 raise ValueError(
                     f"Unexpected dependencies found for IDEX L2A:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"{dependency_list}. Expected three dependencies."
                 )
             science_files = dependencies.get_file_paths(source="idex")
             dependency = load_cdf(science_files[0])
-            datasets = [idex_l2a(dependency)]
+            anc_paths = dependencies.get_file_paths(data_type="ancillary")
+            ancillary_files = {}
+            for path in anc_paths:
+                ancillary_files[path.stem.split("_")[2]] = path
+            datasets = [idex_l2a(dependency, ancillary_files)]
         elif self.data_level == "l2b":
             if len(dependency_list) < 3 or len(dependency_list) > 4:
                 raise ValueError(
@@ -916,16 +972,7 @@ class Idex(ProcessInstrument):
             hk_files = dependencies.get_file_paths(source="idex", descriptor="evt")
             # Remove duplicate housekeeping files
             hk_dependencies = [load_cdf(dep) for dep in list(set(hk_files))]
-            datasets = [idex_l2b(sci_dependencies, hk_dependencies)]
-        elif self.data_level == "l2c":
-            if len(dependency_list) != 1:
-                raise ValueError(
-                    f"Unexpected dependencies found for IDEX L2C:"
-                    f"{dependency_list}. Expected only one dependency."
-                )
-            sci_files = dependencies.get_file_paths(source="idex", descriptor="sci-1mo")
-            dependencies = [load_cdf(f) for f in sci_files]
-            datasets = [idex_l2c(dependencies)]
+            datasets = idex_l2b(sci_dependencies, hk_dependencies)
         return datasets
 
 
@@ -964,16 +1011,19 @@ class Lo(ProcessInstrument):
         elif self.data_level == "l1b":
             data_dict = {}
             science_files = dependencies.get_file_paths(source="lo", data_type="l1a")
+            ancillary_files = dependencies.get_file_paths(
+                source="lo", data_type="ancillary"
+            )
             logger.info(f"Science files for L1B: {science_files}")
             for file in science_files:
                 dataset = load_cdf(file)
                 data_dict[dataset.attrs["Logical_source"]] = dataset
-            datasets = lo_l1b.lo_l1b(data_dict)
+            datasets = lo_l1b.lo_l1b(data_dict, ancillary_files)
 
         elif self.data_level == "l1c":
             data_dict = {}
             anc_dependencies: list = dependencies.get_file_paths(
-                source="lo", descriptor="goodtimes"
+                source="lo", data_type="ancillary"
             )
             science_files = dependencies.get_file_paths(source="lo", descriptor="de")
             for file in science_files:
@@ -983,17 +1033,13 @@ class Lo(ProcessInstrument):
 
         elif self.data_level == "l2":
             data_dict = {}
-            # TODO: Add ancillary descriptors when maps using them are
-            #  implemented.
-            anc_dependencies = dependencies.get_file_paths(
-                source="lo",
-            )
             science_files = dependencies.get_file_paths(source="lo", descriptor="pset")
-            psets = []
-            for file in science_files:
-                psets.append(load_cdf(file))
+            anc_dependencies = dependencies.get_file_paths(data_type="ancillary")
+
+            # Load all pset files into datasets
+            psets = [load_cdf(file) for file in science_files]
             data_dict[psets[0].attrs["Logical_source"]] = psets
-            datasets = lo_l2.lo_l2(data_dict, anc_dependencies)
+            datasets = lo_l2.lo_l2(data_dict, anc_dependencies, self.descriptor)
         return datasets
 
 
@@ -1174,8 +1220,8 @@ class Spacecraft(ProcessInstrument):
             The list of processed products.
         """
         print(f"Processing Spacecraft {self.data_level}")
-
-        if self.data_level == "l1a":
+        processed_dataset = []
+        if self.descriptor == "quaternions":
             # File path is expected output file path
             input_files = dependencies.get_file_paths(source="spacecraft")
             if len(input_files) > 1:
@@ -1184,26 +1230,21 @@ class Spacecraft(ProcessInstrument):
                     f"{input_files}. Expected only one dependency."
                 )
             datasets = list(quaternions.process_quaternions(input_files[0]))
-            return datasets
-        elif self.data_level == "spice":
+            processed_dataset.extend(datasets)
+        elif self.descriptor == "pointing-attitude":
             spice_inputs = dependencies.get_file_paths(
                 data_type=SPICESource.SPICE.value
             )
             ah_paths = [path for path in spice_inputs if ".ah" in path.suffixes]
-            if len(ah_paths) != 1:
-                raise ValueError(
-                    f"Unexpected spice dependencies found for Spacecraft "
-                    f"pointing_kernel: {ah_paths}. Expected exactly one "
-                    f"attitude history file."
-                )
             pointing_kernel_paths = pointing_frame.generate_pointing_attitude_kernel(
-                ah_paths[0]
+                ah_paths[-1]
             )
-            return pointing_kernel_paths
+            processed_dataset.extend(pointing_kernel_paths)
         else:
             raise NotImplementedError(
                 f"Spacecraft processing not implemented for level {self.data_level}"
             )
+        return processed_dataset
 
 
 class Swapi(ProcessInstrument):
@@ -1407,7 +1448,10 @@ class Ultra(ProcessInstrument):
             }
             science_files = dependencies.get_file_paths(source="ultra", data_type="l1b")
             l1b_dict = {
-                dataset.attrs["Logical_source"]: dataset
+                # TODO remove
+                dataset.attrs["Logical_source"].replace(
+                    "cullingmask", "goodtimes"
+                ): dataset
                 for dataset in [load_cdf(sci_file) for sci_file in science_files]
             }
             combined = {**l1a_dict, **l1b_dict}
@@ -1416,11 +1460,12 @@ class Ultra(ProcessInstrument):
             for path in anc_paths:
                 ancillary_files[path.stem.split("_")[2]] = path
             spice_paths = dependencies.get_file_paths(data_type="spice")
-            if spice_paths:
-                has_spice = True
+            # Only the helio pset needs IMAP frames
+            if any("imap_frames" in path.as_posix() for path in spice_paths):
+                imap_frames = True
             else:
-                has_spice = False
-            datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, has_spice)
+                imap_frames = False
+            datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, imap_frames)
         elif self.data_level == "l2":
             all_pset_filepaths = dependencies.get_file_paths(
                 source="ultra", descriptor="pset"

imap_processing/codice/codice_l0.py

@@ -39,7 +39,8 @@ def decom_packets(packet_file: Path) -> dict[int, xr.Dataset]:
     # TODO: Currently need to use the 'old' packet definition for housekeeping
     #       because the simulated housekeeping data being used has various
     #       mis-matches from the telemetry definition. This may be updated
-    #       once new simulated housekeeping data are acquired.
+    #       once new simulated housekeeping data are acquired. See GitHub issue
+    #       #2135.
     if "hskp" in str(packet_file):
         xtce_filename = "P_COD_NHK.xml"
     else: