roc-film 1.13.5__py3-none-any.whl → 1.14.1__py3-none-any.whl

roc/film/tasks/export_solo_coord.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Contains task to generate CSV file containing SolO S/C distance to Sun (in AU) for each date.
+This file is needed by MUSIC FAUST app to define the occurrence rate of the Bias sweeps.
+"""
+
+import os
+import uuid
+from datetime import datetime, timedelta
+from glob import glob
+import csv
+
+from poppy.core.logger import logger
+from poppy.core.task import Task
+
+from roc.rpl.time.spice import SpiceHarvester
+
+from roc.film.tools.file_helpers import get_output_dir
+from roc.film.constants import CP_START_TIME, NAIF_SPICE_ID, TIME_DAILY_STRFORMAT
+
+__all__ = ["ExportSoloHeeCoord"]
+
+
+class ExportSoloHeeCoord(Task):
+    """
+    Task to export SolO HEE coordinates in a CSV file
+    with (distance in AU, longitude in deg, latitude in deg).
+    """
+
+    plugin_name = "roc.film"
+    name = "export_solo_hee_coord"
+
+    def add_targets(self):
+        pass
+
+    def setup_inputs(self):
+        # Get/create list of well processed DDS files
+        self.processed_files = self.pipeline.get(
+            "processed_files", default=[], create=True
+        )
+
+        # Get/create list of failed DDS files
+        self.failed_files = self.pipeline.get("failed_files", default=[], create=True)
+
+        # If output directory not found, create it
+        self.output_dir = get_output_dir(self.pipeline)
+        if not os.path.isdir(self.output_dir):
+            logger.debug(f"Making {self.output_dir}...")
+            os.makedirs(self.output_dir)
+
+        # Get path of SPICE kernels
+        self.kernel_path = SpiceHarvester.spice_kernel_path()
+
+        # Load SPICE kernels (only meta kernels for the moment)
+        self.spice = SpiceHarvester.load_spice_kernels(
+            self.kernel_path, only_mk=True, predictive=False, flown=False
+        )
+
+        # Function to convert from radians to degrees
+        self.dpr = self.spice._spiceypy.dpr()
+
+        # Define output file start time
+        self.start_time = self.pipeline.get("start_time", default=[None])[0]
+        logger.debug(f"start_time value is {self.start_time}")
+
+        # Define output file end time
+        self.end_time = self.pipeline.get("end_time", default=[None])[0]
+        logger.debug(f"end_time value is {self.end_time}")
+
+        # Generating list of days for which distance will be computed
+        if self.start_time is None:
+            self.start_time = CP_START_TIME
+
+        if self.end_time is None:
+            self.end_time = datetime.today() + timedelta(days=90)
+
+        # Get output_csv input argument
+        self.output_csv = self.pipeline.get("output_csv", default=[None])[0]
+        if self.output_csv is None:
+            # If not passed, then try to generate automatically the output CSV filename
+            basename = f"solo_ANC_solo-hee-coord_{self.start_time.strftime(TIME_DAILY_STRFORMAT)}T{self.end_time.strftime(TIME_DAILY_STRFORMAT)}"
+            pattern = os.path.join(self.output_dir, basename + "*.csv")
+            existing_files = list(glob(pattern))
+            data_version = f"{len(existing_files) + 1:02d}"
+            self.output_csv = os.path.join(
+                self.output_dir, basename + f"_V{data_version}.csv"
+            )
+
+    def run(self):
+        # Define task job ID (long and short)
+        self.job_id = str(uuid.uuid4())
+        self.job_sid = self.job_id[:8]
+        logger.info(f"[{self.job_sid}]\t Task started")
+        try:
+            self.setup_inputs()
+        except Exception:
+            logger.exception(f"[{self.job_sid}]\t Initializing inputs has failed!")
+            self.pipeline.exit()
+            return
+
+        logger.info(f"Creating {self.output_csv} ...")
+        with open(self.output_csv, "w", newline="") as csvfile:
+            fieldnames = ["DATE", "R_AU", "HEE_LON_DEG", "HEE_LAT_DEG"]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+            writer.writeheader()
+
+            # Compute SolO S/C HEE coordinates [r(AU), long(deg), lat(deg)]
+            # for each day of the mission
+            # and write results in the CSV file
+            current_date = self.start_time.date()
+            while current_date <= self.end_time.date():
+                # Convert time to string then ephemeris time
+                time_str = current_date.strftime("%Y %B %d") + " 12:00:00"
+                et = self.spice._spiceypy.str2et(time_str)
+
+                # Now we need to compute the actual distance between
+                # the Sun and Solar Orbiter. The above spkezp call gives us
+                # the apparent distance, so we need to adjust our
+                # aberration correction appropriately.
+                [solo_hee_pos, ltime] = self.spice._spiceypy.spkezp(
+                    NAIF_SPICE_ID["SUN"],
+                    et,
+                    "SOLO_HEE",
+                    "NONE",
+                    NAIF_SPICE_ID["SOLAR_ORBITER"],
+                )
+                # Convert SOLO HEE coordinates to [radius, longitude, latitude]
+                [r, lon, lat] = self.spice._spiceypy.reclat(solo_hee_pos)
+                lat = -lat * self.dpr
+                lon = 180.0 + (lon * self.dpr) if lon <= 0 else (lon * self.dpr) - 180.0
+
+                # Convert radius to AU using convrt.
+                r_au = self.spice._spiceypy.convrt(r, "KM", "AU")
+                # print(time_str, r_au, lon, lat)
+
+                row_to_write = {
+                    "DATE": current_date.strftime(TIME_DAILY_STRFORMAT),
+                    "R_AU": r_au,
+                    "HEE_LON_DEG": lon,
+                    "HEE_LAT_DEG": lat,
+                }
+                writer.writerow(row_to_write)
+                logger.debug(f"New line {row_to_write} in {self.output_csv}")
+
+                current_date += timedelta(days=1)

roc/film/tasks/file_handler.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-""" Tasks for file handling in FILM plugin."""
+"""Tasks for file handling in FILM plugin."""
 
 import os
 import shutil
+import uuid
 
 from poppy.core.logger import logger
 from poppy.core.task import Task
 
-__all__ = ['MoveToProdDir', 'MoveFailedFiles',
-           'CopyFailedDds', 'CopyProcessedDds']
+__all__ = ["MoveToProdDir", "MoveFailedFiles", "CopyFailedDds", "CopyProcessedDds"]
 
 from roc.film.tools.tools import safe_move
 
@@ -20,22 +20,27 @@ from roc.film.tools.file_helpers import get_output_dir, get_products_dir
 class MoveToProdDir(Task):
     """Task to move output files folder to
     final products directory."""
-    plugin_name = 'roc.film'
-    name = 'move_to_products_dir'
 
-    def run(self):
+    plugin_name = "roc.film"
+    name = "move_to_products_dir"
 
+    def run(self):
         # TODO - add a lock file mechanism but at the task level
         #   (useful here to make sure that the
         #   a folder in the products_dir is not moved/removed while
         #   the pipeline is still working on it
         #   Add a LockFile class instance to the Task class in Poppy ?
 
+        self.job_uuid = str(uuid.uuid4())
+        self.job_id = self.job_uuid[:8]
+        logger.debug(f"Task {self.job_id} is starting")
+
         # See if --no-move keyword is defined
-        no_move = self.pipeline.get('no_move', default=False, args=True)
+        no_move = self.pipeline.get("no_move", default=False, args=True)
         if no_move:
             logger.info(
-                'Skip current task "move_to_products_dir": --no-move is True')
+                f'--no-move is passed: skip current task "move_to_products_dir"\t[{self.job_id}]'
+            )
             return
 
         # Retrieve pipeline output file directory
@@ -46,49 +51,67 @@ class MoveToProdDir(Task):
         products_dir = get_products_dir(self.pipeline)
 
         # Ignore possible lock file in the output directory
-        ignore_patterns = '*.lock'
+        ignore_patterns = "*.lock"
 
         if not products_dir:
             logger.info(
-                'Skip current task "move_to_products_dir": products_dir argument not defined')
+                f'products_dir argument not defined: Skip current task "move_to_products_dir"\t[{self.job_id}]'
+            )
         else:
             output_dirbasename = os.path.basename(output_dir)
             target_dir = os.path.join(products_dir, output_dirbasename)
-            logger.info(f'Moving {output_dir} into {products_dir}')
-            if safe_move(output_dir, target_dir,
-                         ignore_patterns=ignore_patterns):
-                logger.info(f'{output_dir} moved into {products_dir}')
+            logger.info(f"Moving {output_dir} into {products_dir}")
+            if safe_move(output_dir, target_dir, ignore_patterns=ignore_patterns):
+                logger.info(f"{output_dir} moved into {products_dir}")
+
+        logger.debug(f"Task {self.job_id} completed")
 
 
 class MoveFailedFiles(Task):
     """Move any failed files found
     into a 'failed' subdirectory."""
-    plugin_name = 'roc.film'
-    name = 'move_failed_files'
 
-    def run(self):
+    plugin_name = "roc.film"
+    name = "move_failed_files"
 
+    def setup_inputs(self):
         # Retrieve list of failed files
-        failed_file_list = self.pipeline.get('failed_files', default=[])
-        failed_file_count = len(failed_file_list)
+        self.failed_file_list = self.pipeline.get("failed_files", default=[])
+        self.failed_file_count = len(self.failed_file_list)
 
         # Retrieve output directory
-        output_dir = get_output_dir(self.pipeline)
+        self.output_dir = get_output_dir(self.pipeline)
 
-        if failed_file_count == 0:
-            logger.debug('No failed file(s) to move')
-        else:
+        # Retrieve failed_dir
+        self.failed_dir = self.pipeline.get("failed_dir", default=[])
+
+    def run(self):
+        # Define task job ID (long and short)
+        self.job_uuid = str(uuid.uuid4())
+        self.job_id = self.job_uuid[:8]
+        logger.debug(f"Task {self.job_id} is starting")
+        try:
+            self.setup_inputs()
+        except Exception:
+            logger.exception(f"Initializing inputs has failed for {self.job_id}!")
+            self.pipeline.exit()
+            return
 
+        if self.failed_file_count == 0:
+            logger.debug("No failed file(s) to move\t[{self.job_id}")
+        else:
             # Loop over failed files list
-            for failed_file in failed_file_list:
+            for i, failed_file in enumerate(self.failed_file_list):
+                if not self.failed_dir:
+                    failed_dir = os.path.join(os.path.dirname(failed_file), "failed")
+                else:
+                    failed_dir = self.failed_dir
 
                 # Make failed subdir if not exists
-                failed_dir = os.path.join(output_dir, 'failed')
                 os.makedirs(failed_dir, exist_ok=True)
 
                 # if failed item is a file
                 if os.path.isfile(failed_file):
-
                     # Get failed file basename
                     failed_basename = os.path.basename(failed_file)
 
@@ -98,7 +121,11 @@ class MoveFailedFiles(Task):
                     # perform a safe move (i.e., copy, check and delete) into
                     # failed dir
                     if safe_move(failed_file, target_filepath):
-                        logger.info(f'{failed_file} moved into {failed_dir}')
+                        logger.warning(
+                            f"{failed_file} moved into {failed_dir}\t[{self.job_id}]"
+                        )
+
+        logger.debug(f"Task {self.job_id} completed")
 
 
 class CopyProcessedDds(Task):
@@ -106,49 +133,56 @@ class CopyProcessedDds(Task):
     Task to copy processed DDs files into a dedicated directory.
     """
 
-    plugin_name = 'roc.film'
-    name = 'copy_processed_dds'
+    plugin_name = "roc.film"
+    name = "copy_processed_dds"
 
     def run(self):
+        # Define task job ID (long and short)
+        self.job_uuid = str(uuid.uuid4())
+        self.job_id = self.job_uuid[:8]
+        logger.debug(f"Task {self.job_id} is starting")
 
         # Get processed file target directory
-        processed_dir = self.pipeline.get('processed_dds_dir',
-                                          default=[None], args=True)[0]
+        processed_dir = self.pipeline.get(
+            "processed_dds_dir", default=[None], args=True
+        )[0]
 
         # skip task if processed_dir is None
         if processed_dir is None:
             logger.info(
-                'Skip task copy_processed_dds: No processed_dds_dir argument defined')
+                f"No processed_dds_dir argument defined: skip task copy_processed_dds\t[{self.job_id}]"
+            )
             return
         elif not os.path.isdir(processed_dir):
-            logger.debug(f'Creating {processed_dir}...')
+            logger.debug(f"Creating {processed_dir}...\t[{self.job_id}]")
             os.makedirs(processed_dir)
         else:
-            logger.debug(f'process_dir set to {processed_dir}')
+            logger.debug(f"process_dir set to {processed_dir}\t[{self.job_id}]")
 
         # If processed_files list not defined in the pipeline properties,
         # initialize it
-        processed_file_list = self.pipeline.get(
-            'processed_dds_files', default=[])
+        processed_file_list = self.pipeline.get("processed_dds_files", default=[])
         processed_files_count = len(processed_file_list)
         # Skip task if no processed files
         if processed_files_count == 0:
             logger.info(
-                'Skip task copy_processed_dds: No processed file to move')
+                f"No processed file to move: skip task copy_processed_dds\t[{self.job_id}]"
+            )
             return
 
         # Get clear-dds keyword
-        clear_dds = self.pipeline.get('clear_dds', default=False)
+        clear_dds = self.pipeline.get("clear_dds", default=False)
 
         # Get list of failed files too
-        failed_file_list = self.pipeline.get('failed_dds_files', default=[])
+        failed_file_list = self.pipeline.get("failed_dds_files", default=[])
 
         # Loop over processed files to copy
         for processed_file in processed_file_list.copy():
-
             # Check first that processed file is not in failed list
             if processed_file in failed_file_list:
-                logger.warning(f'{processed_file} found in the failed file list!')
+                logger.warning(
+                    f"{processed_file} found in the failed file list!\t[{self.job_id}]"
+                )
                 continue
 
             # Build target filepath
@@ -156,13 +190,20 @@ class CopyProcessedDds(Task):
             target_filepath = os.path.join(processed_dir, basename)
 
             # copy file
-            logger.debug(f'Copying {processed_file} into {processed_dir}')
+            logger.debug(
+                f"Copying {processed_file} into {processed_dir}\t[{self.job_id}]"
+            )
             try:
                 shutil.copyfile(processed_file, target_filepath)
-            except:
-                logger.exception(f'Copying {processed_file} into {processed_dir} has failed!')
+            except Exception as e:
+                logger.exception(
+                    f"Copying {processed_file} into {processed_dir} has failed!\t[{self.job_id}]"
+                )
+                logger.debug(e)
             else:
-                logger.info(f'{processed_file} copied into {target_filepath}')
+                logger.info(
+                    f"{processed_file} copied into {target_filepath}\t[{self.job_id}]"
+                )
 
             # Remove current file from the list in pipeline properties
             processed_file_list.remove(processed_file)
@@ -170,59 +211,70 @@ class CopyProcessedDds(Task):
             # if clear-dds keyword is passed, then remove processed Dds
             if clear_dds:
                 os.remove(processed_file)
-                logger.debug(f'{processed_file} deleted')
+                logger.debug(f"{processed_file} deleted\t[{self.job_id}]")
 
 
 class CopyFailedDds(Task):
     """
-     Task to copy failed DDs files into a dedicated directory.
-     """
-    plugin_name = 'roc.film'
-    name = 'copy_failed_dds'
+    Task to copy failed DDs files into a dedicated directory.
+    """
+
+    plugin_name = "roc.film"
+    name = "copy_failed_dds"
 
     def run(self):
+        # Define task job ID (long and short)
+        self.job_uuid = str(uuid.uuid4())
+        self.job_id = self.job_uuid[:8]
+        logger.debug(f"Task {self.job_id} is starting")
 
         # Get failed file target directory
-        failed_dir = self.pipeline.get('failed_dds_dir',
-                                       default=[None], args=True)[0]
+        failed_dir = self.pipeline.get("failed_dds_dir", default=[None], args=True)[0]
         # skip task if failed_dir is None
         if failed_dir is None:
             logger.info(
-                'Skip task copy_failed_dds: No failed_dds_dir argument defined')
+                f"No failed_dds_dir argument defined: skip task copy_failed_dds\t[{self.job_id}]"
+            )
             return
         elif not os.path.isdir(failed_dir):
-            logger.debug(f'Creating {failed_dir}...')
+            logger.debug(f"Creating {failed_dir}...\t[{self.job_id}]")
             os.makedirs(failed_dir)
         else:
-            logger.debug(f'failed_dir set to {failed_dir}')
+            logger.debug(f"failed_dir set to {failed_dir}\t[{self.job_id}]")
 
         # If failed_files list not defined in the pipeline properties,
         # initialize it
-        failed_file_list = self.pipeline.get('failed_dds_files', default=[])
+        failed_file_list = self.pipeline.get("failed_dds_files", default=[])
         failed_files_count = len(failed_file_list)
         # Skip task if no failed dds files
         if failed_files_count == 0:
-            logger.info('Skip task copy_failed_dds: No failed file to move')
+            logger.info(
+                f"No failed file to move: skip task copy_failed_dds\t[{self.job_id}]"
+            )
             return
 
         # Get clear-dds keyword
-        clear_dds = self.pipeline.get('clear_dds', default=False)
+        clear_dds = self.pipeline.get("clear_dds", default=False)
 
         # Loop over failed files to copy
         for failed_file in failed_file_list.copy():
-
             # Build target filepath
             basename = os.path.basename(failed_file)
             target_filepath = os.path.join(failed_dir, basename)
 
             # copy file
-            logger.debug(f'Copying {failed_file} into {failed_dir}')
+            logger.debug(f"Copying {failed_file} into {failed_dir}\t[{self.job_id}]")
             try:
                 shutil.copyfile(failed_file, target_filepath)
-            except:
-                logger.exception(f'Copying {failed_file} into {failed_dir} has failed!')
+            except Exception as e:
+                logger.exception(
+                    f"Copying {failed_file} into {failed_dir} has failed!\t[{self.job_id}]"
+                )
+                logger.debug(e)
             else:
-                logger.info(f'{failed_file} copied into {target_filepath}')
+                logger.info(
+                    f"{failed_file} copied into {target_filepath}\t[{self.job_id}]"
+                )
 
             # Remove current file from the list in pipeline properties
             failed_file_list.remove(failed_file)
@@ -230,19 +282,21 @@ class CopyFailedDds(Task):
             # if clear-dds keyword is passed, then remove processed Dds
             if clear_dds:
                 os.remove(failed_file)
-                logger.debug(f'{failed_file} deleted')
+                logger.debug(f"{failed_file} deleted\t[{self.job_id}]")
 
         # Get failed tmraw list
-        failed_tmraw_list = self.pipeline.get('failed_tmraw', default=[])
+        failed_tmraw_list = self.pipeline.get("failed_tmraw", default=[])
         failed_tmraw_count = len(failed_tmraw_list)
         # Skip task if no failed tmraw
         if failed_tmraw_count == 0:
-            logger.debug('No failed tmraw to write')
+            logger.debug("No failed tmraw to write\t[{self.job_id}]")
             return
         else:
             # Else save list of failed tmraw into text file
-            tmraw_failed_file = os.path.join(failed_dir, 'tmraw_failed.log')
-            with open(tmraw_failed_file, 'a') as fw:
+            tmraw_failed_file = os.path.join(failed_dir, "tmraw_failed.log")
+            with open(tmraw_failed_file, "a") as fw:
                 fw.writelines(failed_tmraw_list)
-            logger.info(f'{failed_tmraw_count} failed TmRaw entries '
-                        f'saved into {tmraw_failed_file}')
+            logger.info(
+                f"{failed_tmraw_count} failed TmRaw entries "
+                f"saved into {tmraw_failed_file}\t[{self.job_id}]"
+            )