roc-film 1.13.5__py3-none-any.whl → 1.14.0__py3-none-any.whl

roc/film/tasks/merge_tcreport.py

@@ -15,7 +15,13 @@ from poppy.core.logger import logger
 from poppy.core.target import PyObjectTarget, FileTarget
 from poppy.core.task import Task
 
-from roc.film import DATA_VERSION, TIME_DAILY_STRFORMAT, TCREPORT_PREFIX_BASENAME, ARCHIVE_DAILY_DIR, TIME_ISO_STRFORMAT
+from roc.film import (
+    DATA_VERSION,
+    TIME_DAILY_STRFORMAT,
+    TCREPORT_PREFIX_BASENAME,
+    ARCHIVE_DAILY_DIR,
+    TIME_ISO_STRFORMAT,
+)
 from roc.film.tools.file_helpers import get_output_dir
 from roc.film.tools import valid_data_version, sort_dict_list, get_latest_file
 
@@ -25,58 +31,60 @@ class MergeTcReport(Task):
     Task to merge input set of DDS TcReport XML Elements
     into daily XML files.
     """
-    plugin_name = 'roc.film'
-    name = 'merge_tcreport'
+
+    plugin_name = "roc.film"
+    name = "merge_tcreport"
 
     def add_targets(self):
-        self.add_input(identifier='dds_data',
-                       target_class=PyObjectTarget)
-        self.add_output(identifier='tcreport_daily_xml', many=True,
-                        target_class=FileTarget)
+        self.add_input(identifier="dds_data", target_class=PyObjectTarget)
+        self.add_output(
+            identifier="tcreport_daily_xml", many=True, target_class=FileTarget
+        )
 
     def setup_inputs(self):
-
         # Get data_version input keyword (can be used to force version of
         # output file)
         self.data_version = valid_data_version(
-            self.pipeline.get('data_version', default=[DATA_VERSION])[0])
+            self.pipeline.get("data_version", default=[DATA_VERSION])[0]
+        )
 
         # Get/create list of well processed L0 files
         self.processed_files = self.pipeline.get(
-            'processed_files', default=[], create=True)
+            "processed_files", default=[], create=True
+        )
         # Get/create list of failed DDS files
-        self.failed_files = self.pipeline.get(
-            'failed_files', default=[], create=True)
+        self.failed_files = self.pipeline.get("failed_files", default=[], create=True)
 
         # Get/create list of already processed tcreport
         self.processed_tcreport = self.pipeline.get(
-            'processed_tcreport', default=[], create=True)
+            "processed_tcreport", default=[], create=True
+        )
 
         # Get/create TcReport failed
         self.failed_tcreport = self.pipeline.get(
-            'failed_tcreport', default=[], create=True)
+            "failed_tcreport", default=[], create=True
+        )
 
         # Get local archive path
-        self.archive_path = self.pipeline.get(
-            'archive_path', default=[None])[0]
+        self.archive_path = self.pipeline.get("archive_path", default=[None])[0]
 
         # Get list of dates to process
-        self.filter_date = self.pipeline.get('filter_date', default=[])
+        self.filter_date = self.pipeline.get("filter_date", default=[])
         if self.filter_date:
-            self.filter_date = [filter_date.date()
-                                for filter_date in self.filter_date]
+            self.filter_date = [filter_date.date() for filter_date in self.filter_date]
 
         # If output directory not found, create it
         self.output_dir = get_output_dir(self.pipeline)
         if not os.path.isdir(self.output_dir):
-            logger.debug(f'Making {self.output_dir}...')
+            logger.debug(f"Making {self.output_dir}...")
             os.makedirs(self.output_dir)
 
         # Get input TcReport data
         try:
-            dds_data = self.inputs['dds_data'].value
-        except:
+            dds_data = self.inputs["dds_data"].value
+        except Exception as e:
             logger.exception("Cannot retrieve input 'dds_data' input!")
+            logger.debug(e)
             return False
         else:
             self.tcreport_data = self._get_tcreport_elements(dds_data)
@@ -84,8 +92,7 @@ class MergeTcReport(Task):
         return True
 
     def run(self):
-
-        logger.info('Running MergeTcReport Task...')
+        logger.info("Running MergeTcReport Task...")
 
         # get/initialize inputs
         if not self.setup_inputs():
@@ -100,20 +107,24 @@ class MergeTcReport(Task):
         # Loop over each day in the outputs
         output_files = []
         for current_date, output_data in self.tcreport_data.items():
-
             if self.filter_date and current_date not in self.filter_date:
-                logger.info(f'Skipping current date {current_date}')
+                logger.info(f"Skipping current date {current_date}")
                 continue
 
             # Check if existing data and new data are the same
             current_existing_data = existing_packet_data[current_date]
-            if not [True for current_data in output_data
-                    if current_data not in current_existing_data]:
-                logger.info(f'No new tcreport element for {current_date}')
+            if not [
+                True
+                for current_data in output_data
+                if current_data not in current_existing_data
+            ]:
+                logger.info(f"No new tcreport element for {current_date}")
                 continue
             else:
-                logger.debug(f'Creating a new daily TcReport XML file in {self.output_dir} '
-                             f'for {current_date}')
+                logger.debug(
+                    f"Creating a new daily TcReport XML file in {self.output_dir} "
+                    f"for {current_date}"
+                )
 
             # If existing data list is not empty ...
             new_data = output_data.copy()
@@ -122,41 +133,39 @@ class MergeTcReport(Task):
                 new_data.extend(current_existing_data)
 
             # Make sure it has unique elements
-            new_data = [i for n, i in enumerate(new_data)
-                        if i not in new_data[n + 1:]]
+            new_data = [i for n, i in enumerate(new_data) if i not in new_data[n + 1 :]]
 
             # Make sure new data is time sorted
-            new_data = sort_dict_list(new_data, 'ExecutionTime')
+            new_data = sort_dict_list(new_data, "ExecutionTime")
 
             # define format of data version
-            data_version = f'V{int(self.data_version):02d}'
+            data_version = f"V{int(self.data_version):02d}"
 
             # Build output TcReport file basename
             packet_date_str = current_date.strftime(TIME_DAILY_STRFORMAT)
-            file_basename = '_'.join([
-                TCREPORT_PREFIX_BASENAME,
-                packet_date_str]
-            )
+            file_basename = "_".join([TCREPORT_PREFIX_BASENAME, packet_date_str])
 
             # Build full new output file basename
-            file_basename = '_'.join([file_basename, data_version]) + '.xml'
+            file_basename = "_".join([file_basename, data_version]) + ".xml"
 
             # Build output file path
             output_target_path = os.path.join(self.output_dir, file_basename)
 
             # Write output file
-            logger.info(f'Writing {len(output_data)} TcReport elements '
-                        f'into {output_target_path}')
-            if make_tcreport_xml(new_data, output_target_path,
-                                 overwrite=True,
-                                 logger=logger):
+            logger.info(
+                f"Writing {len(output_data)} TcReport elements "
+                f"into {output_target_path}"
+            )
+            if make_tcreport_xml(
+                new_data, output_target_path, overwrite=True, logger=logger
+            ):
                 self.processed_files.append(output_target_path)
                 output_files.append(output_target_path)
             else:
-                logger.error(f'Writing {output_target_path} has failed!')
+                logger.error(f"Writing {output_target_path} has failed!")
                 self.failed_files.append(output_target_path)
 
-        self.outputs['tcreport_daily_xml'].filepath = output_files
+        self.outputs["tcreport_daily_xml"].filepath = output_files
 
     def _get_existing_data(self, packet_date):
         """
@@ -174,44 +183,39 @@ class MergeTcReport(Task):
         # Build list of directories where to check for existing TM files
         dir_list = [self.output_dir]
         if self.archive_path:
-            dir_list.append(os.path.join(self.archive_path,
-                                         packet_date.strftime(ARCHIVE_DAILY_DIR)))
+            dir_list.append(
+                os.path.join(self.archive_path, packet_date.strftime(ARCHIVE_DAILY_DIR))
+            )
 
         # Loop over directories where to check for existing file
         # (start to check in output directory then in archive dir if provided)
         latest_existing_file = None
         for current_dir in dir_list:
-            latest_existing_file = self._check_existence(
-                current_dir, packet_date)
+            latest_existing_file = self._check_existence(current_dir, packet_date)
             if latest_existing_file:
                 break
 
         # Then, if latest existing file was found, parse it to retrieve data
         if latest_existing_file:
-            logger.info(f'Loading existing TcReport data from {latest_existing_file}...')
-            output_data = self._get_tcreport_elements(
-                xml_to_dict(latest_existing_file))
+            logger.info(
+                f"Loading existing TcReport data from {latest_existing_file}..."
+            )
+            output_data = self._get_tcreport_elements(xml_to_dict(latest_existing_file))
         else:
-            logger.info(f'No existing TcReport data file found '
-                        f' for {packet_date}')
+            logger.info(f"No existing TcReport data file found  for {packet_date}")
 
         return output_data
 
     def _check_existence(self, dir_path, packet_date):
-
         # Convert input packet date into string
         packet_date_str = packet_date.strftime(TIME_DAILY_STRFORMAT)
 
         # Build output TmRaw file basename
-        file_basename = '_'.join([
-            TCREPORT_PREFIX_BASENAME,
-            packet_date_str]
-        )
+        file_basename = "_".join([TCREPORT_PREFIX_BASENAME, packet_date_str])
 
-        existing_files = glob(os.path.join(
-            dir_path, file_basename + '_V??.xml'))
+        existing_files = glob(os.path.join(dir_path, file_basename + "_V??.xml"))
         if existing_files:
-            logger.debug(f'{len(existing_files)} already exists for {packet_date}')
+            logger.debug(f"{len(existing_files)} already exists for {packet_date}")
             # If files found then get latest version
             latest_existing_file = get_latest_file(existing_files)
         else:
@@ -228,9 +232,9 @@ class MergeTcReport(Task):
         :return: list of TcReport XML elements
         """
 
-        output_tcreport_list = tcreport_xml_dict['ns2:ResponsePart'] \
-            ['Response']['PktTcReportResponse'] \
-            ['PktTcReportList']['PktTcReportListElement']
+        output_tcreport_list = tcreport_xml_dict["ns2:ResponsePart"]["Response"][
+            "PktTcReportResponse"
+        ]["PktTcReportList"]["PktTcReportListElement"]
 
         # Make sure that returned output_tcreport_list is a list
         # (If only one PktTcReportListElement is found in the XML
@@ -242,10 +246,12 @@ class MergeTcReport(Task):
         output_tcreport_dict = collections.defaultdict(list)
         for current_tcreport in output_tcreport_list:
             try:
-                key = datetime.strptime(current_tcreport['ExecutionTime'],
-                                        TIME_ISO_STRFORMAT).date()
-            except:
-                logger.warning(f'Cannot get ExecutionTime for {current_tcreport}')
+                key = datetime.strptime(
+                    current_tcreport["ExecutionTime"], TIME_ISO_STRFORMAT
+                ).date()
+            except Exception as e:
+                logger.warning(f"Cannot get ExecutionTime for {current_tcreport}")
+                logger.debug(e)
                 if current_tcreport not in self.failed_tcreport:
                     self.failed_tcreport.append(current_tcreport)
             else:

roc/film/tasks/merge_tmraw.py

@@ -9,51 +9,56 @@ import collections
 import os
 from glob import glob
 
-from edds_process.response import \
-    xml_to_dict, remove_scos_header
+from edds_process.response import xml_to_dict, remove_scos_header
 
 from poppy.core.task import Task
 from poppy.core.logger import logger
 from poppy.core.target import FileTarget, PyObjectTarget
 
-from roc.film import TM_PACKET_CATEG, ARCHIVE_DAILY_DIR, TIME_DAILY_STRFORMAT, TMRAW_PREFIX_BASENAME
+from roc.film import (
+    TM_PACKET_CATEG,
+    ARCHIVE_DAILY_DIR,
+    TIME_DAILY_STRFORMAT,
+    TMRAW_PREFIX_BASENAME,
+)
 from roc.film.tools.file_helpers import get_output_dir
 from roc.film.tools import get_latest_file
 
 from roc.rpl.time import Time
 from roc.rpl.packet_structure.data import Data
 
+
 class MergeTmRaw(Task):
     """
     Task to merge input set of DDS TmRaw Packet XML Elements
     into daily XML files.
     """
-    plugin_name = 'roc.film'
-    name = 'merge_tmraw'
+
+    plugin_name = "roc.film"
+    name = "merge_tmraw"
 
     def add_targets(self):
-        self.add_input(identifier='dds_xml',
-                       target_class=FileTarget)
-        self.add_input(identifier='dds_data',
-                       target_class=PyObjectTarget)
+        self.add_input(identifier="dds_xml", target_class=FileTarget)
+        self.add_input(identifier="dds_data", target_class=PyObjectTarget)
 
     def get_dds_xml(self):
-        return self.pipeline.get('dds_xml', default=[])
+        return self.pipeline.get("dds_xml", default=[])
 
     def setup_inputs(self):
-
         # Get/create list of failed DDS files
         self.failed_dds_files = self.pipeline.get(
-            'failed_dds_files', default=[], create=True)
+            "failed_dds_files", default=[], create=True
+        )
 
         # Get input DDS XML file
         dds_file = None
         try:
-            dds_file = self.inputs['dds_xml'].filepath
+            dds_file = self.inputs["dds_xml"].filepath
             if not os.path.isfile(dds_file):
                 raise FileNotFoundError
-        except:
+        except Exception as e:
             logger.exception(f"Cannot load input DDS XML file '{dds_file}'")
+            logger.debug(e)
             self.failed_dds_files.append(dds_file)
             return False
         else:
@@ -62,34 +67,33 @@ class MergeTmRaw(Task):
         # Get packet category from input DDS filename
         # (Convention should be "RPW_<packet_category>_*.xml"
         try:
-            self.packet_category = os.path.basename(
-                self.dds_file).split('_')[1].lower()
-        except:
-            logger.exception(f'Packet category cannot be extracted from DDS filename!')
+            self.packet_category = os.path.basename(self.dds_file).split("_")[1].lower()
+        except Exception as e:
+            logger.exception("Packet category cannot be extracted from DDS filename!")
+            logger.debug(e)
             self.failed_dds_files.append(self.dds_file)
             return False
 
         # If output directory not found, create it
         self.output_dir = get_output_dir(self.pipeline)
         if not os.path.isdir(self.output_dir):
-            logger.debug(f'Making {self.output_dir}...')
+            logger.debug(f"Making {self.output_dir}...")
             os.makedirs(self.output_dir)
 
         # Get local archive path
-        self.archive_path = self.pipeline.get(
-            'archive_path', default=[None])[0]
+        self.archive_path = self.pipeline.get("archive_path", default=[None])[0]
 
         # Get list of dates to process
-        self.filter_date = self.pipeline.get('filter_date', default=[])
+        self.filter_date = self.pipeline.get("filter_date", default=[])
         if self.filter_date:
-            self.filter_date = [filter_date.date()
-                                for filter_date in self.filter_date]
+            self.filter_date = [filter_date.date() for filter_date in self.filter_date]
 
         # Get input TmRaw data
         try:
-            dds_data = self.inputs['dds_data'].value
-        except:
+            dds_data = self.inputs["dds_data"].value
+        except Exception as e:
             logger.exception("Cannot retrieve input 'dds_data' input!")
+            logger.debug(e)
             self.failed_dds_files.append(self.dds_file)
             return False
         else:
@@ -98,25 +102,26 @@ class MergeTmRaw(Task):
 
         # Get packet cache
         self.packet_cache = self.pipeline.get(
-            'packet_cache', default={tm_cat: {}
-                                     for tm_cat in TM_PACKET_CATEG},
-            create=True)
+            "packet_cache",
+            default={tm_cat: {} for tm_cat in TM_PACKET_CATEG},
+            create=True,
+        )
 
         # And flag to indicated if there are new packets
         self.has_new_packet = self.pipeline.get(
-            'has_new_packet', default={tm_cat: {}
-                                       for tm_cat in TM_PACKET_CATEG},
-            create=True)
+            "has_new_packet",
+            default={tm_cat: {} for tm_cat in TM_PACKET_CATEG},
+            create=True,
+        )
 
         return True
 
     def run(self):
-
-        logger.debug('Running MergeTmRaw Task...')
+        logger.debug("Running MergeTmRaw Task...")
 
         # get/initialize inputs
         if not self.setup_inputs():
-            logger.warning('Missing inputs for MergeTmRaw Task!')
+            logger.warning("Missing inputs for MergeTmRaw Task!")
             return
 
         # Check/Get existing data for previous TmRaw daily files (if any)
@@ -127,9 +132,8 @@ class MergeTmRaw(Task):
 
         # Loop over each day in the outputs
         for current_date, output_data in self.tmraw_data.items():
-
             if self.filter_date and current_date not in self.filter_date:
-                logger.info(f'Skipping current date {current_date}')
+                logger.info(f"Skipping current date {current_date}")
                 continue
 
             # Check if existing data and new data are the same
@@ -138,13 +142,14 @@ class MergeTmRaw(Task):
             # If current_date.year <= 2000, output_data contains not synchronized packets
             # in this case, always write packets into an output file
             new_data = list(set(output_data) - set(current_existing_data))
-            if (current_date.year > 2000 and
-                    not new_data):
-                logger.info(f'No new packet for {current_date} in {self.dds_file}')
+            if current_date.year > 2000 and not new_data:
+                logger.info(f"No new packet for {current_date} in {self.dds_file}")
                 self.has_new_packet[self.packet_category][current_date] = False
             else:
-                logger.info(f'{len(new_data)} new TmRaw Packet elements found '
-                             f'for {current_date} in {self.dds_file}...')
+                logger.info(
+                    f"{len(new_data)} new TmRaw Packet elements found "
+                    f"for {current_date} in {self.dds_file}..."
+                )
 
                 # If existing data list is not empty ...
                 if current_existing_data:
@@ -162,11 +167,11 @@ class MergeTmRaw(Task):
 
             # Store new packets for given category and date in the packet_cache
             if current_existing_data:
-                self.packet_cache[self.packet_category][
-                    current_date] = current_existing_data
+                self.packet_cache[self.packet_category][current_date] = (
+                    current_existing_data
+                )
 
-    def _get_packet_time(self, packet_data,
-                         scos_header=True):
+    def _get_packet_time(self, packet_data, scos_header=True):
         """
         Extract CUC time from current packet binary.
 
@@ -182,15 +187,15 @@ class MergeTmRaw(Task):
             else:
                 packet_raw_data = packet_data
             packet_bytes_data = bytearray.fromhex(packet_raw_data)
-            packet_data = Data(bytes(packet_bytes_data),
-                               len(packet_bytes_data)
-                               )
+            packet_data = Data(bytes(packet_bytes_data), len(packet_bytes_data))
             data_field_header = packet_data.extract_tm_header()
             packet_time = Time.cuc_to_datetime(data_field_header.time[:2])[0]
-        except:
-            logger.exception(f'Packet CUC time cannot be retrieved '
-                             f'from Packet element {packet_data}, skip it '
-                             )
+        except Exception as e:
+            logger.exception(
+                f"Packet CUC time cannot be retrieved "
+                f"from Packet element {packet_data}, skip it "
+            )
+            logger.debug(e)
             if packet_data not in self.failed_tmraw:
                 self.failed_tmraw.append(packet_data)
             return None, None
@@ -209,9 +214,11 @@ class MergeTmRaw(Task):
 
         # First check if there are already packets store in packet_cache
         if packet_date in self.packet_cache[self.packet_category]:
-            logger.info(f'Retrieving existing {self.packet_category} '
-                        f'data for {packet_date} from pipeline cache '
-                        f'({len(self.packet_cache[self.packet_category][packet_date])} packets) ...')
+            logger.info(
+                f"Retrieving existing {self.packet_category} "
+                f"data for {packet_date} from pipeline cache "
+                f"({len(self.packet_cache[self.packet_category][packet_date])} packets) ..."
+            )
             return self.packet_cache[self.packet_category]
 
         # If not initialize output data
@@ -221,43 +228,43 @@ class MergeTmRaw(Task):
         # Build list of directories where to check for existing TM files
         dir_list = [self.output_dir]
         if self.archive_path:
-            dir_list.append(os.path.join(self.archive_path,
-                                         packet_date.strftime(ARCHIVE_DAILY_DIR)))
+            dir_list.append(
+                os.path.join(self.archive_path, packet_date.strftime(ARCHIVE_DAILY_DIR))
+            )
 
         # Loop over directories where to check for existing file
         # (start to check in output directory then in archive dir if provided)
         for current_dir in dir_list:
-            latest_existing_file = self._check_existence(
-                current_dir, packet_date)
+            latest_existing_file = self._check_existence(current_dir, packet_date)
             if latest_existing_file:
                 break
 
         # Then, if latest existing file was found, parse it to retrieve data
         if latest_existing_file:
-            logger.info(f'Loading existing TmRaw data from {latest_existing_file}...')
-            output_data = self._get_tmraw_packets(xml_to_dict(latest_existing_file),
-                                                  scos_header=False)
+            logger.info(f"Loading existing TmRaw data from {latest_existing_file}...")
+            output_data = self._get_tmraw_packets(
+                xml_to_dict(latest_existing_file), scos_header=False
+            )
         else:
-            logger.info(f'No existing {self.packet_category} TmRaw data file found '
-                        f' for {packet_date}')
+            logger.info(
+                f"No existing {self.packet_category} TmRaw data file found "
+                f" for {packet_date}"
+            )
 
         return output_data
 
     def _check_existence(self, dir_path, packet_date):
-
         # Convert input packet date into string
         packet_date_str = packet_date.strftime(TIME_DAILY_STRFORMAT)
 
         # Build output TmRaw file basename
-        file_basename = '_'.join([
-            TMRAW_PREFIX_BASENAME + '-' + self.packet_category,
-            packet_date_str]
+        file_basename = "_".join(
+            [TMRAW_PREFIX_BASENAME + "-" + self.packet_category, packet_date_str]
         )
 
-        existing_files = glob(os.path.join(
-            dir_path, file_basename + '_V??.xml'))
+        existing_files = glob(os.path.join(dir_path, file_basename + "_V??.xml"))
         if existing_files:
-            logger.debug(f'{len(existing_files)} already exists for {packet_date}')
+            logger.debug(f"{len(existing_files)} already exists for {packet_date}")
             # If files found then get latest version
             latest_existing_file = get_latest_file(existing_files)
         else:
@@ -265,8 +272,7 @@ class MergeTmRaw(Task):
 
         return latest_existing_file
 
-    def _get_tmraw_packets(self, tmraw_xml_dict,
-                           scos_header=True):
+    def _get_tmraw_packets(self, tmraw_xml_dict, scos_header=True):
         """
         Extract TmRaw Packet Element list from a dictionary
         of a input TmRaw XML file
@@ -279,15 +285,15 @@ class MergeTmRaw(Task):
 
         # Get/create list of already processed tmraw packets
         self.processed_tmraw = self.pipeline.get(
-            'processed_tmraw', default=[], create=True)
+            "processed_tmraw", default=[], create=True
+        )
 
         # Get/create list of failed tmraw packets
-        self.failed_tmraw = self.pipeline.get(
-            'failed_tmraw', default=[], create=True)
+        self.failed_tmraw = self.pipeline.get("failed_tmraw", default=[], create=True)
 
-        output_packet_list = tmraw_xml_dict['ns2:ResponsePart']\
-            ['Response']['PktRawResponse'] \
-            ['PktRawResponseElement']
+        output_packet_list = tmraw_xml_dict["ns2:ResponsePart"]["Response"][
+            "PktRawResponse"
+        ]["PktRawResponseElement"]
 
         # Make sure that returned output_tmraw_list is a list
         # (If only one Packet element is found in the XML
@@ -297,14 +303,21 @@ class MergeTmRaw(Task):
 
         # Add packet date and time to output list
         # Make sure to take unique packets
-        logger.debug(f'Extracting packet CUC time from '
-                     f'{len(output_packet_list)} TmRaw Packet elements...')
-
-        output_packet_list = list(set([
-            self._get_packet_time(current_packet['Packet'],
-                              scos_header=scos_header)
-            for current_packet in output_packet_list
-        ]))
+        logger.debug(
+            f"Extracting packet CUC time from "
+            f"{len(output_packet_list)} TmRaw Packet elements..."
+        )
+
+        output_packet_list = list(
+            set(
+                [
+                    self._get_packet_time(
+                        current_packet["Packet"], scos_header=scos_header
+                    )
+                    for current_packet in output_packet_list
+                ]
+            )
+        )
 
         # return output as a dictionary of packet date keywords
         output_packet_dict = collections.defaultdict(list)