roc-film 1.13.4__py3-none-any.whl → 1.14.0__py3-none-any.whl

roc/film/exceptions.py

@@ -7,23 +7,25 @@ Exceptions definition for FILM plugin.
 
 from poppy.core.logger import logger
 
-__all__ = ['FilmException',
-           'MetadataException',
-           'UnknownPipeline',
-           'AncReportProdError',
-           'LoadDataSetError',
-           'NoData',
-           'NoEpochFoundError',
-           'L0ProdFailure',
-           'L1BiaProdError',
-           'L1SurvProdFailure',
-           'L1PostProError',
-           'HkProdFailure',
-           'L1SbmProdError',
-           'AncBiaProdError',
-           'HandlingFileError',
-           'InvalidDataVersion',
-           'EmptyInput']
+__all__ = [
+    "FilmException",
+    "MetadataException",
+    "UnknownPipeline",
+    "AncReportProdError",
+    "LoadDataSetError",
+    "NoData",
+    "NoEpochFoundError",
+    "L0ProdFailure",
+    "L1BiaProdError",
+    "L1SurvProdFailure",
+    "L1PostProError",
+    "HkProdFailure",
+    "L1SbmProdError",
+    "AncBiaProdError",
+    "HandlingFileError",
+    "InvalidDataVersion",
+    "EmptyInput",
+]
 
 
 class FilmException(Exception):
@@ -145,8 +147,7 @@ class NoData(Exception):
     Exception raised when no output data processed
     """
 
-    def __init__(self, message=None, ll=logger.error,
-                 *args, **kwargs):
+    def __init__(self, message=None, ll=logger.error, *args, **kwargs):
         super(NoData, self).__init__(*args, **kwargs)
         if message is not None:
             ll(message)
@@ -167,8 +168,8 @@ class L1SbmProdError(Exception):
         logger.error(message)
         self.message = message
 
-#    logger_level = 'warning'
-#    use_traceback = True
+    #    logger_level = 'warning'
+    #    use_traceback = True
 
     pass
 
@@ -181,8 +182,8 @@ class L1BiaProdError(Exception):
         logger.error(message)
         self.message = message
 
-#    logger_level = 'warning'
-#    use_traceback = True
+    #    logger_level = 'warning'
+    #    use_traceback = True
 
     pass
 
@@ -195,8 +196,8 @@ class AncBiaProdError(Exception):
         logger.error(message)
         self.message = message
 
-#    logger_level = 'warning'
-#    use_traceback = True
+    #    logger_level = 'warning'
+    #    use_traceback = True
 
     pass
 
@@ -209,7 +210,7 @@ class InvalidDataVersion(Exception):
         logger.error(message)
         self.message = message
 
-#    logger_level = 'warning'
-#    use_traceback = True
+    #    logger_level = 'warning'
+    #    use_traceback = True
 
     pass

roc/film/tasks/__init__.py

@@ -1,19 +1,19 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from roc.film.tasks.check_dds import *
-from roc.film.tasks.cat_solo_hk import *
-from roc.film.tasks.make_daily_tm import *
-from roc.film.tasks.merge_tmraw import *
-from roc.film.tasks.merge_tcreport import *
-from roc.film.tasks.parse_dds_xml import *
-from roc.film.tasks.dds_to_l0 import *
-from roc.film.tasks.set_l0_utc import *
-from roc.film.tasks.l0_to_hk import *
-from roc.film.tasks.l0_to_l1_surv import *
-from roc.film.tasks.l0_to_l1_sbm import *
-from roc.film.tasks.l0_to_l1_bia_sweep import *
-from roc.film.tasks.l0_to_anc_bia_sweep_table import *
-from roc.film.tasks.l0_to_l1_bia_current import *
-from roc.film.tasks.cdf_postpro import *
-from roc.film.tasks.file_handler import *
+from roc.film.tasks.check_dds import *  # noqa: F403
+from roc.film.tasks.cat_solo_hk import *  # noqa: F403
+from roc.film.tasks.make_daily_tm import *  # noqa: F403
+from roc.film.tasks.merge_tmraw import *  # noqa: F403
+from roc.film.tasks.merge_tcreport import *  # noqa: F403
+from roc.film.tasks.parse_dds_xml import *  # noqa: F403
+from roc.film.tasks.dds_to_l0 import *  # noqa: F403
+from roc.film.tasks.set_l0_utc import *  # noqa: F403
+from roc.film.tasks.l0_to_hk import *  # noqa: F403
+from roc.film.tasks.l0_to_l1_surv import *  # noqa: F403
+from roc.film.tasks.l0_to_l1_sbm import *  # noqa: F403
+from roc.film.tasks.l0_to_l1_bia_sweep import *  # noqa: F403
+from roc.film.tasks.db_to_anc_bia_sweep_table import *  # noqa: F403
+from roc.film.tasks.l0_to_l1_bia_current import *  # noqa: F403
+from roc.film.tasks.cdf_postpro import *  # noqa: F403
+from roc.film.tasks.file_handler import *  # noqa: F403

roc/film/tasks/cat_solo_hk.py

@@ -15,12 +15,18 @@ from poppy.core.logger import logger
 from poppy.core.target import FileTarget
 from poppy.core.task import Task
 
-from roc.film import DATA_VERSION, TIME_DAILY_STRFORMAT, TIME_ISO_STRFORMAT, ARCHIVE_DAILY_DIR
+from roc.film import (
+    DATA_VERSION,
+    TIME_DAILY_STRFORMAT,
+    TIME_ISO_STRFORMAT,
+    ARCHIVE_DAILY_DIR,
+)
 from roc.film.constants import SOLOHK_PREFIX_BASENAME
 from roc.film.tools.file_helpers import get_output_dir
 from roc.film.tools import valid_data_version, sort_dict_list, get_latest_file
 
-__all__ = ['CatSoloHk']
+__all__ = ["CatSoloHk"]
+
 
 class CatSoloHk(Task):
     """
@@ -29,68 +35,72 @@ class CatSoloHk(Task):
     ParamSampleListElement XML elements are loaded from a
     set of input SOLO EDDS Param XML files
     """
-    plugin_name = 'roc.film'
-    name = 'cat_solo_hk'
+
+    plugin_name = "roc.film"
+    name = "cat_solo_hk"
 
     def add_targets(self):
-        self.add_input(identifier='dds_xml', many=True,
-                       target_class=FileTarget,
-                       filepath=self.get_dds_xml())
-        self.add_output(identifier='solohk_daily_xml', many=True,
-                        target_class=FileTarget)
+        self.add_input(
+            identifier="dds_xml",
+            many=True,
+            target_class=FileTarget,
+            filepath=self.get_dds_xml(),
+        )
+        self.add_output(
+            identifier="solohk_daily_xml", many=True, target_class=FileTarget
+        )
 
     def get_dds_xml(self):
-        return self.pipeline.get('dds_files', default=[])
+        return self.pipeline.get("dds_files", default=[])
 
     def setup_inputs(self):
-
         # Get data_version input keyword (can be used to force version of
         # output file)
         self.data_version = valid_data_version(
-            self.pipeline.get('data_version', default=[DATA_VERSION])[0])
+            self.pipeline.get("data_version", default=[DATA_VERSION])[0]
+        )
 
         # Get input list of DDS XML files
-        self.dds_file_list = self.inputs['dds_xml'].filepath
+        self.dds_file_list = self.inputs["dds_xml"].filepath
         self.dds_file_num = len(self.dds_file_list)
         if self.dds_file_num == 0:
-            logger.warning('No input DDS XML file passed as input argument!')
+            logger.warning("No input DDS XML file passed as input argument!")
             return False
 
         # Get/create list of well processed L0 files
         self.processed_files = self.pipeline.get(
-            'processed_files', default=[], create=True)
+            "processed_files", default=[], create=True
+        )
         # Get/create list of failed DDS files
-        self.failed_files = self.pipeline.get(
-            'failed_files', default=[], create=True)
+        self.failed_files = self.pipeline.get("failed_files", default=[], create=True)
 
         # Get/create list of already processed DDS
-        self.processed_dds_files = self.pipeline.get('processed_dds_files',
-                                                     default=[], create=True)
+        self.processed_dds_files = self.pipeline.get(
+            "processed_dds_files", default=[], create=True
+        )
 
         # Get/create list of failed DDS
-        self.failed_dds_files = self.pipeline.get('failed_dds_files',
-                                                  default=[], create=True)
+        self.failed_dds_files = self.pipeline.get(
+            "failed_dds_files", default=[], create=True
+        )
 
         # Get local archive path
-        self.archive_path = self.pipeline.get(
-            'archive_path', default=[None])[0]
+        self.archive_path = self.pipeline.get("archive_path", default=[None])[0]
 
         # Get list of dates to process
-        self.filter_date = self.pipeline.get('filter_date', default=[])
+        self.filter_date = self.pipeline.get("filter_date", default=[])
         if self.filter_date:
-            self.filter_date = [filter_date.date()
-                                for filter_date in self.filter_date]
+            self.filter_date = [filter_date.date() for filter_date in self.filter_date]
 
         # If output directory not found, create it
         self.output_dir = get_output_dir(self.pipeline)
         if not os.path.isdir(self.output_dir):
-            logger.debug(f'Making {self.output_dir}...')
+            logger.debug(f"Making {self.output_dir}...")
             os.makedirs(self.output_dir)
 
         return True
 
     def run(self):
-
         # Initialize task inputs
         if not self.setup_inputs():
             self.pipeline.exit()
@@ -102,15 +112,15 @@ class CatSoloHk(Task):
 
         # Loop over list of input files
         for input_dds_file in self.dds_file_list:
-
             # Retrieve list of SOLO HK PARAMS inside the XML
             try:
                 xml_param_list = CatSoloHk.parse_dds_param_xml(input_dds_file)
             except FileNotFoundError:
-                logger.warning(f'Input file {input_dds_file} not found!')
+                logger.warning(f"Input file {input_dds_file} not found!")
                 continue
-            except:
-                logger.exception(f'Cannot parse {input_dds_file}')
+            except Exception as e:
+                logger.exception(f"Cannot parse {input_dds_file}")
+                logger.debug(e)
                 if input_dds_file not in self.failed_dds_files:
                     self.failed_dds_files.append(input_dds_file)
                 continue
@@ -124,70 +134,71 @@ class CatSoloHk(Task):
         # Loop over days in solo_hk_dict
         output_files = []
         for current_date, current_param_list in solo_hk_dict.items():
-
             # Check if output file already exists for the current
             # day
             existing_data = self._get_existing_data(current_date)
             existing_num = len(existing_data)
-            logger.info(f'{existing_num} Solo HK elements already '
-                        f'found for {current_date}')
+            logger.info(
+                f"{existing_num} Solo HK elements already found for {current_date}"
+            )
 
             # Append new elements in the list of existing data
             new_data = existing_data.copy()
-            [new_data.append(current_data)
-             for current_data in current_param_list
-                if current_data not in existing_data]
+            [
+                new_data.append(current_data)
+                for current_data in current_param_list
+                if current_data not in existing_data
+            ]
 
             # If no difference with existing data, no need to save new output
             # file
             new_data_num = len(new_data)
             if existing_num == new_data_num:
-                logger.info(f'No new SOLO HK element for {current_date}')
+                logger.info(f"No new SOLO HK element for {current_date}")
                 continue
             else:
-                logger.debug(f'Creating a new SOLO HK daily file in {self.output_dir} '
-                             f'for {current_date}')
+                logger.debug(
+                    f"Creating a new SOLO HK daily file in {self.output_dir} "
+                    f"for {current_date}"
+                )
 
             # Always make sure that elements are unique and sorted
             # by ascending timestamp
             # Make sure it has unique elements
-            new_data = [i for n, i in enumerate(new_data)
-                        if i not in new_data[n + 1:]]
+            new_data = [i for n, i in enumerate(new_data) if i not in new_data[n + 1 :]]
 
             # Make sure new data is time sorted
-            new_data = sort_dict_list(new_data, 'TimeStampAsciiA')
+            new_data = sort_dict_list(new_data, "TimeStampAsciiA")
 
             # define format of data version
-            data_version = f'V{int(self.data_version):02d}'
+            data_version = f"V{int(self.data_version):02d}"
 
             # Convert input date into string
             date_str = current_date.strftime(TIME_DAILY_STRFORMAT)
 
             # Build output solo hk daily file basename
-            file_basename = '_'.join([
-                SOLOHK_PREFIX_BASENAME,
-                date_str]
-            )
+            file_basename = "_".join([SOLOHK_PREFIX_BASENAME, date_str])
 
             # Build full new output file basename
-            file_basename = '_'.join([file_basename, data_version]) + '.xml'
+            file_basename = "_".join([file_basename, data_version]) + ".xml"
 
             # Build output file path
             output_target_path = os.path.join(self.output_dir, file_basename)
 
             # Write output file
-            logger.info(f'Writing {len(new_data)} Solo HK elements '
-                        f'into {output_target_path}')
-            if make_param_xml(new_data, output_target_path,
-                              overwrite=True,
-                              logger=logger):
+            logger.info(
+                f"Writing {len(new_data)} Solo HK elements into {output_target_path}"
+            )
+            if make_param_xml(
+                new_data, output_target_path, overwrite=True, logger=logger
+            ):
                 self.processed_files.append(output_target_path)
                 output_files.append(output_target_path)
             else:
-                logger.error(f'Writing {output_target_path} has failed!')
+                logger.error(f"Writing {output_target_path} has failed!")
                 self.failed_files.append(output_target_path)
 
-        self.outputs['solohk_daily_xml'].filepath = output_files
+        self.outputs["solohk_daily_xml"].filepath = output_files
 
     @staticmethod
     def parse_dds_param_xml(xml_file):
@@ -201,9 +212,9 @@ class CatSoloHk(Task):
         ParamSampleListElement = []
 
         if os.path.isfile(xml_file):
-            ParamSampleListElement = xml_to_dict(xml_file)['ns2:ResponsePart'] \
-                ['Response']['ParamResponse'] \
-                ['ParamSampleList']['ParamSampleListElement']
+            ParamSampleListElement = xml_to_dict(xml_file)["ns2:ResponsePart"][
+                "Response"
+            ]["ParamResponse"]["ParamSampleList"]["ParamSampleListElement"]
 
             # Make sure that returned output is a list
             # (If only one PktTcReportListElement is found in the XML
@@ -228,7 +239,6 @@ class CatSoloHk(Task):
 
         # Loop over ParamSampleListElement in the input list
         for current_param in param_list:
-
             # Get element timestamp
             current_timestamp = self._extract_timestamp(current_param)
             # Get date
@@ -245,8 +255,9 @@ class CatSoloHk(Task):
         :return: ParamSampleListElement TimeStampAsciiA as datetime object
         """
 
-        return datetime.strptime(param_element['TimeStampAsciiA'],
-                                 TIME_ISO_STRFORMAT[:-1])
+        return datetime.strptime(
+            param_element["TimeStampAsciiA"], TIME_ISO_STRFORMAT[:-1]
+        )
 
     def _get_existing_data(self, date):
         """
@@ -266,25 +277,26 @@ class CatSoloHk(Task):
 
         # Get SOLO HK daily file local archive directory path
         if self.archive_path:
-            solo_hk_data_dir = os.path.join(self.archive_path,
-                                            date.strftime(ARCHIVE_DAILY_DIR))
+            solo_hk_data_dir = os.path.join(
+                self.archive_path, date.strftime(ARCHIVE_DAILY_DIR)
+            )
         else:
             # Otherwise get output directory
             solo_hk_data_dir = self.output_dir
 
         # Build SOLO HK daily file basename
-        file_basename = '_'.join([
-            SOLOHK_PREFIX_BASENAME,
-            date_str]
-        )
+        file_basename = "_".join([SOLOHK_PREFIX_BASENAME, date_str])
 
         # Check if daily file(s) already exists in the target directory
-        logger.debug(f'Checking for SOLO HK daily file existence '
-                     f'on {solo_hk_data_dir} for {date} ...')
-        existing_files = glob(os.path.join(
-            solo_hk_data_dir, file_basename + '_V??.xml'))
+        logger.debug(
+            f"Checking for SOLO HK daily file existence "
+            f"on {solo_hk_data_dir} for {date} ..."
+        )
+        existing_files = glob(
+            os.path.join(solo_hk_data_dir, file_basename + "_V??.xml")
+        )
         if existing_files:
-            logger.debug(f'{len(existing_files)} files already exist for {date}')
+            logger.debug(f"{len(existing_files)} files already exist for {date}")
             # If files found then get latest version
             latest_existing_file = get_latest_file(existing_files)
         else: