roc-film 1.13.4__py3-none-any.whl → 1.14.0__py3-none-any.whl

roc/film/tools/metadata.py

@@ -11,20 +11,25 @@ from poppy.pop.plugins import Plugin
 
 from roc.film.exceptions import UnknownPipeline, InvalidDataVersion
 from roc.film.tools import valid_data_version
-from roc.film.constants import PLUGIN, TIME_DAILY_STRFORMAT, \
-    CDF_TRANGE_STRFORMAT, UNKNOWN_IDB, DATA_VERSION
-
-__all__ = ['init_l0_meta',
-           'init_cdf_global',
-           'get_data_version',
-           'set_logical_file_id',
-           'get_logical_file_id',
-           'get_spice_kernels'
-           ]
-
-
-def init_cdf_global(l0_attrs, task, master_path,
-                    overwrite=None):
+from roc.film.constants import (
+    PLUGIN,
+    TIME_DAILY_STRFORMAT,
+    CDF_TRANGE_STRFORMAT,
+    UNKNOWN_IDB,
+    DATA_VERSION,
+)
+
+__all__ = [
+    "init_l0_meta",
+    "init_cdf_global",
+    "get_data_version",
+    "set_logical_file_id",
+    "get_logical_file_id",
+    "get_spice_kernels",
+]
+
+
+def init_cdf_global(l0_attrs, task, master_path, overwrite=None):
     """
     Define global attributes data to save into the CDF from the content of the L0 file and task.
     See roc.film.tasks.l0.init_l0_meta for the list of specific L0 meta
@@ -42,79 +47,80 @@ def init_cdf_global(l0_attrs, task, master_path,
     # by import master CDF global attributes
     meta = dict(CDF(master_path).attrs)
 
-    pipeline_id = Configuration.manager['descriptor']['pipeline.identifier'].upper()
-    pipeline_version = Configuration.manager['descriptor']['pipeline.release.version']
+    pipeline_id = Configuration.manager["descriptor"]["pipeline.identifier"].upper()
+    pipeline_version = Configuration.manager["descriptor"]["pipeline.release.version"]
 
-    if pipeline_id == 'RGTS':
+    if pipeline_id == "RGTS":
         # Specific to RGTS
-        meta['Pipeline_name'] = pipeline_id + '>ROC Ground Test SGSE'
+        pipeline_name = pipeline_id + ">ROC Ground Test SGSE"
 
         try:
-            meta['Test_name'] = l0_attrs['Test_name'].encode('utf-8')
-            meta['Test_uuid'] = l0_attrs['Test_uuid']
-            meta['Test_description'] = l0_attrs[
-                'Test_description'].encode('utf-8')
-            meta['Test_creation_date'] = l0_attrs['Test_creation_date']
-            meta['Test_launched_date'] = l0_attrs['Test_launched_date']
-            meta['Test_terminated_date'] = l0_attrs['Test_terminated_date']
-            meta['Test_log_file'] = l0_attrs['Test_log_file']
+            meta["Test_name"] = l0_attrs["Test_name"].encode("utf-8")
+            meta["Test_uuid"] = l0_attrs["Test_uuid"]
+            meta["Test_description"] = l0_attrs["Test_description"].encode("utf-8")
+            meta["Test_creation_date"] = l0_attrs["Test_creation_date"]
+            meta["Test_launched_date"] = l0_attrs["Test_launched_date"]
+            meta["Test_terminated_date"] = l0_attrs["Test_terminated_date"]
+            meta["Test_log_file"] = l0_attrs["Test_log_file"]
+
+            if "Free_field" in l0_attrs and len(l0_attrs["Free_field"].strip()) > 0:
+                meta["Free_field"] = l0_attrs["Free_field"]
+
+            # provider in the good format
+            meta["Provider"] = l0_attrs["Provider"]
 
             # ID of the test for the ROC internal use
-            meta['Test_id'] = l0_attrs['Test_id']
+            meta["Test_id"] = l0_attrs["Test_id"]
         except Exception:
             logger.warning('No "Test_*" attribute found for the input l0')
 
-    elif pipeline_id == 'RODP':
-        # TODO - Complete specific global attributes for RODP
-        meta['Pipeline_name'] = pipeline_id + '>RPW Operation and Data Pipeline'
-        pass
+    elif pipeline_id == "RODP":
+        pipeline_name = pipeline_id + ">RPW Operation and Data Pipeline"
     else:
-        raise UnknownPipeline(f'UNKNOWN PIPELINE TYPE:'
-                              f' {pipeline_id}, ABORTING!')
+        raise UnknownPipeline(f"UNKNOWN PIPELINE TYPE: {pipeline_id}, ABORTING!")
 
     # Common global attributes
     try:
         # Perform some verifications on metadata
-        if str(meta['Pipeline_name']) != str(l0_attrs['Pipeline_name']):
-            logger.warning('Pipeline_name is inconsistent '
-                           f"between the pipeline ({meta['Pipeline_name']})"
-                           f"and the input L0 file ({l0_attrs['Pipeline_name']})!")
-
-        meta['Pipeline_version'] = pipeline_version
-        meta['Parents'] = ['CDF>' + l0_attrs['Logical_file_id']]
-        meta['Parent_version'] = valid_data_version(l0_attrs['Data_version'])
-        meta['Free_field'] = l0_attrs['Free_field']
-        meta['Software_version'] = Plugin.manager[PLUGIN].version
+        if pipeline_name != str(l0_attrs["Pipeline_name"]):
+            logger.warning(
+                "Pipeline_name is inconsistent "
+                f"between the pipeline ({meta['Pipeline_name']})"
+                f"and the input L0 file ({l0_attrs['Pipeline_name']})!"
+            )
+
+        meta["Pipeline_name"] = pipeline_name
+        meta["Pipeline_version"] = pipeline_version
+        meta["Parents"] = ["CDF>" + l0_attrs["Logical_file_id"]]
+        # meta['Parent_version'] = valid_data_version(l0_attrs['Data_version'])
+        meta["Software_version"] = Plugin.manager[PLUGIN].version
 
         # Use for building filename
-        meta['Datetime'] = l0_attrs['Datetime']
-
-        # provider in the good format
-        meta['Provider'] = l0_attrs['Provider']
+        meta["Datetime"] = l0_attrs["Datetime"]
 
         # Get file naming convention
-        meta['File_naming_convention'] = l0_attrs['File_naming_convention']
+        meta["File_naming_convention"] = l0_attrs["File_naming_convention"]
     except Exception:
-        logger.error('Missing attributes in l0 file!')
+        logger.error("Missing attributes in l0 file!")
 
     # the name of the software (plugin) that generated the file, from the
     # descriptor information
-    meta['Software_name'] = PLUGIN
+    meta["Software_name"] = PLUGIN
 
     # Initialize Validate (0 = no validation)
-    meta['Validate'] = '0'
+    meta["Validate"] = "0"
 
     # Initialize data_version to "01"
-    meta['Data_version'] = get_data_version(task)
+    meta["Data_version"] = get_data_version(task)
 
     # If overwrite keyword, then replace g.attrs value
     if overwrite:
         for key, val in overwrite.items():
             meta[key] = val
-            logger.debug(f'{key} g.attribute value set to {val}')
+            logger.debug(f"{key} g.attribute value set to {val}")
 
     # Initialize logical_file_id
-    meta['Logical_file_id'] = set_logical_file_id(meta)
+    meta["Logical_file_id"] = set_logical_file_id(meta)
 
     return meta
 
@@ -127,13 +133,16 @@ def get_idb_version(task, **kwargs):
     :return: string with idb_version
     """
 
-    idb_version = task.pipeline.get('idb_version',
-                                    default=kwargs.get('idb_version', UNKNOWN_IDB))
+    idb_version = task.pipeline.get(
+        "idb_version", default=kwargs.get("idb_version", UNKNOWN_IDB)
+    )
     try:
-        idb_version = task.inputs['raw_data'].value.packet_parser.idb_version
+        idb_version = task.inputs["raw_data"].value.packet_parser.idb_version
     except Exception:
-        logger.debug('No IDB version found in the input raw_data:\n'
-                     f'attempting to retrieve value from pipeline properties: {idb_version}')
+        logger.debug(
+            "No IDB version found in the input raw_data:\n"
+            f"attempting to retrieve value from pipeline properties: {idb_version}"
+        )
 
     return idb_version
 
@@ -146,19 +155,21 @@ def get_idb_source(task, **kwargs):
     :return: string with idb_source
     """
 
-    idb_source = task.pipeline.get('idb_source',
-                                   default=kwargs.get('idb_source', UNKNOWN_IDB))
+    idb_source = task.pipeline.get(
+        "idb_source", default=kwargs.get("idb_source", UNKNOWN_IDB)
+    )
     try:
-        idb_source = task.inputs['raw_data'].value.packet_parser.idb_source
+        idb_source = task.inputs["raw_data"].value.packet_parser.idb_source
     except Exception:
-        logger.debug('No IDB source found in the input raw_data:\n'
-                     f'attempting to retrieve value from pipeline properties: {idb_source}')
+        logger.debug(
+            "No IDB source found in the input raw_data:\n"
+            f"attempting to retrieve value from pipeline properties: {idb_source}"
+        )
 
     return idb_source
 
 
-def init_l0_meta(task,
-                 extra_attrs={}):
+def init_l0_meta(task, extra_attrs={}):
     """
     Initialize RPW L0 metadata
 
@@ -172,100 +183,110 @@ def init_l0_meta(task,
 
     # Retrieve required values from the pipeline properties
     # Get pipeline ID ("RGTS" or "RODP")
-    pipeline_id = task.pipeline.properties.configuration['environment.ROC_PIP_NAME'].upper()
+    pipeline_id = task.pipeline.properties.configuration[
+        "environment.ROC_PIP_NAME"
+    ].upper()
 
     # Get input RawData value
     try:
-        raw_data = task.inputs['raw_data'].value
+        raw_data = task.inputs["raw_data"].value
     except Exception:
         raw_data = None
 
     # Get metadata specific to ROC-SGSE
-    if pipeline_id == 'RGTS':
-
-        meta['Pipeline_name'] = pipeline_id + '>ROC Ground Test SGSE'
+    if pipeline_id == "RGTS":
+        meta["Pipeline_name"] = pipeline_id + ">ROC Ground Test SGSE"
 
         try:
             # Get the 7 first characters of the test log SHA
             test_sha = raw_data.sha1
             test_short_sha = raw_data.short_sha1
 
-            meta['Test_name'] = raw_data.name
-            meta['Test_uuid'] = raw_data.uuid
-            meta['Test_description'] = raw_data.description
-            meta['Test_creation_date'] = str(raw_data.creation_date)
-            meta['Test_launched_date'] = str(raw_data.date)
-            meta['Test_terminated_date'] = str(raw_data.terminated_date)
-            meta['Test_log_file'] = osp.basename(raw_data.file_path)
-            meta['Test_id'] = test_short_sha + '>' + test_sha
-
-            meta['Free_field'] = '-'.join([task.pipeline.provider[:3].lower(),
-                                           test_short_sha])
-            meta['Datetime'] = '-'.join([raw_data.time_min.strftime(CDF_TRANGE_STRFORMAT),
-                                         raw_data.time_max.strftime(CDF_TRANGE_STRFORMAT)])
+            meta["Test_name"] = raw_data.name
+            meta["Test_uuid"] = raw_data.uuid
+            meta["Test_description"] = raw_data.description
+            meta["Test_creation_date"] = str(raw_data.creation_date)
+            meta["Test_launched_date"] = str(raw_data.date)
+            meta["Test_terminated_date"] = str(raw_data.terminated_date)
+            meta["Test_log_file"] = osp.basename(raw_data.file_path)
+            meta["Test_id"] = test_short_sha + ">" + test_sha
+
+            meta["Free_field"] = "-".join(
+                [task.pipeline.provider[:3].lower(), test_short_sha]
+            )
+            meta["Datetime"] = "-".join(
+                [
+                    raw_data.time_min.strftime(CDF_TRANGE_STRFORMAT),
+                    raw_data.time_max.strftime(CDF_TRANGE_STRFORMAT),
+                ]
+            )
         except Exception:
-            logger.warning('No input test log found!')
-            meta['Free_field'] = ''
-            meta['Datetime'] = datetime.now().strftime(TIME_DAILY_STRFORMAT)
+            logger.warning("No input test log found!")
+            meta["Free_field"] = ""
+            meta["Datetime"] = datetime.now().strftime(TIME_DAILY_STRFORMAT)
 
-        meta['File_naming_convention'] = '<Source_name>_<LEVEL>_<Descriptor>_' \
-                                         '<Datetime>_V<Data_version>_' \
-                                         '<Free_field>'
+        meta["File_naming_convention"] = (
+            "<Source_name>_<LEVEL>_<Descriptor>_<Datetime>_V<Data_version>_<Free_field>"
+        )
 
-    elif pipeline_id == 'RODP':
+    elif pipeline_id == "RODP":
         # Get metadata specific to RODP
         # TODO - Complete metadata for RPW L0
 
-        meta['File_naming_convention'] = '<Source_name>_<LEVEL>_<Descriptor>_' \
-                                         '<Datetime>_V<Data_version>'
+        meta["File_naming_convention"] = (
+            "<Source_name>_<LEVEL>_<Descriptor>_<Datetime>_V<Data_version>"
+        )
 
-        meta['Pipeline_name'] = pipeline_id + '>RPW Operation and Data Pipeline'
-        meta['Free_field'] = ''
+        meta["Pipeline_name"] = pipeline_id + ">RPW Operation and Data Pipeline"
+        meta["Free_field"] = ""
 
         # Define Datetime value
-        datetime_attr = extra_attrs.pop('Datetime', None)
+        datetime_attr = extra_attrs.pop("Datetime", None)
         if datetime_attr is None:
-            if raw_data is not None and hasattr(raw_data, 'datetime') and raw_data.datetime is not None:
+            if (
+                raw_data is not None
+                and hasattr(raw_data, "datetime")
+                and raw_data.datetime is not None
+            ):
                 datetime_attr = raw_data.datetime.strftime(TIME_DAILY_STRFORMAT)
             else:
-                datetime_attr = task.pipeline.get('datetime')
+                datetime_attr = task.pipeline.get("datetime")
                 if datetime_attr is None:
-                    logger.warning('Unknown Datetime attribute value')
+                    logger.warning("Unknown Datetime attribute value")
                 else:
                     datetime_attr = datetime_attr.strftime(TIME_DAILY_STRFORMAT)
 
-        meta['Datetime'] = datetime_attr
+        meta["Datetime"] = datetime_attr
 
     else:
-        raise UnknownPipeline(f'UNKNOWN PIPELINE TYPE:'
-                              f' {pipeline_id}, ABORTING!')
+        raise UnknownPipeline(f"UNKNOWN PIPELINE TYPE: {pipeline_id}, ABORTING!")
 
     # Common metadata
-    meta['Project'] = 'SOLO>Solar Orbiter'
-    meta['Source_name'] = 'SOLO>Solar Orbiter'
-    meta['Software_name'] = PLUGIN
-    meta['Software_version'] = Plugin.manager[PLUGIN].version
-    meta['Dataset_ID'] = 'SOLO_L0_RPW'
-    meta['Descriptor'] = 'RPW>Radio and Plasma Waves instrument'
-    meta['LEVEL'] = 'L0>Level 0 data processing'
-    meta['Provider'] = '>'.join(
+    meta["Project"] = "SOLO>Solar Orbiter"
+    meta["Source_name"] = "SOLO>Solar Orbiter"
+    meta["Software_name"] = PLUGIN
+    meta["Software_version"] = Plugin.manager[PLUGIN].version
+    meta["Dataset_ID"] = "SOLO_L0_RPW"
+    meta["Descriptor"] = "RPW>Radio and Plasma Waves instrument"
+    meta["LEVEL"] = "L0>Level 0 data processing"
+    meta["Provider"] = ">".join(
         [
             task.pipeline.provider[:3].upper(),
             task.pipeline.provider,
         ]
     )
 
-    meta['Pipeline_version'] = Configuration.manager['descriptor'][
-        'pipeline.release.version'
+    meta["Pipeline_version"] = Configuration.manager["descriptor"][
+        "pipeline.release.version"
     ]
 
     # Initialize data_version
-    data_version = extra_attrs.pop('Data_version', None)
+    data_version = extra_attrs.pop("Data_version", None)
     if data_version is None:
         # Initialize data_Version to "01"
-        meta['Data_version'] = get_data_version(task)
+        meta["Data_version"] = get_data_version(task)
     else:
-        meta['Data_version'] = data_version
+        meta["Data_version"] = data_version
 
     # Add extra attributes (if any)
     if extra_attrs:
@@ -273,7 +294,7 @@ def init_l0_meta(task,
             meta[key] = val
 
     # Initialize logical_file_id
-    meta['Logical_file_id'] = set_logical_file_id(meta)
+    meta["Logical_file_id"] = set_logical_file_id(meta)
 
     return meta
 
@@ -286,7 +307,7 @@ def get_data_version(task):
     :return: string containing Data_version value
     """
 
-    data_version = task.pipeline.get('data_version', default=None, args=True)
+    data_version = task.pipeline.get("data_version", default=None, args=True)
     if data_version is None:
         data_version = DATA_VERSION
     else:
@@ -296,7 +317,7 @@ def get_data_version(task):
     try:
         return valid_data_version(data_version)
     except Exception:
-        raise InvalidDataVersion(f'Input data version is invalid: {data_version}')
+        raise InvalidDataVersion(f"Input data version is invalid: {data_version}")
 
 
 def set_logical_file_id(metadata):
@@ -308,16 +329,16 @@ def set_logical_file_id(metadata):
     """
 
     # Get expected fields in the file_naming_convention
-    logical_file_id = str(metadata['File_naming_convention'])
-    for field in re.findall(r'<([A-Za-z0-9_\-]+)>', logical_file_id):
+    logical_file_id = str(metadata["File_naming_convention"])
+    for field in re.findall(r"<([A-Za-z0-9_\-]+)>", logical_file_id):
         # Extract value from metadata
-        value = str(metadata[field]).split('>')[0]
-        if field == 'Datetime' or field == 'LEVEL':
+        value = str(metadata[field]).split(">")[0]
+        if field == "Datetime" or field == "LEVEL":
             value = value.upper()
         else:
             value = value.lower()
 
-        logical_file_id = logical_file_id.replace('<' + field + '>', value)
+        logical_file_id = logical_file_id.replace("<" + field + ">", value)
 
     return logical_file_id
 
@@ -334,20 +355,19 @@ def get_logical_file_id(filename):
     return osp.basename(osp.splitext(filename)[0])
 
 
-def get_spice_kernels(time_instance=None,
-                      pattern=None):
+def get_spice_kernels(time_instance=None, pattern=None):
     # If time_instance not passed as input argument,
     # then initialize it from Time class (singleton)
     if time_instance is None:
         from roc.rpl.time import Time
+
         time_instance = Time()
 
     # get all loaded kernels
     loaded_kernels = time_instance.spice.kall()
 
     if pattern is not None:
-        loaded_kernels = [kfile for kfile in loaded_kernels.keys()
-                          if pattern in kfile]
+        loaded_kernels = [kfile for kfile in loaded_kernels.keys() if pattern in kfile]
     else:
         loaded_kernels = list(loaded_kernels.keys())
 

roc/film/tools/skeleton.py

@@ -19,12 +19,12 @@ from roc.film.tools import DESCRIPTOR
 __all__ = ["IDBToExcel"]
 
 
-
 class IDBToExcel(object):
     """
     Class used to convert the IDB in XML format to the excel format needed by
     the converter from excel to CDF skeleton format by Xavier.
     """
+
     def __init__(self, args):
         """
         Store arguments from the command line.
@@ -94,8 +94,8 @@ class IDBToExcel(object):
 
             # save the excel file in the output directory
             outfile = osp.join(
-                    self.args.directory,
-                    data_descr["template"].replace(".cdf", ".xlsx"))
+                self.args.directory, data_descr["template"].replace(".cdf", ".xlsx")
+            )
             logger.info("Saving {0}".format(outfile))
             workbook.save(outfile)
 
@@ -119,8 +119,7 @@ class IDBToExcel(object):
 
                 if version is None:
                     raise ValueError(
-                        "Skeleton version is not set in the HK skeleton " +
-                        "template."
+                        "Skeleton version is not set in the HK skeleton " + "template."
                     )
 
                 # remove surrounds
@@ -130,8 +129,8 @@ class IDBToExcel(object):
                 if re.match("^[0-9]{2}$", version) is None:
                     raise ValueError(
                         (
-                            "Bad format for the skeleton version in the HK " +
-                            "template with value {0}"
+                            "Bad format for the skeleton version in the HK "
+                            + "template with value {0}"
                         ).format(version)
                     )
 
@@ -164,13 +163,11 @@ class IDBToExcel(object):
 
             # logical source
             elif name == "Logical_source":
-                row[3].value = "_".join([items[0].lower(),
-                                         "_HK_",
-                                         items[2].lower()])
+                row[3].value = "_".join([items[0].lower(), "_HK_", items[2].lower()])
 
             # descriptor
             elif name == "Descriptor":
-                row[3].value = (items[2].upper() + ">" + items[2].upper())
+                row[3].value = items[2].upper() + ">" + items[2].upper()
 
             elif name == "MODS":
                 row[3].value = dt.datetime.utcnow().isoformat()
@@ -178,8 +175,8 @@ class IDBToExcel(object):
             # TEXT field
             elif name == "TEXT":
                 row[3].value = (
-                    "This file contains RPW {0} housekeeping." +
-                    "Parameters are returned in the TM_{0}_HK packets."
+                    "This file contains RPW {0} housekeeping."
+                    + "Parameters are returned in the TM_{0}_HK packets."
                 ).format(items[2].split("-")[1])
 
     def base_workbook(self):
@@ -217,7 +214,7 @@ class IDBToExcel(object):
         for row in rows:
             for cell in row:
                 cell.value = None
-        #sheet._garbage_collect()
+        # sheet._garbage_collect()
 
         return wb, attributes
 
@@ -278,9 +275,7 @@ class IDBToExcel(object):
             attributes["LABLAXIS"]["value"] = parameter.definition.name
             attributes["SRDB_PARAM_ID"]["value"] = str(parameter)
             if self.idb_parser.is_enumeration(parameter):
-                attributes["SRDB_ENUM_ID"][
-                    "value"
-                ] = self.idb_parser.enumeration_srdb(
+                attributes["SRDB_ENUM_ID"]["value"] = self.idb_parser.enumeration_srdb(
                     parameter.definition.data_type,
                     parameter.type,
                 )