roc-film 1.13.4__py3-none-any.whl → 1.14.0__py3-none-any.whl

roc/film/tools/tools.py

@@ -17,32 +17,38 @@ from poppy.core.generic.metaclasses import Singleton
 from poppy.core.configuration import Configuration
 from poppy.core.generic.paths import Paths
 from poppy.core.logger import logger
-from roc.film.constants import _ROOT_DIRECTORY, INPUT_DATETIME_STRFTIME, \
-    DATA_VERSION, TIME_DAILY_STRFORMAT
+from roc.film.constants import (
+    _ROOT_DIRECTORY,
+    INPUT_DATETIME_STRFTIME,
+    DATA_VERSION,
+    TIME_DAILY_STRFORMAT,
+)
 
 from roc.film.exceptions import FilmException, HandlingFileError
 
-__all__ = ['paths', 'DESCRIPTOR',
-           'raise_error',
-           'valid_time',
-           'valid_date',
-           'valid_data_version',
-           'valid_single_file',
-           'valid_dir',
-           'extract_datetime',
-           'extract_file_fields',
-           'get_datasets',
-           'sort_indices',
-           'unique_dict_list',
-           'sort_dict_list',
-           'safe_move',
-           'setup_lock',
-           'get_latest_file',
-           'Map',
-           'glob_list',
-           'move_to_trash',
-           'decode',
-           ]
+__all__ = [
+    "paths",
+    "DESCRIPTOR",
+    "raise_error",
+    "valid_time",
+    "valid_date",
+    "valid_data_version",
+    "valid_single_file",
+    "valid_dir",
+    "extract_datetime",
+    "extract_file_fields",
+    "get_datasets",
+    "sort_indices",
+    "unique_dict_list",
+    "sort_dict_list",
+    "safe_move",
+    "setup_lock",
+    "get_latest_file",
+    "Map",
+    "glob_list",
+    "move_to_trash",
+    "decode",
+]
 
 # ________________ Global Variables _____________
 # (define here the global variables)
@@ -65,20 +71,18 @@ def get_descriptor():
     """
 
     class Descriptor(object, metaclass=Singleton):
-
         def __init__(self):
-
-            descriptor = paths.from_root('descriptor.json')
+            descriptor = paths.from_root("descriptor.json")
 
             # Get descriptor content
-            with open(descriptor, 'r') as file_buffer:
+            with open(descriptor, "r") as file_buffer:
                 for key, val in json.load(file_buffer).items():
                     setattr(self, key, val)
 
             # Re-organize task section
             tasks = dict()
             for task in self.tasks:
-                tasks[task['name']] = task
+                tasks[task["name"]] = task
 
             self.tasks = tasks
 
@@ -107,8 +111,9 @@ def valid_time(t, format=INPUT_DATETIME_STRFTIME):
         try:
             return datetime.strptime(t, format)
         except ValueError:
-            raise_error(f"Not a valid datetime: '{t}'.",
-                        exception=argparse.ArgumentTypeError)
+            raise_error(
+                f"Not a valid datetime: '{t}'.", exception=argparse.ArgumentTypeError
+            )
 
 
 def valid_date(t, format=TIME_DAILY_STRFORMAT):
@@ -123,8 +128,9 @@ def valid_date(t, format=TIME_DAILY_STRFORMAT):
         try:
             return datetime.strptime(t, format)
         except ValueError:
-            raise_error(f"Not a valid date: '{t}'.",
-                        exception=argparse.ArgumentTypeError)
+            raise_error(
+                f"Not a valid date: '{t}'.", exception=argparse.ArgumentTypeError
+            )
 
 
 def valid_data_version(data_version):
@@ -138,9 +144,9 @@ def valid_data_version(data_version):
         if isinstance(data_version, list):
             data_version = data_version[0]
         data_version = int(data_version)
-        return f'{data_version:02d}'
+        return f"{data_version:02d}"
     except ValueError:
-        raise_error(f'Input value for --data-version is not valid! ({data_version})')
+        raise_error(f"Input value for --data-version is not valid! ({data_version})")
 
 
 def valid_single_file(file):
@@ -158,11 +164,11 @@ def valid_single_file(file):
         else:
             raise FileNotFoundError
     except FileNotFoundError:
-        raise_error(f'Input file not found! ({file})', exception=FileNotFoundError)
+        raise_error(f"Input file not found! ({file})", exception=FileNotFoundError)
     except ValueError:
-        raise_error(f'Input file is not valid! ({file})', exception=ValueError)
+        raise_error(f"Input file is not valid! ({file})", exception=ValueError)
     except Exception as e:
-        raise_error(f'Problem with input file! ({file})', exception=e)
+        raise_error(f"Problem with input file! ({file})", exception=e)
 
 
 def valid_dir(dir):
@@ -180,11 +186,11 @@ def valid_dir(dir):
         else:
             raise IsADirectoryError
     except IsADirectoryError:
-        raise_error(f'Input directory not found! ({dir})', exception=IsADirectoryError)
+        raise_error(f"Input directory not found! ({dir})", exception=IsADirectoryError)
     except ValueError:
-        raise_error(f'Input directory is not valid! ({dir})', exception=ValueError)
+        raise_error(f"Input directory is not valid! ({dir})", exception=ValueError)
     except Exception as e:
-        raise_error(f'Problem with input directory! ({dir})', exception=e)
+        raise_error(f"Problem with input directory! ({dir})", exception=e)
 
 
 def unique_dates(utc_times):
@@ -212,14 +218,15 @@ def extract_datetime(str_datetime):
     :return: 2-elements list containing Datetime start/end time (if input Datetime has a daily format, return the day twice).
     """
 
-    str_datetime_list = str_datetime.split('-')
+    str_datetime_list = str_datetime.split("-")
     if len(str_datetime_list) == 1:
-        out_datetime = [datetime.strptime(str_datetime, '%Y%m%d')] * 2
+        out_datetime = [datetime.strptime(str_datetime, "%Y%m%d")] * 2
     elif len(str_datetime_list) == 2:
-        out_datetime = [datetime.strptime(
-            dt, '%Y%m%dT%H%M%S') for dt in str_datetime_list]
+        out_datetime = [
+            datetime.strptime(dt, "%Y%m%dT%H%M%S") for dt in str_datetime_list
+        ]
     else:
-        logger.error(f'Wrong input datetime format: {str_datetime}')
+        logger.error(f"Wrong input datetime format: {str_datetime}")
         return None
 
     return out_datetime
@@ -251,30 +258,29 @@ def get_datasets(task, task_name):
     """
 
     # Get dataset JSON file provided as an input argument (if any)
-    dataset_file = task.pipeline.get(
-        'dataset_file', default=[None], args=True)[0]
+    dataset_file = task.pipeline.get("dataset_file", default=[None], args=True)[0]
     # Get --dataset input keyword value (if any)
-    dataset_names = task.pipeline.get('dataset', default=[None], args=True)
+    dataset_names = task.pipeline.get("dataset", default=[None], args=True)
     # Get --data-version input keyword value (if any)
-    data_version = task.pipeline.get('data_version', default=[
-                                     DATA_VERSION], args=True)[0]
+    data_version = task.pipeline.get("data_version", default=[DATA_VERSION], args=True)[
+        0
+    ]
 
     # Get task output dataset description list from the descriptor.json file
-    task_output_list = DESCRIPTOR.tasks[task_name]['outputs']
+    task_output_list = DESCRIPTOR.tasks[task_name]["outputs"]
 
     # If dataset JSON file passed as an value of the --dataset_file input
     # keyword, load list of datasets to create and related data_version
     # (optional)
     if dataset_file and os.path.isfile(dataset_file):
-        with open(dataset_file, 'r') as file_buff:
+        with open(dataset_file, "r") as file_buff:
             # Loop over dataset array in the JSON file to get the name and
             # optionally the version of the output file
             dataset_to_create = []
             data_versions = []
-            for current_dataset_obj in json.load(file_buff)['dataset']:
-                dataset_to_create.append(current_dataset_obj['name'])
-                data_versions.append(
-                    current_dataset_obj.get('version', data_version))
+            for current_dataset_obj in json.load(file_buff)["dataset"]:
+                dataset_to_create.append(current_dataset_obj["name"])
+                data_versions.append(current_dataset_obj.get("version", data_version))
     elif dataset_names[0]:
         # Else if dataset list passed as values of the --dataset input keyword
         dataset_to_create = dataset_names
@@ -282,22 +288,25 @@ def get_datasets(task, task_name):
     else:
         # Else load all the output datasets listed in descriptor for the given
         # task by default
-        dataset_to_create = list(DESCRIPTOR.tasks[task_name]['outputs'].keys())
+        dataset_to_create = list(DESCRIPTOR.tasks[task_name]["outputs"].keys())
         data_versions = [data_version] * len(dataset_to_create)
 
     # Retrieve dataset info from descriptor.json
     dataset_list = []
     for i, dataset_name in enumerate(dataset_to_create):
-
         # Check if current dataset is indeed a output dataset of the task (as
         # described in the descriptor.json file)
         if dataset_name not in task_output_list:
-            logger.warning(f'{dataset_name} is not a valid dataset of the task {task_name}!')
+            logger.warning(
+                f"{dataset_name} is not a valid dataset of the task {task_name}!"
+            )
         else:
             # if yes, get description and store info in the dataset_list list
-            current_dataset = {'name': dataset_name,
-                               'version': data_versions[i],
-                               'descr': task_output_list[dataset_name]}
+            current_dataset = {
+                "name": dataset_name,
+                "version": data_versions[i],
+                "descr": task_output_list[dataset_name],
+            }
             dataset_list.append(current_dataset)
 
     return dataset_list
@@ -310,7 +319,7 @@ def unique_dict_list(list_of_dict):
     :param list_of_dict: List of dict to unify
     :return: return list inside which each dict is unique
     """
-    return [i for n, i in enumerate(list_of_dict) if i not in list_of_dict[n + 1:]]
+    return [i for n, i in enumerate(list_of_dict) if i not in list_of_dict[n + 1 :]]
 
 
 def sort_dict_list(list_of_dict, key):
@@ -333,8 +342,7 @@ def sort_indices(list_to_sort):
     :return: list of sorted indices
     """
 
-    return sorted(range(len(list_to_sort)),
-                  key=lambda k: list_to_sort[k])
+    return sorted(range(len(list_to_sort)), key=lambda k: list_to_sort[k])
 
 
 def safe_move(src, dst, ignore_patterns=[]):
@@ -355,11 +363,14 @@ def safe_move(src, dst, ignore_patterns=[]):
         if os.path.isfile(src):
             shutil.copy(src, dst, follow_symlinks=True)
         elif os.path.isdir(src):
-            shutil.copytree(src, dst,
-                            ignore=shutil.ignore_patterns(ignore_patterns),
-                            dirs_exist_ok=True)
+            shutil.copytree(
+                src,
+                dst,
+                ignore=shutil.ignore_patterns(ignore_patterns),
+                dirs_exist_ok=True,
+            )
     except Exception:
-        raise HandlingFileError(f'Cannot move {src} into {dst}!')
+        raise HandlingFileError(f"Cannot move {src} into {dst}!")
     else:
         # then delete if the file has well copied
         if os.path.exists(dst):
@@ -382,10 +393,11 @@ def setup_lock(pipeline):
     """
 
     # Retrieve lock_file input argument value
-    lock_file = pipeline.get('lock_file', default=[None], args=True)[0]
+    lock_file = pipeline.get("lock_file", default=[None], args=True)[0]
     if lock_file is not None:
         # Retrieve output directory path
         from roc.film.tools.file_helpers import get_output_dir
+
         output_dir = get_output_dir(pipeline)
 
         # Set the value of Pipeline.lock attribute filename
@@ -403,9 +415,9 @@ def sort_cdf_by_epoch(cdf, descending=False, zvar_list=[]):
     """
 
     try:
-        epoch = cdf['Epoch']
+        epoch = cdf["Epoch"]
     except Exception:
-        logger.error('Cannot get Epoch zVariable from input CDF!')
+        logger.error("Cannot get Epoch zVariable from input CDF!")
         return cdf
 
     sorted_idx = np.argsort(epoch[...])
@@ -417,18 +429,20 @@ def sort_cdf_by_epoch(cdf, descending=False, zvar_list=[]):
 
     for zvar in zvar_list:
         current_zvar = cdf[zvar][...]
-        current_zvar = current_zvar[sorted_idx, ]
+        current_zvar = current_zvar[sorted_idx,]
         cdf[zvar] = current_zvar
 
     return cdf
 
 
-def extract_file_fields(rpw_file,
-                        get_source=False,
-                        get_level=False,
-                        get_descriptor=False,
-                        get_datetime=False,
-                        get_version=False):
+def extract_file_fields(
+    rpw_file,
+    get_source=False,
+    get_level=False,
+    get_descriptor=False,
+    get_datetime=False,
+    get_version=False,
+):
     """
     Extract RPW file fields (assuming SolO file naming standards)
 
@@ -440,7 +454,7 @@ def extract_file_fields(rpw_file,
     :param get_version: return only data version field
     :return: list of file fields (or a scalar with expected field)
     """
-    fields = os.path.splitext(os.path.basename(rpw_file))[0].split('_')
+    fields = os.path.splitext(os.path.basename(rpw_file))[0].split("_")
 
     if len(fields) < 5:
         logger.warning(f'Cannot extract file fields: invalid input file "{rpw_file}"!')
@@ -497,12 +511,14 @@ class Map(dict):
         del self.__dict__[key]
 
 
-def glob_list(list_of_files):
+def glob_list(list_of_files: list) -> list:
     """
     Perform glob.glob on a list of input files.
 
-    :param list_of_files: List of input files (passed as strings)
+    :param list_of_files: List of input files (strings)
+    :type:list
     :return: list of files globbed
+    :rtype: list
     """
     output_list = []
     for current_file in list_of_files:
@@ -526,7 +542,7 @@ def move_to_trash(file_or_dir):
         is_file = False
 
     try:
-        logger.debug(f'Moving {file_or_dir} into {trash_dir}')
+        logger.debug(f"Moving {file_or_dir} into {trash_dir}")
         target_path = os.path.join(trash_dir, os.path.basename(file_or_dir))
         shutil.copyfile(file_or_dir, target_path)
         if os.path.exists(target_path):
@@ -536,7 +552,7 @@ def move_to_trash(file_or_dir):
             else:
                 shutil.rmtree(file_or_dir)
     except Exception:
-        logger.exception(f'Moving {file_or_dir} into {trash_dir} has failed!')
+        logger.exception(f"Moving {file_or_dir} into {trash_dir} has failed!")
         target_path = None
 
     return target_path
@@ -551,18 +567,17 @@ def get_trash_dir():
     """
 
     # Get trash folder path
-    if 'ROC_PIP_TRASH_DIR' in Configuration.manager['pipeline']['environment']:
-        trash_dir = Configuration.manager['pipeline'][
-            'environment.ROC_PIP_TRASH_DIR']
-    elif 'ROC_PIP_TRASH_DIR' in os.environ:
-        trash_dir = os.environ['ROC_PIP_TRASH_DIR']
+    if "ROC_PIP_TRASH_DIR" in Configuration.manager["pipeline"]["environment"]:
+        trash_dir = Configuration.manager["pipeline"]["environment.ROC_PIP_TRASH_DIR"]
+    elif "ROC_PIP_TRASH_DIR" in os.environ:
+        trash_dir = os.environ["ROC_PIP_TRASH_DIR"]
     else:
-        raise MissingProperty('ROC_PIP_TRASH_DIR variable is not defined!')
+        raise MissingProperty("ROC_PIP_TRASH_DIR variable is not defined!")
 
     return trash_dir
 
 
-def decode(binary, encoding='UTF-8'):
+def decode(binary, encoding="UTF-8"):
     """
     Decode input binary into string.
 
@@ -575,10 +590,12 @@ def decode(binary, encoding='UTF-8'):
     elif isinstance(binary, list):
         return [element.decode(encoding) for element in binary]
     elif isinstance(binary, np.ndarray):
+
         def f(x):
             return x.decode(encoding)
+
         return np.vectorize(f)(binary)
     elif isinstance(binary, bytes):
         return binary.decode(encoding)
     else:
-        raise ValueError(f'Input binary type ({type(binary)}) is not valid!')
+        raise ValueError(f"Input binary type ({type(binary)}) is not valid!")