roc-film 1.14.0__tar.gz → 1.14.1__tar.gz

{roc_film-1.14.0 → roc_film-1.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: roc-film
-Version: 1.14.0
+Version: 1.14.1
 Summary: RPW FILe Maker (FILM): Plugin to make RPW L0, L1 and HK data files
 License: CeCILL
 Author: Xavier Bonnin

{roc_film-1.14.0 → roc_film-1.14.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "roc-film"
-version = "1.14.0"
+version = "1.14.1"
 readme = "README.md"
 license = "CeCILL"
 repository = "https://gitlab.obspm.fr/ROC/Pipelines/Plugins/FILM"
@@ -26,7 +26,7 @@ xmltodict = ">=0.12,<1.0"
 poppy-core = ">0.12.0"
 poppy-pop = ">0.12.0"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 pytest = "<9"
 pytest-cov = "<6"
 pytest-timeout = "<3"

{roc_film-1.14.0 → roc_film-1.14.1}/roc/film/descriptor.json

@@ -6,10 +6,10 @@
     "description": "RPW FILe Maker (FILM): Plugin to make RPW L0, L1 and HK data files"
   },
   "release": {
-    "version": "1.14.0",
-    "date": "2024-12-06",
+    "version": "1.14.1",
+    "date": "2025-01-29",
     "author": "Xavier Bonnin <xavier.bonnin@obspm.fr>, ROC Team <roc.support@sympa.obspm.fr>",
-    "modification": "Update python dev deps and README",
+    "modification": "Hotfix in file_helper + minor updates",
     "url": "https://gitlab.obspm.fr/ROC/Pipelines/Plugins/FILM"
   },
   "tasks": [

{roc_film-1.14.0 → roc_film-1.14.1}/roc/film/tasks/file_handler.py

@@ -31,10 +31,16 @@ class MoveToProdDir(Task):
         #   the pipeline is still working on it
         #   Add a LockFile class instance to the Task class in Poppy ?
 
+        self.job_uuid = str(uuid.uuid4())
+        self.job_id = self.job_uuid[:8]
+        logger.debug(f"Task {self.job_id} is starting")
+
         # See if --no-move keyword is defined
         no_move = self.pipeline.get("no_move", default=False, args=True)
         if no_move:
-            logger.info('Skip current task "move_to_products_dir": --no-move is True')
+            logger.info(
+                f'--no-move is passed: skip current task "move_to_products_dir"\t[{self.job_id}]'
+            )
             return
 
         # Retrieve pipeline output file directory
@@ -49,7 +55,7 @@ class MoveToProdDir(Task):
 
         if not products_dir:
             logger.info(
-                'Skip current task "move_to_products_dir": products_dir argument not defined'
+                f'products_dir argument not defined: Skip current task "move_to_products_dir"\t[{self.job_id}]'
             )
         else:
             output_dirbasename = os.path.basename(output_dir)
@@ -58,6 +64,8 @@ class MoveToProdDir(Task):
             if safe_move(output_dir, target_dir, ignore_patterns=ignore_patterns):
                 logger.info(f"{output_dir} moved into {products_dir}")
 
+        logger.debug(f"Task {self.job_id} completed")
+
 
 class MoveFailedFiles(Task):
     """Move any failed files found
@@ -81,7 +89,7 @@ class MoveFailedFiles(Task):
         # Define task job ID (long and short)
         self.job_uuid = str(uuid.uuid4())
         self.job_id = self.job_uuid[:8]
-        logger.info(f"Task {self.job_id} is starting")
+        logger.debug(f"Task {self.job_id} is starting")
         try:
             self.setup_inputs()
         except Exception:
@@ -90,7 +98,7 @@ class MoveFailedFiles(Task):
             return
 
         if self.failed_file_count == 0:
-            logger.debug("No failed file(s) to move")
+            logger.debug("No failed file(s) to move\t[{self.job_id}")
         else:
             # Loop over failed files list
             for i, failed_file in enumerate(self.failed_file_list):
@@ -113,7 +121,11 @@ class MoveFailedFiles(Task):
                     # perform a safe move (i.e., copy, check and delete) into
                     # failed dir
                     if safe_move(failed_file, target_filepath):
-                        logger.info(f"{failed_file} moved into {failed_dir}")
+                        logger.warning(
+                            f"{failed_file} moved into {failed_dir}\t[{self.job_id}]"
+                        )
+
+        logger.debug(f"Task {self.job_id} completed")
 
 
 class CopyProcessedDds(Task):
@@ -125,6 +137,11 @@ class CopyProcessedDds(Task):
     name = "copy_processed_dds"
 
     def run(self):
+        # Define task job ID (long and short)
+        self.job_uuid = str(uuid.uuid4())
+        self.job_id = self.job_uuid[:8]
+        logger.debug(f"Task {self.job_id} is starting")
+
         # Get processed file target directory
         processed_dir = self.pipeline.get(
             "processed_dds_dir", default=[None], args=True
@@ -133,14 +150,14 @@ class CopyProcessedDds(Task):
         # skip task if processed_dir is None
         if processed_dir is None:
             logger.info(
-                "Skip task copy_processed_dds: No processed_dds_dir argument defined"
+                f"No processed_dds_dir argument defined: skip task copy_processed_dds\t[{self.job_id}]"
             )
             return
         elif not os.path.isdir(processed_dir):
-            logger.debug(f"Creating {processed_dir}...")
+            logger.debug(f"Creating {processed_dir}...\t[{self.job_id}]")
             os.makedirs(processed_dir)
         else:
-            logger.debug(f"process_dir set to {processed_dir}")
+            logger.debug(f"process_dir set to {processed_dir}\t[{self.job_id}]")
 
         # If processed_files list not defined in the pipeline properties,
         # initialize it
@@ -148,7 +165,9 @@ class CopyProcessedDds(Task):
         processed_files_count = len(processed_file_list)
         # Skip task if no processed files
         if processed_files_count == 0:
-            logger.info("Skip task copy_processed_dds: No processed file to move")
+            logger.info(
+                f"No processed file to move: skip task copy_processed_dds\t[{self.job_id}]"
+            )
             return
 
         # Get clear-dds keyword
@@ -161,7 +180,9 @@ class CopyProcessedDds(Task):
         for processed_file in processed_file_list.copy():
             # Check first that processed file is not in failed list
             if processed_file in failed_file_list:
-                logger.warning(f"{processed_file} found in the failed file list!")
+                logger.warning(
+                    f"{processed_file} found in the failed file list!\t[{self.job_id}]"
+                )
                 continue
 
             # Build target filepath
@@ -169,16 +190,20 @@ class CopyProcessedDds(Task):
             target_filepath = os.path.join(processed_dir, basename)
 
             # copy file
-            logger.debug(f"Copying {processed_file} into {processed_dir}")
+            logger.debug(
+                f"Copying {processed_file} into {processed_dir}\t[{self.job_id}]"
+            )
             try:
                 shutil.copyfile(processed_file, target_filepath)
             except Exception as e:
                 logger.exception(
-                    f"Copying {processed_file} into {processed_dir} has failed!"
+                    f"Copying {processed_file} into {processed_dir} has failed!\t[{self.job_id}]"
                 )
                 logger.debug(e)
             else:
-                logger.info(f"{processed_file} copied into {target_filepath}")
+                logger.info(
+                    f"{processed_file} copied into {target_filepath}\t[{self.job_id}]"
+                )
 
             # Remove current file from the list in pipeline properties
             processed_file_list.remove(processed_file)
@@ -186,7 +211,7 @@ class CopyProcessedDds(Task):
             # if clear-dds keyword is passed, then remove processed Dds
             if clear_dds:
                 os.remove(processed_file)
-                logger.debug(f"{processed_file} deleted")
+                logger.debug(f"{processed_file} deleted\t[{self.job_id}]")
 
 
 class CopyFailedDds(Task):
@@ -198,17 +223,24 @@ class CopyFailedDds(Task):
     name = "copy_failed_dds"
 
     def run(self):
+        # Define task job ID (long and short)
+        self.job_uuid = str(uuid.uuid4())
+        self.job_id = self.job_uuid[:8]
+        logger.debug(f"Task {self.job_id} is starting")
+
         # Get failed file target directory
         failed_dir = self.pipeline.get("failed_dds_dir", default=[None], args=True)[0]
         # skip task if failed_dir is None
         if failed_dir is None:
-            logger.info("Skip task copy_failed_dds: No failed_dds_dir argument defined")
+            logger.info(
+                f"No failed_dds_dir argument defined: skip task copy_failed_dds\t[{self.job_id}]"
+            )
             return
         elif not os.path.isdir(failed_dir):
-            logger.debug(f"Creating {failed_dir}...")
+            logger.debug(f"Creating {failed_dir}...\t[{self.job_id}]")
             os.makedirs(failed_dir)
         else:
-            logger.debug(f"failed_dir set to {failed_dir}")
+            logger.debug(f"failed_dir set to {failed_dir}\t[{self.job_id}]")
 
         # If failed_files list not defined in the pipeline properties,
         # initialize it
@@ -216,7 +248,9 @@ class CopyFailedDds(Task):
         failed_files_count = len(failed_file_list)
         # Skip task if no failed dds files
         if failed_files_count == 0:
-            logger.info("Skip task copy_failed_dds: No failed file to move")
+            logger.info(
+                f"No failed file to move: skip task copy_failed_dds\t[{self.job_id}]"
+            )
             return
 
         # Get clear-dds keyword
@@ -229,14 +263,18 @@ class CopyFailedDds(Task):
             target_filepath = os.path.join(failed_dir, basename)
 
             # copy file
-            logger.debug(f"Copying {failed_file} into {failed_dir}")
+            logger.debug(f"Copying {failed_file} into {failed_dir}\t[{self.job_id}]")
             try:
                 shutil.copyfile(failed_file, target_filepath)
             except Exception as e:
-                logger.exception(f"Copying {failed_file} into {failed_dir} has failed!")
+                logger.exception(
+                    f"Copying {failed_file} into {failed_dir} has failed!\t[{self.job_id}]"
+                )
                 logger.debug(e)
             else:
-                logger.info(f"{failed_file} copied into {target_filepath}")
+                logger.info(
+                    f"{failed_file} copied into {target_filepath}\t[{self.job_id}]"
+                )
 
             # Remove current file from the list in pipeline properties
             failed_file_list.remove(failed_file)
@@ -244,14 +282,14 @@ class CopyFailedDds(Task):
             # if clear-dds keyword is passed, then remove processed Dds
             if clear_dds:
                 os.remove(failed_file)
-                logger.debug(f"{failed_file} deleted")
+                logger.debug(f"{failed_file} deleted\t[{self.job_id}]")
 
         # Get failed tmraw list
         failed_tmraw_list = self.pipeline.get("failed_tmraw", default=[])
         failed_tmraw_count = len(failed_tmraw_list)
         # Skip task if no failed tmraw
         if failed_tmraw_count == 0:
-            logger.debug("No failed tmraw to write")
+            logger.debug("No failed tmraw to write\t[{self.job_id}]")
             return
         else:
             # Else save list of failed tmraw into text file
@@ -260,5 +298,5 @@ class CopyFailedDds(Task):
                 fw.writelines(failed_tmraw_list)
             logger.info(
                 f"{failed_tmraw_count} failed TmRaw entries "
-                f"saved into {tmraw_failed_file}"
+                f"saved into {tmraw_failed_file}\t[{self.job_id}]"
             )

{roc_film-1.14.0 → roc_film-1.14.1}/roc/film/tasks/l0_to_hk.py

@@ -57,7 +57,7 @@ def L0ToHk(task):
 
     # Define task job ID (long and short)
     job_uuid = str(uuid.uuid4())
-    job_id = f"L0ToHk-{job_uuid[:8]}"
+    job_id = f"{job_uuid[:8]}"
     logger.info(f"Task {job_id} is starting")
 
     try:

{roc_film-1.14.0 → roc_film-1.14.1}/roc/film/tasks/l0_to_l1_bia_current.py

@@ -4,6 +4,7 @@
 """Contains task to create the RPW L1 Bias current CDF files."""
 
 import os
+import uuid
 
 from poppy.core.logger import logger
 from poppy.core.target import FileTarget
@@ -31,6 +32,11 @@ def L0ToL1BiaCurrent(task):
     :return:
     """
 
+    # Define task job ID (long and short)
+    job_uuid = str(uuid.uuid4())
+    task.job_id = f"{job_uuid[:8]}"
+    logger.info(f"Task {task.job_id} is starting")
+
     # Get list of input l0 file(s)
     l0_file_list = task.inputs["l0_file"].filepath
 

{roc_film-1.14.0 → roc_film-1.14.1}/roc/film/tools/file_helpers.py

@@ -572,19 +572,28 @@ def l0_to_trange_cdf(
     l0_file_len = len(l0_file_list)
 
     # Get L0 files time_min/time_max
-    l0_time_min, l0_time_max = get_l0_trange(l0_file_list)
+    try:
+        l0_time_min, l0_time_max = get_l0_trange(l0_file_list)
+        if None in l0_time_min or None in l0_time_max:
+            raise ValueError
+    except TypeError:
+        logger.error(f'Input "l0_files" must be a list!\t[{task.job_id}]')
+        return []
+    except ValueError:
+        logger.error(f"Output L0 time min. or max. list is not valid!\t[{task.job_id}]")
+        return []
 
     # Get start_time for output CDF (use time min of L0 files if not defined)
     if not start_time:
         start_time = task.pipeline.get("start_time", default=[min(l0_time_min)])[0]
 
-    logger.debug(f"start_time value is {start_time}    [{task.job_id}]")
+    logger.debug(f"start_time value is {start_time}\t[{task.job_id}]")
 
     # Get end_time for output CDF (use time max of L0 files if not defined)
     if not end_time:
         end_time = task.pipeline.get("end_time", default=[max(l0_time_max)])[0]
 
-    logger.debug(f"end_time value is {end_time}    [{task.job_id}]")
+    logger.debug(f"end_time value is {end_time}\t[{task.job_id}]")
 
     # Loops over each output dataset to produce for the current task
     for current_dataset in dataset_list:
@@ -594,7 +603,7 @@ def l0_to_trange_cdf(
 
         logger.debug(
             "Running file production for the dataset "
-            f"{dataset_name} (V{data_version})    [{task.job_id}]"
+            f"{dataset_name} (V{data_version})\t[{task.job_id}]"
         )
 
         # get the path to the master CDF file of this dataset
@@ -612,14 +621,14 @@ def l0_to_trange_cdf(
         # Check existence
         if not master_path:
             raise FileNotFoundError(
-                "{0} master CDF not found for the dataset {1}!    [{2}]".format(
+                "{0} master CDF not found for the dataset {1}!\t[{2}]".format(
                     master_pattern, dataset_name, task.job_id
                 )
             )
         else:
             master_path = sorted(master_path)[-1]
             logger.info(
-                'Producing dataset "{0}" with the master CDF "{1}"    [{2}]'.format(
+                'Producing dataset "{0}" with the master CDF "{1}"\t[{2}]'.format(
                     dataset_name, master_path, task.job_id
                 )
             )
@@ -635,12 +644,12 @@ def l0_to_trange_cdf(
                 if l0_time_max[i] < start_time or l0_time_min[i] > end_time:
                     logger.debug(
                         f"{l0_file} is outside the time range: "
-                        f"[{start_time}, {end_time}], skip it    [{task.job_id}]"
+                        f"[{start_time}, {end_time}], skip it\t[{task.job_id}]"
                     )
                     continue
                 else:
                     logger.debug(
-                        f"Processing {l0_file} [{l0_file_len - i - 1}]    [{task.job_id}]"
+                        f"Processing {l0_file} [{l0_file_len - i - 1}]\t[{task.job_id}]"
                     )
 
                 # Append current l0 file to parent list
@@ -655,7 +664,7 @@ def l0_to_trange_cdf(
                 ):
                     logger.info(
                         f"No expected packet found for {dataset_name}"
-                        f" in {l0_file} [{','.join(expected_packet)}]    [{task.job_id}]"
+                        f" in {l0_file} [{','.join(expected_packet)}]\t[{task.job_id}]"
                     )
                     continue
 
@@ -664,18 +673,20 @@ def l0_to_trange_cdf(
                 # as the dataset alias in the descriptor
                 func = dataset_func.get(dataset_name)
                 if func is None:
-                    logger.error(f"No function found for {dataset_name}")
+                    logger.error(
+                        f"No function found for {dataset_name}\t[{task.job_id}]"
+                    )
                     failed_files.append(l0_file)
                     break
 
                 # call the dataset-related function
                 try:
-                    logger.debug(f"Running {func}    [{task.job_id}]")
+                    logger.debug(f"Running {func}\t[{task.job_id}]")
                     result = func(l0, task)
                 except Exception as e:
                     # TODO catch exception in the ROC database
                     logger.exception(
-                        f'Running "{func}" function has failed    [{task.job_id}]: \n{e}'
+                        f'Running "{func}" function has failed\t[{task.job_id}]: \n{e}'
                     )
                     # TODO - Add the current failed dataset processing to failed_files
                     failed_files.append(l0_file)
@@ -685,12 +696,12 @@ def l0_to_trange_cdf(
                 if result is None or result.shape[0] == 0:
                     logger.debug(
                         f"Returned {dataset_name} dataset array"
-                        f" is empty for {l0_file}    [{task.job_id}]"
+                        f" is empty for {l0_file}\t[{task.job_id}]"
                     )
                 else:
                     logger.debug(
                         f"{result.shape[0]} {dataset_name} dataset samples"
-                        f" returned from {l0_file}    [{task.job_id}]"
+                        f" returned from {l0_file}\t[{task.job_id}]"
                     )
 
                     # If data is empty
@@ -706,7 +717,7 @@ def l0_to_trange_cdf(
         if nrec == 0:
             logger.warning(
                 "No data for dataset"
-                f" {dataset_name}: skip output cdf creation    [{task.job_id}]"
+                f" {dataset_name}: skip output cdf creation\t[{task.job_id}]"
             )
             continue
 
@@ -716,12 +727,12 @@ def l0_to_trange_cdf(
         # Generation date
         generation_date = datetime.utcnow().strftime(INPUT_DATETIME_STRFTIME)
         logger.debug(
-            f'Set "Generation_date" attr. value to {generation_date}    [{task.job_id}]'
+            f'Set "Generation_date" attr. value to {generation_date}\t[{task.job_id}]'
         )
 
         # file ID
         file_id = str(uuid.uuid4())
-        logger.debug(f'Set "File_ID" attr. value to {file_id}    [{task.job_id}]')
+        logger.debug(f'Set "File_ID" attr. value to {file_id}\t[{task.job_id}]')
 
         # Re-define datetime and parents g.attribute for time range CDF data
         # products
@@ -781,7 +792,7 @@ def l0_to_trange_cdf(
             metadata["SPICE_KERNELS"] = sclk_file[-1]
         else:
             logger.warning(
-                f"No SPICE SCLK kernel saved for {filepath}    [{task.job_id}]"
+                f"No SPICE SCLK kernel saved for {filepath}\t[{task.job_id}]"
             )
 
         # open the target to update its status according to errors etc
@@ -808,27 +819,27 @@ def l0_to_trange_cdf(
                     else:
                         logger.warning(
                             f"No data found between {start_time} and {end_time}"
-                            f" to be written into {filepath}    [{task.job_id}]"
+                            f" to be written into {filepath}\t[{task.job_id}]"
                         )
 
                 except Exception as e:
                     logger.exception(
-                        f"{filepath} production has failed    [{task.job_id}]:\n{e}"
+                        f"{filepath} production has failed\t[{task.job_id}]:\n{e}"
                     )
                     cdf.attrs["Validate"] = "-1"
                     failed_files.append(filepath)
 
             if nrec == 0:
-                logger.info(f"Removing empty file {filepath}...    [{task.job_id}]")
+                logger.info(f"Removing empty file {filepath}...\t[{task.job_id}]")
                 os.remove(filepath)
                 filepath = ""
             elif os.path.isfile(filepath):
                 processed_files.append(filepath)
-                logger.info(f"{filepath} saved    [{task.job_id}]")
+                logger.info(f"{filepath} saved\t[{task.job_id}]")
                 output_file_list.append(filepath)
             else:
                 failed_files.append(filepath)
-                logger.error(f"Writing {filepath} has failed!    [{task.job_id}]")
+                logger.error(f"Writing {filepath} has failed!\t[{task.job_id}]")
 
             # Set output target filepath
             target.filepath = filepath
@@ -849,8 +860,7 @@ def get_l0_trange(l0_files: list, minmax: bool = False) -> tuple:
     """
 
     if not isinstance(l0_files, list):
-        logger.error('Input "l0_files" must be a list!')
-        return None, None
+        raise TypeError
 
     # Get number of l0_files
     nl0 = len(l0_files)