roc-film 1.14.1__tar.gz → 1.14.2__tar.gz

{roc_film-1.14.1 → roc_film-1.14.2}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.3
 Name: roc-film
-Version: 1.14.1
+Version: 1.14.2
 Summary: RPW FILe Maker (FILM): Plugin to make RPW L0, L1 and HK data files
-License: CeCILL
+License: CECILL-2.1
 Author: Xavier Bonnin
 Author-email: xavier.bonnin@obspm.fr
 Requires-Python: >=3.9,<4
-Classifier: License :: Other/Proprietary License
+Classifier: License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10

{roc_film-1.14.1 → roc_film-1.14.2}/pyproject.toml

@@ -4,9 +4,9 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "roc-film"
-version = "1.14.1"
+version = "1.14.2"
 readme = "README.md"
-license = "CeCILL"
+license = "CECILL-2.1"
 repository = "https://gitlab.obspm.fr/ROC/Pipelines/Plugins/FILM"
 description = "RPW FILe Maker (FILM): Plugin to make RPW L0, L1 and HK data files"
 authors = [ "Xavier Bonnin <xavier.bonnin@obspm.fr>", "ROC Team <roc.support@sympa.obspm.fr>",]

{roc_film-1.14.1 → roc_film-1.14.2}/roc/film/commands.py

@@ -1142,24 +1142,6 @@ class CdfPostProCommand(Command):
             required=True,
         )
 
-        parser.add_argument(
-            "--rpw-obs-json",
-            help="""
-            List of RPW SoopKitchen export JSON files. Pattern can also be passed.
-            """,
-            type=str,
-            nargs="+",
-        )
-
-        parser.add_argument(
-            "--rpw-ior-xml",
-            help="""
-            List of RPW IOR XML files. Pattern or ZIP archive can be also passed.
-            """,
-            type=str,
-            nargs="+",
-        )
-
         parser.add_argument(
             "--options",
             help=f"""

{roc_film-1.14.1 → roc_film-1.14.2}/roc/film/constants.py

@@ -141,7 +141,6 @@ TIMEOUT = 14400
 
 # Allowed values for keyword --options in l1_post_pro command
 CDF_POST_PRO_OPTS_ARGS = [
-    "soop_type",
     "obs_id",
     "resize_wf",
     "update_cdf",

{roc_film-1.14.1 → roc_film-1.14.2}/roc/film/descriptor.json

@@ -6,10 +6,10 @@
     "description": "RPW FILe Maker (FILM): Plugin to make RPW L0, L1 and HK data files"
   },
   "release": {
-    "version": "1.14.1",
-    "date": "2025-01-29",
+    "version": "1.14.2",
+    "date": "2025-01-30",
     "author": "Xavier Bonnin <xavier.bonnin@obspm.fr>, ROC Team <roc.support@sympa.obspm.fr>",
-    "modification": "Hotfix in file_helper + minor updates",
+    "modification": "Update cdf_postpro",
     "url": "https://gitlab.obspm.fr/ROC/Pipelines/Plugins/FILM"
   },
   "tasks": [

{roc_film-1.14.1 → roc_film-1.14.2}/roc/film/tasks/cdf_postpro.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-"""Module to create the RPW L1 SBM1/SBM2 CDF files."""
+"""Module to perform some post-processing on the RPW CDF files."""
 
 import json
 import os
@@ -22,13 +22,11 @@ from poppy.core.target import FileTarget
 
 from roc.rpl.time import Time
 from roc.dingo.models.data import EventLog
+from roc.dingo.models.packet import TcLog
 from roc.dingo.tools import query_db
 from roc.dingo.constants import PIPELINE_DATABASE
 
 from roc.film import (
-    TIME_JSON_STRFORMAT,
-    TIME_DOY1_STRFORMAT,
-    TIME_DOY2_STRFORMAT,
     INPUT_DATETIME_STRFTIME,
 )
 from roc.film.tools.file_helpers import is_output_dir, get_output_dir
@@ -80,14 +78,6 @@ class CdfPostPro(Task):
             self.pipeline.exit()
             return
 
-        # Get list of RPW Soopkitchen Observations JSON files
-        self.rpw_obs_json_list = glob_list(
-            self.pipeline.get("rpw_obs_json", default=[])
-        )
-
-        # Get list of RPW IOR XML files
-        self.rpw_ior_xml_list = glob_list(self.pipeline.get("rpw_ior_xml", default=[]))
-
         # Get post-processing options
         self.options = [
             opt.lower()
@@ -167,7 +157,6 @@ class CdfPostPro(Task):
         self.session = Connector.manager[PIPELINE_DATABASE].session
 
         # Initialize some class variables
-        self.soop_type_list = []
         self.obs_id_list = []
         self.event_log = None
 
@@ -227,21 +216,16 @@ class CdfPostPro(Task):
                     # Get time range of the input L1 CDF
                     self.epoch = cdf["Epoch"][...]
                     self.nrec = self.epoch.shape[0]
-                    self.time_min = self.epoch[0]
-                    self.time_max = self.epoch[-1]
+                    self.time_min = min(self.epoch)
+                    self.time_max = max(self.epoch)
                     logger.info(
                         f"{self.current_file} has {self.nrec} records "
                         f"between {self.time_min} "
                         f"and {self.time_max}\t[{self.job_id}]"
                     )
 
-                    # Set SOOP_TYPE global attribute from RPW SOOPKitchen
-                    # export observation JSON files
-                    if "soop_type" in self.options:
-                        self._set_soop_type(cdf)
-
-                    # Set OBS_ID global attribute from IOR XML files (get
-                    # observationID)
+                    # Set OBS_ID global attribute
+                    # from unique_id entries in pipeline.tc_log database table
                     if "obs_id" in self.options:
                         self._set_obs_id(cdf)
 
@@ -410,88 +394,6 @@ class CdfPostPro(Task):
 
         return is_succeeded
 
-    def _set_soop_type(self, cdf_obj):
-        """
-        Set input CDF file with expected value for SOOP_TYPE g.attribute.
-
-        :param cdf_obj: CDF to update (passed as a spacepy.pycdf.CDF class instance)
-        :return: True if SOOP_TYPE has been set, False otherwise
-        """
-
-        logger.info(
-            "Setting SOOP_TYPE global attribute "
-            f"in {self.current_file} ...\t[{self.job_id}]"
-        )
-
-        # Get list of SOOP type from RPW soopkitchen observation json files
-        if not self.soop_type_list:
-            logger.info(
-                "Extracting soopType elements from input "
-                f"list of {len(self.rpw_obs_json_list)} RPW SoopKitchen JSON files...\t[{self.job_id}]"
-            )
-            self.soop_type_list = CdfPostPro.get_soop_type(self.rpw_obs_json_list)
-
-        # Only keep list of soop type betwen time_min and time_max
-        soop_type_list = [
-            current_soop_type["soopType"]
-            for current_soop_type in self.soop_type_list
-            if (
-                datetime.strptime(current_soop_type["startDate"], TIME_JSON_STRFORMAT)
-                <= self.time_max
-                and datetime.strptime(current_soop_type["endDate"], TIME_JSON_STRFORMAT)
-                >= self.time_min
-            )
-        ]
-
-        soop_type_len = len(soop_type_list)
-        if soop_type_len == 0:
-            logger.info(
-                "No Soop Type value found "
-                f"between {self.time_min} and {self.time_max}\t[{self.job_id}]"
-            )
-            cdf_obj.attrs["SOOP_TYPE"] = "none"
-            return False
-        else:
-            cdf_obj.attrs["SOOP_TYPE"] = list(set(soop_type_list))
-            logger.debug(f"SOOP_TYPE = {soop_type_list} in {self.current_file}")
-            logger.info(
-                f"{soop_type_len} entries set for "
-                f"SOOP_TYPE in {self.current_file}\t[{self.job_id}]"
-            )
-
-        # make sure to save the change
-        cdf_obj.save()
-
-        return True
-
-    @staticmethod
-    def get_soop_type(rpw_obs_json_list):
-        """
-        Return list of SOOP_TYPE values for a given set of input RPW SoopKitchen observation JSON files
-
-        :param rpw_obs_json_list: List of input RPW SK JSON files
-        :return: list of SOOP_TYPE values found
-        """
-
-        # Define sub-method
-        def extract_soop_type(json_file):
-            """Extract soopType from input JSON"""
-
-            # Open JSON file
-            with open(json_file, "r") as json_buff:
-                data = json.load(json_buff)
-
-            # Retrieve all "soopType" field from file
-            return data["soops"]
-
-        # Initialize output list
-        soop_type_list = []
-
-        for current_json in rpw_obs_json_list:
-            soop_type_list.extend(extract_soop_type(current_json))
-
-        return soop_type_list
-
     def _set_obs_id(self, cdf_obj):
         """
         Set input CDF file with expected value for OBS_ID g.attribute.
@@ -507,19 +409,19 @@ class CdfPostPro(Task):
 
         # Get list of RPW TC obs id values
         if not self.obs_id_list:
-            logger.info(
-                f"Extracting uniqueID elements from "
-                f"input list of {len(self.rpw_ior_xml_list)} RPW IOR files...\t[{self.job_id}]"
+            logger.debug(
+                f"Requesting pipeline.tc_log table entries from database...\t[{self.job_id}]"
             )
-            self.obs_id_list = CdfPostPro.get_ior_obs_id(self.rpw_ior_xml_list)
+            self.obs_id_list = self.get_tc_log_data()
 
         # Keep only obs_id between time_min and time_max
         obs_id_list = list(
             set(
                 [
-                    current_tc[1]
+                    current_tc["unique_id"]
                     for current_tc in self.obs_id_list
-                    if self.time_max >= current_tc[0] >= self.time_min and current_tc[1]
+                    if current_tc["unique_id"]
+                    and self.time_max >= current_tc["utc_time"] >= self.time_min
                 ]
             )
         )
@@ -546,87 +448,22 @@ class CdfPostPro(Task):
 
         return True
 
-    @staticmethod
-    def get_ior_obs_id(rpw_ior_xml_list):
+    def get_tc_log_data(self):
         """
-        Return list of OBS_ID values from
-        an input list of RPW IOR XML files
+        Return list of [unique_id, utc_time] entries from
+        pipeline.tc_log database table.
 
-        :param rpw_ior_xml_list: List of input RPW TC XML files.
-                                 (ZIP files containing IOR XML can be also passed).
-        :return: list of OBS_ID values found
+        :return: tc_log data entries found
         """
-        import zipfile
-
-        # Define sub-method
-        def extract_obs_id(xml):
-            """
-            Extract OBS_ID from input XML
-
-            :param xml: input IOR XML stream
-            :return: List of (time, observationID) values extracted from IOR
-            """
-            import xmltodict
-
-            # Convert input IOR XML stream into dictionary
-            logger.debug(f"Parsing {xml.name} ...")
-            data = xmltodict.parse(xml.read())
 
-            # Extract list of sequences
-            sequence_list = data["planningData"]["commandRequests"]["occurrenceList"][
-                "sequence"
-            ]
-
-            # Make sure that returned sequence_list is a list
-            # (If only one sequence tag is found in the XML
-            # the xml_to_dict method returns a collections.OrderedDict()
-            # instance).
-            if not isinstance(sequence_list, list):
-                sequence_list = [sequence_list]
-
-            # Retrieve all "observationID" field from input TC XML file
-            # Return as a list of tuple (ExecutionTime, observationID)
-            ior_seq_list = []
-            for current_seq in sequence_list:
-                # Make sure to retrieve executionTime with the
-                # right time format (two are possible)
-                for current_strtformat in [TIME_DOY1_STRFORMAT, TIME_DOY2_STRFORMAT]:
-                    current_time = cast_ior_seq_datetime(
-                        current_seq, current_strtformat
-                    )
-                    if current_time is not None:
-                        break
-                current_obsid = current_seq["observationID"]
-
-                ior_seq_list.append((current_time, current_obsid))
-
-            return ior_seq_list
-
-        # Initialize output list
-        obs_id_list = []
-
-        for current_file in rpw_ior_xml_list:
-            if not os.path.basename(current_file).startswith("IOR"):
-                logger.debug(f"{current_file} not a valid RPW IOR file, skip it")
-                continue
-
-            if zipfile.is_zipfile(current_file):
-                with zipfile.ZipFile(current_file, "r") as zip_stream:
-                    for current_xml in zip_stream.namelist():
-                        with zip_stream.open(current_xml, "r") as ior_xml:
-                            if ior_xml.name.startswith("IOR") and ior_xml.name.endswith(
-                                ".SOL"
-                            ):
-                                obs_id_list.extend(extract_obs_id(ior_xml))
-                            else:
-                                logger.debug(
-                                    f"{current_xml} is not a valid RPW IOR XML file, skip it"
-                                )
-            else:
-                with open(current_file, "r") as ior_xml:
-                    obs_id_list.extend(extract_obs_id(ior_xml))
+        tc_log_data = query_db(
+            self.session,
+            [TcLog.unique_id, TcLog.utc_time],
+            filters=(TcLog.tc_exe_state == "PASSED"),
+            to_dict="records",
+        )
 
-        return obs_id_list
+        return tc_log_data
 
     def _update_cdf(self, cdf_obj):
         """
@@ -824,11 +661,11 @@ class CdfPostPro(Task):
                     # Set 7th bit (X000000)
                     bitmask[w] = bitmask[w] | 64
 
-            logger.debug(
-                f"Set {len(w)} QUALITY_BITMASK records for {row['label']} "
-                f"between {row['start_time']} "
-                f"and {row['end_time']}\t[{self.job_id}]"
-            )
+                logger.debug(
+                    f"Set {len(w)} QUALITY_BITMASK records for {row['label']} "
+                    f"between {row['start_time']} "
+                    f"and {row['end_time']}\t[{self.job_id}]"
+                )
 
         # Save quality_bitmask
         cdf_obj["QUALITY_BITMASK"] = bitmask[...]