roc-film 1.13.4__py3-none-any.whl → 1.14.0__py3-none-any.whl

roc/film/tools/l0.py

@@ -9,11 +9,18 @@ from poppy.core.logger import logger
 from roc.idb.parsers.idb_parser import IDBParser
 from roc.rpl.constants import VALID_PACKET
 
-from roc.film import L0ProdFailure, MIN_DATETIME, MAX_DATETIME, PACKET_TYPE, TIME_L0_STRFORMAT, TIME_ISO_STRFORMAT
+from roc.film import (
+    L0ProdFailure,
+    MIN_DATETIME,
+    MAX_DATETIME,
+    PACKET_TYPE,
+    TIME_L0_STRFORMAT,
+    TIME_ISO_STRFORMAT,
+)
 from roc.film.tools.file_helpers import is_packet
 from roc.film.tools import valid_data_version, Map, decode
 
-__all__ = ['L0']
+__all__ = ["L0"]
 
 
 class L0:
@@ -22,24 +29,30 @@ class L0:
     # Class attributes
     h5py_str_dtype = h5py.special_dtype(vlen=str)
 
-    l0_file_prefix = 'solo_L0_rpw'
-
-    L0_FIELDS = {'TM': ['utc_time',
-                        'binary',
-                        'idb',
-                        'compressed',
-                        'data_field_header',
-                        'packet_header',
-                        'source_data'],
-                 'TC': ['utc_time',
-                        'binary',
-                        'idb',
-                        'tc_ack_state',
-                        'unique_id',
-                        'sequence_name',
-                        'packet_header',
-                        'data_field_header',
-                        'application_data']}
+    l0_file_prefix = "solo_L0_rpw"
+
+    L0_FIELDS = {
+        "TM": [
+            "utc_time",
+            "binary",
+            "idb",
+            "compressed",
+            "data_field_header",
+            "packet_header",
+            "source_data",
+        ],
+        "TC": [
+            "utc_time",
+            "binary",
+            "idb",
+            "tc_ack_state",
+            "unique_id",
+            "sequence_name",
+            "packet_header",
+            "data_field_header",
+            "application_data",
+        ],
+    }
 
     def __init__(self, filepath=None):
         self.filepath = filepath
@@ -67,7 +80,7 @@ class L0:
     @staticmethod
     def extract_header(filepath):
         if filepath and os.path.isfile(filepath):
-            with h5py.File(filepath, 'r') as file:
+            with h5py.File(filepath, "r") as file:
                 header = {key: val for key, val in file.attrs.items()}
         else:
             header = {}
@@ -83,9 +96,9 @@ class L0:
         :return: data version as a string of the form "XX", where XX are 2 digit integers
         """
         try:
-            return os.path.splitext(os.path.basename(filepath))[0].split('_')[4][1:]
+            return os.path.splitext(os.path.basename(filepath))[0].split("_")[4][1:]
         except Exception:
-            return ''
+            return ""
 
     @staticmethod
     def version_from_header(filepath):
@@ -97,15 +110,13 @@ class L0:
         :return: data version as a string of the form "XX", where XX are 2 digit integers
         """
         try:
-            with h5py.File(filepath, 'r') as file:
-                return valid_data_version(file.attrs['Data_version'])
+            with h5py.File(filepath, "r") as file:
+                return valid_data_version(file.attrs["Data_version"])
         except Exception:
-            return ''
+            return ""
 
     @staticmethod
-    def to_hdf5(filepath,
-                packet_parser=None,
-                metadata={}):
+    def to_hdf5(filepath, packet_parser=None, metadata={}):
         """
         Given an PacketParser object like and the target on which to put the packets data,
         generate the L0 HDF5 file in format with all the necessary meta information.
@@ -114,21 +125,21 @@ class L0:
         """
 
         if os.path.isfile(filepath):
-            action = 'a'
+            action = "a"
         else:
-            action = 'w'
+            action = "w"
 
         # Create instance of L0 class
         l0 = L0()
 
         try:
             with h5py.File(filepath, action) as hdf5:
-
                 # Add l0 root metadata
                 for key, val in metadata.items():
                     if key in hdf5.attrs:
-                        logger.debug(f'{key} L0 metadata already defined: '
-                                     f'{hdf5.attrs[key]}!')
+                        logger.debug(
+                            f"{key} L0 metadata already defined: {hdf5.attrs[key]}!"
+                        )
                     hdf5.attrs[key] = val
 
                 # write packets data into the file
@@ -136,11 +147,11 @@ class L0:
                     l0._to_hdf5(hdf5, packet_parser)
 
         except Exception:
-            logger.exception(f'{filepath} production has failed!')
+            logger.exception(f"{filepath} production has failed!")
             raise L0ProdFailure()
 
         if not L0.is_l0(filepath):
-            logger.exception(f'{filepath} not saved correctly!')
+            logger.exception(f"{filepath} not saved correctly!")
             raise L0ProdFailure()
 
         return filepath
@@ -156,23 +167,22 @@ class L0:
 
         # create a group for TM and an other for TC packets
         # and initialize counters
-        if 'TM' not in self.hdf5.keys():
-            tm = self.hdf5.create_group('TM')
+        if "TM" not in self.hdf5.keys():
+            tm = self.hdf5.create_group("TM")
             tm_count = 0
         else:
-            tm = self.hdf5['TM']
-            tm_count = tm.attrs['COUNT']
+            tm = self.hdf5["TM"]
+            tm_count = tm.attrs["COUNT"]
 
-        if 'TC' not in self.hdf5.keys():
-            tc = self.hdf5.create_group('TC')
+        if "TC" not in self.hdf5.keys():
+            tc = self.hdf5.create_group("TC")
             tc_count = 0
         else:
-            tc = self.hdf5['TC']
-            tc_count = tc.attrs['COUNT']
+            tc = self.hdf5["TC"]
+            tc_count = tc.attrs["COUNT"]
 
         # loop over registered packets
         for packet in packet_parser.packet.manager.instances:
-
             # if the packet is empty do not try to add it
             # same thing if it is a compressed one
             if packet.is_compressed or packet.counter == 0:
@@ -182,7 +192,9 @@ class L0:
             criterion = packet.status == VALID_PACKET
             new_counter = len(packet.status[criterion])
             if not packet.counter == new_counter:
-                logger.warning(f'{packet.name} has {packet.counter - new_counter} invalid packets')
+                logger.warning(
+                    f"{packet.name} has {packet.counter - new_counter} invalid packets"
+                )
                 # Loop over packet attributes
                 for key, val in packet.__dict__.items():
                     try:
@@ -195,7 +207,7 @@ class L0:
                 packet.counter = len(packet.status)
 
             # send the packet to the good type
-            logger.debug('Insert {0} into L0 file'.format(packet))
+            logger.debug("Insert {0} into L0 file".format(packet))
             if packet.type == 0:
                 # call the method to store into hdf5 for each packet
                 tm_count += self.packet_to_hdf5(packet, tm)
@@ -204,8 +216,8 @@ class L0:
                 tc_count += self.packet_to_hdf5(packet, tc, tc=True)
 
         # Set TM / TC packet counters
-        tm.attrs['COUNT'] = tm_count
-        tc.attrs['COUNT'] = tc_count
+        tm.attrs["COUNT"] = tm_count
+        tc.attrs["COUNT"] = tc_count
 
         return tm, tc
 
@@ -227,85 +239,95 @@ class L0:
         # Store some metadata of current packet
         # as H5 datasets in L0 packet group
         # Mapping dict of the attributes to add
-        attrs_dict = {'SRDB_ID': (packet.__dict__, 'srdb_id'),
-                      'PACKET_CATEGORY': (packet.__dict__, 'category')}
+        attrs_dict = {
+            "SRDB_ID": (packet.__dict__, "srdb_id"),
+            "PACKET_CATEGORY": (packet.__dict__, "category"),
+        }
         self.add_packet_attrs(packet, attrs_dict, packet_group)
 
         # Store number of packets
         try:
-            if 'COUNT' in packet_group.attrs:
-                packet_count = packet_group.attrs['COUNT']
+            if "COUNT" in packet_group.attrs:
+                packet_count = packet_group.attrs["COUNT"]
             else:
                 packet_count = 0
 
-            packet_group.attrs['COUNT'] = packet_count + packet.counter
+            packet_group.attrs["COUNT"] = packet_count + packet.counter
         except Exception:
-            logger.warning(f'No counter found for packet {packet.name}')
+            logger.warning(f"No counter found for packet {packet.name}")
 
         # packet_header, data_field_header, source_data ou application_data
 
         # inside it, create groups for the packet_header,
         # data_field_header and the source_data/application_data
-        self._packet_to_hdf5('packet_header', packet.header, packet_group)
-        self._packet_to_hdf5('data_field_header',
-                             packet.data_header, packet_group)
+        self._packet_to_hdf5("packet_header", packet.header, packet_group)
+        self._packet_to_hdf5("data_field_header", packet.data_header, packet_group)
 
         if tc:
-            self._packet_to_hdf5('application_data', packet.data, packet_group)
+            self._packet_to_hdf5("application_data", packet.data, packet_group)
 
             # Add TC specific info:
             # array of Ack. TC acceptance/execution state
-            self._packet_to_hdf5('tc_ack_state',
-                                 packet.tc_ack_state.astype(
-                                     self.h5py_str_dtype),
-                                 packet_group)
+            self._packet_to_hdf5(
+                "tc_ack_state",
+                packet.tc_ack_state.astype(self.h5py_str_dtype),
+                packet_group,
+            )
 
             # unique ID as a string
-            self._packet_to_hdf5('unique_id',
-                                 packet.unique_id.astype(self.h5py_str_dtype),
-                                 packet_group)
+            self._packet_to_hdf5(
+                "unique_id", packet.unique_id.astype(self.h5py_str_dtype), packet_group
+            )
 
             # sequence name ID as a string
-            self._packet_to_hdf5('sequence_name',
-                                 packet.sequence_name.astype(
-                                     self.h5py_str_dtype),
-                                 packet_group)
+            self._packet_to_hdf5(
+                "sequence_name",
+                packet.sequence_name.astype(self.h5py_str_dtype),
+                packet_group,
+            )
 
         else:
-            self._packet_to_hdf5('source_data', packet.data, packet_group)
+            self._packet_to_hdf5("source_data", packet.data, packet_group)
 
         # Add binary
-        self._packet_to_hdf5('binary',
-                             packet.binary.astype(self.h5py_str_dtype),
-                             packet_group)
+        self._packet_to_hdf5(
+            "binary", packet.binary.astype(self.h5py_str_dtype), packet_group
+        )
 
         # Add idb_source and idb_version
-        self._packet_to_hdf5('idb',
-                             packet.idb.astype(self.h5py_str_dtype),
-                             packet_group)
+        self._packet_to_hdf5(
+            "idb", packet.idb.astype(self.h5py_str_dtype), packet_group
+        )
 
         # save utc time as string
         # Note: datetime64 could be also saved as int64
-        self._packet_to_hdf5('utc_time',
-                             np.datetime_as_string(packet.utc_time, timezone='UTC').astype(
-                                 self.h5py_str_dtype),
-                             packet_group)
+        self._packet_to_hdf5(
+            "utc_time",
+            np.datetime_as_string(packet.utc_time, timezone="UTC").astype(
+                self.h5py_str_dtype
+            ),
+            packet_group,
+        )
 
         # Save compression status (only if any compressed packet)
-        if not np.all(packet.compressed == 0) or 'compressed' in packet_group.keys():
-            self._packet_to_hdf5('compressed',
-                                 packet.compressed,
-                                 packet_group)
+        if not np.all(packet.compressed == 0) or "compressed" in packet_group.keys():
+            self._packet_to_hdf5("compressed", packet.compressed, packet_group)
             # Mapping dict of the attributes to add for compressed packets
-            packet_name_c = packet.name + '_C'
-            attrs_dict = {'SRDB_ID_C': (self.packet_parser.palisade_metadata[packet_name_c],
-                                        'srdb_id'),
-                          'PACKET_CATEGORY_C': (self.packet_parser.palisade_metadata[packet_name_c],
-                                                'packet_category')}
+            packet_name_c = packet.name + "_C"
+            attrs_dict = {
+                "SRDB_ID_C": (
+                    self.packet_parser.palisade_metadata[packet_name_c],
+                    "srdb_id",
+                ),
+                "PACKET_CATEGORY_C": (
+                    self.packet_parser.palisade_metadata[packet_name_c],
+                    "packet_category",
+                ),
+            }
             self.add_packet_attrs(packet, attrs_dict, packet_group)
 
         # Return number of packets
-        return packet_group.attrs['COUNT']
+        return packet_group.attrs["COUNT"]
 
     def _packet_to_hdf5(self, name, data, group):
         """
@@ -337,16 +359,19 @@ class L0:
                     if delta_size > 0:
                         # If delta size is positive, it means that we need to increase
                         # the size of dataset array along axis 1
-                        logger.debug(f'Resizing axis 1 of {name} from {dataset.shape[1]} to {dataset.shape[1] + delta_size}')
+                        logger.debug(
+                            f"Resizing axis 1 of {name} from {dataset.shape[1]} to {dataset.shape[1] + delta_size}"
+                        )
                         dataset.resize(dataset.shape[1] + delta_size, axis=1)
 
                     # Than save 2D array data
-                    dataset[old_dset_shape[0]:, :data.shape[
-                        1]] = data[:, :data.shape[1]]
+                    dataset[old_dset_shape[0] :, : data.shape[1]] = data[
+                        :, : data.shape[1]
+                    ]
 
                 else:
                     # Saving data along axis 0 if dim = 1
-                    dataset[old_dset_shape[0]:] = data[:]
+                    dataset[old_dset_shape[0] :] = data[:]
             else:
                 # Else if the dataset does not exist, create it and insert input data
                 # Make sure that dataset is resizable (None == unlimited)
@@ -357,8 +382,9 @@ class L0:
                     maxshape = (None, None)
 
                 # Create and fill new chunked dataset
-                dataset = group.create_dataset(name, data.shape, dtype=data.dtype,
-                                               maxshape=maxshape, chunks=True)
+                dataset = group.create_dataset(
+                    name, data.shape, dtype=data.dtype, maxshape=maxshape, chunks=True
+                )
                 dataset[:] = data
 
         # in this case, this is a part of the record array, so treating
@@ -405,7 +431,9 @@ class L0:
         :param filepath: Path of the file to check
         :return: True if input file exists and has a valid RPW L0 file name, False otherwise
         """
-        return os.path.isfile(filepath) and L0.l0_file_prefix in os.path.basename(filepath)
+        return os.path.isfile(filepath) and L0.l0_file_prefix in os.path.basename(
+            filepath
+        )
 
     @staticmethod
     def set_attr(l0_attrs, metadata):
@@ -421,21 +449,24 @@ class L0:
 
         for key, val in metadata.items():
             if key in l0_attrs:
-                logger.debug(f'{key} L0 attribute already defined: '
-                             f'Replace by {l0_attrs[key]}')
+                logger.debug(
+                    f"{key} L0 attribute already defined: Replace by {l0_attrs[key]}"
+                )
             l0_attrs[key] = val
 
     @staticmethod
-    def l0_to_packet_list(files,
-                          include=[],
-                          exclude=[],
-                          start_time=MIN_DATETIME,
-                          end_time=MAX_DATETIME,
-                          packet_type=PACKET_TYPE,
-                          no_header=False,
-                          no_data=False,
-                          ascending=True,
-                          to_dataframe=False):
+    def l0_to_packet_list(
+        files,
+        include=[],
+        exclude=[],
+        start_time=MIN_DATETIME,
+        end_time=MAX_DATETIME,
+        packet_type=PACKET_TYPE,
+        no_header=False,
+        no_data=False,
+        ascending=True,
+        to_dataframe=False,
+    ):
         """
         Parse list of input RPW L0 files
         and return list of TM/TC packets
@@ -457,7 +488,6 @@ class L0:
         import scipy.sparse as sparse
 
         def load_param(current_group, current_df):
-
             if not isinstance(current_group, h5py.Group):
                 return False
 
@@ -476,15 +506,14 @@ class L0:
         total_sample_num = 0
         file_num = len(files)
         for file_idx, file in enumerate(files):
-
-            logger.debug(f'Opening {file}... ({file_idx + 1}/{file_num})')
-            with h5py.File(file, 'r') as l0:
+            logger.debug(f"Opening {file}... ({file_idx + 1}/{file_num})")
+            with h5py.File(file, "r") as l0:
                 for current_type in packet_type:
                     for current_packet in l0[current_type].keys():
-
-                        if ((current_packet in exclude)
-                                or (include and current_packet not in include)):
-                            logger.debug(f'{current_packet} excluded')
+                        if (current_packet in exclude) or (
+                            include and current_packet not in include
+                        ):
+                            logger.debug(f"{current_packet} excluded")
                             continue
                         # else:
                         #    logger.debug(f'{current_packet} included')
@@ -492,87 +521,108 @@ class L0:
                         # Initialize DataFrame instance for the current packet
                         current_df = pd.DataFrame()
 
-                        current_times = decode(l0[current_type][
-                            current_packet]['utc_time'][()])
+                        current_times = decode(
+                            l0[current_type][current_packet]["utc_time"][()]
+                        )
 
-                        current_binary = decode(l0[current_type][
-                            current_packet]['binary'][()])
+                        current_binary = decode(
+                            l0[current_type][current_packet]["binary"][()]
+                        )
 
                         # Get number of
                         current_nsamp = len(current_times)
-                        current_counter = l0[current_type][
-                            current_packet].attrs['COUNT']
+                        current_counter = l0[current_type][current_packet].attrs[
+                            "COUNT"
+                        ]
                         try:
                             assert current_nsamp == current_counter
                         except Exception:
-                            logger.warning(f'Something goes wrong for {current_packet} COUNT in {file}')
+                            logger.warning(
+                                f"Something goes wrong for {current_packet} COUNT in {file}"
+                            )
                         else:
-                            logger.debug(f'{current_counter} {current_packet} found in {file}')
+                            logger.debug(
+                                f"{current_counter} {current_packet} found in {file}"
+                            )
 
                         # Store packet utc time values
-                        current_df['utc_time'] = current_times
+                        current_df["utc_time"] = current_times
                         # Convert utc_time strings into datetime objects
-                        current_df['utc_time'] = current_df['utc_time'].apply(
-                            lambda x: datetime.strptime(x, TIME_L0_STRFORMAT))
+                        current_df["utc_time"] = current_df["utc_time"].apply(
+                            lambda x: datetime.strptime(x, TIME_L0_STRFORMAT)
+                        )
 
                         # Convert binary hexa packet data to string hexa
-                        current_df['binary'] = current_binary
-                        current_df['palisade_id'] = [
-                            current_packet] * current_nsamp
-                        current_df['srdb_id'] = [l0[current_type][current_packet].attrs[
-                            'SRDB_ID']] * current_nsamp
-                        current_df['category'] = [l0[current_type][current_packet].attrs[
-                            'PACKET_CATEGORY']] * current_nsamp
+                        current_df["binary"] = current_binary
+                        current_df["palisade_id"] = [current_packet] * current_nsamp
+                        current_df["srdb_id"] = [
+                            l0[current_type][current_packet].attrs["SRDB_ID"]
+                        ] * current_nsamp
+                        current_df["category"] = [
+                            l0[current_type][current_packet].attrs["PACKET_CATEGORY"]
+                        ] * current_nsamp
 
                         # Get IDB source/version
-                        current_idb = l0[current_type][
-                            current_packet]['idb'][()]
-                        current_df['idb_source'] = decode(current_idb[:, 0])
-                        current_df['idb_version'] = decode(current_idb[:, 1])
+                        current_idb = l0[current_type][current_packet]["idb"][()]
+                        current_df["idb_source"] = decode(current_idb[:, 0])
+                        current_df["idb_version"] = decode(current_idb[:, 1])
 
                         # Case of compressed TM packets
-                        if 'compressed' in l0[current_type][current_packet].keys():
+                        if "compressed" in l0[current_type][current_packet].keys():
                             # Add compressed array to the current dataframe
-                            current_df['compressed'] = l0[current_type][
-                                current_packet]['compressed'][()]
+                            current_df["compressed"] = l0[current_type][current_packet][
+                                "compressed"
+                            ][()]
 
                             # Be sure that compressed TM have the right
                             # PALISADE_ID, SDRB_ID and CATEGORY
-                            where_compressed = (current_df.compressed == '1')
-                            current_df.loc[where_compressed, 'srdb_id'] = l0[current_type][current_packet].attrs[
-                                'SRDB_ID_C']
-                            current_df.loc[where_compressed, 'category'] = l0[current_type][current_packet].attrs[
-                                'PACKET_CATEGORY_C']
-                            current_df.loc[where_compressed,
-                                           'palisade_id'] = current_packet + '_C'
-
-                        if current_type == 'TM':
-                            data_grp = 'source_data'
-                        elif current_type == 'TC':
-                            data_grp = 'application_data'
-                            current_df['unique_id'] = decode(l0[current_type][
-                                current_packet]['unique_id'][()])
-                            current_df['sequence_name'] = decode(l0[current_type][
-                                current_packet]['sequence_name'][()])
-                            current_tc_state = l0[current_type][
-                                current_packet]['tc_ack_state'][()]
-                            current_df['tc_acc_state'] = decode(current_tc_state[:, 0])
-                            current_df['tc_exe_state'] = decode(current_tc_state[:, 1])
+                            where_compressed = current_df.compressed == "1"
+                            current_df.loc[where_compressed, "srdb_id"] = l0[
+                                current_type
+                            ][current_packet].attrs["SRDB_ID_C"]
+                            current_df.loc[where_compressed, "category"] = l0[
+                                current_type
+                            ][current_packet].attrs["PACKET_CATEGORY_C"]
+                            current_df.loc[where_compressed, "palisade_id"] = (
+                                current_packet + "_C"
+                            )
+
+                        if current_type == "TM":
+                            data_grp = "source_data"
+                        elif current_type == "TC":
+                            data_grp = "application_data"
+                            current_df["unique_id"] = decode(
+                                l0[current_type][current_packet]["unique_id"][()]
+                            )
+                            current_df["sequence_name"] = decode(
+                                l0[current_type][current_packet]["sequence_name"][()]
+                            )
+                            current_tc_state = l0[current_type][current_packet][
+                                "tc_ack_state"
+                            ][()]
+                            current_df["tc_acc_state"] = decode(current_tc_state[:, 0])
+                            current_df["tc_exe_state"] = decode(current_tc_state[:, 1])
 
                         if not no_header:
-                            load_param(l0[current_type][current_packet][
-                                'packet_header'], current_df)
-                            load_param(l0[current_type][current_packet][
-                                'data_field_header'], current_df)
+                            load_param(
+                                l0[current_type][current_packet]["packet_header"],
+                                current_df,
+                            )
+                            load_param(
+                                l0[current_type][current_packet]["data_field_header"],
+                                current_df,
+                            )
 
                         if not no_data:
-                            load_param(l0[current_type][current_packet][
-                                data_grp], current_df)
+                            load_param(
+                                l0[current_type][current_packet][data_grp], current_df
+                            )
 
                         # Filter by start_time/end_time
-                        current_mask = (current_df['utc_time'] >= start_time)
+                        current_mask = current_df["utc_time"] >= start_time
                         current_mask = current_mask & (
-                            current_df['utc_time'] <= end_time)
+                            current_df["utc_time"] <= end_time
+                        )
                         current_df = current_df.loc[current_mask]
                         # Get actual number of samples
                         current_nsamp = len(current_df.index)
@@ -583,38 +633,42 @@ class L0:
                         if current_packet not in packet_data:
                             packet_data[current_packet] = current_df
                         else:
-                            packet_data[current_packet] = pd.concat([packet_data[
-                                current_packet], current_df])
+                            packet_data[current_packet] = pd.concat(
+                                [packet_data[current_packet], current_df]
+                            )
 
                         logger.debug(
-                            f'{current_nsamp} samples extracted from {current_packet} (total samples extracted: {total_sample_num})')
+                            f"{current_nsamp} samples extracted from {current_packet} (total samples extracted: {total_sample_num})"
+                        )
 
                         # Free memory
                         current_df = None
 
-        logger.info(f'RPW packet loading completed '
-                    f'({total_sample_num} samples extracted)')
+        logger.info(
+            f"RPW packet loading completed ({total_sample_num} samples extracted)"
+        )
 
         if to_dataframe:
-            logger.debug('Return packet data as a Pandas DataFrame object')
+            logger.debug("Return packet data as a Pandas DataFrame object")
             return packet_data
 
-        logger.info('Converting to list of RPW packets...')
+        logger.info("Converting to list of RPW packets...")
         packet_list = [None] * total_sample_num
         counter = 0
         for key, val in packet_data.items():
-            current_samples = val.to_dict('Records')
+            current_samples = val.to_dict("Records")
             current_nsamp = len(current_samples)
-            packet_list[counter:counter + current_nsamp] = current_samples
+            packet_list[counter : counter + current_nsamp] = current_samples
             counter += current_nsamp
-            perc = '{: .2f}'.format(100.0 * counter / total_sample_num)
-            logger.debug(f'{current_nsamp} {key} samples added to the packet list ({perc}% completed)')
+            perc = "{: .2f}".format(100.0 * counter / total_sample_num)
+            logger.debug(
+                f"{current_nsamp} {key} samples added to the packet list ({perc}% completed)"
+            )
 
         # Sort by ascending packet creation times
         if ascending:
-            logger.info(
-                'Sorting packet list by ascending packet creation time...')
-            key = 'utc_time'
+            logger.info("Sorting packet list by ascending packet creation time...")
+            key = "utc_time"
             sorted_packet_list = sorted(packet_list, key=lambda i: i[key])
         else:
             sorted_packet_list = packet_list
@@ -630,20 +684,27 @@ class L0:
         :return: True if valid, False otherwise
         """
 
-        for mandatory_key in ['palisade_id', 'srdb_id',
-                              'utc_time', 'binary',
-                              'category', 'idb_version',
-                              'idb_source']:
+        for mandatory_key in [
+            "palisade_id",
+            "srdb_id",
+            "utc_time",
+            "binary",
+            "category",
+            "idb_version",
+            "idb_source",
+        ]:
             if not packet.get(mandatory_key):
                 return False
 
-        if packet['palisade_id'].startswith('TM'):
+        if packet["palisade_id"].startswith("TM"):
             pass
-        elif packet['palisade_id'].startswith('TC'):
-            for mandatory_key in ['unique_id',
-                                  'sequence_name',
-                                  'tc_acc_state',
-                                  'tc_exe_state']:
+        elif packet["palisade_id"].startswith("TC"):
+            for mandatory_key in [
+                "unique_id",
+                "sequence_name",
+                "tc_acc_state",
+                "tc_exe_state",
+            ]:
                 if not packet.get(mandatory_key):
                     return False
         else:
@@ -652,11 +713,13 @@ class L0:
         return True
 
     @staticmethod
-    def l0_to_raw(l0_file_list,
-                  expected_packet_list=[],
-                  start_time=None,
-                  end_time=None,
-                  increasing_time=True):
+    def l0_to_raw(
+        l0_file_list,
+        expected_packet_list=[],
+        start_time=None,
+        end_time=None,
+        increasing_time=True,
+    ):
         """
         Extract packet data from a list of L0 files.
 
@@ -676,59 +739,63 @@ class L0:
             l0_file_list = [l0_file_list]
 
         if len(l0_file_list) == 0:
-            logger.warning('Input list of L0 file is empty!')
-            return {'packet_list': []}
+            logger.warning("Input list of L0 file is empty!")
+            return {"packet_list": []}
 
         # loop over L0 file(s) to get data and save packet data
         for i, l0_file in enumerate(l0_file_list):
-            logger.debug(f'Parsing {l0_file}...')
-            with h5py.File(l0_file, 'r') as l0:
-
+            logger.debug(f"Parsing {l0_file}...")
+            with h5py.File(l0_file, "r") as l0:
                 # Get L0 time minimum/maximum
                 l0_time_min = datetime.strptime(
-                    l0.attrs['TIME_MIN'], TIME_ISO_STRFORMAT)
+                    l0.attrs["TIME_MIN"], TIME_ISO_STRFORMAT
+                )
                 l0_time_max = datetime.strptime(
-                    l0.attrs['TIME_MAX'], TIME_ISO_STRFORMAT)
+                    l0.attrs["TIME_MAX"], TIME_ISO_STRFORMAT
+                )
                 # If start_time/end_time passed, then filter L0 data outside
                 # time range
                 if start_time and l0_time_max < start_time:
-                    logger.info(f'{l0_file} time max. is less than {start_time}, skip it')
+                    logger.info(
+                        f"{l0_file} time max. is less than {start_time}, skip it"
+                    )
                     continue
 
                 if end_time and l0_time_min > end_time:
-                    logger.info(f'{l0_file} time min. is greater than {end_time}, skip it')
+                    logger.info(
+                        f"{l0_file} time min. is greater than {end_time}, skip it"
+                    )
                     continue
 
-                file_id = l0.attrs['File_ID']
-                creation_date = l0.attrs['Generation_date']
+                file_id = l0.attrs["File_ID"]
+                creation_date = l0.attrs["Generation_date"]
 
                 # if expected_packet_list not passed as an input
                 # then load all packets
                 if not expected_packet_list:
                     l0_expected_packet_list = []
-                    if 'TM' in l0.keys():
-                        l0_expected_packet_list.extend(list(l0['TM'].keys()))
-                    if 'TC' in l0.keys():
-                        l0_expected_packet_list.extend(list(l0['TC'].keys()))
+                    if "TM" in l0.keys():
+                        l0_expected_packet_list.extend(list(l0["TM"].keys()))
+                    if "TC" in l0.keys():
+                        l0_expected_packet_list.extend(list(l0["TC"].keys()))
                 else:
                     l0_expected_packet_list = expected_packet_list
 
                 # Loop over list of expected packets
                 for expected_packet in l0_expected_packet_list:
-
-                    if expected_packet.startswith('TM'):
-                        current_type = 'TM'
-                        current_data_group = 'source_data'
-                    elif expected_packet.startswith('TC'):
-                        current_type = 'TC'
-                        current_data_group = 'application_data'
+                    if expected_packet.startswith("TM"):
+                        current_type = "TM"
+                        current_data_group = "source_data"
+                    elif expected_packet.startswith("TC"):
+                        current_type = "TC"
+                        current_data_group = "application_data"
                     else:
-                        logger.warning(f'Unknown packet type: {expected_packet}')
+                        logger.warning(f"Unknown packet type: {expected_packet}")
                         continue
 
                     # Check that packet(s) are found in the present l0 file
                     if not is_packet(expected_packet, l0[current_type]):
-                        logger.debug(f'No {expected_packet} packet found in {l0_file}')
+                        logger.debug(f"No {expected_packet} packet found in {l0_file}")
                         # if not continue
                         continue
 
@@ -739,16 +806,16 @@ class L0:
                     current_packet_data = current_packet[current_data_group]
 
                     # Get list of TC execution UTC times
-                    utc_time_list = decode(current_packet['utc_time'][()])
+                    utc_time_list = decode(current_packet["utc_time"][()])
 
                     # Get number of packets
                     current_packet_num = len(utc_time_list)
 
                     # Loop over packets in l0
-                    logger.debug(f'Extracting {current_packet_num} '
-                                 f'{expected_packet} packets...')
+                    logger.debug(
+                        f"Extracting {current_packet_num} {expected_packet} packets..."
+                    )
                     for j, utc_time in enumerate(utc_time_list):
-
                         try:
                             # Put current packet parameter values into a
                             # dictionary
@@ -763,93 +830,101 @@ class L0:
 
                         # Get current packet header
                         current_packet_header = L0.l0_packet_to_dotdict(
-                            l0[current_type][expected_packet]['packet_header'], j)
+                            l0[current_type][expected_packet]["packet_header"], j
+                        )
 
                         # Get current packet data field header
                         current_packet_data_header = L0.l0_packet_to_dotdict(
-                            l0[current_type][expected_packet]['data_field_header'], j)
+                            l0[current_type][expected_packet]["data_field_header"], j
+                        )
 
                         # Compute packet APID
-                        current_apid = IDBParser.compute_apid(current_packet_header['process_id'],
-                                                              current_packet_header['packet_category'])
+                        current_apid = IDBParser.compute_apid(
+                            current_packet_header["process_id"],
+                            current_packet_header["packet_category"],
+                        )
 
                         # Get some packet attributes in L0
                         try:
-                            srdb_id = current_packet.attrs['SRDB_ID']
-                            packet_cat = current_packet.attrs[
-                                'PACKET_CATEGORY']
+                            srdb_id = current_packet.attrs["SRDB_ID"]
+                            packet_cat = current_packet.attrs["PACKET_CATEGORY"]
                         except Exception:
-                            logger.warning(f'Attributes missing in {l0_file}!')
+                            logger.warning(f"Attributes missing in {l0_file}!")
                             srdb_id = None
                             packet_cat = None
 
                         # Get compression flag
                         try:
-                            current_compressed = current_packet[
-                                'compressed'][:]
+                            current_compressed = current_packet["compressed"][:]
                         except Exception:
                             current_compressed = None
 
                         # Add current packet parameters to lists
                         # utc_time, name of packet and parameters
                         current_packet_fields = {
-                            'utc_time': L0.l0_utc_time_to_dt(utc_time),
-                            'palisade_id': expected_packet,
-                            'srdb_id': srdb_id,
-                            'data': parameters,
-                            'type': current_type,
-                            'category': packet_cat,
-                            'binary': decode(current_packet['binary'][j]),
-                            'idb_source': decode(current_packet['idb'][j][0]),
-                            'idb_version': decode(current_packet['idb'][j][1]),
-                            'length': current_packet_header['packet_length'] + 7,
-                            'apid': current_apid,
-                            'header': current_packet_header,
-                            'data_header': current_packet_data_header,
-                            'compressed': current_compressed,
+                            "utc_time": L0.l0_utc_time_to_dt(utc_time),
+                            "palisade_id": expected_packet,
+                            "srdb_id": srdb_id,
+                            "data": parameters,
+                            "type": current_type,
+                            "category": packet_cat,
+                            "binary": decode(current_packet["binary"][j]),
+                            "idb_source": decode(current_packet["idb"][j][0]),
+                            "idb_version": decode(current_packet["idb"][j][1]),
+                            "length": current_packet_header["packet_length"] + 7,
+                            "apid": current_apid,
+                            "header": current_packet_header,
+                            "data_header": current_packet_data_header,
+                            "compressed": current_compressed,
                         }
 
                         # If TC or TM, add also specific attributes
-                        if current_type == 'TC':
-                            current_packet_fields[
-                                'unique_id'] = current_packet['unique_id'][j]
-                            current_packet_fields['sequence_name'] = current_packet[
-                                'sequence_name'][j]
-                            current_packet_fields['ack_acc_state'] = current_packet[
-                                'tc_ack_state'][j][0]
-                            current_packet_fields['ack_exe_state'] = current_packet[
-                                'tc_ack_state'][j][1]
-                        elif current_type == 'TM':
-                            current_packet_fields[
-                                'sync_flag'] = current_packet_data_header['time'][2]
-                            current_packet_fields['obt_time'] = f'1/' \
-                                f"{current_packet_data_header['time'][0]}:" \
+                        if current_type == "TC":
+                            current_packet_fields["unique_id"] = current_packet[
+                                "unique_id"
+                            ][j]
+                            current_packet_fields["sequence_name"] = current_packet[
+                                "sequence_name"
+                            ][j]
+                            current_packet_fields["ack_acc_state"] = current_packet[
+                                "tc_ack_state"
+                            ][j][0]
+                            current_packet_fields["ack_exe_state"] = current_packet[
+                                "tc_ack_state"
+                            ][j][1]
+                        elif current_type == "TM":
+                            current_packet_fields["sync_flag"] = (
+                                current_packet_data_header["time"][2]
+                            )
+                            current_packet_fields["obt_time"] = (
+                                f"1/"
+                                f"{current_packet_data_header['time'][0]}:"
                                 f"{current_packet_data_header['time'][1]}"
+                            )
 
                         output_packet_list.append(current_packet_fields)
 
         output_packet_num = len(output_packet_list)
         if output_packet_num == 0:
-            logger.info('No packet has been returned')
-            return {'packet_list': []}
+            logger.info("No packet has been returned")
+            return {"packet_list": []}
         else:
-            logger.debug(f'{output_packet_num} packets returned')
+            logger.debug(f"{output_packet_num} packets returned")
             # Sort list by increasing time
             if increasing_time:
                 output_packet_list = sorted(
-                    output_packet_list,
-                    key=lambda
-                    k: k['utc_time'])
+                    output_packet_list, key=lambda k: k["utc_time"]
+                )
 
         # FIXME - Strange output combining data from one L0
         #  (l0_time_min, l0_time_max, file_id, creation_date)
         #   and for several L0 (packet_list)
         l0_data = {
-            'time_min': l0_time_min,
-            'time_max': l0_time_max,
-            'file_id': file_id,
-            'creation_date': creation_date,
-            'packet_list': output_packet_list
+            "time_min": l0_time_min,
+            "time_max": l0_time_max,
+            "file_id": file_id,
+            "creation_date": creation_date,
+            "packet_list": output_packet_list,
         }
 
         return l0_data
@@ -874,18 +949,19 @@ class L0:
             l0_file_list = [l0_file_list]
 
         if len(l0_file_list) == 0:
-            logger.warning('Input list of L0 file is empty!')
+            logger.warning("Input list of L0 file is empty!")
             return output_file_list
 
         # loop over L0 file(s) to filter by time range
         for i, l0_file in enumerate(l0_file_list):
-            with h5py.File(l0_file, 'r') as l0:
-
+            with h5py.File(l0_file, "r") as l0:
                 # Get L0 time minimum/maximum
                 l0_time_min = datetime.strptime(
-                    l0.attrs['TIME_MIN'], TIME_ISO_STRFORMAT)
+                    l0.attrs["TIME_MIN"], TIME_ISO_STRFORMAT
+                )
                 l0_time_max = datetime.strptime(
-                    l0.attrs['TIME_MAX'], TIME_ISO_STRFORMAT)
+                    l0.attrs["TIME_MAX"], TIME_ISO_STRFORMAT
+                )
                 # If start_time/end_time passed, then filter L0 data outside
                 # time range
                 if start_time and l0_time_max < start_time:
@@ -930,13 +1006,10 @@ class L0:
         """
         # Nanoseconds not supported by datetime and must removed from input l0
         # utc time
-        return datetime.strptime(decode(l0_utc_time)[:-4] + 'Z',
-                                 TIME_ISO_STRFORMAT)
+        return datetime.strptime(decode(l0_utc_time)[:-4] + "Z", TIME_ISO_STRFORMAT)
 
     @staticmethod
-    def count_packets(l0_file,
-                      tm=True,
-                      tc=True):
+    def count_packets(l0_file, tm=True, tc=True):
         """
         Count number of Packet elements in the input RPW L0 file.
 
@@ -948,13 +1021,12 @@ class L0:
         # Initialize output
         packet_num = 0
 
-        with h5py.File(l0_file, 'r') as l0:
-            tm_num = l0['TM'].attrs['COUNT']
-            tc_num = l0['TC'].attrs['COUNT']
+        with h5py.File(l0_file, "r") as l0:
+            tm_num = l0["TM"].attrs["COUNT"]
+            tc_num = l0["TC"].attrs["COUNT"]
 
             if not tm and not tc:
-                logger.warning(
-                    'At least tm and/or tc keyword must be set to True')
+                logger.warning("At least tm and/or tc keyword must be set to True")
                 return packet_num
 
             if tm:
@@ -965,9 +1037,7 @@ class L0:
             return packet_num
 
     @staticmethod
-    def order_by_utc(l0_file,
-                     unique=False,
-                     update_time_minmax=False):
+    def order_by_utc(l0_file, unique=False, update_time_minmax=False):
         """
         Order data in an input RPW L0 HDF5 file
         by packet creation UTC times.
@@ -978,6 +1048,7 @@ class L0:
                                  root attributes with actual values.
         :return:
         """
+
         # Internal function to sort a dataset according to input indices
         def _sort_dataset(group, item, unique_idx, sorted_idx):
             # If not dtype.names, then we have a dataset
@@ -988,15 +1059,14 @@ class L0:
                 # Only update dataset if no empty data
                 if current_dataset.shape[0] > 0:
                     # Extract sorted/unique dataset array
-                    sorted_dataset = (current_dataset[()][unique_idx])[
-                        sorted_idx]
+                    sorted_dataset = (current_dataset[()][unique_idx])[sorted_idx]
 
                     # If there duplicate values, then resize L0 HDF5 dataset
                     if sorted_dataset.shape[0] < current_dataset.shape[0]:
-                        delta_shape = current_dataset.shape[
-                            0] - sorted_dataset.shape[0]
-                        current_dataset.resize(current_dataset.shape[
-                                               0] - delta_shape, axis=0)
+                        delta_shape = current_dataset.shape[0] - sorted_dataset.shape[0]
+                        current_dataset.resize(
+                            current_dataset.shape[0] - delta_shape, axis=0
+                        )
 
                     # Then update it with sorted indices
                     current_dataset[...] = sorted_dataset
@@ -1007,54 +1077,61 @@ class L0:
                     _sort_dataset(group[item], field, unique_idx, sorted_idx)
 
         # Initialize time_min / time_max
-        time_min = '9999-12-31T23:59:59.999999999Z'
-        time_max = '0000-01-01T00:00:00.000000000Z'
+        time_min = "9999-12-31T23:59:59.999999999Z"
+        time_max = "0000-01-01T00:00:00.000000000Z"
 
         # Open L0 file
-        with h5py.File(l0_file, 'a') as l0:
-
+        with h5py.File(l0_file, "a") as l0:
             # Loop over TM/TC groups in L0
-            for cat in ['TM', 'TC']:
+            for cat in ["TM", "TC"]:
                 # Check that TM/TC group exists
                 if cat in l0.keys():
                     # Loop over each TM packet in L0
                     new_packet_count = []
                     for packet_name in l0[cat].keys():
-
                         # Get packet utc_time values
-                        original_utc_time = l0[cat][
-                            packet_name]['utc_time'][()]
+                        original_utc_time = l0[cat][packet_name]["utc_time"][()]
 
                         if unique:
                             # If unique input keyword is True,
-                            if cat == 'TM':
+                            if cat == "TM":
                                 # then if TM
                                 # apply the uniqueness of the tuple (packet,
                                 # binary) over packets in l0 file
-                                binary = l0[cat][
-                                    packet_name]['binary'][()]
+                                binary = l0[cat][packet_name]["binary"][()]
                                 binary, unique_idx = np.unique(
-                                    binary, return_index=True)
+                                    binary, return_index=True
+                                )
                                 utc_time = original_utc_time[unique_idx]
                             else:
                                 # then if TC
                                 # apply the uniqueness of the tuple (packet,
                                 # utc_time) over packets in l0 file
                                 utc_time, unique_idx = np.unique(
-                                    original_utc_time, return_index=True)
+                                    original_utc_time, return_index=True
+                                )
 
                             # Update packet COUNT attribute
-                            l0[cat][packet_name].attrs[
-                                'COUNT'] = utc_time.shape[0]
+                            l0[cat][packet_name].attrs["COUNT"] = utc_time.shape[0]
                             new_packet_count.append(utc_time.shape[0])
 
                         else:
                             unique_idx = np.arange(original_utc_time.shape[0])
 
-                        if cat == 'TM' and utc_time.shape[0] != original_utc_time.shape[0]:
-                            duplicated_times = [item for item, count in collections.Counter(
-                                original_utc_time).items() if count > 1]
-                            logger.warning(f'There are duplicated times ({duplicated_times}) for packet {packet_name} in {l0_file}!')
+                        if (
+                            cat == "TM"
+                            and utc_time.shape[0] != original_utc_time.shape[0]
+                        ):
+                            duplicated_times = [
+                                item
+                                for item, count in collections.Counter(
+                                    original_utc_time
+                                ).items()
+                                if count > 1
+                            ]
+                            logger.warning(
+                                f"There are duplicated times ({duplicated_times}) for packet {packet_name} in {l0_file}!"
+                            )
 
                             # Code block to get indices of duplicated elements
                             # duplicated_indices = []
@@ -1068,7 +1145,8 @@ class L0:
                         for item in L0.L0_FIELDS[cat]:
                             if item in l0[cat][packet_name].keys():
                                 _sort_dataset(
-                                    l0[cat][packet_name], item, unique_idx, sorted_idx)
+                                    l0[cat][packet_name], item, unique_idx, sorted_idx
+                                )
 
                         if np.min(utc_time):
                             time_min = min([decode(np.min(utc_time)), time_min])
@@ -1078,9 +1156,9 @@ class L0:
                     # If unique is True, then update the COUNT attribute of
                     # TM/TC group
                     if unique:
-                        l0[cat].attrs['COUNT'] = sum(new_packet_count)
+                        l0[cat].attrs["COUNT"] = sum(new_packet_count)
 
             if update_time_minmax:
                 # Save microsecond resolution
-                l0.attrs['TIME_MIN'] = time_min[:-4] + 'Z'
-                l0.attrs['TIME_MAX'] = time_max[:-4] + 'Z'
+                l0.attrs["TIME_MIN"] = time_min[:-4] + "Z"
+                l0.attrs["TIME_MAX"] = time_max[:-4] + "Z"