roc-film 1.13.5__py3-none-any.whl → 1.14.0__py3-none-any.whl

roc/film/tests/hdf5_compare.py

@@ -11,6 +11,7 @@ import sys
 import h5py
 import numpy
 
+
 class Hdf5Reader:
     def __init__(self, logger=logging):
         self.logger = logger
@@ -22,7 +23,7 @@ class Hdf5Reader:
         :return: HDF5 file
         """
         try:
-            hdf5_file = h5py.File(filepath, 'r')
+            hdf5_file = h5py.File(filepath, "r")
             return hdf5_file
         except IOError:
             self.logger.error("Unable to open file '%s'" % filepath)
@@ -46,7 +47,7 @@ class Hdf5Reader:
         :return: Summary of the group
         """
 
-        return {'attr': self.read_attributes(hval), 'htype': 'group'}
+        return {"attr": self.read_attributes(hval), "htype": "group"}
 
     def read_data(self, hval):
         """
@@ -55,10 +56,12 @@ class Hdf5Reader:
         :return: Summary of the dataset
         """
         data_value = hval[()]  # hval.value has been deprecated
-        return {'attr': self.read_attributes(hval),
-                'value': data_value,
-                'htype': 'dataset',
-                'dtype': type(data_value)}
+        return {
+            "attr": self.read_attributes(hval),
+            "value": data_value,
+            "htype": "dataset",
+            "dtype": type(data_value),
+        }
 
     #
     def evaluate_group(self, path, grp):
@@ -76,13 +79,20 @@ class Hdf5Reader:
             elif isinstance(v, h5py.Group):
                 result[k] = self.read_group(v)
             else:
-                raise Exception('Unknown h5py type: %s (%s -- %s)' % (type(v), path, k))
+                raise Exception("Unknown h5py type: %s (%s -- %s)" % (type(v), path, k))
         return result
 
 
 class Hdf5Diff(Hdf5Reader):
-    def __init__(self, filepath1, filepath2, group_path1, group_path2,
-        exclude_attr=[], logger=logging):
+    def __init__(
+        self,
+        filepath1,
+        filepath2,
+        group_path1,
+        group_path2,
+        exclude_attr=[],
+        logger=logging,
+    ):
         # set up the logger
         super().__init__(logger)
 
@@ -100,103 +110,108 @@ class Hdf5Diff(Hdf5Reader):
 
     def compare_dataset(self, path, name, dataset1, dataset2):
         # compare dtypes
-        if dataset1[name]['dtype'] != dataset2[name]['dtype']:
-            d1 = dataset1[name]['dtype']
-            d2 = dataset2[name]['dtype']
-            self.register_diff(path, 'DIFF_DTYPE', (d1, d2), )
+        if dataset1[name]["dtype"] != dataset2[name]["dtype"]:
+            d1 = dataset1[name]["dtype"]
+            d2 = dataset2[name]["dtype"]
+            self.register_diff(
+                path,
+                "DIFF_DTYPE",
+                (d1, d2),
+            )
 
         # compare values
-        v1 = dataset1[name]['value']
-        v2 = dataset2[name]['value']
+        v1 = dataset1[name]["value"]
+        v2 = dataset2[name]["value"]
         try:
             if not numpy.allclose(v1, v2, equal_nan=True):
                 raise ValueError
         except ValueError:
-            self.register_diff(path + name, 'DIFF_DATA', (v1, v2))
+            self.register_diff(path + name, "DIFF_DATA", (v1, v2))
         except TypeError:
             # Try to compare row by row (only if same length)
             if v1.shape != v2.shape:
-                self.register_diff(path + name, 'DIFF_DATA', (v1, v2))
+                self.register_diff(path + name, "DIFF_DATA", (v1, v2))
             else:
                 for i in range(v1.shape[0]):
                     if len(numpy.array(v1[i]).shape) > 0:
                         if any(v1[i] != v2[i]):
-                            self.register_diff(path + name, 'DIFF_DATA', (v1, v2))
+                            self.register_diff(path + name, "DIFF_DATA", (v1, v2))
                             break
                     else:
                         if v1[i] != v2[i]:
-                            self.register_diff(path + name, 'DIFF_DATA', (v1, v2))
+                            self.register_diff(path + name, "DIFF_DATA", (v1, v2))
                             break
-        except:
-            raise Exception(f'Datasets {path+name} cannot be compared')
+        except Exception as e:
+            self.logger.exception(f"Datasets {path + name} cannot be compared:\n{e}")
+            raise
         else:
             pass
 
         # compare attributes
-        for k in dataset1[name]['attr']:
+        for k in dataset1[name]["attr"]:
             if self.exclude_attr and k in self.exclude_attr:
                 continue
-            if k not in dataset2[name]['attr']:
-                self.register_diff(path + name, 'DIFF_UNIQ_ATTR', (k,), file_id=1)
+            if k not in dataset2[name]["attr"]:
+                self.register_diff(path + name, "DIFF_UNIQ_ATTR", (k,), file_id=1)
 
-        for k in dataset2[name]['attr']:
+        for k in dataset2[name]["attr"]:
             if self.exclude_attr and k in self.exclude_attr:
                 continue
-            if k not in dataset1[name]['attr']:
-                self.register_diff(path + name, 'DIFF_UNIQ_ATTR', (k,), file_id=2)
+            if k not in dataset1[name]["attr"]:
+                self.register_diff(path + name, "DIFF_UNIQ_ATTR", (k,), file_id=2)
 
-        for k in dataset1[name]['attr']:
+        for k in dataset1[name]["attr"]:
             if self.exclude_attr and k in self.exclude_attr:
                 continue
-            if k in dataset2[name]['attr']:
-                v1 = dataset1[name]['attr'][k]
-                v2 = dataset2[name]['attr'][k]
+            if k in dataset2[name]["attr"]:
+                v1 = dataset1[name]["attr"][k]
+                v2 = dataset2[name]["attr"][k]
                 if v1 != v2:
-                    self.register_diff(path + name, 'DIFF_ATTR_DTYPE', (k, v1, v2))
+                    self.register_diff(path + name, "DIFF_ATTR_DTYPE", (k, v1, v2))
 
     def compare_group(self, path, name, desc1, desc2):
-
         # compare attributes
-        for k in desc1[name]['attr']:
+        for k in desc1[name]["attr"]:
             if self.exclude_attr and k in self.exclude_attr:
                 continue
-            if k not in desc2[name]['attr']:
-                self.register_diff(path + name, 'DIFF_UNIQ_ATTR', (k,), file_id=1)
-        for k in desc2[name]['attr']:
+            if k not in desc2[name]["attr"]:
+                self.register_diff(path + name, "DIFF_UNIQ_ATTR", (k,), file_id=1)
+        for k in desc2[name]["attr"]:
             if self.exclude_attr and k in self.exclude_attr:
                 continue
-            if k not in desc1[name]['attr']:
-                self.register_diff(path + name, 'DIFF_UNIQ_ATTR', (k,), file_id=2)
+            if k not in desc1[name]["attr"]:
+                self.register_diff(path + name, "DIFF_UNIQ_ATTR", (k,), file_id=2)
 
     def register_diff(self, path, error_code, values, file_id=None):
-
         error_messages = {
-            'DIFF_OBJECTS': '[{path}] Different element types: {values[0]} and {values[1]}',
-            'DIFF_UNIQUE': '[{path}] Element {values[0]} only in file {file_id}',
-            'DIFF_UNIQ_ATTR': '[{path}] Attribute {values[0]} only in file {file_id}',
-            'DIFF_ATTR_DTYPE': '[{path}] Attribute {values[0]} has different type: {values[1]} and {values[2]}',
-            'DIFF_DATA': '[{path}] Different data: {values[0]} and {values[1]}',
-            'DIFF_DTYPE': '[{path}] Different dtypes: {values[0]} and {values[1]}',
+            "DIFF_OBJECTS": "[{path}] Different element types: {values[0]} and {values[1]}",
+            "DIFF_UNIQUE": "[{path}] Element {values[0]} only in file {file_id}",
+            "DIFF_UNIQ_ATTR": "[{path}] Attribute {values[0]} only in file {file_id}",
+            "DIFF_ATTR_DTYPE": "[{path}] Attribute {values[0]} has different type: {values[1]} and {values[2]}",
+            "DIFF_DATA": "[{path}] Different data: {values[0]} and {values[1]}",
+            "DIFF_DTYPE": "[{path}] Different dtypes: {values[0]} and {values[1]}",
         }
 
         error_message = error_messages.get(error_code, None)
 
         if error_message is None:
-            raise Exception('Unknown error_code %s' % (error_code,))
+            raise Exception("Unknown error_code %s" % (error_code,))
 
-        content = {'path': path, 'values': values, 'error_code': error_code}
+        content = {"path": path, "values": values, "error_code": error_code}
 
         if file_id is not None:
-            content['file_id'] = file_id
+            content["file_id"] = file_id
 
         # store the error
         self.diff_list.append(content)
 
         # and send logs
-        self.logger.info(error_message.format(path=path, values=values, file_id=file_id))
+        self.logger.info(
+            error_message.format(path=path, values=values, file_id=file_id)
+        )
 
-    def diff_groups(self, grp1, grp2, path='/'):
-        self.logger.debug('Examining ' + path)
+    def diff_groups(self, grp1, grp2, path="/"):
+        self.logger.debug("Examining " + path)
 
         # get the groups content
         desc1 = self.evaluate_group(path, grp1)
@@ -210,37 +225,44 @@ class Hdf5Diff(Hdf5Reader):
             if k in desc2:
                 common.append(k)
             else:
-                self.register_diff(path, 'DIFF_UNIQUE', (k,), file_id=1)
+                self.register_diff(path, "DIFF_UNIQUE", (k,), file_id=1)
 
         # get the keys specific to the group 2
         for k in desc2:
             if k not in desc1:
-                self.register_diff(path, 'DIFF_UNIQUE', (k,), file_id=2)
+                self.register_diff(path, "DIFF_UNIQUE", (k,), file_id=2)
 
         # loop over common keys
         for name in common:
-            self.logger.debug('\t' + name)
+            self.logger.debug("\t" + name)
 
             # compare types
-            h1 = desc1[name]['htype']
-            h2 = desc2[name]['htype']
+            h1 = desc1[name]["htype"]
+            h2 = desc2[name]["htype"]
             if h1 != h2:
-                self.register_diff(path, 'DIFF_OBJECTS', (h1, h2), )
+                self.register_diff(
+                    path,
+                    "DIFF_OBJECTS",
+                    (h1, h2),
+                )
                 # different hdf5 types -- don't try to compare further
                 continue
 
             # call the appropriate method(s) depending on the node type
-            if desc1[name]['htype'] == 'dataset':
+            if desc1[name]["htype"] == "dataset":
                 # handle dataset
                 self.compare_dataset(path, name, desc1, desc2)
-            elif desc1[name]['htype'] == 'group':
+            elif desc1[name]["htype"] == "group":
                 # handle groups
                 self.compare_group(path, name, desc1, desc2)
                 # recurse into subgroup
-                self.diff_groups(grp1[name], grp2[name], path=path + name + '/')
+                self.diff_groups(grp1[name], grp2[name], path=path + name + "/")
             else:
                 # handle unrecognized hdf5 objects
-                self.logger.warning("Element is not a recognized type (%s) and isn't being evaluated" % h1)
+                self.logger.warning(
+                    "Element is not a recognized type (%s) and isn't being evaluated"
+                    % h1
+                )
                 continue
 
     def diff_files(self):

roc/film/tests/test_dds_to_l0.py

@@ -7,7 +7,6 @@ Test tm_to_l0 command of the roc.film plugin.
 
 import filecmp
 import os
-import tempfile
 from pathlib import Path
 from pprint import pformat
 
@@ -45,16 +44,16 @@ class TestDdsToL0(CommandTestCase):
         # clear the files produced during test
         shutil.rmtree(self.output_dir_path)
 
-    #@pytest.mark.skip()
+    # @pytest.mark.skip()
     @pytest.mark.parametrize(
-        'date_to_test,idb_source,idb_version',
-        [('20220501', 'MIB', '20200131')],
+        "date_to_test,idb_source,idb_version",
+        [("20220501", "MIB", "20200131")],
     )
     def test_dds_to_l0(self, date_to_test, idb_source, idb_version):
         from poppy.core.conf import Settings
 
         # Name of the command to test
-        cmd = 'dds_to_l0'
+        cmd = "dds_to_l0"
 
         # Retrieve data for current test
         test_data_path = FilmTest().get_test_data(cmd, extract=True)
@@ -64,89 +63,132 @@ class TestDdsToL0(CommandTestCase):
         # Initialize inputs and expected outputs
         input_dir_path, inputs = FilmTest.get_inputs(test_data_dir)
         if not inputs:
-            raise FileNotFoundError(f'No input found in {test_data_dir}!')
-        expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(test_data_dir)
+            raise FileNotFoundError(f"No input found in {test_data_dir}!")
+        expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(
+            test_data_dir
+        )
         if not expected_outputs:
-            raise FileNotFoundError(f'No expected output found in {test_data_dir}!')
+            raise FileNotFoundError(f"No expected output found in {test_data_dir}!")
 
         # Initialize directory where files produced during test will be saved
-        output_dir_path = os.path.join(test_data_dir, 'output')
+        output_dir_path = os.path.join(test_data_dir, "output")
         self.output_dir_path = output_dir_path
 
         # Check that SPICE kernels are present in ./data/spice_kernels folder
         spice_kernels_dir = FilmTest.get_spice_kernel_dir()
         if not os.path.isdir(spice_kernels_dir):
-            raise FileNotFoundError(f'No SPICE kernel set found in {spice_kernels_dir}!')
+            raise FileNotFoundError(
+                f"No SPICE kernel set found in {spice_kernels_dir}!"
+            )
 
         # length in scos header bytes to remove from input binary TM packets
-        scos_header_byte_length = '0'
+        scos_header_byte_length = "0"
 
         # Split inputs between two lists containing RPW TM and TC files respectively
-        tm_inputs = ' '.join([
-            os.path.join(input_dir_path, input_file)
-            for input_file in inputs
-            if os.path.basename(input_file).startswith('solo_TM_')
-        ])
-        tc_inputs = ' '.join([
-            os.path.join(input_dir_path, input_file)
-            for input_file in inputs
-            if os.path.basename(input_file).startswith('solo_TC_')
-        ])
+        tm_inputs = " ".join(
+            [
+                os.path.join(input_dir_path, input_file)
+                for input_file in inputs
+                if os.path.basename(input_file).startswith("solo_TM_")
+            ]
+        )
+        tc_inputs = " ".join(
+            [
+                os.path.join(input_dir_path, input_file)
+                for input_file in inputs
+                if os.path.basename(input_file).startswith("solo_TC_")
+            ]
+        )
 
         # initialize the main command
         # Make sure that start-time/end_time keyword values are consistent with input DDS files
-        command_to_test = ' '.join(['pop', 'film',
-                           '--force',
-                           '--idb-version', idb_version,
-                           '--idb-source', idb_source,
-                           '--scos-header', scos_header_byte_length,
-                           '--cdag',
-                           cmd, date_to_test,
-                           '--dds-tmraw-xml', tm_inputs,
-                           '--dds-tcreport-xml', tc_inputs,
-                           '--output-dir', output_dir_path,
-                           '-ll', 'INFO'])
+        command_to_test = " ".join(
+            [
+                "pop",
+                "film",
+                "--force",
+                "--idb-version",
+                idb_version,
+                "--idb-source",
+                idb_source,
+                "--scos-header",
+                scos_header_byte_length,
+                "--cdag",
+                cmd,
+                date_to_test,
+                "--dds-tmraw-xml",
+                tm_inputs,
+                "--dds-tcreport-xml",
+                tc_inputs,
+                "--output-dir",
+                output_dir_path,
+                "-ll",
+                "INFO",
+            ]
+        )
 
         # define the required plugins
-        plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.dingo', 'roc.film']
+        plugin_list = [
+            "poppy.pop",
+            "roc.idb",
+            "roc.rpl",
+            "roc.rap",
+            "roc.dingo",
+            "roc.film",
+        ]
         #
         # run the command
         # force the value of the plugin list
-        with mock.patch.object(Settings, 'configure',
-                               autospec=True,
-                               side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
+        with mock.patch.object(
+            Settings,
+            "configure",
+            autospec=True,
+            side_effect=self.mock_configure_settings(
+                dictionary={"PLUGINS": plugin_list}
+            ),
+        ):
             # Load IDB in the database
             FilmTest.load_idb(self, idb_version=idb_version)
 
             # Run the command to test
-            logger.info(f'Running {command_to_test}')
+            logger.info(f"Running {command_to_test}")
             self.run_command(shlex.split(command_to_test))
 
         # compare directory content
-        dirs_cmp = filecmp.dircmp(output_dir_path,
-                                  expected_output_dir_path)
+        dirs_cmp = filecmp.dircmp(output_dir_path, expected_output_dir_path)
 
         dirs_cmp.report()
         # ensure that we have the same files in both directories
         assert (len(dirs_cmp.left_only) == 0) and (len(dirs_cmp.right_only) == 0)
 
         # ensure the name of the two expected and generated L0 files is the same
-        produced_l0_path = os.path.join(output_dir_path, f'solo_L0_rpw-cdag_{date_to_test}_V01.h5')
-        expected_l0_path = os.path.join(expected_output_dir_path, f'solo_L0_rpw-cdag_{date_to_test}_V01.h5')
+        produced_l0_path = os.path.join(
+            output_dir_path, f"solo_L0_rpw-cdag_{date_to_test}_V01.h5"
+        )
+        expected_l0_path = os.path.join(
+            expected_output_dir_path, f"solo_L0_rpw-cdag_{date_to_test}_V01.h5"
+        )
         assert Path(produced_l0_path).name == Path(expected_l0_path).name
 
         # Compare content of the two L0 files
         attributes_to_ignore = [
-                         'File_ID',
-                         'Generation_date',
-                         'Pipeline_version',
-                         'Pipeline_name',
-                         'Software_version',
-                         'IDB_version'
-                    ]
-        result = Hdf5Diff(produced_l0_path, expected_l0_path, '/', '/',
-                          exclude_attr=attributes_to_ignore).diff_files()
+            "File_ID",
+            "Generation_date",
+            "Pipeline_version",
+            "Pipeline_name",
+            "Software_version",
+            "IDB_version",
+        ]
+        result = Hdf5Diff(
+            produced_l0_path,
+            expected_l0_path,
+            "/",
+            "/",
+            exclude_attr=attributes_to_ignore,
+        ).diff_files()
         if result:
-            logger.error(f'Differences between expected output and the command output: {pformat(result)}')
+            logger.error(
+                f"Differences between expected output and the command output: {pformat(result)}"
+            )
 
         assert result == []

roc/film/tests/test_dds_to_tc.py

@@ -5,22 +5,16 @@
 Test dds_to_tc command of the roc.film plugin.
 """
 
-import filecmp
-import os
 import tempfile
-from pprint import pformat
 
 import pytest
 import shutil
-import unittest.mock as mock
 
-from poppy.core.logger import logger
 from poppy.core.test import CommandTestCase
 from roc.film.tests.test_film import FilmTest
 
 
 class TestDdsToTc(CommandTestCase):
-
     film = FilmTest()
 
     def setup_method(self, method):
@@ -44,12 +38,15 @@ class TestDdsToTc(CommandTestCase):
         # clear the downloaded files
         shutil.rmtree(self.tmp_dir_path)
 
-    @pytest.mark.skip(reason='Not working')
-    @pytest.mark.parametrize('idb_source,idb_version', [
-        ('MIB', '20200131'),
-    ])
+    @pytest.mark.skip(reason="Not working")
+    @pytest.mark.parametrize(
+        "idb_source,idb_version",
+        [
+            ("MIB", "20200131"),
+        ],
+    )
     def test_classify_tcreport(self, idb_source, idb_version):
-        from poppy.core.conf import Settings
+        pass
 
         # input_dir_path, inputs = self.get_inputs('classify_tcreport')
         # expected_output_dir_path, expected_outputs = self.get_expected_outputs('classify_tcreport')

roc/film/tests/test_dds_to_tm.py

@@ -5,16 +5,11 @@
 Test dds_to_tm command of the roc.film plugin.
 """
 
-import filecmp
-import os
 import tempfile
-from pprint import pformat
 
 import pytest
 import shutil
-import unittest.mock as mock
 
-from poppy.core.logger import logger
 from poppy.core.test import CommandTestCase
 from roc.film.tests.test_film import FilmTest
 
@@ -43,12 +38,15 @@ class TestDdsToTm(CommandTestCase):
         # clear the downloaded files
         shutil.rmtree(self.tmp_dir_path)
 
-    @pytest.mark.skip(reason='Not working')
-    @pytest.mark.parametrize('idb_source,idb_version', [
-        ('MIB', '20200131'),
-    ])
+    @pytest.mark.skip(reason="Not working")
+    @pytest.mark.parametrize(
+        "idb_source,idb_version",
+        [
+            ("MIB", "20200131"),
+        ],
+    )
     def test_classify_tmraw(self, idb_source, idb_version):
-        from poppy.core.conf import Settings
+        pass
 
         # input_dir_path, inputs = self.get_inputs('classify_tmraw')
         # expected_output_dir_path, expected_outputs = self.get_expected_outputs('classify_tmraw')