roc-film 1.13.4__py3-none-any.whl → 1.14.0__py3-none-any.whl

roc/film/tests/test_film.py

@@ -4,6 +4,7 @@
 """
 Tests module for the roc.film plugin.
 """
+
 import glob
 import shlex
 import tarfile
@@ -18,21 +19,25 @@ from poppy.core.conf import settings
 
 class FilmTest:
     # Base URL for downloading test data for FILM plugin
-    base_url = 'https://rpw.lesia.obspm.fr/roc/data/private/devtest/roc/test_data/rodp/film'
+    base_url = (
+        "https://rpw.lesia.obspm.fr/roc/data/private/devtest/roc/test_data/rodp/film"
+    )
 
     # test credentials
-    host = 'roc2-dev.obspm.fr'
-    username = os.environ.get('ROC_TEST_USER', 'roctest')
+    host = "roc2-dev.obspm.fr"
+    username = os.environ.get("ROC_TEST_USER", "roctest")
     password = None
 
     def __init__(self):
-        logger.debug('FilmTest setup_class()')
-        logger.debug(f'base_url = {self.base_url}')
+        logger.debug("FilmTest setup_class()")
+        logger.debug(f"base_url = {self.base_url}")
         try:
-            self.password = os.environ['ROC_TEST_PASSWORD']
+            self.password = os.environ["ROC_TEST_PASSWORD"]
         except KeyError:
-            raise KeyError('You have to define the test user password using'
-                           'the "ROC_TEST_PASSWORD" environment variable')
+            raise KeyError(
+                "You have to define the test user password using"
+                'the "ROC_TEST_PASSWORD" environment variable'
+            )
 
     @staticmethod
     def get_test_data_path(command=None):
@@ -44,16 +49,18 @@ class FilmTest:
         :return: data_test_path, string containing the path
         """
         # Define default value
-        data_test_path = os.path.join(tempfile.gettempdir(), 'roc', 'film')
+        data_test_path = os.path.join(tempfile.gettempdir(), "roc", "film")
 
         # Get pipeline configuration parameters
         conf = FilmTest.load_configuration()
 
         # Check if ROC_TEST_DATA_PATH env. variable is defined
         # in: (1) config file, (2) shell env.
-        for source in [conf['environment'], os.environ]:
+        for source in [conf["environment"], os.environ]:
             try:
-                data_test_path = os.path.join(source['ROC_TEST_DATA_PATH'], 'roc', 'film')
+                data_test_path = os.path.join(
+                    source["ROC_TEST_DATA_PATH"], "roc", "film"
+                )
             except Exception:
                 # logger.debug('Env. variable ROC_TEST_DATA_PATH not set')
                 pass
@@ -63,19 +70,22 @@ class FilmTest:
         if command:
             data_test_path = os.path.join(data_test_path, command.lower())
 
-        logger.info(f'Using {data_test_path} to store test data')
+        logger.info(f"Using {data_test_path} to store test data")
         # Make sure the directory exists (ignore if it does)
         Path(data_test_path).mkdir(parents=True, exist_ok=True)
 
         return data_test_path
 
-    def get_test_data(self, command,
-                      base_url=None,
-                      test_data_dir=None,
-                      test_data_file='test_data.tar.gz',
-                      target_dir=None,
-                      overwrite=False,
-                      extract=True):
+    def get_test_data(
+        self,
+        command,
+        base_url=None,
+        test_data_dir=None,
+        test_data_file="test_data.tar.gz",
+        target_dir=None,
+        overwrite=False,
+        extract=True,
+    ):
         """
         Try to get data for the current FILM command test.
 
@@ -106,25 +116,26 @@ class FilmTest:
         auth = (self.username, self.password)
 
         if not os.path.isfile(test_data_path) or overwrite:
-
             # Build complete URL and download file
-            test_data_url = '/'.join([base_url, command, test_data_file])
+            test_data_url = "/".join([base_url, command, test_data_file])
             if all([val is not None for val in auth]):
-                logger.info(f'Downloading {test_data_url} in {test_data_dir} ...')
+                logger.info(f"Downloading {test_data_url} in {test_data_dir} ...")
                 download_file(test_data_path, test_data_url, auth=auth)
             else:
-                raise IOError(f'At least one of the following arguments missing: [username, password]!')
+                raise IOError(
+                    "At least one of the following arguments missing: [username, password]!"
+                )
 
         else:
-            logger.info(f'{test_data_path} already exists')
+            logger.info(f"{test_data_path} already exists")
 
         if extract:
             if tarfile.is_tarfile(str(test_data_path)):
-                logger.info(f'Extracting {test_data_path} ...')
-                with tarfile.open(str(test_data_path), 'r:*') as tarball:
-                    tarball.extractall(path=target_dir)
+                logger.info(f"Extracting {test_data_path} ...")
+                with tarfile.open(str(test_data_path), "r:*") as tarball:
+                    tarball.extractall(path=target_dir, filter="fully_trusted")
             else:
-                raise tarfile.ReadError(f'{test_data_path} is not a valid tarball!')
+                raise tarfile.ReadError(f"{test_data_path} is not a valid tarball!")
 
         return test_data_path
 
@@ -135,9 +146,14 @@ class FilmTest:
 
         :return: output_dir_path, string containing the path
         """
+        # Initialize output
+        output_dir_path = ""
+
         # Define value using command name
         if command:
-            output_dir_path = os.path.join(FilmTest.get_test_data_path(command), 'output')
+            output_dir_path = os.path.join(
+                FilmTest.get_test_data_path(command), "output"
+            )
         else:
             # Else attempt to get from config file or env variables.
             # Get pipeline configuration parameters
@@ -145,16 +161,20 @@ class FilmTest:
 
             # Check if pipeline.output_path variable is defined in config
             try:
-                output_dir_path = conf['pipeline']['output_path']
+                output_dir_path = conf["pipeline"]["output_path"]
             except KeyError:
-                logger.debug('Variable pipeline.output_path not set in the config. file!')
+                logger.debug(
+                    "Variable pipeline.output_path not set in the config. file!"
+                )
             else:
-                if output_dir_path.startswith('$ROOT_DIRECTORY'):
-                    output_dir_path = output_dir_path.replace('$ROOT_DIRECTORY', settings.ROOT_DIRECTORY)
+                if output_dir_path.startswith("$ROOT_DIRECTORY"):
+                    output_dir_path = output_dir_path.replace(
+                        "$ROOT_DIRECTORY", settings.ROOT_DIRECTORY
+                    )
 
         # Create directory
         Path(output_dir_path).mkdir(exist_ok=True, parents=True)
-        logger.debug(f'Using {output_dir_path} to store data produced during the test')
+        logger.debug(f"Using {output_dir_path} to store data produced during the test")
 
         return output_dir_path
 
@@ -166,15 +186,18 @@ class FilmTest:
         :param test_data_dir: Root path of the test data directory
         :return: tuple (input_dir_path, inputs)
         """
-        input_dir_path = os.path.join(test_data_dir, 'inputs')
+        input_dir_path = os.path.join(test_data_dir, "inputs")
         if not os.path.isdir(input_dir_path):
-            logger.warning(f'{input_dir_path} not found')
+            logger.warning(f"{input_dir_path} not found")
             inputs = list()
         else:
-            inputs = [item for item in os.listdir(input_dir_path)
-                      if os.path.isfile(os.path.join(input_dir_path, item))]
+            inputs = [
+                item
+                for item in os.listdir(input_dir_path)
+                if os.path.isfile(os.path.join(input_dir_path, item))
+            ]
             if len(inputs) == 0:
-                logger.warning(f'No input file found in {input_dir_path}')
+                logger.warning(f"No input file found in {input_dir_path}")
 
         return input_dir_path, inputs
 
@@ -186,15 +209,18 @@ class FilmTest:
         :param test_data_dir: Root path of the test data directory
         :return: tuple (expected_output_dir_path, expected_outputs)
         """
-        expected_output_dir_path = os.path.join(test_data_dir, 'expected_outputs')
+        expected_output_dir_path = os.path.join(test_data_dir, "expected_outputs")
         if not os.path.isdir(expected_output_dir_path):
-            logger.warning(f'{expected_output_dir_path} not found')
+            logger.warning(f"{expected_output_dir_path} not found")
             expected_outputs = list()
         else:
-            expected_outputs = [item for item in os.listdir(expected_output_dir_path)
-                                if os.path.isfile(os.path.join(expected_output_dir_path, item))]
+            expected_outputs = [
+                item
+                for item in os.listdir(expected_output_dir_path)
+                if os.path.isfile(os.path.join(expected_output_dir_path, item))
+            ]
             if len(expected_outputs) == 0:
-                logger.warning(f'No input file found in {expected_output_dir_path}')
+                logger.warning(f"No input file found in {expected_output_dir_path}")
 
         return expected_output_dir_path, expected_outputs
 
@@ -206,18 +232,18 @@ class FilmTest:
         :return: spice_kernels_dir
         """
         # Define default value
-        spice_kernels_dir = os.path.join(Path.cwd(), 'data', 'spice_kernels')
+        spice_kernels_dir = os.path.join(Path.cwd(), "data", "spice_kernels")
 
         # Get pipeline configuration parameters
         conf = FilmTest.load_configuration()
 
         # Check if SPICE_KERNEL_PATH env. variable is defined
         # in: (1) config file, (2) shell env.
-        for source in [conf['environment'], os.environ]:
+        for source in [conf["environment"], os.environ]:
             try:
-                spice_kernels_dir = os.path.join(source['SPICE_KERNEL_PATH'])
-            except Exception:
-                # logger.debug('Env. variable SPICE_KERNEL_PATH not set')
+                spice_kernels_dir = os.path.join(source["SPICE_KERNEL_PATH"])
+            except Exception as e:
+                logger.debug(e)
                 pass
             else:
                 break
@@ -225,18 +251,20 @@ class FilmTest:
         return spice_kernels_dir
 
     @staticmethod
-    def get_diff_files(dirs_cmp, path=''):
+    def get_diff_files(dirs_cmp, path=""):
         for name in dirs_cmp.diff_files:
             yield os.path.join(path, name)
         for parent, sub_dirs_cmp in dirs_cmp.subdirs.items():
-            for filepath in FilmTest.get_diff_files(sub_dirs_cmp, path=os.path.join(path, parent)):
+            for filepath in FilmTest.get_diff_files(
+                sub_dirs_cmp, path=os.path.join(path, parent)
+            ):
                 yield filepath
 
     @staticmethod
     def load_configuration():
         from poppy.core.configuration import Configuration
 
-        configuration = Configuration(os.getenv('PIPELINE_CONFIG_FILE', None))
+        configuration = Configuration(os.getenv("PIPELINE_CONFIG_FILE", None))
         configuration.read()
 
         return configuration
@@ -249,34 +277,36 @@ class FilmTest:
         :return: idb_release_dir_path, string containing the path
         """
         # Define default value
-        idb_release_dir_path = os.path.join(tempfile.gettempdir(), 'roc', 'idb_release')
+        idb_release_dir_path = os.path.join(tempfile.gettempdir(), "roc", "idb_release")
 
         # Get pipeline configuration parameters
         conf = FilmTest.load_configuration()
 
         # Check if IDB_INSTALL_DIR env. variable is defined
         # in: (1) config file, (2) shell env.
-        for source in [conf['environment'], os.environ]:
+        for source in [conf["environment"], os.environ]:
             try:
-                idb_dump_dir_path = os.path.join(source['IDB_INSTALL_DIR'])
-            except Exception:
-                # logger.debug('Env. variable IDB_INSTALL_DIR not set')
+                _ = os.path.join(source["IDB_INSTALL_DIR"])
+            except Exception as e:
+                logger.debug(e)
                 pass
             else:
                 break
 
-        logger.info(f'Using {idb_release_dir_path} to store RPW IDB source files')
+        logger.info(f"Using {idb_release_dir_path} to store RPW IDB source files")
 
         return idb_release_dir_path
 
     @staticmethod
-    def load_idb(test_class_instance,
-                 idb_version=None,
-                 idb_dump_file=None,
-                 user=os.environ.get('ROC_TEST_USER', 'roctest'),
-                 password=os.environ.get('ROC_TEST_PASSWORD', None),
-                 install_dir=None,
-                 log_level='ERROR'):
+    def load_idb(
+        test_class_instance,
+        idb_version=None,
+        idb_dump_file=None,
+        user=os.environ.get("ROC_TEST_USER", "roctest"),
+        password=os.environ.get("ROC_TEST_PASSWORD", None),
+        install_dir=None,
+        log_level="ERROR",
+    ):
         """
         Function that gathers commands to load IDB in the database.
 
@@ -291,7 +321,7 @@ class FilmTest:
         """
 
         # Make sure to have an up-and-running database
-        test_class_instance.run_command('pop db upgrade heads -ll INFO')
+        test_class_instance.run_command("pop db upgrade heads -ll INFO")
 
         if not install_dir:
             install_dir = FilmTest.get_idb_release_dir_path()
@@ -300,50 +330,51 @@ class FilmTest:
             # Loading IDB dump file in the database
             # (required a valid dump file and psql command line tool)
             command = [
-                'pop',
-                '-ll',
+                "pop",
+                "-ll",
                 log_level,
-                'idb',
-                '--force',
-                'load_idb_dump',
-                '--i',
+                "idb",
+                "--force",
+                "load_idb_dump",
+                "--i",
                 install_dir,
-                '-d',
+                "-d",
                 idb_dump_file,
-                '-a',
-                user, password,
+                "-a",
+                user,
+                password,
             ]
             test_class_instance.run_command(command)
         elif idb_version:
             # Load PALISADE IDB first (always)
             idb_palisade_loading = [
-                'pop',
-                'idb',
-                'install',
-                '-i',
+                "pop",
+                "idb",
+                "install",
+                "-i",
                 install_dir,
-                '-s',
-                'PALISADE',
-                '-v',
-                '4.3.5_MEB_PFM',
-                '--load',
-                '-ll',
+                "-s",
+                "PALISADE",
+                "-v",
+                "4.3.5_MEB_PFM",
+                "--load",
+                "-ll",
                 log_level,
             ]
 
             # Then load MIB
             idb_mib_loading = [
-                'pop',
-                'idb',
-                'install',
-                '-i',
+                "pop",
+                "idb",
+                "install",
+                "-i",
                 install_dir,
-                '-s',
-                'MIB',
-                '-v',
+                "-s",
+                "MIB",
+                "-v",
                 idb_version,
-                '--load',
-                '-ll',
+                "--load",
+                "-ll",
                 log_level,
             ]
 
@@ -353,12 +384,16 @@ class FilmTest:
             # Apply IDB loading
             test_class_instance.run_command(idb_mib_loading)
         else:
-            raise ValueError('Missing input argument to load IDB!')
+            raise ValueError("Missing input argument to load IDB!")
 
     @staticmethod
-    def load_l0(test_class_instance, l0_dir_path, idb_version,
-                idb_source='MIB',
-                pattern='solo_L0_rpw*.h5'):
+    def load_l0(
+        test_class_instance,
+        l0_dir_path,
+        idb_version,
+        idb_source="MIB",
+        pattern="solo_L0_rpw*.h5",
+    ):
         """
         Insert L0 data in the database (required to run some tests)
 
@@ -374,32 +409,42 @@ class FilmTest:
         # Get SCLK and LSK SPICE kernels
         kernels_dir = FilmTest.get_spice_kernel_dir()
         try:
-            sclk_file = sorted(list(glob.glob(os.path.join(kernels_dir, 'sclk', '*.tsc'))))[-1]
-        except:
-            raise FileNotFoundError('No SPICE SCLK kernel loaded!')
+            sclk_file = sorted(
+                list(glob.glob(os.path.join(kernels_dir, "sclk", "*.tsc")))
+            )[-1]
+        except Exception as e:
+            logger.exception(f"No SPICE SCLK kernel loaded:\n{e}")
+            raise
         else:
-            logger.info(f'Use {sclk_file}')
+            logger.info(f"Use {sclk_file}")
         try:
-            lsk_file = sorted(list(glob.glob(os.path.join(kernels_dir, 'lsk', '*.tls'))))[-1]
-        except:
-            raise FileNotFoundError('No SPICE LSK kernel loaded!')
+            lsk_file = sorted(
+                list(glob.glob(os.path.join(kernels_dir, "lsk", "*.tls")))
+            )[-1]
+        except Exception as e:
+            logger.exception(f"No SPICE LSK kernel loaded:\n{e}")
+            raise
         else:
-            logger.info(f'Use {lsk_file}')
+            logger.info(f"Use {lsk_file}")
 
         # Inserting L0 data into the database
-        cmd = ' '.join(['pop',
-                        '-ll INFO',
-                        'dingo',
-                        f'--idb-version {idb_version}',
-                        f'--idb-source {idb_source}',
-                        f'--sclk {sclk_file}',
-                        f'--lsk {lsk_file}',
-                        'l0_to_db',
-                        '-l0 ' + ' '.join(l0_files_path)])
+        cmd = " ".join(
+            [
+                "pop",
+                "-ll INFO",
+                "dingo",
+                f"--idb-version {idb_version}",
+                f"--idb-source {idb_source}",
+                f"--sclk {sclk_file}",
+                f"--lsk {lsk_file}",
+                "l0_to_db",
+                "-l0 " + " ".join(l0_files_path),
+            ]
+        )
         logger.debug(cmd)
         test_class_instance.run_command(shlex.split(cmd))
 
         # Inserting HFR time info in the database
-        cmd = 'pop -ll INFO dingo l0_to_hfrtimelog -l0 ' + ' '.join(l0_files_path)
+        cmd = "pop -ll INFO dingo l0_to_hfrtimelog -l0 " + " ".join(l0_files_path)
         logger.debug(cmd)
         test_class_instance.run_command(shlex.split(cmd))

roc/film/tests/test_l0_to_hk.py

@@ -14,7 +14,7 @@ import pytest
 import shutil
 import unittest.mock as mock
 
-from maser.tools.cdf.cdfcompare import cdf_compare
+from roc.film.tests.cdf_compare import cdf_compare
 
 from poppy.core.logger import logger
 from poppy.core.test import CommandTestCase
@@ -25,7 +25,7 @@ from roc.film.tests.test_film import FilmTest
 class TestL0ToHk(CommandTestCase):
     film = FilmTest()
 
-    cmd = 'l0_to_hk'
+    cmd = "l0_to_hk"
 
     def setup_method(self, method):
         super().setup_method(method)
@@ -46,15 +46,18 @@ class TestL0ToHk(CommandTestCase):
         # clear the files produced during test
         shutil.rmtree(self.output_dir_path)
 
-    #@pytest.mark.skip()
-    @pytest.mark.parametrize('idb_source,idb_version', [
-        ('MIB', '20200131'),
-    ])
+    # @pytest.mark.skip()
+    @pytest.mark.parametrize(
+        "idb_source,idb_version",
+        [
+            ("MIB", "20200131"),
+        ],
+    )
     def test_l0_to_hk(self, idb_source, idb_version):
         from poppy.core.conf import Settings
 
         # Name of the command to test
-        cmd = 'l0_to_hk'
+        cmd = "l0_to_hk"
 
         # Retrieve data for current test
         test_data_path = FilmTest().get_test_data(cmd, extract=True)
@@ -64,46 +67,69 @@ class TestL0ToHk(CommandTestCase):
         # Initialize inputs and expected outputs
         input_dir_path, inputs = FilmTest.get_inputs(test_data_dir)
         if not inputs:
-            raise FileNotFoundError(f'No input found in {test_data_dir}!')
-        expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(test_data_dir)
+            raise FileNotFoundError(f"No input found in {test_data_dir}!")
+        expected_output_dir_path, expected_outputs = FilmTest.get_expected_outputs(
+            test_data_dir
+        )
         if not expected_outputs:
-            raise FileNotFoundError(f'No expected output found in {test_data_dir}!')
+            raise FileNotFoundError(f"No expected output found in {test_data_dir}!")
 
         # Initialize directory where files produced during test will be saved
-        output_dir_path = os.path.join(test_data_dir, 'output')
+        output_dir_path = os.path.join(test_data_dir, "output")
         self.output_dir_path = output_dir_path
 
         # Check that SPICE kernels are present in ./data/spice_kernels folder
         spice_kernels_dir = FilmTest.get_spice_kernel_dir()
         if not os.path.isdir(spice_kernels_dir):
-            raise FileNotFoundError(f'No SPICE kernel set found in {spice_kernels_dir}!')
+            raise FileNotFoundError(
+                f"No SPICE kernel set found in {spice_kernels_dir}!"
+            )
 
         # initialize the main command
-        command_to_test = ' '.join(['pop',
-                                    'film',
-                                    '--force',
-                           '--idb-version', idb_version,
-                           '--idb-source', idb_source,
-                           cmd,
-                           os.path.join(input_dir_path, inputs[0]),
-                           '--output-dir', output_dir_path,
-                           '-ll', 'INFO'])
+        command_to_test = " ".join(
+            [
+                "pop",
+                "film",
+                "--force",
+                "--idb-version",
+                idb_version,
+                "--idb-source",
+                idb_source,
+                cmd,
+                os.path.join(input_dir_path, inputs[0]),
+                "--output-dir",
+                output_dir_path,
+                "-ll",
+                "INFO",
+            ]
+        )
 
         # define the required plugins
-        plugin_list = ['poppy.pop', 'roc.idb', 'roc.rpl', 'roc.rap', 'roc.dingo', 'roc.film']
+        plugin_list = [
+            "poppy.pop",
+            "roc.idb",
+            "roc.rpl",
+            "roc.rap",
+            "roc.dingo",
+            "roc.film",
+        ]
 
         # run the command
         # force the value of the plugin list
-        with mock.patch.object(Settings, 'configure',
-                               autospec=True,
-                               side_effect=self.mock_configure_settings(dictionary={'PLUGINS': plugin_list})):
+        with mock.patch.object(
+            Settings,
+            "configure",
+            autospec=True,
+            side_effect=self.mock_configure_settings(
+                dictionary={"PLUGINS": plugin_list}
+            ),
+        ):
             # Load IDB in the database
             FilmTest.load_idb(self, idb_version=idb_version)
             self.run_command(shlex.split(command_to_test))
 
         # compare directory content
-        dirs_cmp = filecmp.dircmp(output_dir_path,
-                                  expected_output_dir_path)
+        dirs_cmp = filecmp.dircmp(output_dir_path, expected_output_dir_path)
 
         dirs_cmp.report()
 
@@ -112,23 +138,25 @@ class TestL0ToHk(CommandTestCase):
 
         for filename in FilmTest.get_diff_files(dirs_cmp):
             # compare only cdf files with differences
-            if filename.endswith('.cdf'):
+            if filename.endswith(".cdf"):
                 # use cdf compare to compute the differences between expected output and the command output
                 result = cdf_compare(
                     os.path.join(output_dir_path, filename),
                     os.path.join(expected_output_dir_path, filename),
                     list_ignore_gatt=[
-                        'File_ID',
-                        'Generation_date',
-                        'Pipeline_version',
-                        'Pipeline_name',
-                        'Software_version',
-                        'IDB_version'
-                    ]
+                        "File_ID",
+                        "Generation_date",
+                        "Pipeline_version",
+                        "Pipeline_name",
+                        "Software_version",
+                        "IDB_version",
+                    ],
                 )
 
                 # compare the difference dict with the expected one
                 if result:
-                    logger.error(f'Differences between expected output and the command output: {pformat(result)}')
+                    logger.error(
+                        f"Differences between expected output and the command output: {pformat(result)}"
+                    )
 
                 assert result == {}