SankeyExcelParser 1.0.0b0__py3-none-any.whl

SankeyExcelParser/su_trace.py

@@ -0,0 +1,239 @@
+import os
+import time
+import logging
+import logging.handlers
+import psutil
+
+import pandas as pd
+
+logger = logging.getLogger()
+
+
+def logger_init(
+    logname,
+    mode
+):
+    """
+    Initialize global logger
+
+    Parameters
+    ----------
+    logname : string
+        Log file
+    mode : string
+        File open function (see https://www.manpagez.com/man/3/fopen/)
+    """
+    global logger
+    logger = logging.getLogger("sumoptimisation")  # root logger
+    if len(logger.handlers) > 0:
+        logger.handlers[0].close()
+        logger.removeHandler(logger.handlers[0])
+    logger.setLevel(logging.DEBUG)
+    hdlr = logging.FileHandler(logname, mode)
+    fmt = logging.Formatter("%(levelname)-5s %(message)s", "%x %X")
+    hdlr.setFormatter(fmt)
+    logger.addHandler(hdlr)
+
+
+def base_filename():
+    """
+    Returns filename from which logger comes from.
+
+    Returns
+    -------
+    string
+        Base bilename
+    """
+    return logging.getLogger("sumoptimisation").handlers[0].baseFilename
+
+
+def run_log(myfile):
+    """
+    Start logging.
+
+    Parameters
+    ----------
+    myfile : string
+        Log file
+    """
+    # Check if file will be created
+    IsNew = not os.path.isfile(myfile)
+    # Log format
+    logging.basicConfig(
+        filename=myfile,
+        format='%(asctime)s,%(msecs)03d - %(levelname)-8s - %(funcName)-20s (%(lineno)04d): %(message)s',
+        datefmt='%H:%M:%S',
+        level=logging.INFO)
+    # Define a Handler which writes INFO messages or higher to the sys.stderr
+    console = logging.StreamHandler()
+    # Set a format which is simpler for console use
+    co_formatter = logging.Formatter('%(funcName)-20s (%(lineno)04d): %(message)s')
+    # Tell the handler to use this format
+    console.setFormatter(co_formatter)
+    # Add the handler to the root logger
+    logging.getLogger().addHandler(console)
+    if IsNew:
+        strnow = time.strftime('%Y-%m-%d')
+        logging.info(f'Log file just created. Date of creation : {strnow}')
+    else:
+        logging.info('*****************************')
+        logging.info('********** New run **********')
+        logging.info('*****************************')
+
+
+def log_level(StrLevel="INFO"):
+    '''
+    Change the level information of the current logger.
+
+    Parameters
+    ----------
+    StrLevel : string
+        Level of Information.
+        Possible values are (All calls with a higher value than the selected one are logged)
+        - "NOTSET"(value 0)
+        - "DEBUG"(value 10)
+        - "INFO"(20)
+        - "WARNING"(30)
+        - "ERROR"(40)
+        - "CRITICAL"(50)
+    '''
+    switcher = {
+        "NOTSET": 0,
+        "DEBUG": 10,
+        "INFO": 20,
+        "WARNING": 30,
+        "ERROR": 40,
+        "CRITICAL": 50}
+    NewLevel = 20
+    if StrLevel in switcher:
+        NewLevel = switcher[StrLevel]
+    logging.getLogger().setLevel(NewLevel)
+
+
+def check_log(nbmax=20):
+    '''
+    Create a new filename for logs.
+    Remove older files, keep only the 'nbmax' latest files.
+
+    Parameters
+    ----------
+    nbmax : int, optionnal (default=20)
+        Maximum of latest log files to keep.
+
+    Returns
+    -------
+    string
+        Log filename as 'logs/log_YYMMDD.log'.
+    '''
+    log_def = 'log_' + time.strftime('%Y%m%d') + '.log'
+    dir_fi = 'logs' + os.path.sep
+    if not os.path.isdir('logs'):
+        os.makedirs('logs')
+    else:
+        li_file = os.listdir('logs')
+        df_file = pd.DataFrame(li_file, columns=['files'])
+        df_file['date'] = [os.path.getctime(dir_fi + fi) for fi in li_file]
+        df_sort = df_file.sort_values(by=['date'], ascending=False)
+        if len(df_sort['date'] > nbmax):
+            df_del = df_sort['files'][nbmax:]
+            for fi in df_del:
+                os.remove(dir_fi + fi)
+    log_def = dir_fi + log_def
+    return log_def
+
+
+def timems(
+    t_input: float,
+    f_out='',
+    b_full=False,
+):
+    '''
+    Convert input as millisecs or milisec, as int or full timestamp.
+
+    Parameters
+    ----------
+    t_input : float
+        Time is secs to convert.
+    f_out : string {'milli', 'micro', ''}, optionnal (default='')
+        Maximum of latest log files to keep.
+        - 'milli' : convert input time in milisecs
+        - 'micro' : convert input time in microsecs
+        -  other : keep as secs
+    b_full : bool, optionnal (default=False)
+        Chose type of output
+        - true : Return full timestamp with date as string
+        - false : Return only converted data as int
+
+    Returns
+    -------
+    string or int
+        full timestamp with date as string / converted data as int
+    '''
+    if b_full:
+        st0 = time.localtime(t_input)
+        comp = 0
+        if f_out == 'milli':
+            comp = int(round((t_input-int(t_input))*1000))
+        elif f_out == 'micro':
+            comp = int(round((t_input-int(t_input))*1000000))
+        return time.strftime(f'%Y-%m-%d %H:%M:%S,{comp}', st0)  # return a string
+    else:
+        if f_out == 'milli':
+            comp = int(round(t_input*1000))
+        elif f_out == 'micro':
+            comp = int(round(t_input*1000000))
+        else:
+            comp = int(round(t_input))  # time in sec
+        return comp  # return an integer
+
+
+def perf_process(procname='python'):
+    '''
+    Gives process and system infos.
+
+    Parameters
+    ----------
+    procname : string, optionnal (default='python')
+        Process from which we need informations
+
+    Returns
+    -------
+    array-like[3]
+        Process infos [PID, Process Info, System info].
+        - PID : id of process
+        - Process info : CPU charge of process, MEM consumtion,
+                         MEM dedicated to sub-process, MEM dedicated to main process
+        - System info : CPU charge, MEM used, MEM available
+    '''
+    first_pid_found = False
+    procname = procname.lower()
+    for proc in psutil.process_iter():
+        pname = proc.name().lower()
+        if procname in pname:
+            if not first_pid_found:
+                first_pid_found = True
+                pyid = proc.pid
+                stproc_id = f'process id is : {pyid}'
+            else:
+                stproc_id += f', id_add: {proc.pid}'
+    # PROCESS INFOS
+    ppy = psutil.Process(pyid)
+    stproc_inf = f'PROCESS - cpu_percent: {ppy.cpu_percent()}, '
+    mem_val = str(round(ppy.memory_percent(), 2))
+    stproc_inf += f'mem_percent: {mem_val}, '
+    ppym = ppy.memory_full_info()
+    mem_val = str(round(ppym.uss/1024/1024))
+    stproc_inf += f'mem_dedic_PID: {mem_val} Mo, '
+    ppym = psutil.Process().memory_full_info()
+    mem_val = str(round(ppym.uss/1024/1024))
+    stproc_inf += f'mem_dedic: {mem_val} Mo'
+    sysm = psutil.virtual_memory()
+    # SYSTEM INFOS
+    stsys_inf = f'SYSTEM - cpu_percent: {psutil.cpu_percent()}, '
+    mem_val = str(round(sysm.percent, 2))
+    stsys_inf += f'mem_percent: {mem_val}, '
+    mem_val = str(round(sysm.used/1024/1024))
+    stsys_inf += f'mem_used: {mem_val} Mo, '
+    mem_val = str(round(sysm.available/1024/1024))
+    stsys_inf += f'mem_avail: {mem_val} Mo'
+    return [stproc_id, stproc_inf, stsys_inf]

SankeyExcelParser/tests/integration/test_base.py

@@ -0,0 +1,356 @@
+"""
+Auteur : Vincent LE DOZE
+Date : 07/12/23
+"""
+
+# External libs ---------------------------------------------------------------
+import unittest
+import os
+import json
+import tempfile
+
+# External modules ------------------------------------------------------------
+from os import listdir
+from shutil import copyfile
+
+# Local modules ---------------------------------------------------------------
+from SankeyExcelParser.io_excel import load_sankey_from_excel_file
+from SankeyExcelParser.sankey import Sankey
+from SankeyExcelParser import su_trace as su_trace
+
+# Constants -------------------------------------------------------------------
+TESTS_DIR = os.environ.get('TESTS_DIR')
+XLPARSER_TESTS_REFS_DIR = 'ref_tests__SankeyExcelParser'
+SCMFA_TESTS_REFS_DIR = 'ref_tests__SCMFA'
+OPENSANKEY_TESTS_REFS_DIR = 'ref_tests__OpenSankey'
+MFASANKEY_TESTS_REFS_DIR = 'ref_tests__MFASankey'
+
+DIRS_TO_EXCLUDE = [
+    '.git',
+    '.md',
+    XLPARSER_TESTS_REFS_DIR,
+    SCMFA_TESTS_REFS_DIR,
+    OPENSANKEY_TESTS_REFS_DIR,
+    MFASANKEY_TESTS_REFS_DIR
+]
+
+MAXSIZE = 200000
+
+REF_TEST_PREFIX = 'expected_'
+REF_SANKEY_SUFFIX = '_sankey-dict'
+REF_LOG_SUFFIX = '_parsing_logs'
+
+EXPECTED_RESULTS = {}
+EXPECTED_RESULTS[os.path.relpath('Tests/create_empty_ter/pommes_poires') + ' create empty ter'] = None
+EXPECTED_RESULTS[os.path.relpath('Tests/create_empty_ter/simplified_example_fr') + ' create empty ter'] = None
+EXPECTED_RESULTS[os.path.relpath('Projets/AlimentationAnimale/orge check input')] = None
+
+LOGS_KEY = 'Logs'
+SANKEY_KEY = 'Sankey'
+
+TEST_PARAMETERS = []
+
+
+# Functions -------------------------------------------------------------------
+def keep_exploring_file_or_folder(file_or_folder):
+    # Exclude certains files or folders
+    for _ in DIRS_TO_EXCLUDE:
+        if _ in file_or_folder:
+            return False
+    return True
+
+
+def parse_folder(current_dir):
+    folder_content = listdir(current_dir)
+    for file_or_folder in folder_content:
+        # Folder exclusion conditions
+        if not keep_exploring_file_or_folder(file_or_folder):
+            continue
+        # Does test file exists ?
+        if os.path.isfile(os.path.join(current_dir, file_or_folder)):
+            # It it something like <file>.xlsx ?
+            if ('xlsx' in file_or_folder) and \
+               ('reconciled' not in file_or_folder) and \
+               ('converted' not in file_or_folder) and \
+               ('old' not in file_or_folder) and \
+               ('solution' not in file_or_folder):
+                # If file is too large, don't test
+                file_stats = os.stat(os.path.join(current_dir, file_or_folder))
+                if file_stats.st_size > MAXSIZE:
+                    continue
+                # Get file name
+                file_relpath = os.path.relpath(
+                    os.path.join(current_dir, file_or_folder),
+                    TESTS_DIR)
+                test_name = os.path.splitext(file_relpath)[0]
+                test_dir, test_subname = os.path.split(test_name)
+                # Update expected results
+                if test_name not in EXPECTED_RESULTS:
+                    EXPECTED_RESULTS[test_name] = {}
+                # Create results output dir
+                test_refs_dir = os.path.join(current_dir, XLPARSER_TESTS_REFS_DIR)
+                if XLPARSER_TESTS_REFS_DIR not in listdir(current_dir):
+                    os.mkdir(test_refs_dir)
+                # Get related logs
+                test_ref_log_path = os.path.join(
+                    test_refs_dir,
+                    REF_TEST_PREFIX + test_subname + REF_LOG_SUFFIX + '.json')
+                EXPECTED_RESULTS[test_name][LOGS_KEY] = {}
+                if os.path.isfile(test_ref_log_path):
+                    with open(test_ref_log_path, "r") as test_ref_log_file:
+                        test_ref_log = json.load(test_ref_log_file)
+                        EXPECTED_RESULTS[test_name][LOGS_KEY] = test_ref_log
+                # Get related sankey as dict expected result
+                test_ref_sankey_path = os.path.join(
+                    test_refs_dir,
+                    REF_TEST_PREFIX + test_subname + REF_SANKEY_SUFFIX + '.json')
+                EXPECTED_RESULTS[test_name][SANKEY_KEY] = {}
+                if os.path.isfile(test_ref_sankey_path):
+                    with open(test_ref_sankey_path, "r") as test_ref_sankey_file:
+                        test_ref_sankey = json.load(test_ref_sankey_file)
+                        EXPECTED_RESULTS[test_name][SANKEY_KEY] = test_ref_sankey
+                # Finish updating test
+                TEST_PARAMETERS.append((
+                    test_name,
+                    file_relpath,
+                    EXPECTED_RESULTS[test_name]))
+            continue
+        # Recursivly parse sub-directories
+        parse_folder(os.path.join(current_dir, file_or_folder))
+
+
+# Fill constants values
+parse_folder(TESTS_DIR)
+
+
+# Class -----------------------------------------------------------------------
+class MFAProblemsTests(unittest.TestCase):
+    generate_results = False
+
+    @classmethod
+    def set_generate_results(cls):
+        cls.generate_results = True
+        cls.new_results = {}
+
+    def prepare_test(
+        self,
+        file_name: str,
+        use_reconciled: bool = False
+    ):
+        """
+        Read and check input excel file to use for tests.
+
+        Parameters
+        ----------
+        :param file_name: Input excel file to read
+        :type file_name: str
+
+        Optional parameters
+        -------------------
+        :param use_reconciled: Use reconciled version of given file
+        :type use_reconciled: bool, optional (defaults to False)
+
+        Returns
+        -------
+        :return: sankey, sheet_to_remove_names, excel_filepath, tmp_dir
+        :rtype: tuple as (Sankey, list, str, str)
+        """
+        # Make temp directory
+        tmp_dir = tempfile.mkdtemp()
+        # Init log file
+        logname = tmp_dir + os.path.sep + "rollover.log"
+        su_trace.logger_init(logname, "w")
+        # Get path of file to test
+        tests_dir = os.environ.get('TESTS_DIR')
+        excel_filepath = os.path.join(tests_dir, file_name)
+        # Get reconcilied version of file to test
+        excel_rec_filename = os.path.splitext(file_name)[0]+'_reconciled.xlsx'
+        excel_rec_filepath = os.path.join(tests_dir, excel_rec_filename)
+        # Can we test reconciled file ?
+        if os.path.isfile(excel_rec_filepath):
+            file_stats = os.stat(excel_rec_filepath)
+            if file_stats.st_size > MAXSIZE:
+                use_reconciled = False
+        # Read files
+        sheet_to_remove_names = []
+        sankey = Sankey()
+        if use_reconciled and os.path.isfile(excel_rec_filepath):
+            excel_rectmp_filepath = os.path.join(tmp_dir, os.path.basename(excel_rec_filename))
+            excel_filepath = excel_rec_filepath
+            copyfile(excel_rec_filepath, excel_rectmp_filepath)
+            ok, msg = load_sankey_from_excel_file(
+                excel_rectmp_filepath,
+                sankey,
+                sheet_to_remove_names)
+            if not ok:
+                su_trace.logger.error(msg)
+                return (sankey, sheet_to_remove_names, excel_filepath, tmp_dir)
+            # Remove results if needed
+            if sankey.has_at_least_one_result():
+                sankey.reset_all_results()
+        else:
+            sankey = Sankey()
+            ok, msg = load_sankey_from_excel_file(
+                excel_filepath,
+                sankey,
+                sheet_to_remove_names)
+            if not ok:
+                su_trace.logger.error(msg)
+                return (sankey, sheet_to_remove_names, excel_filepath, tmp_dir)
+        # Verify structure that we got
+        ok, msg = sankey.check_overall_sankey_structure()
+        if not ok:
+            su_trace.logger.error(msg)
+            return (sankey, sheet_to_remove_names, excel_filepath, tmp_dir)
+        # Return
+        return (sankey, sheet_to_remove_names, excel_filepath, tmp_dir)
+
+    def _compare_recursively(
+        self,
+        test_entry: dict,
+        ref_entry: dict
+    ):
+        """
+        Compare two dicts recursively
+
+        Parameters
+        ----------
+        :param test_entry: _description_
+        :type test_entry: dict
+
+        :param ref_entry: _description_
+        :type ref_entry: dict
+
+        Optional parameters
+        -------------------
+        """
+        # Dict -> Recurs
+        if isinstance(test_entry, dict):
+            for key in test_entry.keys():
+                if key != LOGS_KEY:
+                    self._compare_recursively(
+                        test_entry[key],
+                        ref_entry[key])
+            return
+        # List -> loop on all items
+        if isinstance(test_entry, list):
+            for item_to_test, item_as_ref in zip(test_entry, ref_entry):
+                self._compare_recursively(
+                    item_to_test,
+                    item_as_ref)
+            return
+        # Float -> test approx
+        if isinstance(test_entry, float):
+            self.assertAlmostEqual(
+                test_entry,
+                ref_entry, 2)
+            return
+        # Normal test
+        self.assertEqual(
+            test_entry,
+            ref_entry)
+
+    def check_logs(
+        self,
+        expected_results: dict,
+        test_name: str
+    ):
+        """
+        Compare logs with expected logs
+
+        Parameters
+        ----------
+        :param expected_results: Dict that contains results to check with
+        :type expected_results: dict
+
+        :param test_name: Name of current test
+        :type test_name: str
+        """
+        # Read current logs
+        base_filename = su_trace.base_filename()
+        with open(base_filename, "r") as f:
+            results = f.read()
+        # Splited lines
+        results_array = results.split('\n')
+        # Splited line but filtered
+        filter_result_array = []
+        for row in results_array:
+            if ('Main Problem' in row) or \
+               ('DEBUG' in row) or \
+               ('PERF' in row) or \
+               ('SOLVED in' in row):
+                continue
+            if 'Entering variables classification at' in row:
+                continue
+            if ('took' in row) or \
+               ('Took' in row) or \
+               ('done in' in row) or \
+               ('matrix reduction done' in row) or \
+               ('Output (matrix_reduction)' in row) or \
+               ('Interval' in row):
+                continue
+            filter_result_array.append(row)
+        # Check if logs are the same
+        if not self.generate_results:
+            self.assertEqual(
+                filter_result_array,
+                expected_results[LOGS_KEY])
+        else:
+            if test_name not in self.new_results:
+                self.new_results[test_name] = {}
+            self.new_results[test_name][LOGS_KEY] = filter_result_array
+
+    def compare_sankey(
+        self,
+        test_name: str,
+        sankey: Sankey,
+        expected_results: dict
+    ):
+        """
+        Compare a Sankey struct to expected results
+
+        Parameters
+        ----------
+        :param test_name: name of current test
+        :type test_name: str
+
+        :param sankey: current sankey struct to check
+        :type sankey: Sankey
+
+        :param expected_results:  Dict that contains results to check with
+        :type expected_results: dict
+        """
+        # Get sankey equivalent dict
+        sankey_dict = sankey.get_as_dict()
+        # Run tests
+        if not self.generate_results:
+            # Compare with ref
+            self._compare_recursively(
+                sankey_dict,
+                expected_results[SANKEY_KEY])
+            return
+        # Create test
+        if test_name not in self.new_results:
+            self.new_results[test_name] = {}
+        self.new_results[test_name][SANKEY_KEY] = sankey_dict
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.generate_results:
+            for test_name in cls.new_results:
+                test_dir, test_subname = os.path.split(test_name)
+                test_refs_dir = os.path.join(TESTS_DIR, test_dir, XLPARSER_TESTS_REFS_DIR)
+                # Save logs
+                if LOGS_KEY in cls.new_results[test_name].keys():
+                    test_logs_filename = REF_TEST_PREFIX + test_subname + REF_LOG_SUFFIX + '.json'
+                    test_logs_filepath = os.path.join(test_refs_dir, test_logs_filename)
+                    logs_as_json = json.dumps(cls.new_results[test_name][LOGS_KEY], indent=2)
+                    with open(test_logs_filepath, "w") as test_logs_file:
+                        test_logs_file.write(logs_as_json)
+                # Save sankey struct as json if present (if reconciliation did not failed)
+                if SANKEY_KEY in cls.new_results[test_name].keys():
+                    test_sankey_filename = REF_TEST_PREFIX + test_subname + REF_SANKEY_SUFFIX + '.json'
+                    test_sankey_filepath = os.path.join(test_refs_dir, test_sankey_filename)
+                    sankey_as_json = json.dumps(cls.new_results[test_name][SANKEY_KEY], indent=2)
+                    with open(test_sankey_filepath, "w") as test_sankey_file:
+                        test_sankey_file.write(sankey_as_json)

SankeyExcelParser/tests/integration/test_run_check_input.py

@@ -0,0 +1,100 @@
+"""
+Auteur : Vincent LE DOZE
+Date : 07/12/23
+"""
+
+# External libs ---------------------------------------------------------------
+import argparse
+# import os
+import unittest
+from parameterized import parameterized
+
+# Local modules ---------------------------------------------------------------
+from SankeyExcelParser.tests.integration.test_base import MFAProblemsTests
+from SankeyExcelParser.tests.integration.test_base import TEST_PARAMETERS
+from SankeyExcelParser.tests.integration.test_base import SANKEY_KEY
+
+
+# Class -----------------------------------------------------------------------
+class MFAProblemTestCheckInput(MFAProblemsTests):
+
+    # @parameterized.expand([(
+    #     os.path.relpath('Projets/AlimentationAnimale/orge check')+' input',
+    #     os.path.relpath('Projets/AlimentationAnimale/orge_new_segm.xlsx'),
+    #     EXPECTED_RESULTS[os.path.relpath('Projets/AlimentationAnimale/orge')+' check input'])])
+    @parameterized.expand(TEST_PARAMETERS, skip_on_empty=True)
+    def test_check_input(
+        self,
+        test_name: str,
+        file_name: str,
+        expected_results: dict
+    ):
+        """
+        Check if Sankey extraction from Excel file is OK.
+
+
+        Parameters
+        ----------
+        :param test_name: Name of current test
+        :type test_name: str
+
+        :param file_name: Name of current excel file to test
+        :type file_name: str
+
+        :param ouput_log: Logs expected for sankey extraction and checks
+        :type ouput_log: list
+
+        Optional parameters
+        -------------------
+        """
+        # For Debug
+        print('\n{}'.format(self._testMethodName), end=' -> ', flush=True)
+        # Read and check file
+        (
+            sankey,
+            sheet_to_remove_names,
+            excel_filepath,
+            _
+        ) = self.prepare_test(file_name)
+        # Check logs
+        self.check_logs(expected_results, test_name)
+        # Delete useless entries to avoid erasing previous results
+        if self.generate_results:
+            del expected_results[SANKEY_KEY]
+
+
+# Main ------------------------------------------------------------------------
+if __name__ == '__main__':
+    # Get args
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--generate_results",
+        action='store_true',
+        required=False,
+        help="Option to regenerate tests results")
+    parser.add_argument(
+        'filenames',
+        metavar='F',
+        type=str,
+        nargs='*',
+        help='Specific files to test')
+    args = parser.parse_args()
+    # Generate test if needed
+    if args.generate_results:
+        MFAProblemsTests.set_generate_results()
+    # Get tests names to run
+    if len(args.filenames) == 0:
+        loader = unittest.TestLoader()
+        names = loader.getTestCaseNames(MFAProblemTestCheckInput)
+    else:
+        names = args.filenames
+    # Append tests to test suite
+    suite = unittest.TestSuite()
+    for name in names:
+        try:
+            suite.addTest(MFAProblemTestCheckInput(name))
+        except Exception:
+            print("Error when adding {} to test base".format(name))
+    # Run tests
+    runner = unittest.TextTestRunner()
+    runner.run(suite)