ParUtils 1.2.3__py3-none-any.whl → 1.2.5__py3-none-any.whl

parutils/changelog.py

@@ -1,7 +1,16 @@
-__VERSION__ = '1.2.3'
-# close_logger returns the closed logger object
-# no more stip('\n') in save_list
-# empty_log_every_buffer
+__VERSION__ = '1.2.5'
+# Faster file.load_txt
+# Corrected comment typos
+
+# __VERSION__ = '1.2.4'
+# # !dir -> log_dir
+# # const to default kwargs
+# # Logger kwargs comments
+
+# __VERSION__ = '1.2.3'
+# # close_logger returns the closed logger object
+# # no more strip('\n') in save_list
+# # empty_log_every_buffer
 
 # __VERSION__ = '1.2.2'
 # # log_every

parutils/csvl.py

@@ -8,7 +8,7 @@ E_WRONG_TYPE_LIST = "List elements must be of type list (if you want to save a l
 
 
 def get_csv_fields_dict(in_path):
-    """Returns a dictionary whose keys are the csv fields of the 'in_path' file
+    """Returns a dictionary whose keys are the CSV fields of the 'in_path' file
     and elements are the columns index.
     """
 
@@ -21,10 +21,10 @@ def get_csv_fields_dict(in_path):
 
 
 def load_csv(in_path, quote=False):
-    """Loads a csv file and returns a list whose elements correspond to the
+    """Loads a CSV file and returns a list whose elements correspond to the
     lines of the 'in_path' file.
-    Each element is also a list whose elements correspond to the csv fields.
-    When quote is True, the buitin csv package is used and separators between quote char
+    Each element is also a list whose elements correspond to the CSV fields.
+    When quote is True, the builtin CSV package is used and separators between quote char
     are ignored (less performant).
     """
 
@@ -43,16 +43,16 @@ def load_csv(in_path, quote=False):
 
 
 def csv_to_list(line_in: str):
-    """Converts a csv line to a list using SEPARATOR as separator"""
+    """Converts a CSV line to a list using SEPARATOR as separator"""
 
     return line_in.strip('\n').split(SEPARATOR)
 
 
 def save_csv(array_in, out_path, mode='w', quote=False):
-    """Saves a list to a csv file
+    """Saves a list to a CSV file
 
     - mode: mode for the open methode.
-    - quote: determines whether each field should be wrapped into double quotes or not (default=False)
+    - quote: determines whether each field should be wrapped in double quotes or not (default=False)
     """
     import os.path as p
 
@@ -63,9 +63,9 @@ def save_csv(array_in, out_path, mode='w', quote=False):
 
 
 def write_csv_line(row, out_file: TextIOWrapper, quote=False):
-    """Writes a line to a csv file
+    """Writes a line to a CSV file
 
-    - row: has to be a list
+    - row: must be a list
     """
 
     if not isinstance(row, list):
@@ -78,7 +78,7 @@ def write_csv_line(row, out_file: TextIOWrapper, quote=False):
 
 
 def csv_clean(s: str):
-    """Cleans a csv field by removing csv separators and new line characters"""
+    """Cleans a CSV field by removing CSV separators and new line characters"""
 
     out = s.replace('\r', '')
     out = out.replace('\n', '')
@@ -89,7 +89,7 @@ def csv_clean(s: str):
 def get_header(in_path, csv=False):
     """Returns the header of a file
 
-    - csv: if True, the returned header is a list containing each csv field
+    - csv: if True, the returned header is a list containing each CSV field
     """
 
     with open(in_path, 'r', encoding='utf-8') as in_file:

parutils/file.py

@@ -15,7 +15,7 @@ def delete_folder(dir):
 
 
 def mkdirs(dir, delete=False):
-    """Same as os.makedirs but
+    """Same as os.makedirs but:
     - Input can also be a path
     - With a 'delete' option which (if True) deletes the folder if it already exists."""
 
@@ -38,7 +38,7 @@ def list_files(in_dir,
                ignore_list=[]):
     """Lists the files of the 'in_dir' directory
 
-    - incl_root: if True, the root directory is included in each paths
+    - incl_root: if True, the root directory is included in each path
     - walk: if True, the files of all the subdirectories are listed as well
     - only_list: list of wanted patterns. e.g. ['*.py'] (only these patterns will be output)
     - ignore_list: list of unwanted patterns. e.g. ['*.pyc'] (these patterns won't be output)
@@ -66,22 +66,16 @@ def list_files(in_dir,
 def load_txt(in_path, list_out=True):
     """Loads a text file
 
-    - list_out: if True, a list es output, each element representing a line a the file. If False, a string is output.
+    - list_out: if True, a list is output, each element representing a line of the file. If False, a string is output.
     """
 
+    with open(in_path, 'r', encoding='utf-8') as f:
+        data = f.read()
+
     if list_out:
-        out = []
+        return data.split('\n')
     else:
-        out = ''
-
-    with open(in_path, 'r', encoding='utf-8') as in_file:
-        for line in in_file:
-            if list_out:
-                out.append(line.strip('\n'))
-            else:
-                out += line
-
-    return out
+        return data
 
 
 def save_list(in_list, out_path, mode='w'):

parutils/logging/const.py

@@ -1,5 +1,2 @@
-DEFAULT_LEVEL = 0
 DEFAULT_DIR = 'log'
-DEFAULT_LOG_FORMAT = '%H:%M:%S - '
-DEFAULT_FILE_FORMAT = '%Y%m%d_%H%M%S'
 MAX_ERR_COUNT = 10

parutils/logging/logger.py

@@ -16,12 +16,12 @@ class Logger:
         self,
         file_label='',
         force_new_logger=False,
-        level=None,
-        log_format=None,
-        file_write=True,
-        dir=None,
-        file_format=None,
-        log_every=1,
+        level=0,
+        log_format='%H:%M:%S - ',
+        file_write=True,  # Logs are written in a file or not
+        log_dir=None,  # Directory where the log file is saved (default is 'log')
+        file_format='%Y%m%d_%H%M%S',
+        log_every=1,  # If equals to n, the log file will be written only every n logs. That can significantly improve perfs when writing logs to high latency places (eg. network drives)
     ) -> None:
         from . import g
 
@@ -32,8 +32,8 @@ class Logger:
         self.logs = []
         self.buffer = ''
         self.err_count = 0
-        self.level = level if level else const.DEFAULT_LEVEL
-        self.log_format = log_format if log_format else const.DEFAULT_LOG_FORMAT
+        self.level = level
+        self.log_format = log_format
         self.file_write = file_write
         self.start_time = time()
 
@@ -44,8 +44,8 @@ class Logger:
         if not file_write:
             return
         self.file_label = file_label
-        self.dir = dir if dir else const.DEFAULT_DIR
-        self.file_format = file_format if file_format else const.DEFAULT_FILE_FORMAT
+        self.dir = log_dir if log_dir else const.DEFAULT_DIR
+        self.file_format = file_format
         file_base_name = datetime.now().strftime(self.file_format)
         if self.file_label:
             file_base_name += '_' + self.file_label

parutils/logging/main.py

@@ -6,7 +6,7 @@ def log(*args, level=0, c_out=True):
     """Logs 'str_in' in the current log file (log_path)
 
     - level: log level. Current log level is the attribute level of the current logger.
-    You can get the current loger by using the get_logger function. Nothing is logged if logger level < level
+    You can get the current logger by using the get_logger function. Nothing is logged if logger level < level
     - c_out: specifies if something should be printed in the console or not
     """
 
@@ -16,7 +16,7 @@ def log_print(*args, level=0, c_out=True, nb_tab=0, dashes=0, tab_char='    ', s
     """Prints something in the current log file (log_path)
 
     - level: log level. Current log level is the attribute level of the current logger.
-    You can get the current loger by using the get_logger function. Nothing is logged if logger level < level
+    You can get the current logger by using the get_logger function. Nothing is logged if logger level < level
     - c_out: specifies if something should be printed in the console or not
     - nb_tab: number of tab indentations
     - dashes: total length of the input string extended with dashes ('-')

parutils/msc.py

@@ -2,7 +2,7 @@ from typing import List
 
 
 def list_to_dict(list_in: List[str], separator='='):
-    """Transforms 'list_in' to a dictionary using the 'separator'"""
+    """Transforms 'list_in' into a dictionary using the 'separator'"""
 
     out = {}
     for elt in list_in:
@@ -14,8 +14,8 @@ def list_to_dict(list_in: List[str], separator='='):
 
 
 def replace_from_dict(str_in: str, dict_in, var_del='@@'):
-    """Replaces the variables (delimited by '@@') in 'str_in' with the values
-    of 'dict_in'.
+    """Replaces the variables (delimited by '@@') in 'str_in' with values
+    from 'dict_in'.
 
     Example:
     - replace_from_dict('Hello @@VAR@@', {'VAR': 'world'}) returns 'Hello world'

parutils/strg.py

@@ -74,7 +74,7 @@ def like_dict(in_str, like_dict, case_sensitive=True, skey='', exact=False):
 
 
 def hash512(in_str: str, length=10):
-    """Contrary to hash, this hash function is not randomised, meaning it
+    """Contrary to hash, this hash function is not randomized, meaning it
     always outputs the same string for the same input string"""
     import hashlib
 
@@ -147,7 +147,7 @@ def get_duration_string(start_time, return_dms=False, end_time=None):
 
 
 def big_number(int_in):
-    """Converts a potentially big number into a lisible string.
+    """Converts a potentially big number into a readable string.
 
     Example:
     - big_number(10000000) returns '10 000 000'.

parutils/tests/conftest.py

@@ -0,0 +1,20 @@
+TESTS_LOG_DIR = 'log\\tests'
+TESTS_OUT_DIR = 'out\\tests'
+
+
+def init():
+    import parutils as u
+    import parutils.logging.const as const
+
+    const.DEFAULT_DIR = TESTS_LOG_DIR
+    u.dq.OUT_DIR = TESTS_OUT_DIR
+    u.mkdirs(TESTS_LOG_DIR, True)
+    u.mkdirs(TESTS_OUT_DIR, True)
+
+
+def pytest_sessionstart(session):
+    """
+    Called after the Session object has been created and
+    before performing collection and entering the run test loop.
+    """
+    init()

parutils/tests/logging/check_log.py

@@ -0,0 +1,76 @@
+LOG_FILE = [
+    "Log file initialised (*)",
+    "Python interpreter path: ",
+    "Python version: *",
+    "ParUtils version: *",
+    "This will be logged in a file",
+]
+LOG_FILE_NOT = [
+    "This won't be logged in a file",
+]
+
+N_W = len(LOG_FILE) + len(LOG_FILE_NOT)
+WARNINGS = [
+    "Expression 'This won't be logged in a file' couldn't be found in log file",
+    "Expression 'Log file initialised (*)' was found in log file",
+    "Expression 'Python interpreter path: ' was found in log file",
+    "Expression 'Python version: *' was found in log file",
+    "Expression 'ParUtils version: *' was found in log file",
+    "Expression 'This will be logged in a file' was found in log file",
+]
+WARN = WARNINGS + [f"check_log LOG_FILE ended with {N_W} warnings"]
+LEVEL_WARN_ERR = WARNINGS + [
+    f"check_log LOG_FILE nok, too many warnings ({N_W} warnings)",
+    f"[ttry] Exception caught match expected ('check_log LOG_FILE nok, too many warnings ({N_W} warnings)')",
+]
+
+LOG_INPUT = [
+    "Test log input",
+    "user command",
+]
+
+STEP_LOG = [
+    "5 elements appended in * ms. 20 elements appended in total.",
+    "Examples of duplicates (limited to 5)",
+    "out_list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]",
+]
+
+LOG_DICT = [
+    "key1: value1",
+    "key2: value2",
+    "key3:",
+    "  skey1: value1",
+    "  skey2:",
+    "    sskey1: value1",
+    "    sskey2: value2",
+    "    skey1: value1",
+    "    skey2:",
+    "        sskey1: value1",
+    "        sskey2: value2",
+    "	skey1: value1",
+    "	skey2:",
+    "		sskey1: value1",
+    "		sskey2: value2",
+]
+
+ERR_HANDLING = [
+    "Warning: the following message couldn't be logged because of [Errno 22] Invalid argument: ':.:.': * - test error handling normal 1",
+    "Warning: the following message couldn't be logged because of [Errno 22] Invalid argument: ':.:.': * - test error handling normal 2",
+    "* - test error handling normal 3",
+]
+ERR_HANDLING_NOT = [
+    "test error handling max limit",
+]
+
+END_1 = [
+    "check_log LOG_FILE ok",
+    "check_log WARN ok",
+    "check_log LOG_INPUT ok",
+    "check_log UPDATE_LOGS ok",
+]
+END_2 = [
+    "check_log LEVEL_WARN_ERR ok",
+    "check_log STEP_LOG ok",
+    "check_log LOG_DICT ok",
+    "check_log ERR_HANDLING ok",
+]

parutils/tests/logging/test_logging.py

@@ -0,0 +1,177 @@
+import pytest
+import parutils as u
+from parutils.logging import const
+from parutils.tests.logging import check_log as cl
+
+
+def t_log_every():
+    u.g.logs = []
+    u.close_logger()
+    u.Logger('TEST_LOG_EVERY', log_every=4)
+    u.log("log_elt_1")
+    assert u.g.logs == []
+    u.check_log(in_list_not=["log_elt_1"], name='LOG_EVERY_1')
+
+    u.log("log_elt_2")
+    u.log("log_elt_3")
+    u.log("log_elt_4")
+    u.check_log(["log_elt_1", "log_elt_2", "log_elt_3", "log_elt_4", 'check_log LOG_EVERY_1 ok'])
+    assert len(u.g.logs) == 2
+
+    u.log("log_elt_5")
+    logs_txt = u.load_txt(u.get_logger().log_path, False)
+    assert "log_elt_5" not in logs_txt
+    assert len(u.g.logs) == 2
+
+    log_path = u.get_logger().log_path
+    logs = u.load_txt(log_path)
+    assert len(logs) == 14 
+    logs0 = u.close_logger().logs
+    assert "log_elt_5" in logs0[2]
+    logs = u.load_txt(log_path)
+    assert len(logs) == 17 
+    logs_txt = u.load_txt(log_path, False)
+    assert "log_elt_5" in logs_txt
+    assert len(u.g.logs) == 3
+    u.log_print()
+
+
+def t_log_file():
+    u.g.logs = []
+    u.close_logger()
+    u.log("This won't be logged in a file\n")
+    e_ref = "No log file has been initialised"
+    u.ttry(u.check_log, e_ref, cl.LOG_FILE)
+    u.log_print()
+
+    u.Logger('TEST_LOGGING_1')
+    u.Logger()
+    u.set_logger(u.get_logger())
+    u.log("This will be logged", "in a file\n")
+    u.get_logger().empty_log_every_buffer()
+    u.check_log(cl.LOG_FILE, cl.LOG_FILE_NOT, log_matches=True, name='LOG_FILE')
+    u.check_log(["Expression matched: Log file initialised (*)"], name='LOG_MTACHES')
+    u.log_print()
+
+
+def t_warn():
+    with pytest.warns(UserWarning):  # disables the warnings
+        u.check_log(cl.LOG_FILE_NOT, cl.LOG_FILE, name='LOG_FILE', max_warn=10)
+    u.check_log(cl.WARN, name='WARN')
+    u.log_print()
+
+
+def t_input(monkeypatch):
+    monkeypatch.setattr('builtins.input', mock_input)
+    assert u.log_input("Test log input") == "user command"
+    u.check_log(cl.LOG_INPUT, name='LOG_INPUT')
+    u.log_print()
+
+
+def t_update_logs():
+    s = "test update_logs"
+    logs = u.get_logs()
+    logs.append(s)
+    u.update_logs(logs)
+    new_logs = u.get_logs()
+    assert logs == new_logs
+    u.check_log([s], name='UPDATE_LOGS')
+    u.log_print()
+
+
+def t_level_warn_err():
+    u.Logger('TEST_LOGGING_2')
+    u.log("This will be logged", "in a file\n")
+    u.log("This won't be logged in a file\n", level=1)
+    u.log_print("This won't be logged in a file\n", level=1)
+    with pytest.warns(UserWarning):
+        e_ref = f"check_log LOG_FILE nok, too many warnings ({cl.N_W} warnings)"
+        u.ttry(u.check_log, e_ref, cl.LOG_FILE_NOT, cl.LOG_FILE, name='LOG_FILE')
+    u.check_log(cl.LEVEL_WARN_ERR, name='LEVEL_WARN_ERR')
+    u.log_print()
+
+
+def t_step_log():
+    out_list = []
+    u.init_sl_timer()
+    for i in range(1, 21):
+        # time.sleep(0.05)  # simulates io / calculation
+        out_list.append(i)
+        u.step_log(i, 5, "elements appended")
+
+    u.log_print('\nout_list:', out_list)
+    u.log_example(out_list)
+    u.log_example([])
+    u.check_log(cl.STEP_LOG, name='STEP_LOG')
+    u.log_print()
+
+
+def t_log_dict():
+    ssd = {'sskey1': 'value1', 'sskey2': 'value2'}
+    sd = {'skey1': 'value1', 'skey2': ssd}
+    d = {'key1': 'value1', 'key2': 'value2', 'key3': sd}
+    u.log_dict(d, depth=2, tab_char='  ')
+    u.log_dict(d, depth=2, tab_char='    ')
+    u.log_dict(d, depth=2, tab_char='\t')
+    u.check_log(cl.LOG_DICT, name='LOG_DICT')
+    u.log_print()
+
+
+def t_err_handling():
+    # Error handling - normal case
+    logger = u.get_logger()
+    back = logger.log_path
+    logger.log_path = ':.:.'
+    u.log('test error handling normal 1')
+    u.log('test error handling normal 2')
+    logger.log_path = back
+    u.log('test error handling normal 3')
+
+    # Error handling - max limit reatched case
+    const.MAX_ERR_COUNT = 2
+    logger.log_path = ':.:.'
+    u.log('test error handling max limit 1')
+    u.log('test error handling max limit 2')
+    u.log('test error handling max limit 3')
+    u.log('test error handling max limit 4')
+    logger.log_path = back
+    logger.file_write = True
+    logger.buffer = ''
+    assert "test error handling max limit 4" in logger.logs[-1]
+    assert "The number of logging errors in a row reached" in logger.logs[-2]
+    u.check_log(cl.ERR_HANDLING, cl.ERR_HANDLING_NOT, name='ERR_HANDLING')
+    u.log_print()
+
+
+def test_logging(monkeypatch):
+    t_log_every()
+    t_log_file()
+    t_warn()
+    t_input(monkeypatch)
+    t_update_logs()
+    u.check_log(cl.END_1, name='END_1')
+    u.close_logger()
+    u.log_print()
+
+    t_level_warn_err()
+    t_step_log()
+    t_log_dict()
+    t_err_handling()
+    assert "This won't be logged in a file" in u.g.logs[0]
+    u.check_log(cl.END_2, name='END_2')
+    u.close_logger()
+    u.log_print()
+
+
+def mock_input(txt):
+    out = "user command"
+    print(txt + out)
+    return out
+
+
+if __name__ == '__main__':  # pragma: no cover
+    from parutils.tests.conftest import init
+    from _pytest.monkeypatch import MonkeyPatch
+
+    init()
+    test_logging(MonkeyPatch())

parutils/tests/string/check_log.py

@@ -0,0 +1,10 @@
+CL = [
+    "Log file initialised",
+    "Python version:",
+    "Test of like functions",
+    "like simple ok",
+    "like m ok",
+    "like_list ok",
+    "like_dict ok",
+    "Test get_duration_string",
+]

parutils/tests/string/test_string.py

@@ -0,0 +1,74 @@
+import parutils as u
+from parutils.tests.string.check_log import CL
+
+
+def get_duration():
+    u.log_print("Test get_duration_string", dashes=100)
+
+    dstr = u.get_duration_string(0, end_time=0.35)
+    u.log(dstr)
+    assert dstr == "350 ms"
+
+    (dms, dstr) = u.get_duration_string(0, end_time=5.369, return_dms=True)
+    u.log(dstr, dms)
+    assert (dstr, dms) == ("5.3 s", 5369)
+
+    dstr = u.get_duration_string(0, end_time=150)
+    u.log(dstr)
+    assert dstr == "2 minutes and 30 seconds"
+
+    u.log_print()
+
+
+def like():
+    u.log_print("Test of like functions", dashes=100)
+
+    s = '2 test ok?'
+    assert u.like(s, 'test')
+    assert not u.like(s, 'test', exact=True)
+    assert u.like(s, '*test*', exact=True)
+    assert u.like(s, 'TEST', case_sensitive=False)
+    assert u.like(s, 'TEST') is False
+    u.log("like simple ok")
+
+    m = u.like(s, '2 * ok?')
+    assert m.group(1) == 'test'
+    u.log("like m ok")
+
+    lst = ['1', 'test']
+    e_ref = u.strg.E_WRONG_TYPE_LIST
+    u.ttry(u.like_list, e_ref, s, 'test')
+    assert u.like_list(s, lst)
+    assert u.like_list(s, lst, exact=True) is False
+    assert u.like_list('TEST', lst) is False
+    u.log("like_list ok")
+
+    dct = {'1': ['a', 'b'], '2': 'test'}
+    e_ref = u.strg.E_WRONG_TYPE_DICT
+    u.ttry(u.like_dict, e_ref, s, 'test')
+    assert u.like_dict(s, dct) == '2'
+    assert u.like_dict(s, dct, exact=True) is False
+    assert u.like_dict('b', dct) == '1'
+    assert u.like_dict('TEST', dct) is False
+    u.log("like_dict ok")
+
+    assert u.hash512('TEST', 4) == '7bfa'
+    assert len(u.gen_random_string(10)) == 10
+    u.log_print()
+
+
+def test_string():
+
+    u.Logger('TEST_STRING', True)
+    assert u.big_number(1000) == '1 000'
+    assert u.truncate('test_test', 4) == 't...'
+    get_duration()
+    like()
+    u.check_log(CL)
+    u.close_logger()
+
+
+if __name__ == '__main__':  # pragma: no cover
+    from parutils.tests.conftest import init
+    init()
+    test_string()

parutils/tests/test_csv.py

@@ -0,0 +1,27 @@
+import parutils as u
+
+IN1_PATH = 'parutils\\tests\\files\\in1.csv'
+IN3_PATH = 'parutils\\tests\\files\\in3.csv'
+OUT_PATH = 'parutils\\tests\\files\\out.csv'
+
+
+def test_csv():
+
+    ar1 = u.load_csv(IN3_PATH, True)
+    assert ar1[1][1] == 'comment1;comment2'
+    u.save_csv(ar1, IN3_PATH, quote=True)
+    ar2 = u.load_csv(IN3_PATH, True)
+    assert ar1 == ar2
+
+    d = u.get_csv_fields_dict(IN1_PATH)
+    assert d == {'ID': 0, 'NAME': 1}
+
+    s = u.csv_clean('FIELD1;\n')
+    assert s == 'FIELD1'
+
+    e_ref = u.csvl.E_WRONG_TYPE_LIST
+    u.ttry(u.save_csv, e_ref, ['1'], OUT_PATH)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    test_csv()

parutils/tests/test_dq.py

@@ -0,0 +1,35 @@
+import parutils.tests.conftest as conftest
+import parutils as u
+
+FILES_DIR = 'parutils\\tests\\files'
+OUT_DIR = conftest.TESTS_OUT_DIR
+DUP_IN = FILES_DIR + '\\dup_in.csv'
+DUP_OUT = OUT_DIR + '\\out_dup.csv'
+DUP_OUT_REF = FILES_DIR + '\\dup_out_ref.csv'
+IN_1 = FILES_DIR + '\\in1.csv'
+IN_2 = FILES_DIR + '\\in2.csv'
+
+
+def test_dq():
+
+    u.Logger('TEST_DQ', True)
+    u.log_print("Test toolDup - find_dup_list", dashes=100)
+    list_in = u.load_csv(DUP_IN)
+    dup_list = u.find_dup_list(list_in)
+    u.log_example(dup_list)
+    u.save_csv(dup_list, DUP_OUT)
+    u.file_match(DUP_OUT, DUP_OUT_REF, del_dup=True)
+    u.diff_list(['1'], ['2'])
+
+    e_ref = "Files don't match"
+    u.ttry(u.file_match, e_ref, IN_1, IN_2)
+
+    assert u.find_dup_list([]) == []
+    assert u.del_dup_list([]) == []
+    u.close_logger()
+
+
+if __name__ == '__main__':  # pragma: no cover
+    from parutils.tests.conftest import init
+    init()
+    test_dq()

parutils/tests/test_file.py

@@ -0,0 +1,27 @@
+import parutils as u
+
+
+def test_file():
+    u.mkdirs('')
+
+    assert u.list_files('not exist') == []
+    assert len(u.list_files('parutils\\tests')) > 1
+
+    out = u.list_files('parutils\\tests', only_list=['test'], ignore_list=['file', 'msc', '0'])
+    assert out == ['parutils\\tests\\conftest.py', 'parutils\\tests\\test_csv.py', 'parutils\\tests\\test_dq.py']
+
+    out = u.list_files('parutils\\tests', only_list=['in*.csv'], ignore_list=['file', 'msc', 'init'], incl_root=False, walk=True)
+    assert out == ['in1.csv', 'in2.csv', 'in3.csv']
+
+    out = u.list_files('parutils\\tests', only_list=['test'], ignore_list=['file', 'msc', 'init', '0'], abspath=True)
+    lst = ['c:\\*\\tests\\conftest.py', 'c:\\*\\tests\\test_csv.py', 'c:\\*\\tests\\test_dq.py']
+    for elt in out:
+        assert u.like_list(elt, lst, case_sensitive=False)
+
+    path = 'out\\tests\\out.txt'
+    u.save_list(out, path)
+    assert u.count_lines(path) == 3
+
+
+if __name__ == '__main__':  # pragma: no cover
+    test_file()

parutils/tests/test_msc.py

@@ -0,0 +1,29 @@
+import parutils as u
+
+
+def test_msc():
+    lst = ['key1=value1', 'key2=value2']
+    out = u.list_to_dict(lst)
+
+    d = {'key1': 'value1', 'key2': 'value2'}
+    assert out == d
+
+    out = u.replace_from_dict('Hello @@VAR@@', {'VAR': 'world'})
+    assert out == 'Hello world'
+
+    u.ttry(nok_func, 'test_error')
+    err = "[ttry] Exception caught ('test_error') don't match expected ('test_error_1')"
+    u.ttry(u.ttry, err, nok_func, 'test_error_1')
+    u.ttry(u.ttry, "[ttry] No exception was caught", ok_func, 'test_error')
+
+
+def nok_func():
+    raise Exception('test_error')
+
+
+def ok_func():
+    pass
+
+
+if __name__ == '__main__':  # pragma: no cover
+    test_msc()

{ParUtils-1.2.3.dist-info → parutils-1.2.5.dist-info}/METADATA

@@ -1,17 +1,16 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ParUtils
-Version: 1.2.3
-Summary: This package contains a bunch of Python utils developed for Support, Test and Automation IT Engineers
-Home-page: https://github.com/paularnaud2/ParUtils
-Author: Paul ARNAUD
-Author-email: paularnaud2@gmail.com
-Project-URL: Bug Tracker, https://github.com/paularnaud2/ParUtils/issues
+Version: 1.2.5
+Summary: This package contains a bunch of Python utilities developed for Support, Test and Automation IT Engineers
+Author-email: Paul ARNAUD <paularnaud2@gmail.com>
+Project-URL: Homepage, https://github.com/paularnaud2/ParUtils
+Project-URL: Issues, https://github.com/paularnaud2/ParUtils/issues
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Dynamic: license-file
 
 # ParUtils
  
@@ -76,8 +75,9 @@ Data quality features:
 - `del_dup_list`: removes duplicates from a list
 
  
-### Logging with parutils
+## Logging with parutils
  
+### Basic usage guide
 The `log` function and the `Logger` class are directly available from the parutils package. So you can do:
 
     import parutils as u
@@ -98,6 +98,7 @@ Note that the default constants for the logging sub package are stored in __paru
 
     u.logging.const.DEFAULT_DIR = '<my_custom_dir>'
  
+### About the step_log function
 The `step_log` function allows you to log some information only when the input ``counter`` is a multiple of the input ``step``. Thus, `step_log` is to be used in loops to __track the progress of long processes__ such as reading or writing millions of lines in a file. The ``what`` input expects a description of what is being counted. It's default value is ``'lines written'``.  
 In order to correctly measure the elapsed time for the first log line, the ``step_log`` function has to be initialised by running ``init_sl_timer()``.  
 So for example, if you input ``step=500`` and don't input any ``what`` value, you should get something like this:
@@ -108,3 +109,14 @@ So for example, if you input ``step=500`` and don't input any ``what`` value, yo
  
 Checkout the __test_logging.py__ file in __tests/logging__ for simple examples of use.
  
+ ### About the retry mechanism
+ If the logger is initialized, the log information is written in a file each time you call the log function. This file writing can fail, especially if your log file is located on a network drive.
+
+ Parutils has a built-in retry mechanism related to log file writing failures. Here is how it works:  
+ Whenever writing some log fails, a warning is printed, looking like this
+
+    Warning: the following message couldn't be logged because of <error>: <logged message>
+This warning is stored in a buffer and the logger will try to log this buffer to the log file next time it has something to log. If the next try also fails, another warning is printed and added to the buffer, etc, up to a certain limit. When the size of the buffer exceeds 10, the logger stops trying to log into a file, and just prints the logs in the currently available console.
+
+### About the log_every argument
+Because each logged message implies opening a file, writing to this file, and closing the file, logging can severely affect performance, especially when logging to some network locations, or more generally to a slow drive. The `log_every` argument of the Logger constructor allows to mitigate this by logging to the file only every <log_every> log messages, using a buffer mechanism. By default, log_every is set to one, so the log file is writing for every log messages. For example, if you set log_every to 10, the log file is only written every 10 log entries, while every log entry will still be printed live on the console.

parutils-1.2.5.dist-info/RECORD

@@ -0,0 +1,32 @@
+parutils/__init__.py,sha256=EaFtUmU0kYgnjvAFwPmoKt7I5zn9bxY02R7TUtcE6M4,1434
+parutils/changelog.py,sha256=-zSqn-8tI_VdDjCEenmTdpBN9br5-meNPj-lY8l0AW4,1920
+parutils/csvl.py,sha256=JgJPnN8IPrdMD8OOlPtDSEhJULMkPkqnDnB-D4z9F-E,2871
+parutils/dq.py,sha256=kdvzOo5FLCumXagY05kGSseYlbR-zvjjmsMtGVliVc8,2302
+parutils/file.py,sha256=prENNneiQmKKRS5BxGH7tN23x5e6XSfyrhpO0lOLs0g,2739
+parutils/g.py,sha256=n99gQbLs2rmszBYw_wy43P4iQo3lqpBHz3-Q7sm9n_U,11
+parutils/msc.py,sha256=6Yx-VJss0xRqQAiuVURu0L4oDHkcsF7T5EK6eXrm9TQ,745
+parutils/strg.py,sha256=g4FjVWlUBsQO3xFoMe-ZYskCjVEQVyUCdDGliS_kbz4,5564
+parutils/testing.py,sha256=kXrMz8Kze4mZejaePmV_OIca4iuNcG_dhGVHCgVuF-k,610
+parutils/wrap.py,sha256=PqMyKodgEjWDXoIVZZXRaBSdkGFL5OWmVJttl2Crrg8,421
+parutils/logging/__init__.py,sha256=rSNpgjuYer-Hhn6zKzKwKSn_KfcajEXlk6cJnPC9eJU,461
+parutils/logging/cl.py,sha256=NCLuxLHvqRTypZlAgwnegkByLTnTirKntnwguJHBelA,2086
+parutils/logging/const.py,sha256=9GrSonZu9YXz0U1dVHIiAaAjGcVQs_cnKc__6-C_CIc,41
+parutils/logging/core.py,sha256=POLKsQaaKYZOR7IhWtAaF5a8noZL7hKdssmrOtle4S4,850
+parutils/logging/g.py,sha256=ZSrgZw63kwxIW66A7-9_iYeDt4AstNZ_XXQgK8MglyQ,47
+parutils/logging/logger.py,sha256=4jBb7fjZBQKKNwat46BjTrcSV20uKX3-hcyBMTFk34M,5011
+parutils/logging/main.py,sha256=va2jaQXN6TO1ycajswyJzRXzlMB2BVQ_CervUnnQE70,2129
+parutils/logging/sl.py,sha256=3-sj_o33cZmOqeFxlTl5HyHOvSAhn9glYcc-BmTUpZc,1164
+parutils/tests/conftest.py,sha256=iIyQvgVV8GzUwJ0tlN2KUJHX5ZO7lp8bKFK3VmRx4DU,485
+parutils/tests/test_csv.py,sha256=vHs_CT1oLWAGkRvKdF-Ilz62MczEDKqQfmmQe-nt-Wg,671
+parutils/tests/test_dq.py,sha256=Qb5lldyg-UAOvNbViNLnchkG93v_-DQ96XkIPgOIXWo,967
+parutils/tests/test_file.py,sha256=iYaSm0YU7qQIq9p7Sea4KPdoKRpqWcbgbkZG4u0DR_s,1031
+parutils/tests/test_msc.py,sha256=C2Q_88i2sIbz52KhvdkY0-3Aa9Yk2FSHFxVYUqoQ_Fs,696
+parutils/tests/logging/check_log.py,sha256=Se-EuWuwxlcksjIkLZU4d77-o3eyFdppJCxGJ-GcorY,2332
+parutils/tests/logging/test_logging.py,sha256=bceIEO7W7rhLm_YrqRcXSA0NaQIAU1x6XZMWU3uNeFk,5260
+parutils/tests/string/check_log.py,sha256=m2RRJIorA6jg7uV_8nxWfKHUFZ98YPDn-DqzcTw3cQ0,211
+parutils/tests/string/test_string.py,sha256=jhfGkrHENUERIG2UAVIlGYDpeecQS-_-JtmMUR7qDMA,2023
+parutils-1.2.5.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
+parutils-1.2.5.dist-info/METADATA,sha256=EMs4o4amP6SV9U0Uhj15UpTte2ETmkAVke9hYCpzwgo,6868
+parutils-1.2.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+parutils-1.2.5.dist-info/top_level.txt,sha256=1MDobUcroeYEvdupZCAFvA5hJjm7LSDUV5A4jHySNis,9
+parutils-1.2.5.dist-info/RECORD,,

{ParUtils-1.2.3.dist-info → parutils-1.2.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ParUtils-1.2.3.dist-info/RECORD

@@ -1,23 +0,0 @@
-parutils/__init__.py,sha256=EaFtUmU0kYgnjvAFwPmoKt7I5zn9bxY02R7TUtcE6M4,1434
-parutils/changelog.py,sha256=DC3AEcRVeDeJl62Ri4eM5_0TIxG5znRhBpvcZAvAjZY,1730
-parutils/csvl.py,sha256=UBqw0lyhTNqFLSNIE-JM7TAQYCRgHftYQmHi21ru8-A,2874
-parutils/dq.py,sha256=kdvzOo5FLCumXagY05kGSseYlbR-zvjjmsMtGVliVc8,2302
-parutils/file.py,sha256=dedO84jZVXrE0x_31Wy_QhM6wTPOAshhp4SuT92rrPQ,2869
-parutils/g.py,sha256=n99gQbLs2rmszBYw_wy43P4iQo3lqpBHz3-Q7sm9n_U,11
-parutils/msc.py,sha256=0hf-WrYhDquaDtfHXINNs9tROwJ7iYV2J6y0umkaGRA,745
-parutils/strg.py,sha256=_8NQJ1iY58Z_xACTDVcyonvRHDRuy9Ai3Kv3CvVXRh4,5563
-parutils/testing.py,sha256=kXrMz8Kze4mZejaePmV_OIca4iuNcG_dhGVHCgVuF-k,610
-parutils/wrap.py,sha256=PqMyKodgEjWDXoIVZZXRaBSdkGFL5OWmVJttl2Crrg8,421
-parutils/logging/__init__.py,sha256=rSNpgjuYer-Hhn6zKzKwKSn_KfcajEXlk6cJnPC9eJU,461
-parutils/logging/cl.py,sha256=NCLuxLHvqRTypZlAgwnegkByLTnTirKntnwguJHBelA,2086
-parutils/logging/const.py,sha256=L2OppBDAZYKUTIjsLPt6hTUh8UWH_1mX9YaGWlwzpj4,135
-parutils/logging/core.py,sha256=POLKsQaaKYZOR7IhWtAaF5a8noZL7hKdssmrOtle4S4,850
-parutils/logging/g.py,sha256=ZSrgZw63kwxIW66A7-9_iYeDt4AstNZ_XXQgK8MglyQ,47
-parutils/logging/logger.py,sha256=4x_F98soVd-PO5y2uQlDDyHI50E58m4VTfpjLB0OjJ0,4843
-parutils/logging/main.py,sha256=arUvgB0YTZpidt4wl2GZsR4bRUj_rrSYMrTnCMKt4gA,2127
-parutils/logging/sl.py,sha256=3-sj_o33cZmOqeFxlTl5HyHOvSAhn9glYcc-BmTUpZc,1164
-ParUtils-1.2.3.dist-info/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
-ParUtils-1.2.3.dist-info/METADATA,sha256=RY72J9pjWc6KdNdC_EzYqvNhM0_-5bmzE5wORFlf2wQ,5285
-ParUtils-1.2.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ParUtils-1.2.3.dist-info/top_level.txt,sha256=1MDobUcroeYEvdupZCAFvA5hJjm7LSDUV5A4jHySNis,9
-ParUtils-1.2.3.dist-info/RECORD,,