custom-profiler 0.1.0__py3-none-any.whl

custom_profiler/__init__.py

@@ -0,0 +1,11 @@
+
+from . import custum_profiler
+from . import collecteur
+
+from functools import partial
+
+profiler = partial(custum_profiler.profiler, linePerline=False)
+profiler_lbl = partial(custum_profiler.profiler, linePerline=True)
+magic_profiler = custum_profiler.magic_profiler
+profiler_collecteur = collecteur.profiler_collecteur
+INTERACTIVITY_OPT_ENUM = collecteur.INTERACTIVITY_OPT_ENUM

custom_profiler/collecteur.py

@@ -0,0 +1,140 @@
+import time
+import sys
+if sys.platform == 'linux':
+    import resource
+import logging
+from collections import OrderedDict
+
+import psutil
+from psutil._common import bytes2human
+process = psutil.Process()
+
+from custom_profiler.custum_logger import add_logging_level
+from custom_profiler.human_readable_time import human_time_duration as htd
+
+
+class INTERACTIVITY_OPT_ENUM :
+    ENABLE        = "ENABLE"        # thread (for memory peak follow) and interactive print
+    MF_NO_INTERAC = "MF_NO_INTERAC" # memory peak follow (with thread), no interacrtif print
+    DISABLE       = "DISABLE"       # no thread, no memory peak follow, no interacrtif print
+    AUTO          = "AUTO"          # if console is redirect in file (sys.stdout.isatty() == false) AUTO is equivalente to MF_NO_INTERAC else is equivalente to ENABLE
+
+def get_ENUM_list(ENUM):
+    return [key for key in ENUM.__dict__ if key not in ["__main__", "__module__", "__doc__", '__dict__', '__weakref__']]
+
+
+class profiler_collecteur(object):
+    _instance = None
+
+    def __new__(self):
+        if self._instance is None:
+            self._instance = super(profiler_collecteur, self).__new__(self)
+            self.profData = OrderedDict()
+            self.profThread = OrderedDict()
+            self.interractivity = INTERACTIVITY_OPT_ENUM.ENABLE
+            self.logger = None
+            self.start_time = time.perf_counter()
+        return self._instance
+
+    def options(self, interractivity = INTERACTIVITY_OPT_ENUM.ENABLE
+                , useLogger=False
+                , loggername = " ⚡"
+                , addCustumLvl= True
+                , profilerlvl = 25):
+        
+        assert interractivity in get_ENUM_list(INTERACTIVITY_OPT_ENUM), f'interractivity {interractivity} must be in INTERACTIVITY_OPT_ENUM : {getEnumList(INTERACTIVITY_OPT_ENUM)}'
+        
+        if interractivity == INTERACTIVITY_OPT_ENUM.AUTO :
+            if sys.stdout.isatty():
+                self.interractivity = INTERACTIVITY_OPT_ENUM.ENABLE
+            else :
+                self.interractivity = INTERACTIVITY_OPT_ENUM.MF_NO_INTERAC
+
+        if useLogger:
+            if self.interractivity == INTERACTIVITY_OPT_ENUM.ENABLE :
+                self.interractivity = INTERACTIVITY_OPT_ENUM.MF_NO_INTERAC
+            if addCustumLvl :
+                add_logging_level('PROFILER', profilerlvl)
+                logging.getLogger().setLevel("PROFILER")  
+                self.logger = logging.getLogger(loggername).profiler
+            else :
+                logging.getLogger().setLevel("INFO")  
+                self.logger = logging.getLogger(loggername).info
+        else :
+            self.logger = None
+
+    def save(self, fname, deltaTime, deltaMem, long_fname=None):
+        if fname in self.profData.keys():
+            self.profData[fname]["dt"] += deltaTime
+            self.profData[fname]["dm"] += deltaMem
+            self.profData[fname]["dm_list"].append(deltaMem)
+            self.profData[fname]["nbCall"] += 1
+        else :
+            self.profData[fname] = {"dt": deltaTime ,"dm": deltaMem, "dm_list": [deltaMem], "nbCall": 1}
+
+        t_str = htd(deltaTime)
+        value = f"{t_str}"
+        strmen = bytes2human(deltaMem)
+        if long_fname == None :
+            long_fname = fname
+        self.print_line(long_fname, value, strmen)
+
+    def thread_view(self, fname, deltaMem):
+        if fname in self.profThread.keys():
+            if deltaMem > self.profThread[fname]:
+                self.profThread[fname] = deltaMem
+        else :
+            self.profThread[fname] = deltaMem
+
+    def get_global_info(self):
+        run_time = htd(time.perf_counter() - self.start_time)
+        if sys.platform == 'win32':
+            mem_peack = bytes2human(process.memory_info().peak_wset)
+        else :
+            mem_peack = bytes2human(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 0.0009765625) # in bytes
+        return run_time, mem_peack
+    
+    def _print(self, toprint, end='\n'):
+        if self.logger:
+            self.logger(toprint)
+        else:
+            print(toprint, end=end)
+
+    def print_line(self, fname, delta_time, delta_mem, end='\n', color=""):
+        delta_mem = " Δ " + f"{delta_mem:>7}"
+        if fname in self.profThread.keys():
+            mmax = 0.
+            if fname in self.profData.keys(): 
+                mmax = max(self.profData[fname]["dm_list"])
+            if mmax < self.profThread[fname] :
+                mmax = self.profThread[fname]
+            delta_mem += " / peak " +  f"{bytes2human(mmax):>7}"
+        toprint = f"{color} ⚡ {fname: ^21} took : {delta_time:<10} consumes : {delta_mem} \033[0m"
+        self._print(toprint, end)
+    
+    def __str__(self):
+        run_time, mem_peack = self.get_global_info()
+        str = ("\n " + "⚡" * 6 + f" customProfiler log : global timer {run_time} / max memory use {mem_peack:^10}"+ "⚡" * 6)
+        str += "\n " + "⚡" * 50
+        str += "\n ⚡ {:^31} | {:8} | {:<29} | {:^17} ⚡".format("fct name"
+                                                    , "Nb call"
+                                                    , "  time : mean / global"
+                                                    , "mem : mean / max")
+        str += "\n ⚡ "+ "="*95 + "⚡"
+        for key, val in self.profData.items():
+            t_str = htd(val["dt"])
+            t_p_call_str = htd(val["dt"]/val['nbCall'])
+            str += f"\n ⚡ {key: ^31.31} | {val['nbCall']:^8} "
+            str += f"| {t_p_call_str} / {t_str} "
+            strmen = bytes2human(self.profData[key]["dm"]/val['nbCall'])
+            mmax = max(self.profData[key]["dm_list"])
+            if key in self.profThread.keys():
+                if mmax < self.profThread[key] :
+                    mmax = self.profThread[key]
+            strmaxmem = bytes2human(mmax)
+            str += f"| {strmen:>7} / {strmaxmem:>7} ⚡"
+        str += "\n " + "⚡" * 50
+        return str
+
+    def __del__(self):
+        print(self)

custom_profiler/custum_logger.py

@@ -0,0 +1,63 @@
+import logging
+
+#https://stackoverflow.com/questions/2183233/how-to-add-a-custom-loglevel-to-pythons-logging-facility/35804945#35804945
+def add_logging_level(levelName, levelNum, methodName=None):
+    """
+    Comprehensively adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    `levelName` becomes an attribute of the `logging` module with the value
+    `levelNum`. `methodName` becomes a convenience method for both `logging`
+    itself and the class returned by `logging.getLoggerClass()` (usually just
+    `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is
+    used.
+
+    To avoid accidental clobberings of existing attributes, this method will
+    raise an `AttributeError` if the level name is already an attribute of the
+    `logging` module or if the method name is already present 
+
+    Example
+    -------
+    >>> addLoggingLevel('TRACE', logging.DEBUG - 5)
+    >>> logging.getLogger(__name__).setLevel("TRACE")
+    >>> logging.getLogger(__name__).trace('that worked')
+    >>> logging.trace('so did this')
+    >>> logging.TRACE
+    5
+
+    """
+    if not methodName:
+        methodName = levelName.lower()
+
+    if hasattr(logging, levelName):
+       raise AttributeError('{} already defined in logging module'.format(levelName))
+    if hasattr(logging, methodName):
+       raise AttributeError('{} already defined in logging module'.format(methodName))
+    if hasattr(logging.getLoggerClass(), methodName):
+       raise AttributeError('{} already defined in logger class'.format(methodName))
+
+    # This method was inspired by the answers to Stack Overflow post
+    # http://stackoverflow.com/q/2183233/2988730, especially
+    # http://stackoverflow.com/a/13638084/2988730
+    def logForLevel(self, message, *args, **kwargs):
+        if self.isEnabledFor(levelNum):
+            self._log(levelNum, message, args, **kwargs)
+    def logToRoot(message, *args, **kwargs):
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
+
+if __name__ == "__main__":
+
+    add_logging_level('PROFILER', 25)
+    logging.getLogger().setLevel("PROFILER")  
+    logger = logging.getLogger("⚡")
+
+    ch = logging.StreamHandler()
+    logger.addHandler(ch)
+
+    logger.profiler("test print")
+

custom_profiler/custum_profiler.py

@@ -0,0 +1,95 @@
+import sys
+import time
+
+from functools import wraps
+
+import threading
+from threading import Thread
+from threading import Event
+
+import psutil
+from psutil._common import bytes2human
+process = psutil.Process()
+
+from custom_profiler.line_by_line import trace_calls
+from custom_profiler.collecteur import profiler_collecteur, INTERACTIVITY_OPT_ENUM
+from custom_profiler.human_readable_time import human_time_duration as htd
+
+
+profC = profiler_collecteur()
+
+
+def task(event, fname, start_time, start_mem):
+    i = 0
+    while True :
+        time.sleep(0.01)
+        if i % 100 == True :
+            t_str = htd(time.perf_counter() - start_time)
+            dm = process.memory_info().rss - start_mem
+            profC.thread_view(fname, dm) #sauvegarde delta mem max
+            strmen = bytes2human(dm)
+            if profC.interractivity == INTERACTIVITY_OPT_ENUM.ENABLE :
+                if threading.active_count() < 3:
+                    profC.print_line(fname, t_str, strmen, end="\r", color="\033[93m")
+
+        i += 1
+        if event.is_set():
+            break
+
+
+class thread_mananger:
+    def __init__(self, fname, start_time, start_mem):
+        self.event = Event()
+        self.t = Thread(target=task, args=(self.event, fname, start_time, start_mem))
+        self.t.daemon = True
+        self.t.start()
+
+    def end(self):
+        self.event.set()
+        self.t.join()
+
+#https://stackoverflow.com/questions/5929107/decorators-with-parameters
+def profiler(func, linePerline):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        if profC.interractivity != INTERACTIVITY_OPT_ENUM.DISABLE and linePerline == False:
+            tm = thread_mananger(func.__name__, time.perf_counter(), process.memory_info().rss)
+        
+        start_mem = process.memory_info().rss
+        start_time = time.perf_counter()
+
+        if linePerline :
+            sys.settrace(trace_calls)
+        result = func(*args, **kwargs)
+        if linePerline :
+            sys.settrace(None)
+
+        end_time = time.perf_counter()
+        end_mem = process.memory_info().rss
+
+        if profC.interractivity != INTERACTIVITY_OPT_ENUM.DISABLE and linePerline == False :
+            tm.end()
+
+        profC.save(func.__name__, end_time - start_time, end_mem - start_mem)
+
+        return result
+    return wrapper
+
+
+class magic_profiler():
+
+    def __init__(self, func_name):
+        self.func_name = func_name
+
+    def __enter__(self):
+        if profC.interractivity != INTERACTIVITY_OPT_ENUM.DISABLE :
+            self.tm = thread_mananger(self.func_name, time.perf_counter(), process.memory_info().rss)
+        self.start_mem = process.memory_info().rss
+        self.start_time = time.perf_counter()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        end_time = time.perf_counter()
+        end_mem = process.memory_info().rss
+        if profC.interractivity != INTERACTIVITY_OPT_ENUM.DISABLE :
+            self.tm.end()
+        profC.save(self.func_name, end_time - self.start_time, end_mem - self.start_mem)

custom_profiler/human_readable_time.py

@@ -0,0 +1,49 @@
+import time
+from decimal import Decimal
+
+
+TIME_UTS_DURATION_UNITS = (
+    ('d', 60*60*24),
+    ('h', 60*60),
+    ('min', 60),
+    ('s', 1),
+)
+
+
+TIME_LTS_DURATION_UNITS = (
+    ('s ', 1),
+    ('ms', 1e-3),
+    ('µs', 1e-6),
+    ('ns', 1e-9),
+)
+
+
+def human_time_duration(seconds):
+    assert type(seconds) in [float, int], f'seconds ({type(seconds)}) must be a float or int'
+    if seconds == 0:
+        return 0
+    if seconds <= 60 :
+        for unit, div in TIME_LTS_DURATION_UNITS:
+            if seconds >= div :
+                return f"{seconds/div:11.2f}{unit}"
+        return f"{Decimal(seconds):11.2E}s "
+    else :
+        if seconds < 3600 :
+            return f"{seconds//60:6.0f}min{seconds%60:2.0f}s"
+        else :
+            h = seconds // 3600
+            m = (seconds - h * 3600) // 60
+            s = seconds - (h * 3600) - (m * 60)
+            return f"{h:3}h{m:2}min{s:2}s"
+
+if __name__ == "__main__":
+    
+    for i in range(0, 11):
+        print(f"{10**-i:8}", human_time_duration(10**-i) \
+            , human_time_duration(2*10**-i)
+            , human_time_duration(10**-i - 10**(-i-3))
+            , human_time_duration(1/3*10**-i))
+
+    for i in [1, 2, 59, 60, 61, 144, 3599, 3600, 3601, 3660, 3661, 3600*120]:
+        print(f"{i:8}",human_time_duration(i))
+    

custom_profiler/line_by_line.py

@@ -0,0 +1,75 @@
+import inspect
+import time
+import sys
+
+import psutil
+from psutil._common import bytes2human
+process = psutil.Process()
+
+from custom_profiler.collecteur import profiler_collecteur
+from custom_profiler.human_readable_time import human_time_duration as htd
+
+profC = profiler_collecteur()
+
+
+def trace_lines(frame, event, arg):
+
+    code_source, lineStart = inspect.getsourcelines(frame.f_code)
+    co_name  = frame.f_code.co_name
+
+    if event != "line":
+        fname = f"l {frame.f_lineno:<3} {profC.__src[-1]:40}"
+        fnameSave  = f"{co_name} l {frame.f_lineno:<3}"
+        t     = time.perf_counter() - profC.__tic
+        men   = process.memory_info().rss - profC.__tic_mem
+        profC.save(fnameSave, t, men, fname)
+        profC._print( " " + "⚡"*20 + f" line per line : end") 
+        return
+
+    profC.__src.append(code_source[frame.f_lineno - lineStart].split('\n')[0])
+
+    if len(profC.__src) == 1 :
+        head = code_source[1].split('\n')[0]
+        profC._print( " " + "⚡"*20 + f" line per line : {head.split()[1].split('(')[0]} from {frame.f_code.co_filename}") 
+    else :
+        fname  = f"l {frame.f_lineno-1:<3} {profC.__src[-2]:40}"
+        fnameSave  = f"{co_name} l {frame.f_lineno-1:<3}"
+        t  = time.perf_counter() - profC.__tic
+        men = process.memory_info().rss - profC.__tic_mem
+        profC.save(fnameSave, t, men, fname)
+
+    profC.__tic = time.perf_counter()
+    profC.__tic_mem = process.memory_info().rss
+
+  
+def trace_calls(frame, event, arg):
+    profC.__tic = 0.
+    profC.__tic_mem = 0.
+    profC.__src = []
+
+    if event != "call":
+        return
+    return trace_lines
+
+
+def lpl(fonction):
+    def wrapper(*args, **kwargs):
+        sys.settrace(trace_calls)
+        resultat = fonction(*args, **kwargs)
+        sys.settrace(None)
+        return resultat
+    return wrapper
+
+
+if __name__ == "__main__":
+
+    @lpl
+    def my_func():
+        a = [1] * (10 ** 6)
+        b = [2] * (2 * 10 ** 7)  
+        time.sleep(1)
+        del b
+        # time.sleep(5)
+        return a
+
+    my_func()

custom_profiler-0.1.0.dist-info/METADATA

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: custom_profiler
+Version: 0.1.0
+Summary: time and memory profiler
+Author: Karim Ammar
+Author-email: karim.ammar@cea.fr
+Maintainer: Karim Ammar
+Keywords: profiler
+Requires-Dist: psutil
+
+time and memory profiler

custom_profiler-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+custom_profiler/__init__.py,sha256=uZ_u96uNulrFnkRWXzk8kLYC1QMU3U_Zn1nC6eOA2c8,378
+custom_profiler/collecteur.py,sha256=LhwCfbIA7rP7izCQN1Ry4JISvzHRQj-CXuXtOfMho8o,5967
+custom_profiler/custum_logger.py,sha256=x6AV6PI3XNFszhPdJ08BynRhF5R5tJFaWvB09PvM-Dw,2450
+custom_profiler/custum_profiler.py,sha256=nCDIATRqKm2-AzqeAeY0yhJwYfbVmLYqOsBzzgkpqpM,3103
+custom_profiler/human_readable_time.py,sha256=xtYYVqfdbhag8L1ajhCjv8tq1YDxZEoq9a9ck_voVqM,1291
+custom_profiler/line_by_line.py,sha256=MOSBGhrvStWm0PC-ByNbxKfkR9OwW6QVaC-c_wlNX64,2043
+custom_profiler-0.1.0.dist-info/METADATA,sha256=CnpVw6SFKVoXdtaLlFBiJFkCG90UFzxXP2Kh25veJuI,237
+custom_profiler-0.1.0.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+custom_profiler-0.1.0.dist-info/top_level.txt,sha256=ublWGGbVR2ckXpSymkDluEn2-sP8i6LVpql4DZOs1YY,16
+custom_profiler-0.1.0.dist-info/RECORD,,

custom_profiler-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (70.2.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

custom_profiler-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+custom_profiler