easy-utils-dev 2.138__py3-none-any.whl → 2.140__py3-none-any.whl

easy_utils_dev/EasySsh.py

@@ -69,7 +69,7 @@ class CREATESSH :
         ch.disconnect = disconnect
         return ch
 
-    def ssh_execute(self ,command ,  merge_output=False , hide_output=False) :
+    def ssh_execute(self ,command ,  merge_output=False , hide_output=True) :
         self.logger.info(f"executing {command}")
         try :
             stdin_ , stdout_ , stderr_ = self.ssh.exec_command(command)

easy_utils_dev/debugger-C-PF4PAMMP.py

@@ -0,0 +1,486 @@
+import logging
+import os
+import  sys
+from datetime import datetime
+from logging.handlers import RotatingFileHandler
+from .utils import getRandomKey , convert_mb_to_bytes , getTimestamp , start_thread
+from .custom_env import custom_env , setupEnvironment
+from .Events import EventEmitter
+from threading import Thread
+from time import sleep
+
+gEvent = EventEmitter()
+logging.addLevelName(25, "SCREEN")
+
+def setGlobalHomePath( path ) :
+    env = custom_env()
+    env['debugger_homepath'] = path
+    gEvent.dispatchEvent('update_home_path')
+
+def setGlobalDisableOnScreen(on_screen=False) :
+    env = custom_env()
+    env['debugger_on_screen'] = on_screen
+    if not on_screen :
+        gEvent.dispatchEvent('disable_global_printing')
+    else :
+        gEvent.dispatchEvent('enable_global_printing')
+        
+    
+def setGlobalDebugLevel(level='info') :
+    env = custom_env()
+    env['debugger_global_level'] = level
+    gEvent.dispatchEvent('update_debug_level')
+
+
+class DEBUGGER:
+    def __init__(self 
+            , name
+            , level='info', 
+            fullSyntax=True, 
+            onscreen=True,
+            log_rotation=3,
+            homePath=None,
+            id=getRandomKey(9) , 
+            global_debugger=None,
+            disable_log_write=False,
+            file_name=None,
+            seperate_files=True
+        ):
+        env = custom_env()
+        setupEnvironment( 'debugger' )
+        debugger_on_screen = env.get('debugger_on_screen' , True)
+        env['debugger_on_screen'] = debugger_on_screen
+        self.env = env
+        self.events = gEvent
+        self.debuggerLabel = f"{name}"
+        self.logger = logging.getLogger(self.debuggerLabel)
+        self.set_level(level)
+        self.file_handler_class=None
+        self.LOG_SIZE_THRESHOLD_IN_BYTES = 10 * 1024 * 1024
+        self.BACKUP_COUNT = log_rotation
+        self.homePath = homePath
+        self.lastAbsoluteHomePath= None
+        self.fullSyntax=fullSyntax
+        self.onScreen= onscreen
+        self.id = id
+        self.how_many_times_write= 0
+        self.stream_service = None
+        if not env['debugger'].get(name) :
+            self.console = console_handler = logging.StreamHandler()
+        else :
+            self.console = console_handler = env['debugger'].get(name).console
+        if not self.logger.hasHandlers() :
+            self.logger.addHandler(self.console)
+        self.name = name
+        self.rotate_disabled=False
+        self.isInPyinstaller = False
+        self.log_iterations=0
+        self.log_iterations_threshold = 200
+        self.global_debugger = global_debugger
+        self.isLogWriteDisabled = disable_log_write
+        self.type = "CUSTOM_DEBUGGER"
+        self.seperate_files=seperate_files
+        if fullSyntax :
+            f = f"[%(asctime)s]-[{self.name}]-[%(levelname)s]: %(message)s"
+        else :
+            f = f"[{self.name}]-[%(levelname)s]: %(message)s"
+        self.syntax = f
+        self.formatter = logging.Formatter(f , datefmt='%Y-%m-%d %H:%M:%S' )
+        self.filename = file_name
+        path = self.homepath(homePath)
+        if not env['debugger'].get(name) :
+            console_handler.setFormatter(self.formatter)
+        if not disable_log_write :
+            if not env['debugger'].get(name) :
+                self.file_handler_class = self.createRotateFileHandler(path)
+        if onscreen : 
+            self.enable_print()
+        elif not onscreen : 
+            self.disable_print()
+        self.events.addEventListener('disable_global_printing' , self.disable_print )
+        self.events.addEventListener('enable_global_printing' , self.enable_print )
+        self.events.addEventListener('update_home_path' , self.updateGlobalHomePath )
+        self.events.addEventListener('update_debug_level' , self.updateGlobalSetLevel )
+        if env['debugger'].get(name) :
+            self =  env['debugger'].get(name)
+        else:
+            env['debugger'][id] = self
+            env['debugger'][name] = self
+        if not env.get('debugger_on_screen' , True ) :
+            self.disable_print()
+        if env.get('debugger_on_screen' , True ) :
+            self.enable_print()
+        if os.environ.get("EASY_UTILS_DEBUG_LEVEL") :
+            EASY_UTILS_DEBUG_LEVEL = os.environ.get("EASY_UTILS_DEBUG_LEVEL")
+            if not EASY_UTILS_DEBUG_LEVEL.lower() in ['info' , 'debug' , 'warning' , 'error' , 'critical'] :
+                self.logger.error(f'EASY_UTILS_DEBUG_LEVEL ENV must be one of [info,debug,warning,error,critical] | Current Env Variable Is "{EASY_UTILS_DEBUG_LEVEL}". Skipping ')
+            else :
+                self.set_level(EASY_UTILS_DEBUG_LEVEL)
+        if os.environ.get("EASY_UTILS_ENABLE_PRINT" , '' ).lower() == 'true' :
+            self.enable_print()
+
+    def switch_full_syntax(self , toggle) :
+        if toggle :
+            f = f"[%(asctime)s]-[{self.name}]-[%(levelname)s]: %(message)s"
+        else :
+            f = f"[{self.name}]-[%(levelname)s]: %(message)s"
+        self.syntax = f
+        self.formatter = logging.Formatter(f , datefmt='%Y-%m-%d %H:%M:%S' )
+        self.console.setFormatter(self.formatter)    
+
+    def custom_log_syntax(self , syntax) :
+        '''
+        f"[%(asctime)s]-[{self.name}]-[%(levelname)s]: %(message)s"
+        '''
+        f = syntax
+        self.syntax = f
+        self.formatter = logging.Formatter(f , datefmt='%Y-%m-%d %H:%M:%S' )
+        self.console.setFormatter(self.formatter)    
+
+    def updateGlobalHomePath(self ) :
+        if not self.isLogWriteDisabled :
+            getFromEnv = self.env.get('debugger_homepath' , None )
+            self.homepath(getFromEnv)
+            if getFromEnv : 
+                self.file_handler_class = self.createRotateFileHandler(self.homePath)
+
+    def updateGlobalSetLevel( self ) :
+        self.set_level(self.env['debugger_global_level'])
+
+    def advertiseGlobalDebugLevel(self , level) :
+        setGlobalDebugLevel(level)
+
+    def disable_rotate(self) :
+        self.rotate_disabled = True
+
+    def enable_rotate(self) :
+        self.rotate_disabled = False
+
+    def createRotateFileHandler( self , path ) :
+        old = self.file_handler_class
+        if old :
+            self.logger.removeHandler(old)
+        file_handler = RotatingFileHandler(path ,  maxBytes=self.LOG_SIZE_THRESHOLD_IN_BYTES , backupCount=self.BACKUP_COUNT , delay=True )
+        self.file_handler= file_handler.setFormatter(self.formatter)
+        self.logger.addHandler(file_handler)
+        return file_handler
+
+    def update_log_iterantions_threshold(self,threshold : int ):
+        '''
+        set value when rotation should be checked. when every on_log function called.
+        by default rotation will be checked every 200 on_log function call.
+        '''
+        self.log_iterations_threshold = threshold
+
+    def updateGlobalDebugger(self , logger ) :
+        '''
+        this function pass the log message to other logger to write the same log message to it.
+        logger must be debugger class.
+        '''
+        if logger.type != 'CUSTOM_DEBUGGER' :
+            raise Exception(f'Invalid logger type. must pass debugger class.')
+        self.global_debugger = logger
+
+    def getStreamServiceUrlPath(self) :
+        return self.streampath
+
+    def getStreamService(self) :
+        return self.stream_service
+
+    def isStreamServiceAvailable(self) :
+        if self.stream_service :
+            return True
+        return False
+
+    def addStreamService( self , socketio , streampath='/debugger/stream/log' ) :
+        """
+        This function takes a live socketio server. it emit the log message using default path which is /debugger/stream/log
+        """
+        self.stream_service = socketio
+        self.streampath = streampath
+        
+    def updateLogName( self , name ) :
+        self.name = name
+
+    def disable_log_write(self) :
+        '''
+        this function is used to disable the log write to file. if onScreen is enabled, logs will be displayed only on screen.
+        '''
+        self.isLogWriteDisabled = True
+        if self.file_handler_class :
+            self.logger.removeHandler(self.file_handler_class)
+    
+    def enable_log_write(self) :
+        self.createRotateFileHandler(self.homePath)
+        
+    def manage_file_rotation(self, record ) :
+        handler = self.get_rotate_handler()
+        if handler.shouldRollover(record) :
+            handler.doRollover()
+            self.log_iterations = 0
+
+    def get_rotate_handler(self) :
+        return self.file_handler_class
+            
+    def change_log_size(self, size) -> bool:
+        '''
+        change the size of each log file rotation.
+        default is 10M
+        size should be passed as MB
+        '''
+        size = convert_mb_to_bytes(size)
+        self.LOG_SIZE_THRESHOLD_IN_BYTES = size
+        handler = self.get_rotate_handler()
+        handler.maxBytes = size
+        return True
+
+    
+    def checks_in_bg(self) :
+        while True :
+            if self.env.get('GLOBAL_DEBUGGER_STREAM_SERVICE') :
+                self.addStreamService(socketio=self.env.get('GLOBAL_DEBUGGER_STREAM_SERVICE'))
+            if self.env.get('debugger_global_level' , None) : 
+                self.set_level( level=self.env.get('debugger_global_level') )
+            if not self.env.get('debugger_on_screen' , True ) :
+                self.disable_print()
+            if self.env.get('debugger_on_screen' , True ) :
+                self.enable_print()
+            if os.environ.get("EASY_UTILS_DEBUG_LEVEL") :
+                EASY_UTILS_DEBUG_LEVEL = os.environ.get("EASY_UTILS_DEBUG_LEVEL")
+                if not EASY_UTILS_DEBUG_LEVEL.lower() in ['info' , 'debug' , 'warning' , 'error' , 'critical'] :
+                    self.logger.error(f'EASY_UTILS_DEBUG_LEVEL ENV must be one of [info,debug,warning,error,critical] | Current Env Variable Is "{EASY_UTILS_DEBUG_LEVEL}". Skipping ')
+                else :
+                    self.set_level(EASY_UTILS_DEBUG_LEVEL)
+            self.updateGlobalHomePath()
+            if os.environ.get("EASY_UTILS_ENABLE_PRINT" , '' ).lower() == 'true' :
+                self.enable_print()
+            sleep(5)
+
+    def close(self) :
+        try :
+            logging.shutdown()
+        except :
+            pass
+
+    def homepath(self , path=None ) :
+        env = custom_env()
+        getFromEnv = env.get('debugger_homepath' , None )
+        if getFromEnv is not None :
+            self.homePath = getFromEnv
+        else :
+            if path is not None :
+                if self.homePath and '.log' in str(self.homePath): 
+                    self.lastAbsoluteHomePath= self.homePath
+                self.homePath = path
+            else :
+                self.homePath = os.path.join(os.getcwd() , 'debug')
+        if not os.path.exists( self.homePath ) :
+            try : 
+                os.makedirs( self.homePath )
+            except : 
+                pass
+        if self.filename :
+            self.homePath = os.path.join( self.homePath, f'{self.filename}.log' ) 
+        else :
+            self.homePath = os.path.join( self.homePath, f'{self.name}.log' ) 
+        return self.homePath
+
+    def get_current_levels(self):
+        """
+        Returns a list of log levels that will be printed based on the current logging level.
+        """
+        levels_order = [
+            ('debug', logging.DEBUG),
+            ('info', logging.INFO),
+            ('warning', logging.WARNING),
+            ('error', logging.ERROR),
+            ('critical', logging.CRITICAL),
+        ]
+        # Optional custom level
+        if hasattr(logging, 'SCREEN'):
+            levels_order.append(('screen', logging.SCREEN))
+        current_level = self.logger.level
+        # Return all levels with numeric value >= current_level
+        return [name for name, value in levels_order if value >= current_level]
+
+    def enable_print(self) :
+        self.onScreen = True
+        self.logger.addHandler(self.console)
+
+    def disable_print(self) : 
+        self.onScreen = False
+        self.logger.removeHandler(self.console)
+
+    def changeHomePath( self , path ) :
+        p = self.homepath(path)
+        self.file_handler_class = self.createRotateFileHandler(p)
+        if self.lastAbsoluteHomePath :
+            os.remove(self.lastAbsoluteHomePath)
+
+    def isGlobalDebuggerDefined(self) :
+        if self.global_debugger :
+            return True
+        else :
+            return False
+
+    def set_level(self, level : str):
+        if 'info' in level.lower() : lvl = logging.INFO
+        elif 'warn' in level.lower() : lvl = logging.WARNING
+        elif 'warning' in level.lower() : lvl = logging.WARNING
+        elif 'critical' in level.lower() : lvl = logging.CRITICAL
+        elif 'debug' in level.lower() : lvl = logging.DEBUG
+        elif 'error' in level.lower() : lvl = logging.ERROR
+        elif 'screen' in level.lower() : lvl = logging.SCREEN
+        else : raise ValueError('Unknown level, not one of [info,warn,warning,critical,debug,error,screen]')
+        self.currentDebugLevel = level
+        self.logger.setLevel(lvl)
+
+    def get_current_debug_level(self) :
+        return self.currentDebugLevel
+
+    def get_logger(self) : 
+        return self.logger
+    
+    def before_log(self , message , level) :
+
+        def __call_thread__() :
+            if not level in self.get_current_levels() :
+                return
+            if self.isStreamServiceAvailable() :
+                d = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                streamUrl = self.getStreamServiceUrlPath()
+                self.stream_service.emit( streamUrl , {
+                    'message' : message ,
+                    'level' : level ,
+                    'msg' : message,
+                    'date' : d ,
+                    'id' : self.id,
+                    'formate' : 'json' ,
+                    'source' : self.name ,
+                    'getTimestamp' : getTimestamp()
+                })
+        try :
+            t= Thread(target=__call_thread__)
+            # t.daemon=True
+            t.start()
+        except :
+            __call_thread__()
+            
+        params = {
+                'screen' : True  ,
+                'file': True
+            }
+        if self.onScreen and self.env['debugger_on_screen'] == True :
+            params['screen'] = True
+        else :
+            params['screen'] = False
+        return params
+        
+
+    def info(self, message , external_debugger=None,source=None):
+        def __call__(message) :
+            if source :
+                message = f'[{source}]-{message}'
+            self.before_log(message , 'info')
+            self.logger.info(message)
+            if self.isGlobalDebuggerDefined() : 
+                self.global_debugger.info(message)
+            if external_debugger :
+                external_debugger.info(message)
+        try :
+            r=Thread(target=__call__,args=[message])
+            r.daemon=True
+            r.start()
+            r.join()
+        except :
+            __call__(message)
+
+    def debug(self, message , external_debugger=None,source=None):
+        def __call__(message) :
+            if source :
+                message = f'[{source}]-{message}'
+            self.before_log(message , 'debug')
+            self.logger.debug(message)
+            if self.isGlobalDebuggerDefined() : 
+                self.global_debugger.debug(message)
+            if external_debugger :
+                external_debugger.debug(message)
+        try :
+            r=Thread(target=__call__,args=[message])
+            r.daemon=True
+            r.start()
+            r.join()
+        except :
+            __call__(message)
+
+    def warning(self, message , external_debugger=None,source=None):
+        def __call__(message) :
+            if source :
+                message = f'[{source}]-{message}'
+            self.before_log(message , 'warning')
+            self.logger.warning(message)
+            if self.isGlobalDebuggerDefined() : 
+                self.global_debugger.warning(message)
+            if external_debugger :
+                external_debugger.warning(message)
+        try :
+            r=Thread(target=__call__,args=[message])
+            r.daemon=True
+            r.start()
+            r.join()
+        except :
+            __call__(message)
+
+    def error(self, message,external_debugger=None,source=None):
+        def __call__(message) :
+            if source :
+                message = f'[{source}]-{message}'
+            self.before_log(message , 'error')
+            self.logger.error(message)
+            if self.isGlobalDebuggerDefined() : 
+                self.global_debugger.error(message)
+            if external_debugger :
+                external_debugger.error(message)
+        try :
+            r=Thread(target=__call__,args=[message])
+            r.daemon=True
+            r.start()
+            r.join()
+        except :
+            __call__(message)
+
+    def critical(self, message,external_debugger=None,source=None):
+        def __call__() :
+            if source :
+                message = f'[{source}]-{message}'
+            self.before_log(message , 'critical')
+            self.logger.critical(message)
+            if self.isGlobalDebuggerDefined() : 
+                self.global_debugger.critical(message)
+            if external_debugger :
+                external_debugger.critical(message)
+        try :
+            r=Thread(target=__call__,args=[message])
+            r.daemon=True
+            r.start()
+            r.join()
+        except :
+            __call__(message)
+
+    def screen(self, message,external_debugger=None,source=None):
+        def __call__() :
+            if source :
+                message = f'[{source}]-{message}'
+            self.before_log(message , 'critical')
+            print(f"{self.syntax}")
+            if self.isGlobalDebuggerDefined() : 
+                self.global_debugger.critical(message)
+            if external_debugger :
+                external_debugger.critical(message)
+        try :
+            r=Thread(target=__call__,args=[message])
+            r.daemon=True
+            r.start()
+            r.join()
+        except :
+            __call__(message)

easy_utils_dev/debugger.py

@@ -128,6 +128,15 @@ class DEBUGGER:
         self.formatter = logging.Formatter(f , datefmt='%Y-%m-%d %H:%M:%S' )
         self.console.setFormatter(self.formatter)    
 
+    def custom_log_syntax(self , syntax) :
+        '''
+        f"[%(asctime)s]-[{self.name}]-[%(levelname)s]: %(message)s"
+        '''
+        f = syntax
+        self.syntax = f
+        self.formatter = logging.Formatter(f , datefmt='%Y-%m-%d %H:%M:%S' )
+        self.console.setFormatter(self.formatter)    
+
     def updateGlobalHomePath(self ) :
         if not self.isLogWriteDisabled :
             getFromEnv = self.env.get('debugger_homepath' , None )

easy_utils_dev/temp_memory.py

@@ -17,21 +17,28 @@ class TemporaryMemory :
             sleep(5)
             now = getTimestamp()
             for key, value in list(self.store.items()) :
-                if now >= value.get('removeTimestamp' , 0) :
-                    if value.get('store_deleted_key' , False) :
-                        self.deletions.append(key)
-                    self.delete(key)
+                if value.get('removeTimestamp') :
+                    if now >= value.get('removeTimestamp' , 0) :
+                        if value.get('store_deleted_key' , False) :
+                            self.deletions.append(key)
+                        self.delete(key)
 
     def delete(self , key ) :
-        del self.store[key]
+        try :
+            del self.store[key]
+        except :
+            pass
         gc.collect()
 
-    def get(self , key ) :
-        return self.store.get(key , {}).get('item')
+    def get(self , key , default=None ) :
+        return self.store.get(key , {}).get('item' , default)
 
     def save(self, item , custom_key=None ,auto_destroy_period=60 , store_deleted_key=True) :
         now = getTimestamp()
-        later = getTimestamp(after_seconds=auto_destroy_period)
+        if auto_destroy_period :
+            later = getTimestamp(after_seconds=auto_destroy_period)
+        else :
+            later = None
         if not custom_key :
             custom_key = f"{getTimestamp()}-{generateToken(iter=4)}".upper()
         self.store[custom_key] = {

easy_utils_dev/wsnoclib.py

@@ -35,6 +35,7 @@ class KafkaConfig :
         self.kafka_subscription_deleted = False 
         self.base_url = None
         self.kafka_thread=None
+        self.enable_auto_refresh = False
 
 
 
@@ -69,7 +70,8 @@ class WSNOCLIB :
         self.onGoingRequests=0
         self.fastQueue = FastQueue(request_max_count)
         self.queue = []
-        self.tokenRefreshPeriod = 2700
+        self.token = {}
+        self.tokenRefreshPeriod = None
         self.final_results = []
         self.killed=False
         self.nes=[]
@@ -78,6 +80,7 @@ class WSNOCLIB :
         self.refresh_inprogress = False
         self.kafka = kafka
         self.refresh_thread = None
+        self.token_refresh_count = 0
         self.session = WSNOCSession(self)
         if register_atexit :
             atexit.register(self.goodbye)
@@ -124,33 +127,23 @@ class WSNOCLIB :
             return
         if not pingAddress(self.address) :
             raise Exception(f'Address {self.address} is not pingable.')
-        self.logger.info(f'Connecting to {self.address} using username: {self.username} BAuth={BAuth(self.username, self.password)}')
+        # self.logger.info(f'Connecting to {self.address} using username: {self.username}')
         self.logger.debug(f'Connecting to {self.address} using username: {self.username}')
         URL = f"https://{self.address}/rest-gateway/rest/api/v1/auth/token"
-        self.logger.debug(f'Login URL is {URL}')
         headers = {
             "Content-Type": "application/json"
         }
         data = {
             "grant_type": "client_credentials"
         }
-        r = requests.post(url = URL , headers=headers , auth=BAuth(self.username, self.password), verify=False, json=data)
-        self.logger.debug(f"""
-            response_header={r.headers}
-            response_encoding={r.encoding}
-            response_reason={r.reason}
-            body={r.request.body}
-            headers={r.request.headers}
-            URL={URL}
-            content={r.text}
-        """)
-        self.logger.info(f'Request return status code : {r.status_code}')
-        self.logger.debug(f"Login Request response body is {r.text}")
-        if r.status_code != 200 :
+        r = self.session.post(url = URL , headers=headers , auth=BAuth(self.username, self.password), json=data , retries=5)
+        if not r.ok :
             self.logger.debug(f'fail message {r.text}')
             raise Exception(f'Failed to authenticate WSNOC. Return status code : {r.status_code}')
         self.access_token = r.json()["access_token"]
         self.refresh_token = r.json()["refresh_token"]
+        if not self.tokenRefreshPeriod :
+            self.tokenRefreshPeriod = int(r.json()["expires_in"]) - 100
         self.bearer_token = f'Bearer {self.access_token}'
         self.token = r.json()
         self.token.update({'bearer_token' :  self.bearer_token })
@@ -202,12 +195,14 @@ class WSNOCLIB :
         sleep(self.tokenRefreshPeriod)
         self.logger.info(f"Waiting period completed. Starting Revoking/Login process ...")
         self.refresh_inprogress = True
-        self.logout(logout=False)
-        sleep(30)
-        self.logger.info(f"Logout process completed. Starting Reconnecting ...")
-        self.connect(self.auto_refresh_token)
+        self.renew_by_refresh_token()
         self.refresh_inprogress = False
-        self.event.dispatchEvent('onTokenRefresh', self.token)
+        self.kafka.refresh_inprogress = True
+        if self.kafka.enable_auto_refresh :
+            self.renewSubscription()
+        self.kafka.refresh_inprogress = False
+        self.token_refresh_count += 1
+        self.runAutoRefreshThread()
 
     def kafka_connect( self , 
             user , 
@@ -253,7 +248,6 @@ class WSNOCLIB :
         self.logger.debug(f"nsp_address: {nsp_address}")
         self.logger.debug(f"NewRelease: {new_release}")
         self.kafka.kafka_subscription_deleted= False
-
         if not self.external_nsp : 
             if new_release :
                 self.kafka.kafka_nsp_os_name = 'nspos-kafka'
@@ -265,8 +259,6 @@ class WSNOCLIB :
             self.kafka.kafka_address = nsp_address
             self.kafka.kafka_port = 9192
             self.kafka.kafka_nsp_os_name = 'nspos-tomcat'
-
-        self.logger.info('requesting kafka subscription ...')
         if self.loggedOut or self.killed:
             self.logger.error(f"WSNOC API Authentication process loggedout or killed. exit")
             raise Exception('WSNOC API Authentication process loggedout or killed. exit')
@@ -283,21 +275,38 @@ class WSNOCLIB :
         self.nsp_key = f'{self.temp_dir}/nsp.key'
         self.nsp_cert = f'{self.temp_dir}/nsp.pem'
         if not self.external_nsp :
+            ####################
+            ####################
+            #
+            #   IN CASE OF INTERNAL NSP
+            #   
+            ####################
+            ####################
+            checkContainer = self.ssh.ssh_execute(f"docker ps | grep -i 'nspos-kafka' | wc -l")
+            if checkContainer != '0' :
+                self.kafka.kafka_nsp_os_name = 'nspos-kafka'
+            self.kafka.kafka_port = None
             self.logger.debug(f"Working on internal NSP to copy the files ...")
             self.ssh.ssh_execute(f"docker cp {self.kafka.kafka_nsp_os_name}:/opt/nsp/os/ssl/certs/nsp/nsp.pem /tmp/nsp.pem")
             self.ssh.ssh_execute(f"docker cp {self.kafka.kafka_nsp_os_name}:/opt/nsp/os/ssl/nsp.key /tmp/nsp.key")
             sftp.get('/tmp/nsp.pem' , f'{self.temp_dir}/nsp.pem')
             sftp.get('/tmp/nsp.key' , f'{self.temp_dir}/nsp.key')
             self.kafka.ca_cert = f'{self.temp_dir}/nsp.pem'
-            baseUrl = self.kafka.base_url = None
-            port = self.kafka.kafka_api_port
+            self.kafka.base_url = f'https://{self.address}'
         else :
+            ####################
+            ####################
+            #
+            #   IN CASE OF EXTERNAL NSP
+            #   
+            ####################
+            ####################
             self.logger.debug(f"Working on external NSP to copy the files ...")
             CertLoc = f"""find /var/lib/kubelet/pods/ -type d -path "*/volumes/kubernetes.io~empty-dir/shared-tls-volume" | head -n1"""
             CertLoc = self.ssh.ssh_execute(CertLoc).replace('\n','')
             self.logger.debug(f"CertLoc Host: {CertLoc}")
-            baseUrl = self.kafka.base_url = f'https://{nsp_address}'
-            port = self.kafka.kafka_port = None
+            self.kafka.base_url = f'https://{nsp_address}'
+            self.kafka.kafka_port = None
             if len(CertLoc) > 15 :
                 self.logger.debug(f"Copying cert files from nsp host machine ....")
                 copies = [
@@ -334,7 +343,6 @@ class WSNOCLIB :
             self.logger.debug(f'affectedObjectType Filter : filter updated with "{filter}"')
         if custom_filter_expression :
             filter += f" {custom_filter_expression}"
-        URL = '/nbi-notification/api/v1/notifications/subscriptions'
         kafkaForm = {
                 "categories": [
                     {
@@ -344,19 +352,18 @@ class WSNOCLIB :
                 ]
         }
         self.logger.debug(f"Kafka Filter Form : {kafkaForm}")
-        response = self.post(
-            url=URL,
-            baseUrl=baseUrl,
-            return_json=True,
-            port=port,
-            body=json.dumps(kafkaForm)
-        )
-        if response :
+        if self.kafka.kafka_port is not None :
+            URL = f"{self.kafka.base_url}:{self.kafka.kafka_port}/nbi-notification/api/v1/notifications/subscriptions"
+        else :
+            URL = f"{self.kafka.base_url}/nbi-notification/api/v1/notifications/subscriptions"
+        response = self.session.post(URL , json=kafkaForm , retries=3)
+        if response.ok :
+            response = response.json()
             self.kafka.subscriptionId = response['response']['data']['subscriptionId']
             self.kafka.response = response
             self.kafka.topicId = response['response']['data']['topicId']
             if auto_refresh :
-                self.kafka.kafka_thread = start_thread(target=self.renewSubscription)
+                self.kafka.enable_auto_refresh = True
             self.killed=False
         else :
             self.logger.error(f"Failed to create kafka subscription.")
@@ -365,81 +372,84 @@ class WSNOCLIB :
         return self.kafka
 
     def change_kafka_refresh_period(self , period : int =3000) :
-        self.kafka.kafka_refresh_period = period
+        print('Deprecated function change_kafka_refresh_period. Kafka refresh period is now managed by WSNOC API SLEEP PERIOD')
 
     def renewSubscription(self) :
-        while True :
-            try :
-                sleep(self.kafka.kafka_refresh_period) 
-                if self.loggedOut or self.killed:
-                    break
-                self.logger.info('Renewing subscription ...')
-                self.kafka.refresh_inprogress = True
-                URL = f"/nbi-notification/api/v1/notifications/subscriptions/{self.kafka.subscriptionId}/renewals"
-                response = self.post(
-                    url=URL,
-                    baseUrl=self.kafka.base_url,
-                    port=self.kafka.kafka_port,
-                )
-                self.logger.info(f'Renewing subscription [{response.status_code}]')
-                self.logger.info(f'Renewing subscription Response: [{response.text}]')
-            except Exception as error :
-                self.logger.error(f'failed to renew subscription. {error}')
-                self.logger.debug(traceback.format_exc())
-            self.kafka.refresh_inprogress = False
-
-    def deleteKafkaSubscription(self , subscriptionId=None) :
-        self.logger.info(f'Deleting subscription subscriptionId:{subscriptionId}')
-        if not subscriptionId :
-            self.logger.info(f'Deleting subscription subscriptionId:{self.kafka.subscriptionId}')
-            subscriptionId=self.kafka.subscriptionId
-        URL = f"/nbi-notification/api/v1/notifications/subscriptions/{subscriptionId}"
-        self.kafka.kafka_subscription_deleted= True
-        self.delete(
-            url=URL,
-            return_json=True,
-            port=self.kafka.kafka_api_port,
-        )
-        self.killed=True
-        return True
+        self.logger.info('Renewing subscription ...')
+        if self.kafka.kafka_port is not None :
+            URL = f"{self.kafka.base_url}:{self.kafka.kafka_port}/nbi-notification/api/v1/notifications/subscriptions/{self.kafka.subscriptionId}/renewals"
+        else :
+            URL = f"{self.kafka.base_url}/nbi-notification/api/v1/notifications/subscriptions/{self.kafka.subscriptionId}/renewals"
+        response = self.session.post(URL , retries=3)
+        if not response.ok :
+            self.logger.error(f'failed to renew subscription. {response.text}')
+
+    # def renewSubscription(self) :
+    #     while True :
+    #         try :
+    #             sleep(self.kafka.kafka_refresh_period) 
+    #             if self.loggedOut or self.killed:
+    #                 break
+    #             self.logger.info('Renewing subscription ...')
+    #             self.kafka.refresh_inprogress = True
+    #             URL = f"{self.kafka.base_url}:{self.kafka.kafka_port}/nbi-notification/api/v1/notifications/subscriptions/{self.kafka.subscriptionId}/renewals"
+    #             response = self.session.post(URL , retries=3)
+    #             self.logger.debug(f'Renewing subscription Response: [{response.text}]')
+    #         except Exception as error :
+    #             self.logger.error(f'failed to renew subscription. {error}')
+    #             self.logger.debug(traceback.format_exc())
+    #         self.kafka.refresh_inprogress = False
+
+    # def deleteKafkaSubscription(self , subscriptionId=None) :
+    #     self.logger.info(f'Deleting subscription subscriptionId:{subscriptionId}')
+    #     if not subscriptionId :
+    #         self.logger.info(f'Deleting subscription subscriptionId:{self.kafka.subscriptionId}')
+    #         subscriptionId=self.kafka.subscriptionId
+    #     self.kafka.kafka_subscription_deleted= True
+    #     URL = f"{self.kafka.base_url}:{self.kafka.kafka_port}/nbi-notification/api/v1/notifications/subscriptions/{subscriptionId}"
+    #     response = self.session.delete(URL , retries=3)
+    #     return response
     
-    def handle_beautify_alarm(self , alarm ) :
-        oalarm = alarm
-        alarm = alarm['data']['ietf-restconf:notification']
-        if 'create' in str(list(alarm.keys())) :
-            alarmData = alarm['nsp-fault:alarm-create']
-            oalarm['dataEnh'] = {
-                'newAlarm' : True,
-                'alarmChange' : False,
-                'alarmId' : int(alarmData['objectId'].split(':')[-1]),
-                'neId' : alarmData['neId'],
-                'neName	' : alarmData['neName'],
-                'alarmName' : alarmData['alarmName'],
-                'cleared' : False,
-                'aknowledged' : False,
-                **alarmData , 
-            }
-        elif 'change' in str(list(alarm.keys())) :
-            alarmData = alarm['nsp-fault:alarm-change']
-            cleared = False
-            aknowledged = False
-            if 'severity' in list(alarmData.keys()) :
-                if alarmData['severity']['new-value'] == 'cleared' :
-                    cleared = True
-            if 'acknowledged' in list(alarmData.keys()) :
-                    aknowledged = alarmData['acknowledged']['new-value']
-            oalarm['dataEnh'] = {
-                'newAlarm' : False,
-                'alarmChange' : True,
-                'alarmId' : int(alarmData['objectId'].split(':')[-1]),
-                'cleared' : cleared,
-                'aknowledged' : aknowledged,
-                **alarmData ,
-            }
-        return oalarm
-
-
-    def kafka_listen(self) :  
+    # def handle_beautify_alarm(self , alarm ) :
+    #     oalarm = alarm
+    #     alarm = alarm['data']['ietf-restconf:notification']
+    #     if 'create' in str(list(alarm.keys())) :
+    #         alarmData = alarm['nsp-fault:alarm-create']
+    #         oalarm['dataEnh'] = {
+    #             'newAlarm' : True,
+    #             'alarmChange' : False,
+    #             'alarmId' : int(alarmData['objectId'].split(':')[-1]),
+    #             'neId' : alarmData['neId'],
+    #             'neName	' : alarmData['neName'],
+    #             'alarmName' : alarmData['alarmName'],
+    #             'cleared' : False,
+    #             'aknowledged' : False,
+    #             **alarmData , 
+    #         }
+    #     elif 'change' in str(list(alarm.keys())) :
+    #         alarmData = alarm['nsp-fault:alarm-change']
+    #         cleared = False
+    #         aknowledged = False
+    #         if 'severity' in list(alarmData.keys()) :
+    #             if alarmData['severity']['new-value'] == 'cleared' :
+    #                 cleared = True
+    #         if 'acknowledged' in list(alarmData.keys()) :
+    #                 aknowledged = alarmData['acknowledged']['new-value']
+    #         oalarm['dataEnh'] = {
+    #             'newAlarm' : False,
+    #             'alarmChange' : True,
+    #             'alarmId' : int(alarmData['objectId'].split(':')[-1]),
+    #             'cleared' : cleared,
+    #             'aknowledged' : aknowledged,
+    #             **alarmData ,
+    #         }
+    #     return oalarm
+
+
+    def kafka_listen(self) : 
+        def hold_if_kafka_refresh_inprogress() :
+            while self.kafka.refresh_inprogress :
+                sleep(.1)
         self.logger.info('Listening to Kafka Notifications ...')
         if not self.kafka.topicId :
             self.logger.error(f'kafka is not established. exit.')
@@ -464,9 +474,7 @@ class WSNOCLIB :
         )
         try:
             while True:
-                if self.kafka.refresh_inprogress :
-                    self.logger.info(f"Waiting for kafka subscription to be ready. In refresh process...")
-                    sleep(1)
+                hold_if_kafka_refresh_inprogress()
                 if self.kafka.kafka_subscription_deleted :
                     self.logger.info(f"Kafka subscription is deleted. exit.")
                     break
@@ -583,6 +591,27 @@ class WSNOCLIB :
             return r.json()
         return r
 
+    def renew_by_refresh_token(self) :
+        URL = f"https://{self.address}/rest-gateway/rest/api/v1/auth/token"
+        headers = {
+             "Content-Type": "application/json"
+        }
+        data = {
+            "grant_type": "refresh_token",
+            "refresh_token": f"{self.refresh_token}"
+            
+        }
+        r = self.session.post(URL , headers=headers , json=data , auth=BAuth(self.username, self.password) , skip_hold_for_token_refresh=True)
+        if r.ok :
+            if not self.tokenRefreshPeriod :
+                self.tokenRefreshPeriod = int(r.json()["expires_in"]) - 100
+            self.access_token = r.json()["access_token"]
+            self.refresh_token = r.json()["refresh_token"]
+            self.bearer_token = f'Bearer {self.access_token}'
+            self.token = r.json()
+            self.token.update({'bearer_token' :  self.bearer_token })
+        return r
+
     def session_info(self) :
         self.logger.debug('Getting Version ...')
         response = self.get( url='/oms1350/data/common/sessionInfo')
@@ -676,33 +705,48 @@ class WSNOCSession(requests.Session):
         self.headers.update({"Content-Type": "application/json"})  # base defaults
         self._wsnoc = wsnoc
         self.verify = False
+        self.retries = 0
+        self.debug_this_request = False
+        self.skip_hold_for_token_refresh = False
         
     def rebuild_auth(self, prepared_request, response):
         return
 
-    def hold_for_token_refresh(self) :
+    def hold_for_token_refresh(self, url=None) :
         while self._wsnoc.refresh_inprogress :
-            sleep(.1)
+            self._wsnoc.logger.info(f'Waiting for token refresh. {url if url else "No URL"}')
+            sleep(.5)
 
-    def request(self, method, url , retries=0 , **kwargs):
+    def request(self, method, url , retries=0 , skip_hold_for_token_refresh=False , debug_this_request=False , **kwargs):
         self._wsnoc.logger.debug(f'[{method}] : {url}')
-        self.hold_for_token_refresh()
+        if not skip_hold_for_token_refresh :
+            self.hold_for_token_refresh(url)
         self._wsnoc.api_count += 1
         token = self._wsnoc.getLatestToken().get('bearer_token')
         request_headers = kwargs.get('headers' , {})
-        if not request_headers.get('Authorization') :
-            request_headers['Authorization'] = token
+        if token :
+            if not request_headers.get('Authorization') :
+                request_headers['Authorization'] = token
         kwargs['headers'] = request_headers
         request = super().request(method, url, **kwargs  )
+        if debug_this_request :
+            self._wsnoc.logger.info(f'''
+                [DEBUG] [{method}] : {url}
+                [DEBUG] Headers: {request_headers}
+                [DEBUG] Body: {kwargs.get('data' , {})}
+                [DEBUG] Response: {request.text}
+                [DEBUG] OK: {request.ok}
+                [DEBUG] Method: {request.request.method}
+            ''')
         for i in range(retries) :
             if request.ok :
                 break
             if not request.ok :
                 sleep(1)
-                self.hold_for_token_refresh()
+                self.hold_for_token_refresh(url)
                 request = super().request(method, url, **kwargs )
                 self._wsnoc.logger.debug(f'[Try-{i}] [{method}] : {url}- {request.status_code}')
-        self._wsnoc.logger.info(f'[{method}] : {url}- {request.status_code}')
+        self._wsnoc.logger.info(f'[{method}] : {url} - [{request.status_code}]')
         return request
 
 

{easy_utils_dev-2.138.dist-info → easy_utils_dev-2.140.dist-info}/METADATA

@@ -1,21 +1,23 @@
-Metadata-Version: 2.1
-Name: easy-utils-dev
-Version: 2.138
+Metadata-Version: 2.4
+Name: easy_utils_dev
+Version: 2.140
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
 Requires-Dist: ping3
 Requires-Dist: flask
-Requires-Dist: flask-cors
+Requires-Dist: flask_cors
 Requires-Dist: xmltodict
 Requires-Dist: paramiko
 Requires-Dist: oracledb
 Requires-Dist: requests
-Requires-Dist: flask-socketio
+Requires-Dist: flask_socketio
 Requires-Dist: python-dotenv
 Requires-Dist: gevent
 Requires-Dist: pyzipper
 Requires-Dist: pyjwt
 Requires-Dist: authlib
 Requires-Dist: kafka-python
-
+Dynamic: classifier
+Dynamic: keywords
+Dynamic: requires-dist

{easy_utils_dev-2.138.dist-info → easy_utils_dev-2.140.dist-info}/RECORD

@@ -1,4 +1,4 @@
-easy_utils_dev/EasySsh.py,sha256=PbFK1vSBcnt_SfkZXZGh5wg6Sg9R2WlXeGdvjZluJ9E,3592
+easy_utils_dev/EasySsh.py,sha256=9pQaHHQGng8X3fsy4Vvto5RIEObm-iOt52VLiO6m1g0,3591
 easy_utils_dev/Events.py,sha256=MdI53gAyXX_2jmChpayayQM0ZitgjtkyUNrQYGEEZnw,2978
 easy_utils_dev/FastQueue.py,sha256=Drt8B_hEdmg9eAt7OWSgTyoJ3rUHkeJHk9xdaehtEsY,5622
 easy_utils_dev/NameObject.py,sha256=Z4Qp3qfMcQeMPw35PV_xOu8kigRtfSRZ4h7woR0t3Gg,270
@@ -8,7 +8,8 @@ easy_utils_dev/brevosmtp.py,sha256=A5n13MnVQnDuSjYQ91-MLftmqfg3VQ-36Zqw9OtoTB4,3
 easy_utils_dev/check_license.py,sha256=C8vKXwaduoF3FSuDJ-J_j5jStNNyUdS-mOdLNfsCjmc,4825
 easy_utils_dev/cplib.py,sha256=TPCFNQh4jYSGsm_CrMdivmD3G5B1rprco2O4B0BSnzo,19570
 easy_utils_dev/custom_env.py,sha256=vxrjikpSNJlKfoBE-ef88UExlpXucUe-HcwHMn3gfB0,1510
-easy_utils_dev/debugger.py,sha256=Oz4WcT59FbHouTpprslocljDvevFdmP9hc_zNRVmSdE,18318
+easy_utils_dev/debugger-C-PF4PAMMP.py,sha256=5GmZU1l9R3CSZVIGff-l-gYaf-imJGSGTsh-cxNz6og,18637
+easy_utils_dev/debugger.py,sha256=5GmZU1l9R3CSZVIGff-l-gYaf-imJGSGTsh-cxNz6og,18637
 easy_utils_dev/easy_oracle.py,sha256=Jyc3HSl6eyLayjS8NoE4GOaf8otQlonR5_qOg2h1DjE,2157
 easy_utils_dev/encryptor.py,sha256=f5Zjn0DGtXCyhldpVnBtfcTb4h4Wp0eQPHusEYwIags,1512
 easy_utils_dev/ept.py,sha256=X-Z0_XCNfKK1TxQZquPMBqo3ZuxnK4TvWAYPnBUMlPI,25363
@@ -23,15 +24,15 @@ easy_utils_dev/openid_server.py,sha256=_odeg6omuizSUEJLtbAVn2PnG9vkcUAQ7rU3K5yXk
 easy_utils_dev/optics_utils.py,sha256=G-hFX2iiUCSJjk7BICBRGvVoDq0IBONLZSjagoB5FMg,964
 easy_utils_dev/require_auth.py,sha256=UsYAxfLX5wda6hd0nfLR_tl0bGQ4DYIpaTowsYSku-E,881
 easy_utils_dev/simple_sqlite.py,sha256=J-mcTUnHmAn0eCPD8j-WEoA19uzHRXJ4YRJsyx9B-do,13113
-easy_utils_dev/temp_memory.py,sha256=gfC-izYw8Sg1DD5iOdL8R5-fdB5RK0qkXZie9WmFjPA,1431
+easy_utils_dev/temp_memory.py,sha256=0Dx_vNUSFQRMtJZNT8tUZXubcG7T6jxevw2WYuGmVe8,1658
 easy_utils_dev/uiserver.py,sha256=d9jImwgSURMH4MIU4Zkro2b20cAbuV8rO_If42FVyZ4,6826
 easy_utils_dev/utils.py,sha256=BmVnbxc336c6WTeDFcEHN6Mavt7fJrIEyK4GXODV3gI,13345
 easy_utils_dev/winserviceapi.py,sha256=2ZP6jaSt1-5vEJYXqwBhwX-1-eQ3V3YzntsoOoko2cw,18804
-easy_utils_dev/wsnoclib.py,sha256=z18kdPJJeNYAaomqH1t6KfTWm77BNPx6aPAW9P0PSOc,30813
+easy_utils_dev/wsnoclib.py,sha256=kqkOkiafXnrIgdlV__3uyrAoqxG5dmIHnwqm0EkwS2U,33457
 easy_utils_dev/wsselib.py,sha256=YweScnoAAH_t29EeIjBpkQ6HtX0Rp9mQudRsRce2SE8,7920
 easy_utils_dev/ept_sql/create_dirs.sql,sha256=KWfX-Nc6lvr_BC-P6O97NE0idoPW4GNKUKUCgonJhto,3508
 easy_utils_dev/ept_sql/create_ept_tables.sql,sha256=WDHyIyeReV8_QaYBPIpSy-lto3OKvZtex1tWs-FPURQ,67737
-easy_utils_dev-2.138.dist-info/METADATA,sha256=uN_YPuZajKHryq24B8n7zfC3qVYpw9oXaDabo_Vfo1E,510
-easy_utils_dev-2.138.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-easy_utils_dev-2.138.dist-info/top_level.txt,sha256=7vBsrpq7NmilkdU3YUvfd5iVDNBaT07u_-ut4F7zc7A,15
-easy_utils_dev-2.138.dist-info/RECORD,,
+easy_utils_dev-2.140.dist-info/METADATA,sha256=11wRCPmmDl1s7jptVfwy-_ru_jnkTn2nJMcJ-dM07uM,572
+easy_utils_dev-2.140.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+easy_utils_dev-2.140.dist-info/top_level.txt,sha256=7vBsrpq7NmilkdU3YUvfd5iVDNBaT07u_-ut4F7zc7A,15
+easy_utils_dev-2.140.dist-info/RECORD,,

{easy_utils_dev-2.138.dist-info → easy_utils_dev-2.140.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any