easy-utils-dev 2.140__tar.gz → 2.142__tar.gz

{easy_utils_dev-2.140 → easy_utils_dev-2.142}/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.140
+Version: 2.142
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
 Requires-Dist: ping3
+Requires-Dist: snakebite-py3
 Requires-Dist: flask
 Requires-Dist: flask_cors
 Requires-Dist: xmltodict

easy_utils_dev-2.142/easy_utils_dev/uiserver.py

@@ -0,0 +1,227 @@
+import copy
+import gc
+import time
+from werkzeug.serving import ThreadedWSGIServer
+from easy_utils_dev.utils import getRandomKey , generateToken , getTimestamp
+from flask_socketio import SocketIO
+from engineio.async_drivers import gevent
+from flask_cors import CORS
+import logging  , os
+from flask import jsonify, request , current_app
+from flask import Flask
+from threading import Thread
+import threading
+from easy_utils_dev.custom_env import cenv
+from easy_utils_dev.utils import kill_thread
+from multiprocessing import Process
+from werkzeug.serving import make_ssl_devcert
+from time import sleep
+from easy_utils_dev.utils import start_thread , getRandomKeysAndStr , mkdirs
+from easy_utils_dev.temp_memory import TemporaryMemory
+
+def getClassById( id ) :
+    return cenv[id]
+
+def create_ssl(host,output) :
+    '''
+    host : is the IP/Adress of the server which servers the web-server
+    output: the output locaiton to generate the ssl certificate. it should end with filename without extension
+    '''
+    return make_ssl_devcert( output , host=host)
+
+def clone_request(request):
+    """Return a plain dict clone of Flask request data."""
+    return {
+        "method": request.method,
+        "path": request.path,
+        "url": request.url,
+        "headers": dict(request.headers),
+        "args": request.args.to_dict(flat=False),
+        "form": request.form.to_dict(flat=False),
+        "json": request.get_json(silent=True),
+        "data": request.get_data(),   # raw body bytes
+        "files": {k: v.filename for k, v in request.files.items()},
+        "remote_addr": request.remote_addr,
+        "cookies": request.cookies,
+    }
+
+class AbortRequest :
+    def __init__(self, request ) :
+        self.request = clone_request(request)
+        self.abort_id = None
+        self.abortable = False
+        self.thread = None
+        self.cache = None
+        self.start_ts = getTimestamp()
+
+    def abort(self) : 
+        kill_thread(self.thread)
+        self.cache.delete(self.abort_id)
+        try :
+            gc.collect()
+        except :
+            pass
+
+
+class UISERVER :
+    def __init__(self ,
+            id=getRandomKey(n=15),
+            secretkey=generateToken(),
+            address='localhost',
+            port=5312 , 
+            https=False , 
+            ssl_crt=None,
+            ssl_key=None,
+            template_folder='templates/' ,
+            static_folder = 'templates/assets'
+            ,**kwargs
+        ) -> None:
+        self.id = id
+        self.static_folder = static_folder
+        self.app = app = Flask(self.id , template_folder=template_folder  ,  static_folder=self.static_folder )
+        app.config['SECRET_KEY'] = secretkey
+        CORS(app,resources={r"/*":{"origins":"*"}})
+        self.address= address 
+        self.port = port
+        self.thread = None
+        self.ssl_crt=ssl_crt
+        self.ssl_key=ssl_key
+        self.enable_test_url=True
+        self.abort_requests = {}
+        self.abort_base_url = '/request/abort'
+        if https :
+            self.httpProtocol = 'https'
+        else :
+            self.httpProtocol = 'http'
+        self.socketio = SocketIO(app , cors_allowed_origins="*"  ,async_mode='threading' , engineio_logger=False , always_connect=True ,**kwargs )
+        cenv[id] = self
+        self.fullAddress = f"{self.httpProtocol}://{self.address}:{self.port}"
+        self.cache = TemporaryMemory()
+
+    def update_cert(self , crt, ssl ) :
+        self.ssl_crt=crt
+        self.ssl_key=ssl
+
+    def register_abortable_request(self , request , abort_id = None ) :
+        path = request.path
+        Abort = AbortRequest(request)
+        if not path.startswith(self.abort_base_url) :
+            if not abort_id :
+                if not request.headers.get('abortid') :
+                    abort_id = getRandomKeysAndStr(n=20)
+                else :
+                    abort_id = request.headers.get('abortid')
+            
+            Abort.abort_id = abort_id
+            current_thread = threading.current_thread()
+            Abort.thread = current_thread
+            Abort.cache = self.cache
+            Abort.start_ts = getTimestamp()
+            self.cache.set( Abort , custom_key=abort_id , auto_destroy_period=120 , store_deleted_key=False )
+        return Abort
+
+    def start_before_request(self) : 
+
+        @self.app.route(f'{self.abort_base_url}/<id>' , methods=['DELETE'])
+        def abort_request(id : str ) :
+            abort : AbortRequest = self.cache.get(id)
+            timestamp = getTimestamp()
+            if abort :
+                abort.abort()
+                for i in range(30) :
+                    th = abort.thread
+                    alive = th.is_alive()
+                    if not alive :
+                        break
+                    time.sleep(.25)
+                return { 'status' : 200 , 'message' : 'Request aborted' , 'abort_timestamp' : timestamp , 'abort_id' : id , 'alive' : alive  , 'url' : abort.request.get('path')}
+            else :
+                return { 'status' : 404 , 'message' : 'Request not found or request is not abortable. Check request headers for abortable flag.'}
+
+        @self.app.before_request
+        def before_request() :
+            abortable = request.headers.get('abortable')
+            if abortable :
+                abort = self.register_abortable_request(request)
+                request.abortable = True
+                request.abort_id = abort.abort_id
+                # check here if async in the headers
+                # if yes . i will trigger the function in thread 
+                # start_tread(#how to get the target function here ? )
+                # now i want to return response to UI { status : 200 , message : 'request now in running bg' , abort_id : abort.abort_id }
+                # the flask function should not be called again
+                if request.headers.get('async') == 'false' : 
+                    target_func = current_app.view_functions.get(request.endpoint)
+                    if not target_func:
+                        return jsonify({"error": "Route not found"}), 404
+                    th = start_thread(target=target_func, args=request.args, kwargs=request.form)
+                    abort.thread = th
+                    return {"status": 200, "message": "Request now in running bg", "abort_id": abort.abort_id} , 200
+
+        @self.app.after_request
+        def after_request(response) :
+            try :
+                if request.abortable :
+                    response.headers['abortid'] = request.abort_id 
+                    response.headers['abortable'] = True
+            except :
+                response.headers['abortable'] = False
+            return response
+
+
+    def getInstance(self) :
+        return self.getFlask() , self.getSocketio() , self.getWsgi()
+    
+    def getSocketio( self ):
+        return self.socketio
+    
+    def getFlask( self ):
+        return self.app
+    
+    def getWsgi(self) :
+        return self.wsgi_server
+    
+    def shutdownUi(self) :
+        kill_thread(self.thread)
+        self.wsgi_server.server_close()
+        self.wsgi_server.shutdown()
+
+    def _wait_th(self , t ) :
+        t.join()
+        
+    def thrStartUi(self , suppress_prints=True) :
+        if self.enable_test_url :
+            if not suppress_prints :
+                print(f'TEST URL GET-METHOD /connection/test/internal')
+            @self.app.route('/connection/test/internal' , methods=['GET'])
+            def test_connection():
+                return f"Status=200<br> ID={self.id}<br> one-time-token={getRandomKey(20)}"
+        if self.httpProtocol == 'http' :
+            con = None
+        elif self.httpProtocol == 'https' :
+            con=(self.ssl_crt , self.ssl_key)
+        self.wsgi_server = wsgi_server = ThreadedWSGIServer(
+            host = self.address ,
+            ssl_context=con,
+            # ssl_context=('ssl.crt', 'ssl.key'),
+            port = self.port,
+            app = self.app )
+        if not suppress_prints :
+            print(f"web-socket: {self.fullAddress}")
+            print(f"UI URL : {self.fullAddress}")
+        log = logging.getLogger('werkzeug')
+        log.setLevel(logging.ERROR)
+        wsgi_server.serve_forever()
+    
+    def startUi(self ,daemon , suppress_prints=True) :
+        self.start_before_request()
+        self.thread = self.flaskprocess = Thread(target=self.thrStartUi , args=[suppress_prints])
+        self.flaskprocess.daemon = False
+        self.flaskprocess.start()
+        start_thread(target=self._wait_th , args=[self.thread] , daemon=daemon)
+        return self.thread
+    
+    def stopUi(self) :
+        kill_thread(self.thread)
+        return True
+    

{easy_utils_dev-2.140 → easy_utils_dev-2.142}/easy_utils_dev/wsnoclib.py

@@ -1,7 +1,7 @@
 from easy_utils_dev.debugger import DEBUGGER
 import requests , json , subprocess
 from requests.auth import HTTPBasicAuth as BAuth    
-from .utils import pingAddress , fixTupleForSql , start_thread , mkdirs
+from .utils import pingAddress , fixTupleForSql , start_thread , mkdirs  , getTimestamp
 from time import sleep
 from urllib3.exceptions import InsecureRequestWarning
 from urllib3 import disable_warnings
@@ -15,7 +15,8 @@ import tempfile , os
 from kafka import KafkaConsumer
 from easy_utils_dev.utils import kill_thread
 import atexit
-
+from snakebite.client import Client as HdfsClient
+from datetime import datetime
 
 
 class KafkaConfig :
@@ -38,7 +39,6 @@ class KafkaConfig :
         self.enable_auto_refresh = False
 
 
-
 class WSNOCLIB : 
     def __init__( 
         self, 
@@ -51,7 +51,8 @@ class WSNOCLIB :
         request_max_count=30,
         tmp_dir = tempfile.gettempdir() ,
         kafka = KafkaConfig(),
-        register_atexit=True
+        register_atexit=True,
+        trust_env=True
     ): 
         self.logger = DEBUGGER(f'{debug_name}-{ip}',level=debug_level,homePath=debug_homepath)
         self.disabledWarnings = self.disableUrlWarnings()
@@ -59,6 +60,7 @@ class WSNOCLIB :
         self.address = ip
         self.username = username
         self.password = password
+        self.trust_env = trust_env
         self.external_nsp = False
         self.api_count = 0
         self.api_count_limit = 999999999999
@@ -82,6 +84,8 @@ class WSNOCLIB :
         self.refresh_thread = None
         self.token_refresh_count = 0
         self.session = WSNOCSession(self)
+        self.connected = False
+        self.pm_hadoop = PmHadoopClient(self)
         if register_atexit :
             atexit.register(self.goodbye)
 
@@ -150,6 +154,7 @@ class WSNOCLIB :
         if auto_refresh_token :
             self.autoRefreshThread = self.refresh_thread = start_thread(target=self.runAutoRefreshThread)
         self.logger.debug(f'token => {r.text}')
+        self.connected = True
         return self.token
 
 
@@ -182,6 +187,7 @@ class WSNOCLIB :
             r.close()
         except :
             pass
+        self.connected = False
         return True
 
     def goodbye(self):
@@ -610,6 +616,7 @@ class WSNOCLIB :
             self.bearer_token = f'Bearer {self.access_token}'
             self.token = r.json()
             self.token.update({'bearer_token' :  self.bearer_token })
+        self.connected = True
         return r
 
     def session_info(self) :
@@ -706,6 +713,9 @@ class WSNOCSession(requests.Session):
         self._wsnoc = wsnoc
         self.verify = False
         self.retries = 0
+        if not wsnoc.trust_env :
+            os.environ['https_proxy'] = ""
+            os.environ['http_proxy'] = ""
         self.debug_this_request = False
         self.skip_hold_for_token_refresh = False
         
@@ -749,6 +759,96 @@ class WSNOCSession(requests.Session):
         self._wsnoc.logger.info(f'[{method}] : {url} - [{request.status_code}]')
         return request
 
+class PmHadoopClient :
+
+
+    def __init__(self , _wsnoc : WSNOCLIB):
+        self.ip = _wsnoc.address
+        self.wsnoc = _wsnoc
+        self.client : HdfsClient = None
+        self.hdfs_port = 8020
+        self.hdfs_user = 'otn'
+        self.hdfs_root = '/'
+        self.hdfs_use_trash = False
+        self.PM24H = 1
+        self.PM15M = 2
+        self.KPIAGGR = 3
+        self.logger : DEBUGGER = self.wsnoc.logger
+
+    
+    def connect(self) :
+        if not self.wsnoc.connected :
+            raise Exception('WSNOC is not connected')
+        try :
+            self.client = HdfsClient(self.ip, self.hdfs_port, use_trash=self.hdfs_use_trash)
+        except Exception as e:
+            self.logger.warning('PM Hadoop client is using WSNOC port 8020/custom port. Please check if it is not blocked by firewall.' , source='PmHadoopClient')
+            self.logger.error(f'Failed to connect to PM Hadoop: {e}' , source='PmHadoopClient')
+            raise
+
+    def change_date_to_timestamp(self , date_str) :
+        '''
+        this is a helper function to convert date string to timestamp
+        date_str : must be in the format of %Y%m%d example: 20250101
+        return : timestamp
+        '''
+        dt = datetime.strptime(date_str, "%Y%m%d")
+        return int(dt.timestamp())
+
+    def pm_list(self , mode , date_range=[]) :
+        '''
+        mode : must be one of the following:
+            - self.PM24H
+            - self.PM15M
+            - self.KPIAGGR
+        date_range : must be a list of two integers in the format of [start_timestamp, end_timestamp]
+            -  for example: [1718217600, 1718221200]
+        - if date_range is not provided, all available PM dates will be returned
+        '''
+        if mode == self.PM24H :
+            _mode = 'ONE_DAY'
+        elif mode == self.PM15M :
+            _mode = 'FIFTEEN_MINS'
+        elif mode == self.KPIAGGR :
+            _mode = 'KPIAGGR'
+        else :
+            raise Exception(f'Invalid mode: {mode}')
+        self.logger.info(f"Getting available PM dates for {_mode}" , source='PmHadoopClient')
+        dirs = list(self.client.ls([f'/PMDATA/{_mode}']))
+        ts_now = getTimestamp()
+        for index , dir in enumerate(dirs) :
+            self.logger.info(f"Processing {dir.get('path')}" , source='PmHadoopClient')
+            date_str = dir.get('path').split('=')[-1]
+            dir['pm_date'] = int(date_str)
+            dir['mode'] = _mode
+            dt = datetime.strptime(date_str, "%Y%m%d")
+            dir['pm_date_timestamp'] = int(dt.timestamp())
+            if len(date_range) > 0 :
+                if dir['pm_date_timestamp'] < date_range[0] :
+                    del dirs[index]
+                if dir['pm_date_timestamp'] > date_range[1] :
+                    del dirs[index]
+        return dirs
+
+    def _download_dir(self , hdfs_path, local_path):
+        for entry in self.client.ls([hdfs_path]):
+            # print(entry)
+            entry_path = entry['path']
+            entry_type = entry['file_type']
+            if entry_type == 'd':
+                subdir = os.path.join(local_path, os.path.basename(entry_path))
+                mkdirs(subdir)
+                self._download_dir(entry_path, subdir)
+            else:  # FILE
+                self.logger.debug(f"Downloading {entry_path} → {local_path}" , source='PmHadoopClient')
+                self.client.copyToLocal([entry_path], local_path)
+        
+    def download(self , obj , destination_path) :
+        self.wsnoc.logger.info(f'Downloading {obj.get("pm_date")} to {destination_path}')
+        destination_path = f"{destination_path}/{obj.get('mode')}/{obj.get('pm_date')}"
+        mkdirs(destination_path)
+        self._download_dir(obj.get('path'), destination_path)
+
 
 if __name__ == '__main__' :
     # noc = WSNOCLIB('10.20.30.55' , 'admin' , 'Nokia@2024') 

{easy_utils_dev-2.140 → easy_utils_dev-2.142}/easy_utils_dev.egg-info/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.140
+Version: 2.142
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
 Requires-Dist: ping3
+Requires-Dist: snakebite-py3
 Requires-Dist: flask
 Requires-Dist: flask_cors
 Requires-Dist: xmltodict

{easy_utils_dev-2.140 → easy_utils_dev-2.142}/easy_utils_dev.egg-info/requires.txt

@@ -1,5 +1,6 @@
 psutil
 ping3
+snakebite-py3
 flask
 flask_cors
 xmltodict

{easy_utils_dev-2.140 → easy_utils_dev-2.142}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '2.140'
+VERSION = '2.142'
 
 # Setting up
 setup(
@@ -11,6 +11,7 @@ setup(
     install_requires=[
             'psutil' ,
             'ping3' , 
+            'snakebite-py3',
             'flask' , 
             'flask_cors' , 
             'xmltodict' , 

easy_utils_dev-2.140/easy_utils_dev/uiserver.py

@@ -1,193 +0,0 @@
-from werkzeug.serving import ThreadedWSGIServer
-from easy_utils_dev.utils import getRandomKey , generateToken , getTimestamp
-from flask_socketio import SocketIO
-from engineio.async_drivers import gevent
-from engineio.async_drivers import threading
-from flask_cors import CORS
-import logging  , os
-from flask import Flask
-from threading import Thread
-from easy_utils_dev.custom_env import cenv
-from easy_utils_dev.utils import kill_thread
-from multiprocessing import Process
-from werkzeug.serving import make_ssl_devcert
-from time import sleep
-from easy_utils_dev.utils import start_thread , getRandomKeysAndStr , mkdirs
-
-def getClassById( id ) :
-    return cenv[id]
-
-def create_ssl(host,output) :
-    '''
-    host : is the IP/Adress of the server which servers the web-server
-    output: the output locaiton to generate the ssl certificate. it should end with filename without extension
-    '''
-    return make_ssl_devcert( output , host=host)
-    
-
-
-class Abort :
-    def __init__(self, requestId=getRandomKeysAndStr(50)) :
-        self.requestId = requestId
-        self.result= None
-        self.thread = None
-        self.aborted=False
-        self.response = {}
-        self.error = False
-        self.message = ''
-        self.starttimestamp = getTimestamp()
-        self.endtimestamp = 0
-        self.kill = self.api_abort
-        pass
-
-
-    def abortable_process(self , operation , args=[] , kwargs={}) :
-        def thread_run() :
-            try :
-                self.result =  operation( *args, **kwargs )
-                self.error = False
-                self.message = ''
-                self.endtimestamp = getTimestamp()
-            except Exception as error :
-                self.error = True
-                self.message = str(error)
-        thread = self.thread = start_thread( target = thread_run )  
-        while thread.is_alive() :
-            sleep(.1)
-            if self.aborted :
-                self.response = {
-                    'message' : 'request aborted.' , 
-                    'id' : self.requestId ,
-                    'status' : 405 , 
-                    'result' : None, 
-                    'error' : self.error , 
-                    'error_message' : '',
-                    'starttimestamp' : self.starttimestamp,
-                    'endtimestamp' : self.endtimestamp,
-                    'aborted' : True,
-                    'threadIsAlive' : thread.is_alive()
-                    } 
-                return self.response
-        sleep(.2)
-        self.response = {
-            'message' : 'request completed.' , 
-            'id' : self.requestId , 
-            'status' : 200 , 
-            'result' : self.result , 
-            'error' : self.error , 
-            'error_message' : self.message ,
-            'starttimestamp' : self.starttimestamp,
-            'endtimestamp' : self.endtimestamp,
-            'aborted' : False ,
-            'threadIsAlive' : thread.is_alive()
-        }
-        return self.response
-
-
-    def api_abort(self) :
-        self.endtimestamp = getTimestamp()
-        kill_thread(self.thread)
-        sleep(.5)
-        self.aborted=True
-
-
-class UISERVER :
-    def __init__(self ,
-                 id=getRandomKey(n=15),
-                 secretkey=generateToken(),
-                 address='localhost',
-                 port=5312 , 
-                 https=False , 
-                 ssl_crt=None,
-                 ssl_key=None,
-                 template_folder='templates/' ,
-                 static_folder = 'templates/assets'
-                   ,**kwargs
-                ) -> None:
-        self.id = id
-        self.static_folder = static_folder
-        self.app = app = Flask(self.id , template_folder=template_folder  ,  static_folder=self.static_folder )
-        app.config['SECRET_KEY'] = secretkey
-        CORS(app,resources={r"/*":{"origins":"*"}})
-        self.address= address 
-        self.port = port
-        self.thread = None
-        self.ssl_crt=ssl_crt
-        self.ssl_key=ssl_key
-        self.enable_test_url=True
-        self.abort_requests = {}
-        if https :
-            self.httpProtocol = 'https'
-        else :
-            self.httpProtocol = 'http'
-        self.socketio = SocketIO(app , cors_allowed_origins="*"  ,async_mode='threading' , engineio_logger=False , always_connect=True ,**kwargs )
-        cenv[id] = self
-        self.fullAddress = f"{self.httpProtocol}://{self.address}:{self.port}"
-
-    def update_cert(self , crt, ssl ) :
-        self.ssl_crt=crt
-        self.ssl_key=ssl
-
-
-    def getAbort(self , id ) :
-        result : Abort = self.abort_requests.get(id , Abort)
-        return result
-
-    def updateAbort( self , id , abort ) :
-        self.abort_requests[id] = abort
-
-    def getInstance(self) :
-        return self.getFlask() , self.getSocketio() , self.getWsgi()
-    
-    def getSocketio( self ):
-        return self.socketio
-    
-    def getFlask( self ):
-        return self.app
-    
-    def getWsgi(self) :
-        return self.wsgi_server
-    
-    def shutdownUi(self) :
-        kill_thread(self.thread)
-        self.wsgi_server.server_close()
-        self.wsgi_server.shutdown()
-
-    def _wait_th(self , t ) :
-        t.join()
-        
-    def thrStartUi(self , suppress_prints=True) :
-        if self.enable_test_url :
-            if not suppress_prints :
-                print(f'TEST URL GET-METHOD /connection/test/internal')
-            @self.app.route('/connection/test/internal' , methods=['GET'])
-            def test_connection():
-                return f"Status=200<br> ID={self.id}<br> one-time-token={getRandomKey(20)}"
-        if self.httpProtocol == 'http' :
-            con = None
-        elif self.httpProtocol == 'https' :
-            con=(self.ssl_crt , self.ssl_key)
-        self.wsgi_server = wsgi_server = ThreadedWSGIServer(
-            host = self.address ,
-            ssl_context=con,
-            # ssl_context=('ssl.crt', 'ssl.key'),
-            port = self.port,
-            app = self.app )
-        if not suppress_prints :
-            print(f"web-socket: {self.fullAddress}")
-            print(f"UI URL : {self.fullAddress}")
-        log = logging.getLogger('werkzeug')
-        log.setLevel(logging.ERROR)
-        wsgi_server.serve_forever()
-    
-    def startUi(self ,daemon , suppress_prints=True) :
-        self.thread = self.flaskprocess = Thread(target=self.thrStartUi , args=[suppress_prints])
-        self.flaskprocess.daemon = False
-        self.flaskprocess.start()
-        start_thread(target=self._wait_th , args=[self.thread] , daemon=daemon)
-        return self.thread
-    
-    def stopUi(self) :
-        kill_thread(self.thread)
-        return True
-