easy-utils-dev 2.140__py3-none-any.whl → 2.142__py3-none-any.whl

easy_utils_dev/uiserver.py

@@ -1,18 +1,23 @@
+import copy
+import gc
+import time
 from werkzeug.serving import ThreadedWSGIServer
 from easy_utils_dev.utils import getRandomKey , generateToken , getTimestamp
 from flask_socketio import SocketIO
 from engineio.async_drivers import gevent
-from engineio.async_drivers import threading
 from flask_cors import CORS
 import logging  , os
+from flask import jsonify, request , current_app
 from flask import Flask
 from threading import Thread
+import threading
 from easy_utils_dev.custom_env import cenv
 from easy_utils_dev.utils import kill_thread
 from multiprocessing import Process
 from werkzeug.serving import make_ssl_devcert
 from time import sleep
 from easy_utils_dev.utils import start_thread , getRandomKeysAndStr , mkdirs
+from easy_utils_dev.temp_memory import TemporaryMemory
 
 def getClassById( id ) :
     return cenv[id]
@@ -23,87 +28,54 @@ def create_ssl(host,output) :
     output: the output locaiton to generate the ssl certificate. it should end with filename without extension
     '''
     return make_ssl_devcert( output , host=host)
-    
-
 
-class Abort :
-    def __init__(self, requestId=getRandomKeysAndStr(50)) :
-        self.requestId = requestId
-        self.result= None
+def clone_request(request):
+    """Return a plain dict clone of Flask request data."""
+    return {
+        "method": request.method,
+        "path": request.path,
+        "url": request.url,
+        "headers": dict(request.headers),
+        "args": request.args.to_dict(flat=False),
+        "form": request.form.to_dict(flat=False),
+        "json": request.get_json(silent=True),
+        "data": request.get_data(),   # raw body bytes
+        "files": {k: v.filename for k, v in request.files.items()},
+        "remote_addr": request.remote_addr,
+        "cookies": request.cookies,
+    }
+
+class AbortRequest :
+    def __init__(self, request ) :
+        self.request = clone_request(request)
+        self.abort_id = None
+        self.abortable = False
         self.thread = None
-        self.aborted=False
-        self.response = {}
-        self.error = False
-        self.message = ''
-        self.starttimestamp = getTimestamp()
-        self.endtimestamp = 0
-        self.kill = self.api_abort
-        pass
-
-
-    def abortable_process(self , operation , args=[] , kwargs={}) :
-        def thread_run() :
-            try :
-                self.result =  operation( *args, **kwargs )
-                self.error = False
-                self.message = ''
-                self.endtimestamp = getTimestamp()
-            except Exception as error :
-                self.error = True
-                self.message = str(error)
-        thread = self.thread = start_thread( target = thread_run )  
-        while thread.is_alive() :
-            sleep(.1)
-            if self.aborted :
-                self.response = {
-                    'message' : 'request aborted.' , 
-                    'id' : self.requestId ,
-                    'status' : 405 , 
-                    'result' : None, 
-                    'error' : self.error , 
-                    'error_message' : '',
-                    'starttimestamp' : self.starttimestamp,
-                    'endtimestamp' : self.endtimestamp,
-                    'aborted' : True,
-                    'threadIsAlive' : thread.is_alive()
-                    } 
-                return self.response
-        sleep(.2)
-        self.response = {
-            'message' : 'request completed.' , 
-            'id' : self.requestId , 
-            'status' : 200 , 
-            'result' : self.result , 
-            'error' : self.error , 
-            'error_message' : self.message ,
-            'starttimestamp' : self.starttimestamp,
-            'endtimestamp' : self.endtimestamp,
-            'aborted' : False ,
-            'threadIsAlive' : thread.is_alive()
-        }
-        return self.response
-
-
-    def api_abort(self) :
-        self.endtimestamp = getTimestamp()
+        self.cache = None
+        self.start_ts = getTimestamp()
+
+    def abort(self) : 
         kill_thread(self.thread)
-        sleep(.5)
-        self.aborted=True
+        self.cache.delete(self.abort_id)
+        try :
+            gc.collect()
+        except :
+            pass
 
 
 class UISERVER :
     def __init__(self ,
-                 id=getRandomKey(n=15),
-                 secretkey=generateToken(),
-                 address='localhost',
-                 port=5312 , 
-                 https=False , 
-                 ssl_crt=None,
-                 ssl_key=None,
-                 template_folder='templates/' ,
-                 static_folder = 'templates/assets'
-                   ,**kwargs
-                ) -> None:
+            id=getRandomKey(n=15),
+            secretkey=generateToken(),
+            address='localhost',
+            port=5312 , 
+            https=False , 
+            ssl_crt=None,
+            ssl_key=None,
+            template_folder='templates/' ,
+            static_folder = 'templates/assets'
+            ,**kwargs
+        ) -> None:
         self.id = id
         self.static_folder = static_folder
         self.app = app = Flask(self.id , template_folder=template_folder  ,  static_folder=self.static_folder )
@@ -116,6 +88,7 @@ class UISERVER :
         self.ssl_key=ssl_key
         self.enable_test_url=True
         self.abort_requests = {}
+        self.abort_base_url = '/request/abort'
         if https :
             self.httpProtocol = 'https'
         else :
@@ -123,18 +96,78 @@ class UISERVER :
         self.socketio = SocketIO(app , cors_allowed_origins="*"  ,async_mode='threading' , engineio_logger=False , always_connect=True ,**kwargs )
         cenv[id] = self
         self.fullAddress = f"{self.httpProtocol}://{self.address}:{self.port}"
+        self.cache = TemporaryMemory()
 
     def update_cert(self , crt, ssl ) :
         self.ssl_crt=crt
         self.ssl_key=ssl
 
+    def register_abortable_request(self , request , abort_id = None ) :
+        path = request.path
+        Abort = AbortRequest(request)
+        if not path.startswith(self.abort_base_url) :
+            if not abort_id :
+                if not request.headers.get('abortid') :
+                    abort_id = getRandomKeysAndStr(n=20)
+                else :
+                    abort_id = request.headers.get('abortid')
+            
+            Abort.abort_id = abort_id
+            current_thread = threading.current_thread()
+            Abort.thread = current_thread
+            Abort.cache = self.cache
+            Abort.start_ts = getTimestamp()
+            self.cache.set( Abort , custom_key=abort_id , auto_destroy_period=120 , store_deleted_key=False )
+        return Abort
+
+    def start_before_request(self) : 
+
+        @self.app.route(f'{self.abort_base_url}/<id>' , methods=['DELETE'])
+        def abort_request(id : str ) :
+            abort : AbortRequest = self.cache.get(id)
+            timestamp = getTimestamp()
+            if abort :
+                abort.abort()
+                for i in range(30) :
+                    th = abort.thread
+                    alive = th.is_alive()
+                    if not alive :
+                        break
+                    time.sleep(.25)
+                return { 'status' : 200 , 'message' : 'Request aborted' , 'abort_timestamp' : timestamp , 'abort_id' : id , 'alive' : alive  , 'url' : abort.request.get('path')}
+            else :
+                return { 'status' : 404 , 'message' : 'Request not found or request is not abortable. Check request headers for abortable flag.'}
+
+        @self.app.before_request
+        def before_request() :
+            abortable = request.headers.get('abortable')
+            if abortable :
+                abort = self.register_abortable_request(request)
+                request.abortable = True
+                request.abort_id = abort.abort_id
+                # check here if async in the headers
+                # if yes . i will trigger the function in thread 
+                # start_tread(#how to get the target function here ? )
+                # now i want to return response to UI { status : 200 , message : 'request now in running bg' , abort_id : abort.abort_id }
+                # the flask function should not be called again
+                if request.headers.get('async') == 'false' : 
+                    target_func = current_app.view_functions.get(request.endpoint)
+                    if not target_func:
+                        return jsonify({"error": "Route not found"}), 404
+                    th = start_thread(target=target_func, args=request.args, kwargs=request.form)
+                    abort.thread = th
+                    return {"status": 200, "message": "Request now in running bg", "abort_id": abort.abort_id} , 200
+
+        @self.app.after_request
+        def after_request(response) :
+            try :
+                if request.abortable :
+                    response.headers['abortid'] = request.abort_id 
+                    response.headers['abortable'] = True
+            except :
+                response.headers['abortable'] = False
+            return response
 
-    def getAbort(self , id ) :
-        result : Abort = self.abort_requests.get(id , Abort)
-        return result
-
-    def updateAbort( self , id , abort ) :
-        self.abort_requests[id] = abort
 
     def getInstance(self) :
         return self.getFlask() , self.getSocketio() , self.getWsgi()
@@ -181,6 +214,7 @@ class UISERVER :
         wsgi_server.serve_forever()
     
     def startUi(self ,daemon , suppress_prints=True) :
+        self.start_before_request()
         self.thread = self.flaskprocess = Thread(target=self.thrStartUi , args=[suppress_prints])
         self.flaskprocess.daemon = False
         self.flaskprocess.start()

easy_utils_dev/wsnoclib.py

@@ -1,7 +1,7 @@
 from easy_utils_dev.debugger import DEBUGGER
 import requests , json , subprocess
 from requests.auth import HTTPBasicAuth as BAuth    
-from .utils import pingAddress , fixTupleForSql , start_thread , mkdirs
+from .utils import pingAddress , fixTupleForSql , start_thread , mkdirs  , getTimestamp
 from time import sleep
 from urllib3.exceptions import InsecureRequestWarning
 from urllib3 import disable_warnings
@@ -15,7 +15,8 @@ import tempfile , os
 from kafka import KafkaConsumer
 from easy_utils_dev.utils import kill_thread
 import atexit
-
+from snakebite.client import Client as HdfsClient
+from datetime import datetime
 
 
 class KafkaConfig :
@@ -38,7 +39,6 @@ class KafkaConfig :
         self.enable_auto_refresh = False
 
 
-
 class WSNOCLIB : 
     def __init__( 
         self, 
@@ -51,7 +51,8 @@ class WSNOCLIB :
         request_max_count=30,
         tmp_dir = tempfile.gettempdir() ,
         kafka = KafkaConfig(),
-        register_atexit=True
+        register_atexit=True,
+        trust_env=True
     ): 
         self.logger = DEBUGGER(f'{debug_name}-{ip}',level=debug_level,homePath=debug_homepath)
         self.disabledWarnings = self.disableUrlWarnings()
@@ -59,6 +60,7 @@ class WSNOCLIB :
         self.address = ip
         self.username = username
         self.password = password
+        self.trust_env = trust_env
         self.external_nsp = False
         self.api_count = 0
         self.api_count_limit = 999999999999
@@ -82,6 +84,8 @@ class WSNOCLIB :
         self.refresh_thread = None
         self.token_refresh_count = 0
         self.session = WSNOCSession(self)
+        self.connected = False
+        self.pm_hadoop = PmHadoopClient(self)
         if register_atexit :
             atexit.register(self.goodbye)
 
@@ -150,6 +154,7 @@ class WSNOCLIB :
         if auto_refresh_token :
             self.autoRefreshThread = self.refresh_thread = start_thread(target=self.runAutoRefreshThread)
         self.logger.debug(f'token => {r.text}')
+        self.connected = True
         return self.token
 
 
@@ -182,6 +187,7 @@ class WSNOCLIB :
             r.close()
         except :
             pass
+        self.connected = False
         return True
 
     def goodbye(self):
@@ -610,6 +616,7 @@ class WSNOCLIB :
             self.bearer_token = f'Bearer {self.access_token}'
             self.token = r.json()
             self.token.update({'bearer_token' :  self.bearer_token })
+        self.connected = True
         return r
 
     def session_info(self) :
@@ -706,6 +713,9 @@ class WSNOCSession(requests.Session):
         self._wsnoc = wsnoc
         self.verify = False
         self.retries = 0
+        if not wsnoc.trust_env :
+            os.environ['https_proxy'] = ""
+            os.environ['http_proxy'] = ""
         self.debug_this_request = False
         self.skip_hold_for_token_refresh = False
         
@@ -749,6 +759,96 @@ class WSNOCSession(requests.Session):
         self._wsnoc.logger.info(f'[{method}] : {url} - [{request.status_code}]')
         return request
 
+class PmHadoopClient :
+
+
+    def __init__(self , _wsnoc : WSNOCLIB):
+        self.ip = _wsnoc.address
+        self.wsnoc = _wsnoc
+        self.client : HdfsClient = None
+        self.hdfs_port = 8020
+        self.hdfs_user = 'otn'
+        self.hdfs_root = '/'
+        self.hdfs_use_trash = False
+        self.PM24H = 1
+        self.PM15M = 2
+        self.KPIAGGR = 3
+        self.logger : DEBUGGER = self.wsnoc.logger
+
+    
+    def connect(self) :
+        if not self.wsnoc.connected :
+            raise Exception('WSNOC is not connected')
+        try :
+            self.client = HdfsClient(self.ip, self.hdfs_port, use_trash=self.hdfs_use_trash)
+        except Exception as e:
+            self.logger.warning('PM Hadoop client is using WSNOC port 8020/custom port. Please check if it is not blocked by firewall.' , source='PmHadoopClient')
+            self.logger.error(f'Failed to connect to PM Hadoop: {e}' , source='PmHadoopClient')
+            raise
+
+    def change_date_to_timestamp(self , date_str) :
+        '''
+        this is a helper function to convert date string to timestamp
+        date_str : must be in the format of %Y%m%d example: 20250101
+        return : timestamp
+        '''
+        dt = datetime.strptime(date_str, "%Y%m%d")
+        return int(dt.timestamp())
+
+    def pm_list(self , mode , date_range=[]) :
+        '''
+        mode : must be one of the following:
+            - self.PM24H
+            - self.PM15M
+            - self.KPIAGGR
+        date_range : must be a list of two integers in the format of [start_timestamp, end_timestamp]
+            -  for example: [1718217600, 1718221200]
+        - if date_range is not provided, all available PM dates will be returned
+        '''
+        if mode == self.PM24H :
+            _mode = 'ONE_DAY'
+        elif mode == self.PM15M :
+            _mode = 'FIFTEEN_MINS'
+        elif mode == self.KPIAGGR :
+            _mode = 'KPIAGGR'
+        else :
+            raise Exception(f'Invalid mode: {mode}')
+        self.logger.info(f"Getting available PM dates for {_mode}" , source='PmHadoopClient')
+        dirs = list(self.client.ls([f'/PMDATA/{_mode}']))
+        ts_now = getTimestamp()
+        for index , dir in enumerate(dirs) :
+            self.logger.info(f"Processing {dir.get('path')}" , source='PmHadoopClient')
+            date_str = dir.get('path').split('=')[-1]
+            dir['pm_date'] = int(date_str)
+            dir['mode'] = _mode
+            dt = datetime.strptime(date_str, "%Y%m%d")
+            dir['pm_date_timestamp'] = int(dt.timestamp())
+            if len(date_range) > 0 :
+                if dir['pm_date_timestamp'] < date_range[0] :
+                    del dirs[index]
+                if dir['pm_date_timestamp'] > date_range[1] :
+                    del dirs[index]
+        return dirs
+
+    def _download_dir(self , hdfs_path, local_path):
+        for entry in self.client.ls([hdfs_path]):
+            # print(entry)
+            entry_path = entry['path']
+            entry_type = entry['file_type']
+            if entry_type == 'd':
+                subdir = os.path.join(local_path, os.path.basename(entry_path))
+                mkdirs(subdir)
+                self._download_dir(entry_path, subdir)
+            else:  # FILE
+                self.logger.debug(f"Downloading {entry_path} → {local_path}" , source='PmHadoopClient')
+                self.client.copyToLocal([entry_path], local_path)
+        
+    def download(self , obj , destination_path) :
+        self.wsnoc.logger.info(f'Downloading {obj.get("pm_date")} to {destination_path}')
+        destination_path = f"{destination_path}/{obj.get('mode')}/{obj.get('pm_date')}"
+        mkdirs(destination_path)
+        self._download_dir(obj.get('path'), destination_path)
+
 
 if __name__ == '__main__' :
     # noc = WSNOCLIB('10.20.30.55' , 'admin' , 'Nokia@2024') 

{easy_utils_dev-2.140.dist-info → easy_utils_dev-2.142.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.140
+Version: 2.142
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
 Requires-Dist: ping3
+Requires-Dist: snakebite-py3
 Requires-Dist: flask
 Requires-Dist: flask_cors
 Requires-Dist: xmltodict

{easy_utils_dev-2.140.dist-info → easy_utils_dev-2.142.dist-info}/RECORD

@@ -25,14 +25,14 @@ easy_utils_dev/optics_utils.py,sha256=G-hFX2iiUCSJjk7BICBRGvVoDq0IBONLZSjagoB5FM
 easy_utils_dev/require_auth.py,sha256=UsYAxfLX5wda6hd0nfLR_tl0bGQ4DYIpaTowsYSku-E,881
 easy_utils_dev/simple_sqlite.py,sha256=J-mcTUnHmAn0eCPD8j-WEoA19uzHRXJ4YRJsyx9B-do,13113
 easy_utils_dev/temp_memory.py,sha256=0Dx_vNUSFQRMtJZNT8tUZXubcG7T6jxevw2WYuGmVe8,1658
-easy_utils_dev/uiserver.py,sha256=d9jImwgSURMH4MIU4Zkro2b20cAbuV8rO_If42FVyZ4,6826
+easy_utils_dev/uiserver.py,sha256=zR-3DCo2FUF6Z7vpnzrKuWUl1MC0TparS8_LEDOyuuc,8886
 easy_utils_dev/utils.py,sha256=BmVnbxc336c6WTeDFcEHN6Mavt7fJrIEyK4GXODV3gI,13345
 easy_utils_dev/winserviceapi.py,sha256=2ZP6jaSt1-5vEJYXqwBhwX-1-eQ3V3YzntsoOoko2cw,18804
-easy_utils_dev/wsnoclib.py,sha256=kqkOkiafXnrIgdlV__3uyrAoqxG5dmIHnwqm0EkwS2U,33457
+easy_utils_dev/wsnoclib.py,sha256=idg6V2K9CEHq_SvMqqmAihZdl61LAmqvTGDmp4hl4sM,37733
 easy_utils_dev/wsselib.py,sha256=YweScnoAAH_t29EeIjBpkQ6HtX0Rp9mQudRsRce2SE8,7920
 easy_utils_dev/ept_sql/create_dirs.sql,sha256=KWfX-Nc6lvr_BC-P6O97NE0idoPW4GNKUKUCgonJhto,3508
 easy_utils_dev/ept_sql/create_ept_tables.sql,sha256=WDHyIyeReV8_QaYBPIpSy-lto3OKvZtex1tWs-FPURQ,67737
-easy_utils_dev-2.140.dist-info/METADATA,sha256=11wRCPmmDl1s7jptVfwy-_ru_jnkTn2nJMcJ-dM07uM,572
-easy_utils_dev-2.140.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-easy_utils_dev-2.140.dist-info/top_level.txt,sha256=7vBsrpq7NmilkdU3YUvfd5iVDNBaT07u_-ut4F7zc7A,15
-easy_utils_dev-2.140.dist-info/RECORD,,
+easy_utils_dev-2.142.dist-info/METADATA,sha256=YHAkh7rEopSSfFi9AxuqfXERrMEYFPDkcItFbPN8wGs,602
+easy_utils_dev-2.142.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+easy_utils_dev-2.142.dist-info/top_level.txt,sha256=7vBsrpq7NmilkdU3YUvfd5iVDNBaT07u_-ut4F7zc7A,15
+easy_utils_dev-2.142.dist-info/RECORD,,