easy-utils-dev 2.142__tar.gz → 2.144__tar.gz

{easy_utils_dev-2.142 → easy_utils_dev-2.144}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.142
+Version: 2.144
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
@@ -16,6 +16,8 @@ Requires-Dist: flask_socketio
 Requires-Dist: python-dotenv
 Requires-Dist: gevent
 Requires-Dist: pyzipper
+Requires-Dist: shutil
+Requires-Dist: bs4
 Requires-Dist: pyjwt
 Requires-Dist: authlib
 Requires-Dist: kafka-python

{easy_utils_dev-2.142 → easy_utils_dev-2.144}/easy_utils_dev/uiserver.py

@@ -1,19 +1,21 @@
 import copy
 import gc
 import time
+from tkinter import NO
 from werkzeug.serving import ThreadedWSGIServer
 from easy_utils_dev.utils import getRandomKey , generateToken , getTimestamp
 from flask_socketio import SocketIO
 from engineio.async_drivers import gevent
 from flask_cors import CORS
 import logging  , os
-from flask import jsonify, request , current_app
+from flask import jsonify, request , current_app , copy_current_request_context
 from flask import Flask
 from threading import Thread
 import threading
 from easy_utils_dev.custom_env import cenv
 from easy_utils_dev.utils import kill_thread
 from multiprocessing import Process
+import traceback
 from werkzeug.serving import make_ssl_devcert
 from time import sleep
 from easy_utils_dev.utils import start_thread , getRandomKeysAndStr , mkdirs
@@ -53,20 +55,48 @@ class AbortRequest :
         self.thread = None
         self.cache = None
         self.start_ts = getTimestamp()
+        self.result = None
+        self.async_request = False
+        self.internalid = None
+        self.in_progress = False
+        self.start_ts = None
+        self.end_ts = None
+        self.execution = None
+        self.success = None
+        self.traceback = None
+        self.delete_async_request_ts = None
+        self.killed = False
+
 
     def abort(self) : 
         kill_thread(self.thread)
-        self.cache.delete(self.abort_id)
+        self.killed = True
+        self.in_progress = False
+        self.success = False
+        self.traceback = None
+        self.result = None
+        self.delete_async_request_ts = getTimestamp(after_seconds=3600)
+        self.end_ts = getTimestamp()
+        self.execution = round(self.end_ts - self.start_ts, 2)
+        if not self.async_request :
+            self.cache.delete(self.abort_id)
         try :
             gc.collect()
         except :
             pass
 
+class SocketClientObject :
+    def __init__(self ) :
+        self.client : SocketIO
+        self.internalid : str = None
+        self.request = {}
+        self.rooms = None
 
 class UISERVER :
     def __init__(self ,
             id=getRandomKey(n=15),
             secretkey=generateToken(),
+            serve_with_secret_key=False,
             address='localhost',
             port=5312 , 
             https=False , 
@@ -86,9 +116,14 @@ class UISERVER :
         self.thread = None
         self.ssl_crt=ssl_crt
         self.ssl_key=ssl_key
+        self.serve_with_secret_key=serve_with_secret_key
+        self.secretkey=secretkey
         self.enable_test_url=True
         self.abort_requests = {}
+        self.bg_requests = {}
+        self.socketio_clients = []
         self.abort_base_url = '/request/abort'
+        self.request_reply_base_url= '/request/result'
         if https :
             self.httpProtocol = 'https'
         else :
@@ -97,6 +132,9 @@ class UISERVER :
         cenv[id] = self
         self.fullAddress = f"{self.httpProtocol}://{self.address}:{self.port}"
         self.cache = TemporaryMemory()
+        start_thread(target=self.delete_very_old_requests)
+        self.secret_key_execlude_urls = []
+        self.socketio_rooms = {}
 
     def update_cert(self , crt, ssl ) :
         self.ssl_crt=crt
@@ -120,6 +158,29 @@ class UISERVER :
             self.cache.set( Abort , custom_key=abort_id , auto_destroy_period=120 , store_deleted_key=False )
         return Abort
 
+    def delete_very_old_requests(self) :
+        while True :
+            now = getTimestamp()
+            for key, value in list(self.bg_requests.items()) :
+                value : AbortRequest = value
+                if value.delete_async_request_ts :
+                    if value.delete_async_request_ts > now :
+                        del self.bg_requests[key]
+            gc.collect()
+            sleep(320)
+
+    def create_room(self , room_id : str , members : list[str] ) :
+        self.socketio_rooms[room_id] = members
+    
+    def add_member_to_room(self , room_id : str , member : str ) :
+        self.socketio_rooms[room_id].append(member)
+    
+    def remove_member_from_room(self , room_id : str , member : str ) :
+        self.socketio_rooms[room_id].remove(member)
+    
+    def get_room_members(self , room_id : str ) :
+        return self.socketio_rooms[room_id]
+
     def start_before_request(self) : 
 
         @self.app.route(f'{self.abort_base_url}/<id>' , methods=['DELETE'])
@@ -137,10 +198,55 @@ class UISERVER :
                 return { 'status' : 200 , 'message' : 'Request aborted' , 'abort_timestamp' : timestamp , 'abort_id' : id , 'alive' : alive  , 'url' : abort.request.get('path')}
             else :
                 return { 'status' : 404 , 'message' : 'Request not found or request is not abortable. Check request headers for abortable flag.'}
+        
+        @self.app.route(f'{self.request_reply_base_url}/<id>' , methods=['GET'])
+        def get_result_of_async_request(id : str ) :
+            request : AbortRequest = self.bg_requests.get(id)
+            if request :
+                return {
+                    'status' : 200 , 
+                    'internalid' : id , 
+                    'result' : request.result ,
+                    'in_progress' : request.in_progress ,
+                    'async_request' : request.async_request ,
+                    'internalid' : request.internalid ,
+                    'start_ts' : request.start_ts ,
+                    'end_ts' : request.end_ts ,
+                    'execution' : request.execution,
+                    'success' : request.success,
+                    'killed' : request.killed
+                }
 
         @self.app.before_request
         def before_request() :
+            if (self.serve_with_secret_key) and (request.path not in self.secret_key_execlude_urls) and (request.headers.get('secretkey') != self.secretkey):
+                return jsonify({"error": "Secret key is invalid"}), 401
+                        
+            @copy_current_request_context
+            def run_async_job_results( target_func , abort : AbortRequest ) :
+                abort.in_progress = True
+                abort.async_request = True
+                abort.internalid = request.internalid
+                abort.start_ts = getTimestamp()
+                try :
+                    result = target_func(*request.args, **request.form)
+                    abort.success = True
+                except Exception as e :
+                    abort.success = False
+                    abort.result = str(e)
+                    abort.traceback = traceback.format_exc()
+                    raise
+                abort.result = result
+                abort.end_ts = getTimestamp()
+                abort.execution = round(abort.end_ts - abort.start_ts, 2)
+                abort.in_progress = False
+                abort.delete_async_request_ts = getTimestamp(after_seconds=3600)
+
+
             abortable = request.headers.get('abortable')
+            requestId = getRandomKeysAndStr(n=10)
+            request.start_ts = getTimestamp()
+            request.internalid = requestId
             if abortable :
                 abort = self.register_abortable_request(request)
                 request.abortable = True
@@ -154,13 +260,20 @@ class UISERVER :
                     target_func = current_app.view_functions.get(request.endpoint)
                     if not target_func:
                         return jsonify({"error": "Route not found"}), 404
-                    th = start_thread(target=target_func, args=request.args, kwargs=request.form)
+                    th = start_thread(target=run_async_job_results, args=[target_func , abort ])
                     abort.thread = th
+                    self.bg_requests[requestId] = abort
                     return {"status": 200, "message": "Request now in running bg", "abort_id": abort.abort_id} , 200
+    
 
         @self.app.after_request
         def after_request(response) :
             try :
+                now = getTimestamp()
+                response.headers['internalid'] = request.internalid
+                response.headers['start_ts'] = request.start_ts
+                response.headers['end_ts'] = now
+                response.headers['execution'] = round(now - request.start_ts, 2)
                 if request.abortable :
                     response.headers['abortid'] = request.abort_id 
                     response.headers['abortable'] = True
@@ -168,6 +281,35 @@ class UISERVER :
                 response.headers['abortable'] = False
             return response
 
+        # socketio client connected.
+        @self.socketio.on('connect')
+        def handle_client_connect():
+            sid = request.sid
+            client = SocketClientObject()
+            client.internalid = getRandomKeysAndStr(n=20)
+            client.sid = sid
+            client.request = request
+            client.rooms = self.socketio_rooms
+            self.socketio_clients.append(client)
+            self.socketio.emit('/internal/connect', { 
+                'status': 200, 
+                'message': 'client connected' , 
+                'internalid' : client.internalid,
+                'sid' : sid,
+            }, 
+                to=sid
+            )
+            print(f'client connected : {sid} | Clients : {len(self.socketio_clients)}')
+        
+        # socketio client connected.
+        @self.socketio.on('disconnect')
+        def handle_client_disconnect():
+            for i , c in enumerate(list(self.socketio_clients)) :
+                print(f'c.sid : {c.sid} | request.sid : {request.sid}')
+                if c.sid == request.sid :
+                    del self.socketio_clients[i]
+                    print(f'client disconnected : {request.sid} | Clients : {len(self.socketio_clients)}')
+                    break
 
     def getInstance(self) :
         return self.getFlask() , self.getSocketio() , self.getWsgi()
@@ -188,7 +330,8 @@ class UISERVER :
 
     def _wait_th(self , t ) :
         t.join()
-        
+    
+
     def thrStartUi(self , suppress_prints=True) :
         if self.enable_test_url :
             if not suppress_prints :
@@ -196,6 +339,8 @@ class UISERVER :
             @self.app.route('/connection/test/internal' , methods=['GET'])
             def test_connection():
                 return f"Status=200<br> ID={self.id}<br> one-time-token={getRandomKey(20)}"
+
+                
         if self.httpProtocol == 'http' :
             con = None
         elif self.httpProtocol == 'https' :

{easy_utils_dev-2.142 → easy_utils_dev-2.144}/easy_utils_dev/utils.py

@@ -1,4 +1,4 @@
-import datetime , string , subprocess , psutil ,secrets , os ,ping3 , time , sys , argparse , ctypes , math , threading , random , jwt , socket
+import datetime , string , subprocess , psutil , shutil ,secrets , os ,ping3 , time , sys , argparse , ctypes , math , threading , random , jwt , socket
 
 
 def getRandomKey(n=10, numbers=True) :
@@ -463,5 +463,19 @@ def getMachineAddresses() :
     ip_addresses = socket.gethostbyname_ex(socket.gethostname())[2]
     return ip_addresses
 
+
+def get_free_space(path: str , in_mb = True) -> int:
+    """
+    Return available free space in bytes for the given path.
+    Works on Linux, macOS, and Windows.
+    """
+    # Expand ~ and handle drive letters
+    path = os.path.abspath(os.path.expanduser(path))
+    total, used, free = shutil.disk_usage(path)
+    if in_mb :
+        return convert_bytes_to_mb(free)
+    return free  # in bytes
+
+
 if __name__ == "__main__":
     pass

{easy_utils_dev-2.142 → easy_utils_dev-2.144}/easy_utils_dev/wsnoclib.py

@@ -1,7 +1,8 @@
+from tkinter import NO
 from easy_utils_dev.debugger import DEBUGGER
 import requests , json , subprocess
 from requests.auth import HTTPBasicAuth as BAuth    
-from .utils import pingAddress , fixTupleForSql , start_thread , mkdirs  , getTimestamp
+from .utils import get_free_space, pingAddress , fixTupleForSql , start_thread , mkdirs  , getTimestamp
 from time import sleep
 from urllib3.exceptions import InsecureRequestWarning
 from urllib3 import disable_warnings
@@ -378,7 +379,7 @@ class WSNOCLIB :
         return self.kafka
 
     def change_kafka_refresh_period(self , period : int =3000) :
-        print('Deprecated function change_kafka_refresh_period. Kafka refresh period is now managed by WSNOC API SLEEP PERIOD')
+        self.logger.warning('Deprecated, Kafka refresh period is now managed by WSNOC API SLEEP PERIOD. Nothing is applied now.')
 
     def renewSubscription(self) :
         self.logger.info('Renewing subscription ...')
@@ -390,67 +391,6 @@ class WSNOCLIB :
         if not response.ok :
             self.logger.error(f'failed to renew subscription. {response.text}')
 
-    # def renewSubscription(self) :
-    #     while True :
-    #         try :
-    #             sleep(self.kafka.kafka_refresh_period) 
-    #             if self.loggedOut or self.killed:
-    #                 break
-    #             self.logger.info('Renewing subscription ...')
-    #             self.kafka.refresh_inprogress = True
-    #             URL = f"{self.kafka.base_url}:{self.kafka.kafka_port}/nbi-notification/api/v1/notifications/subscriptions/{self.kafka.subscriptionId}/renewals"
-    #             response = self.session.post(URL , retries=3)
-    #             self.logger.debug(f'Renewing subscription Response: [{response.text}]')
-    #         except Exception as error :
-    #             self.logger.error(f'failed to renew subscription. {error}')
-    #             self.logger.debug(traceback.format_exc())
-    #         self.kafka.refresh_inprogress = False
-
-    # def deleteKafkaSubscription(self , subscriptionId=None) :
-    #     self.logger.info(f'Deleting subscription subscriptionId:{subscriptionId}')
-    #     if not subscriptionId :
-    #         self.logger.info(f'Deleting subscription subscriptionId:{self.kafka.subscriptionId}')
-    #         subscriptionId=self.kafka.subscriptionId
-    #     self.kafka.kafka_subscription_deleted= True
-    #     URL = f"{self.kafka.base_url}:{self.kafka.kafka_port}/nbi-notification/api/v1/notifications/subscriptions/{subscriptionId}"
-    #     response = self.session.delete(URL , retries=3)
-    #     return response
-    
-    # def handle_beautify_alarm(self , alarm ) :
-    #     oalarm = alarm
-    #     alarm = alarm['data']['ietf-restconf:notification']
-    #     if 'create' in str(list(alarm.keys())) :
-    #         alarmData = alarm['nsp-fault:alarm-create']
-    #         oalarm['dataEnh'] = {
-    #             'newAlarm' : True,
-    #             'alarmChange' : False,
-    #             'alarmId' : int(alarmData['objectId'].split(':')[-1]),
-    #             'neId' : alarmData['neId'],
-    #             'neName	' : alarmData['neName'],
-    #             'alarmName' : alarmData['alarmName'],
-    #             'cleared' : False,
-    #             'aknowledged' : False,
-    #             **alarmData , 
-    #         }
-    #     elif 'change' in str(list(alarm.keys())) :
-    #         alarmData = alarm['nsp-fault:alarm-change']
-    #         cleared = False
-    #         aknowledged = False
-    #         if 'severity' in list(alarmData.keys()) :
-    #             if alarmData['severity']['new-value'] == 'cleared' :
-    #                 cleared = True
-    #         if 'acknowledged' in list(alarmData.keys()) :
-    #                 aknowledged = alarmData['acknowledged']['new-value']
-    #         oalarm['dataEnh'] = {
-    #             'newAlarm' : False,
-    #             'alarmChange' : True,
-    #             'alarmId' : int(alarmData['objectId'].split(':')[-1]),
-    #             'cleared' : cleared,
-    #             'aknowledged' : aknowledged,
-    #             **alarmData ,
-    #         }
-    #     return oalarm
-
 
     def kafka_listen(self) : 
         def hold_if_kafka_refresh_inprogress() :
@@ -773,13 +713,18 @@ class PmHadoopClient :
         self.PM24H = 1
         self.PM15M = 2
         self.KPIAGGR = 3
+        self.CURRENT = 1
+        self.ARCHIVE = 2
         self.logger : DEBUGGER = self.wsnoc.logger
+        self.FREE_STORAGE_STRICT = True
+        self.FREE_STORAGE_THRESHOLD_MB = 30000
 
     
     def connect(self) :
         if not self.wsnoc.connected :
             raise Exception('WSNOC is not connected')
         try :
+            self.logger.info(f"Connecting to PM Hadoop at {self.ip}:{self.hdfs_port}" , source='PmHadoopClient')
             self.client = HdfsClient(self.ip, self.hdfs_port, use_trash=self.hdfs_use_trash)
         except Exception as e:
             self.logger.warning('PM Hadoop client is using WSNOC port 8020/custom port. Please check if it is not blocked by firewall.' , source='PmHadoopClient')
@@ -795,16 +740,23 @@ class PmHadoopClient :
         dt = datetime.strptime(date_str, "%Y%m%d")
         return int(dt.timestamp())
 
-    def pm_list(self , mode , date_range=[]) :
+    def pm_list(self , mode , target_pm , date_range=[] , date_range_in_days=None) :
         '''
         mode : must be one of the following:
             - self.PM24H
             - self.PM15M
             - self.KPIAGGR
+        target_pm : must be on the following :
+            - self.CURRENT
+            - self.ARCHIVE
         date_range : must be a list of two integers in the format of [start_timestamp, end_timestamp]
             -  for example: [1718217600, 1718221200]
+        date_range_in_days : if provided, date_range will be set to last X days
+            -  for example: 30 days ago to now
+            - if date_range_in_days is not provided, date_range will be used as is
         - if date_range is not provided, all available PM dates will be returned
         '''
+        self.logger.info(f"Getting available PM dates Original Args: {mode}/{target_pm}" , source='PmHadoopClient')
         if mode == self.PM24H :
             _mode = 'ONE_DAY'
         elif mode == self.PM15M :
@@ -813,9 +765,20 @@ class PmHadoopClient :
             _mode = 'KPIAGGR'
         else :
             raise Exception(f'Invalid mode: {mode}')
-        self.logger.info(f"Getting available PM dates for {_mode}" , source='PmHadoopClient')
-        dirs = list(self.client.ls([f'/PMDATA/{_mode}']))
+        if target_pm == self.CURRENT :
+            _target = 'PMDATA'
+        elif target_pm == self.ARCHIVE : 
+            _target = "ARC_PMDATA"
+        else :
+            self.logger.error(f"target_pm arg must be self.CURRENT or self.ARCHIVE.")
+            raise Exception("Invalid TARGET_PM")
+        self.logger.info(f"Getting available PM dates for {_target}/{_mode}" , source='PmHadoopClient')
+        dirs = list(self.client.ls([f'/{_target}/{_mode}']))
         ts_now = getTimestamp()
+        if date_range_in_days :
+            self.logger.info(f"Setting date range to last {date_range_in_days} days" , source='PmHadoopClient')
+            date_range = [ts_now - date_range_in_days * 24 * 60 * 60, ts_now]
+        self.logger.info(f"Date range: {tuple(date_range)}" , source='PmHadoopClient')
         for index , dir in enumerate(dirs) :
             self.logger.info(f"Processing {dir.get('path')}" , source='PmHadoopClient')
             date_str = dir.get('path').split('=')[-1]
@@ -829,10 +792,12 @@ class PmHadoopClient :
                 if dir['pm_date_timestamp'] > date_range[1] :
                     del dirs[index]
         return dirs
+    
+    def use_jump_host(self , jhost) :
+        ...
 
     def _download_dir(self , hdfs_path, local_path):
         for entry in self.client.ls([hdfs_path]):
-            # print(entry)
             entry_path = entry['path']
             entry_type = entry['file_type']
             if entry_type == 'd':
@@ -840,13 +805,19 @@ class PmHadoopClient :
                 mkdirs(subdir)
                 self._download_dir(entry_path, subdir)
             else:  # FILE
-                self.logger.debug(f"Downloading {entry_path} → {local_path}" , source='PmHadoopClient')
+                self.logger.info(f"Downloading {entry_path} → {local_path}" , source='PmHadoopClient')
                 self.client.copyToLocal([entry_path], local_path)
         
     def download(self , obj , destination_path) :
         self.wsnoc.logger.info(f'Downloading {obj.get("pm_date")} to {destination_path}')
         destination_path = f"{destination_path}/{obj.get('mode')}/{obj.get('pm_date')}"
+        self.logger.debug(f"FREE_STORAGE_THRESHOLD_MB={self.FREE_STORAGE_THRESHOLD_MB}  FREE_STORAGE_STRICT={self.FREE_STORAGE_STRICT}")
         mkdirs(destination_path)
+        free_space = get_free_space(destination_path)
+        if self.FREE_STORAGE_STRICT :
+            if free_space < self.FREE_STORAGE_THRESHOLD_MB :
+                self.logger.error(f'Free storage is less than {self.FREE_STORAGE_THRESHOLD_MB} MB. Skipping download. [free_space={free_space} MB]')
+                raise Exception(f'No enough space to download PM data.')
         self._download_dir(obj.get('path'), destination_path)
 
 

{easy_utils_dev-2.142 → easy_utils_dev-2.144}/easy_utils_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.142
+Version: 2.144
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
@@ -16,6 +16,8 @@ Requires-Dist: flask_socketio
 Requires-Dist: python-dotenv
 Requires-Dist: gevent
 Requires-Dist: pyzipper
+Requires-Dist: shutil
+Requires-Dist: bs4
 Requires-Dist: pyjwt
 Requires-Dist: authlib
 Requires-Dist: kafka-python

{easy_utils_dev-2.142 → easy_utils_dev-2.144}/easy_utils_dev.egg-info/requires.txt

@@ -11,6 +11,8 @@ flask_socketio
 python-dotenv
 gevent
 pyzipper
+shutil
+bs4
 pyjwt
 authlib
 kafka-python

{easy_utils_dev-2.142 → easy_utils_dev-2.144}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '2.142'
+VERSION = '2.144'
 
 # Setting up
 setup(
@@ -22,6 +22,8 @@ setup(
             'python-dotenv',
             'gevent',
             'pyzipper',
+            "shutil",
+            "bs4",
             'pyjwt',
             'authlib',
             'kafka-python'