easy-utils-dev 2.141__tar.gz → 2.143__tar.gz

{easy_utils_dev-2.141 → easy_utils_dev-2.143}/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.141
+Version: 2.143
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
 Requires-Dist: ping3
+Requires-Dist: snakebite-py3
 Requires-Dist: flask
 Requires-Dist: flask_cors
 Requires-Dist: xmltodict

{easy_utils_dev-2.141 → easy_utils_dev-2.143}/easy_utils_dev/wsnoclib.py

@@ -1,7 +1,7 @@
 from easy_utils_dev.debugger import DEBUGGER
 import requests , json , subprocess
 from requests.auth import HTTPBasicAuth as BAuth    
-from .utils import pingAddress , fixTupleForSql , start_thread , mkdirs
+from .utils import pingAddress , fixTupleForSql , start_thread , mkdirs  , getTimestamp
 from time import sleep
 from urllib3.exceptions import InsecureRequestWarning
 from urllib3 import disable_warnings
@@ -15,7 +15,8 @@ import tempfile , os
 from kafka import KafkaConsumer
 from easy_utils_dev.utils import kill_thread
 import atexit
-
+from snakebite.client import Client as HdfsClient
+from datetime import datetime
 
 
 class KafkaConfig :
@@ -38,7 +39,6 @@ class KafkaConfig :
         self.enable_auto_refresh = False
 
 
-
 class WSNOCLIB : 
     def __init__( 
         self, 
@@ -51,7 +51,8 @@ class WSNOCLIB :
         request_max_count=30,
         tmp_dir = tempfile.gettempdir() ,
         kafka = KafkaConfig(),
-        register_atexit=True
+        register_atexit=True,
+        trust_env=True
     ): 
         self.logger = DEBUGGER(f'{debug_name}-{ip}',level=debug_level,homePath=debug_homepath)
         self.disabledWarnings = self.disableUrlWarnings()
@@ -59,6 +60,7 @@ class WSNOCLIB :
         self.address = ip
         self.username = username
         self.password = password
+        self.trust_env = trust_env
         self.external_nsp = False
         self.api_count = 0
         self.api_count_limit = 999999999999
@@ -82,6 +84,8 @@ class WSNOCLIB :
         self.refresh_thread = None
         self.token_refresh_count = 0
         self.session = WSNOCSession(self)
+        self.connected = False
+        self.pm_hadoop = PmHadoopClient(self)
         if register_atexit :
             atexit.register(self.goodbye)
 
@@ -150,6 +154,7 @@ class WSNOCLIB :
         if auto_refresh_token :
             self.autoRefreshThread = self.refresh_thread = start_thread(target=self.runAutoRefreshThread)
         self.logger.debug(f'token => {r.text}')
+        self.connected = True
         return self.token
 
 
@@ -182,6 +187,7 @@ class WSNOCLIB :
             r.close()
         except :
             pass
+        self.connected = False
         return True
 
     def goodbye(self):
@@ -610,6 +616,7 @@ class WSNOCLIB :
             self.bearer_token = f'Bearer {self.access_token}'
             self.token = r.json()
             self.token.update({'bearer_token' :  self.bearer_token })
+        self.connected = True
         return r
 
     def session_info(self) :
@@ -706,6 +713,9 @@ class WSNOCSession(requests.Session):
         self._wsnoc = wsnoc
         self.verify = False
         self.retries = 0
+        if not wsnoc.trust_env :
+            os.environ['https_proxy'] = ""
+            os.environ['http_proxy'] = ""
         self.debug_this_request = False
         self.skip_hold_for_token_refresh = False
         
@@ -749,6 +759,108 @@ class WSNOCSession(requests.Session):
         self._wsnoc.logger.info(f'[{method}] : {url} - [{request.status_code}]')
         return request
 
+class PmHadoopClient :
+
+
+    def __init__(self , _wsnoc : WSNOCLIB):
+        self.ip = _wsnoc.address
+        self.wsnoc = _wsnoc
+        self.client : HdfsClient = None
+        self.hdfs_port = 8020
+        self.hdfs_user = 'otn'
+        self.hdfs_root = '/'
+        self.hdfs_use_trash = False
+        self.PM24H = 1
+        self.PM15M = 2
+        self.KPIAGGR = 3
+        self.CURRENT = 1
+        self.ARCHIVE = 2
+        self.logger : DEBUGGER = self.wsnoc.logger
+
+    
+    def connect(self) :
+        if not self.wsnoc.connected :
+            raise Exception('WSNOC is not connected')
+        try :
+            self.client = HdfsClient(self.ip, self.hdfs_port, use_trash=self.hdfs_use_trash)
+        except Exception as e:
+            self.logger.warning('PM Hadoop client is using WSNOC port 8020/custom port. Please check if it is not blocked by firewall.' , source='PmHadoopClient')
+            self.logger.error(f'Failed to connect to PM Hadoop: {e}' , source='PmHadoopClient')
+            raise
+
+    def change_date_to_timestamp(self , date_str) :
+        '''
+        this is a helper function to convert date string to timestamp
+        date_str : must be in the format of %Y%m%d example: 20250101
+        return : timestamp
+        '''
+        dt = datetime.strptime(date_str, "%Y%m%d")
+        return int(dt.timestamp())
+
+    def pm_list(self , mode , target_pm , date_range=[]) :
+        '''
+        mode : must be one of the following:
+            - self.PM24H
+            - self.PM15M
+            - self.KPIAGGR
+        target_pm : must be on the following :
+            - self.CURRENT
+            - self.ARCHIVE
+        date_range : must be a list of two integers in the format of [start_timestamp, end_timestamp]
+            -  for example: [1718217600, 1718221200]
+        - if date_range is not provided, all available PM dates will be returned
+        '''
+        if mode == self.PM24H :
+            _mode = 'ONE_DAY'
+        elif mode == self.PM15M :
+            _mode = 'FIFTEEN_MINS'
+        elif mode == self.KPIAGGR :
+            _mode = 'KPIAGGR'
+        else :
+            raise Exception(f'Invalid mode: {mode}')
+        if target_pm == self.CURRENT :
+            _target = 'PMDATA'
+        elif target_pm == self.ARCHIVE : 
+            _target = "ARC_PMDATA"
+        else :
+            self.logger.error(f"target_pm arg must be self.CURRENT or self.ARCHIVE.")
+            raise Exception("Invalid TARGET_PM")
+        self.logger.info(f"Getting available PM dates for {_target}/{_mode}" , source='PmHadoopClient')
+        dirs = list(self.client.ls([f'/{_target}/{_mode}']))
+        ts_now = getTimestamp()
+        for index , dir in enumerate(dirs) :
+            self.logger.info(f"Processing {dir.get('path')}" , source='PmHadoopClient')
+            date_str = dir.get('path').split('=')[-1]
+            dir['pm_date'] = int(date_str)
+            dir['mode'] = _mode
+            dt = datetime.strptime(date_str, "%Y%m%d")
+            dir['pm_date_timestamp'] = int(dt.timestamp())
+            if len(date_range) > 0 :
+                if dir['pm_date_timestamp'] < date_range[0] :
+                    del dirs[index]
+                if dir['pm_date_timestamp'] > date_range[1] :
+                    del dirs[index]
+        return dirs
+
+    def _download_dir(self , hdfs_path, local_path):
+        for entry in self.client.ls([hdfs_path]):
+            # print(entry)
+            entry_path = entry['path']
+            entry_type = entry['file_type']
+            if entry_type == 'd':
+                subdir = os.path.join(local_path, os.path.basename(entry_path))
+                mkdirs(subdir)
+                self._download_dir(entry_path, subdir)
+            else:  # FILE
+                self.logger.debug(f"Downloading {entry_path} → {local_path}" , source='PmHadoopClient')
+                self.client.copyToLocal([entry_path], local_path)
+        
+    def download(self , obj , destination_path) :
+        self.wsnoc.logger.info(f'Downloading {obj.get("pm_date")} to {destination_path}')
+        destination_path = f"{destination_path}/{obj.get('mode')}/{obj.get('pm_date')}"
+        mkdirs(destination_path)
+        self._download_dir(obj.get('path'), destination_path)
+
 
 if __name__ == '__main__' :
     # noc = WSNOCLIB('10.20.30.55' , 'admin' , 'Nokia@2024') 

{easy_utils_dev-2.141 → easy_utils_dev-2.143}/easy_utils_dev.egg-info/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.141
+Version: 2.143
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
 Requires-Dist: ping3
+Requires-Dist: snakebite-py3
 Requires-Dist: flask
 Requires-Dist: flask_cors
 Requires-Dist: xmltodict

{easy_utils_dev-2.141 → easy_utils_dev-2.143}/easy_utils_dev.egg-info/requires.txt

@@ -1,5 +1,6 @@
 psutil
 ping3
+snakebite-py3
 flask
 flask_cors
 xmltodict

{easy_utils_dev-2.141 → easy_utils_dev-2.143}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '2.141'
+VERSION = '2.143'
 
 # Setting up
 setup(
@@ -11,6 +11,7 @@ setup(
     install_requires=[
             'psutil' ,
             'ping3' , 
+            'snakebite-py3',
             'flask' , 
             'flask_cors' , 
             'xmltodict' ,