easy-utils-dev 2.144__tar.gz → 2.146__tar.gz

{easy_utils_dev-2.144 → easy_utils_dev-2.146}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.144
+Version: 2.146
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
@@ -17,7 +17,6 @@ Requires-Dist: python-dotenv
 Requires-Dist: gevent
 Requires-Dist: pyzipper
 Requires-Dist: shutil
-Requires-Dist: bs4
 Requires-Dist: pyjwt
 Requires-Dist: authlib
 Requires-Dist: kafka-python

{easy_utils_dev-2.144 → easy_utils_dev-2.146}/easy_utils_dev/temp_memory.py

@@ -35,10 +35,9 @@ class TemporaryMemory :
 
     def save(self, item , custom_key=None ,auto_destroy_period=60 , store_deleted_key=True) :
         now = getTimestamp()
+        later = None
         if auto_destroy_period :
             later = getTimestamp(after_seconds=auto_destroy_period)
-        else :
-            later = None
         if not custom_key :
             custom_key = f"{getTimestamp()}-{generateToken(iter=4)}".upper()
         self.store[custom_key] = {

{easy_utils_dev-2.144 → easy_utils_dev-2.146}/easy_utils_dev/utils.py

@@ -476,6 +476,21 @@ def get_free_space(path: str , in_mb = True) -> int:
         return convert_bytes_to_mb(free)
     return free  # in bytes
 
+def can_be_int(string) :
+    """
+    Check if a string can be converted to an integer.
+    
+    Args:
+        string (str): String to check.
+    
+    Returns:
+        bool: True if string can be converted to an integer, False otherwise.
+    """
+    try :
+        int(string)
+        return True
+    except :
+        return False
 
 if __name__ == "__main__":
     pass

{easy_utils_dev-2.144 → easy_utils_dev-2.146}/easy_utils_dev/wsnoclib.py

@@ -9,7 +9,6 @@ from urllib3 import disable_warnings
 from threading import Thread
 from easy_utils_dev.Events import EventEmitter
 from .EasySsh import CREATESSH
-import traceback
 import xmltodict
 from .FastQueue import FastQueue
 import tempfile , os 
@@ -67,7 +66,6 @@ class WSNOCLIB :
         self.api_count_limit = 999999999999
         self.temp_dir = os.path.join(tmp_dir , 'wsnoclib' )
         self.baseUrl = self.createBaseUrl()
-        self.session = requests.Session()
         self.numberOfRequests=0
         self.request_max_count = request_max_count
         self.onGoingRequests=0
@@ -85,6 +83,8 @@ class WSNOCLIB :
         self.refresh_thread = None
         self.token_refresh_count = 0
         self.session = WSNOCSession(self)
+        self.max_concurrent_requests = 40
+        self.current_requests = 0
         self.connected = False
         self.pm_hadoop = PmHadoopClient(self)
         if register_atexit :
@@ -662,15 +662,28 @@ class WSNOCSession(requests.Session):
     def rebuild_auth(self, prepared_request, response):
         return
 
+    def _refactor_url(self , url) :
+        if str(url).startswith('/') :
+            url = f"{self._wsnoc.baseUrl}/{url}"
+        return url
+
     def hold_for_token_refresh(self, url=None) :
         while self._wsnoc.refresh_inprogress :
             self._wsnoc.logger.info(f'Waiting for token refresh. {url if url else "No URL"}')
             sleep(.5)
 
+    def hold_for_max_concurrent_requests(self, url=None) :
+        while self._wsnoc.current_requests >= self._wsnoc.max_concurrent_requests :
+            self._wsnoc.logger.info(f'Max concurrent requests reached. Waiting for a request to complete. {url if url else "No URL"}')
+            self._wsnoc.logger.debug(f'Max concurrent requests reached. Waiting for a request to complete. {url if url else "No URL"} [current_requests={self._wsnoc.current_requests}] [max_concurrent_requests={self._wsnoc.max_concurrent_requests}]')
+            sleep(1)
+
     def request(self, method, url , retries=0 , skip_hold_for_token_refresh=False , debug_this_request=False , **kwargs):
+        url = self._refactor_url(url)
         self._wsnoc.logger.debug(f'[{method}] : {url}')
         if not skip_hold_for_token_refresh :
             self.hold_for_token_refresh(url)
+        self.hold_for_max_concurrent_requests(url)
         self._wsnoc.api_count += 1
         token = self._wsnoc.getLatestToken().get('bearer_token')
         request_headers = kwargs.get('headers' , {})
@@ -678,6 +691,8 @@ class WSNOCSession(requests.Session):
             if not request_headers.get('Authorization') :
                 request_headers['Authorization'] = token
         kwargs['headers'] = request_headers
+        start_ts = getTimestamp()
+        self._wsnoc.current_requests += 1
         request = super().request(method, url, **kwargs  )
         if debug_this_request :
             self._wsnoc.logger.info(f'''
@@ -687,6 +702,7 @@ class WSNOCSession(requests.Session):
                 [DEBUG] Response: {request.text}
                 [DEBUG] OK: {request.ok}
                 [DEBUG] Method: {request.request.method}
+                [DEBUG] StartTs: {start_ts}
             ''')
         for i in range(retries) :
             if request.ok :
@@ -696,7 +712,13 @@ class WSNOCSession(requests.Session):
                 self.hold_for_token_refresh(url)
                 request = super().request(method, url, **kwargs )
                 self._wsnoc.logger.debug(f'[Try-{i}] [{method}] : {url}- {request.status_code}')
-        self._wsnoc.logger.info(f'[{method}] : {url} - [{request.status_code}]')
+        end_ts = getTimestamp()
+        execution_secs = round(end_ts - start_ts, 2)
+        self._wsnoc.logger.info(f'[{method}] : {url} - [{request.status_code}][{execution_secs}sec]')
+        request.start_ts = start_ts
+        request.end_ts = end_ts
+        request.execution = execution_secs
+        self._wsnoc.current_requests -= 1
         return request
 
 class PmHadoopClient :
@@ -718,6 +740,8 @@ class PmHadoopClient :
         self.logger : DEBUGGER = self.wsnoc.logger
         self.FREE_STORAGE_STRICT = True
         self.FREE_STORAGE_THRESHOLD_MB = 30000
+        self.jhost = False
+        self.jhost_obj = None
 
     
     def connect(self) :
@@ -774,27 +798,36 @@ class PmHadoopClient :
             raise Exception("Invalid TARGET_PM")
         self.logger.info(f"Getting available PM dates for {_target}/{_mode}" , source='PmHadoopClient')
         dirs = list(self.client.ls([f'/{_target}/{_mode}']))
-        ts_now = getTimestamp()
         if date_range_in_days :
+            ts_now = getTimestamp() # this is in seconds
             self.logger.info(f"Setting date range to last {date_range_in_days} days" , source='PmHadoopClient')
             date_range = [ts_now - date_range_in_days * 24 * 60 * 60, ts_now]
         self.logger.info(f"Date range: {tuple(date_range)}" , source='PmHadoopClient')
         for index , dir in enumerate(dirs) :
             self.logger.info(f"Processing {dir.get('path')}" , source='PmHadoopClient')
-            date_str = dir.get('path').split('=')[-1]
-            dir['pm_date'] = int(date_str)
-            dir['mode'] = _mode
-            dt = datetime.strptime(date_str, "%Y%m%d")
-            dir['pm_date_timestamp'] = int(dt.timestamp())
-            if len(date_range) > 0 :
-                if dir['pm_date_timestamp'] < date_range[0] :
-                    del dirs[index]
-                if dir['pm_date_timestamp'] > date_range[1] :
-                    del dirs[index]
+            if 'ds' in dir.get('path' , '') :
+                date_str = dir.get('path').split('=')[-1]
+                dir['pm_date'] = int(date_str)
+                dir['mode'] = _mode
+                dt = datetime.strptime(date_str, "%Y%m%d")
+                dir['pm_date_timestamp'] = int(dt.timestamp())
+                if len(date_range) > 0 :
+                    if dir['pm_date_timestamp'] < date_range[0] :
+                        del dirs[index]
+                    if dir['pm_date_timestamp'] > date_range[1] :
+                        del dirs[index]
         return dirs
     
-    def use_jump_host(self , jhost) :
-        ...
+    def use_jump_host(self , jhost , use=True) :
+        '''
+        not yet implemented
+        '''
+        self.jhost = use
+        if use :
+            self.jhost_obj = None
+        else :
+            self.jhost_obj = None
+        
 
     def _download_dir(self , hdfs_path, local_path):
         for entry in self.client.ls([hdfs_path]):

{easy_utils_dev-2.144 → easy_utils_dev-2.146}/easy_utils_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easy_utils_dev
-Version: 2.144
+Version: 2.146
 Keywords: python3
 Classifier: Programming Language :: Python :: 3
 Requires-Dist: psutil
@@ -17,7 +17,6 @@ Requires-Dist: python-dotenv
 Requires-Dist: gevent
 Requires-Dist: pyzipper
 Requires-Dist: shutil
-Requires-Dist: bs4
 Requires-Dist: pyjwt
 Requires-Dist: authlib
 Requires-Dist: kafka-python

{easy_utils_dev-2.144 → easy_utils_dev-2.146}/easy_utils_dev.egg-info/requires.txt

@@ -12,7 +12,6 @@ python-dotenv
 gevent
 pyzipper
 shutil
-bs4
 pyjwt
 authlib
 kafka-python

{easy_utils_dev-2.144 → easy_utils_dev-2.146}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '2.144'
+VERSION = '2.146'
 
 # Setting up
 setup(
@@ -23,7 +23,6 @@ setup(
             'gevent',
             'pyzipper',
             "shutil",
-            "bs4",
             'pyjwt',
             'authlib',
             'kafka-python'