ipulse-shared-core-ftredge 2.51__tar.gz → 2.52__tar.gz

{ipulse_shared_core_ftredge-2.51/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.51
+Version: 2.52
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.51 → ipulse_shared_core_ftredge-2.52}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.51',
+    version='2.52',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[

{ipulse_shared_core_ftredge-2.51 → ipulse_shared_core_ftredge-2.52}/src/ipulse_shared_core_ftredge/__init__.py

@@ -2,7 +2,7 @@ from .models import (Organisation, UserAuth, UserProfile,
                      UserStatus, UserProfileUpdate, pulse_enums)
 from .utils_gcp import (setup_gcp_logger_and_error_report, 
                         read_csv_from_gcs, read_json_from_gcs, 
-                        write_csv_to_gcs, write_data_to_gcs)
+                        write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           update_check_with_schema_template)
 from .utils_common import (Notice,  NoticesManager)

{ipulse_shared_core_ftredge-2.51 → ipulse_shared_core_ftredge-2.52}/src/ipulse_shared_core_ftredge/utils_common.py

@@ -11,7 +11,7 @@ from contextlib import contextmanager
 from typing import List
 from google.cloud import logging as cloudlogging
 from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeLevel, NoticeManagerCategory, NoticeStatus
-from ipulse_shared_core_ftredge.utils_gcp import write_data_to_gcs
+from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
 
 
 # ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
@@ -72,7 +72,7 @@ class Notice:
         # Determine the number of lines to keep from the start and end
         keep_lines_start = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines))
         keep_lines_end = min(self.MAX_TRACEBACK_LINES // 2, len(combined_lines) - keep_lines_start)
-        
+
         if len(combined_lines) > self.MAX_TRACEBACK_LINES:
             # Include the first few and last few lines, and an indicator of truncation
             formatted_traceback = '\n'.join(
@@ -82,8 +82,8 @@ class Notice:
             )
         else:
             formatted_traceback = '\n'.join(combined_lines)
-        
-        return formatted_traceback  
+
+        return formatted_traceback 
 
     @property
     def start_context(self):
@@ -123,12 +123,12 @@ class NoticesManager:
     SUCCESS_CODE_START_VALUE = NoticeLevel.SUCCESS.value
 
     def __init__(self, start_context: str, category: NoticeManagerCategory = NoticeManagerCategory.NOTICES, logger_name=None):
-        self._notice_manager_id = str(uuid.uuid4())
+        self._id = str(uuid.uuid4())
         self._notices = []
         self._early_stop = False
-        self._error_count = 0
-        self._warning_count = 0
-        self._success_count = 0
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._successes_count = 0
         self._level_counts = {level.name: 0 for level in NoticeLevel}
         self._start_context = start_context
         self._context_stack = []
@@ -157,23 +157,31 @@ class NoticesManager:
         if self._context_stack:
             self._context_stack.pop()
 
-    def get_current_context(self):
+    @property
+    def current_context(self):
         return " >> ".join(self._context_stack)
 
-    def get_start_context(self):
+    @property
+    def start_context(self):
         return self._start_context
-    
-    def get_notice_manager_id(self):
-        return self._notice_manager_id
-    
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def early_stop(self):
+        return self._early_stop
+
     def set_early_stop(self, max_errors_tolerance:int, create_error_notice=True,pop_context=False):
-        self._early_stop = True
+        self.early_stop = True
         if create_error_notice:
             if pop_context:
                 self.pop_context()
             self.add_notice(Notice(level=NoticeLevel.ERROR,
                     subject="EARLY_STOP",
                     description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+
     def reset_early_stop(self):
         self._early_stop = False
 
@@ -184,9 +192,9 @@ class NoticesManager:
         if (self._category == NoticeManagerCategory.SUCCESSES.value and notice.level != NoticeLevel.SUCCESS) or \
            (self._category == NoticeManagerCategory.WARN_ERRS.value and notice.level.value < self.WARNING_CODE_START_VALUE):
             raise ValueError(f"Invalid notice level {notice.level.name} for category {self._category}")
-        notice.start_context = self.get_start_context()
-        notice.context = self.get_current_context()
-        notice.notice_manager_id = self._notice_manager_id
+        notice.start_context = self.start_context
+        notice.context = self.current_context
+        notice.notice_manager_id = self.id
         notice_dict = notice.to_dict()
         self._notices.append(notice_dict)
         self._update_counts(notice_dict)
@@ -203,9 +211,9 @@ class NoticesManager:
 
     def clear_notices_and_counts(self):
         self._notices = []
-        self._error_count = 0
-        self._warning_count = 0
-        self._success_count = 0
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._successes_count = 0
         self._level_counts = {level.name: 0 for level in NoticeLevel}
 
     def clear_notices(self):
@@ -222,24 +230,24 @@ class NoticesManager:
             notice for notice in self._notices
             if context_substring in notice["context"]
         ]
-
+    
     def contains_errors(self):
-        return self._error_count > 0
+        return self._errors_count > 0
 
     def count_errors(self):
-        return self._error_count
+        return self._errors_count
 
     def contains_warnings_or_errors(self):
-        return self._warning_count > 0 or self._error_count > 0
+        return self._warnings_count > 0 or self._errors_count > 0
 
     def count_warnings_and_errors(self):
-        return self._warning_count + self._error_count
+        return self._warnings_count + self._errors_count
 
     def count_warnings(self):
-        return self._warning_count
+        return self._warnings_count
 
     def count_successes(self):
-        return self._success_count
+        return self._successes_count
 
     def count_all_notices(self):
         return len(self._notices)
@@ -256,40 +264,40 @@ class NoticesManager:
         )
 
     def count_notices_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True)
+        return self._count_notices(self.current_context, exact_match=True)
 
     def count_notices_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context())
+        return self._count_notices(self.current_context)
 
     def count_notices_by_level_for_current_context(self, level: NoticeLevel):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=level.value, level_code_max=level.value)
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
 
     def count_notices_by_level_for_current_and_nested_contexts(self, level: NoticeLevel):
-        return self._count_notices(self.get_current_context(), level_code_min=level.value, level_code_max=level.value)
+        return self._count_notices(self.current_context, level_code_min=level.value, level_code_max=level.value)
 
     def count_errors_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.ERROR_CODE_START_VALUE)
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.ERROR_CODE_START_VALUE)
 
     def count_errors_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.ERROR_CODE_START_VALUE)
+        return self._count_notices(self.current_context, level_code_min=self.ERROR_CODE_START_VALUE)
 
     def count_warnings_and_errors_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE)
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE)
 
     def count_warnings_and_errors_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.WARNING_CODE_START_VALUE)
+        return self._count_notices(self.current_context, level_code_min=self.WARNING_CODE_START_VALUE)
 
     def count_warnings_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
 
     def count_warnings_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
+        return self._count_notices(self.current_context, level_code_min=self.WARNING_CODE_START_VALUE, level_code_max=self.ERROR_CODE_START_VALUE - 1)
 
     def count_successes_for_current_context(self):
-        return self._count_notices(self.get_current_context(), exact_match=True, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+        return self._count_notices(self.current_context, exact_match=True, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
 
     def count_successes_for_current_and_nested_contexts(self):
-        return self._count_notices(self.get_current_context(), level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
+        return self._count_notices(self.current_context, level_code_min=self.SUCCESS_CODE_START_VALUE, level_code_max=self.SUCCESS_CODE_START_VALUE)
 
     def export_notices_to_gcs_file(self, bucket_name, storage_client, file_prefix=None, file_name=None, top_level_context=None, save_locally=False, local_path=None, logger=None, max_retries=2):
         def log_message(message):
@@ -309,10 +317,9 @@ class NoticesManager:
             else:
                 file_name = f"{file_prefix}_{timestamp}_len{len(self._notices)}.json"
 
-        cloud_path = None
-        local_path = None
+        result=None
         try:
-            cloud_path, local_path = write_data_to_gcs(
+            result= write_json_to_gcs(
                 bucket_name=bucket_name,
                 storage_client=storage_client,
                 data=self._notices,
@@ -320,13 +327,14 @@ class NoticesManager:
                 save_locally=save_locally,
                 local_path=local_path,
                 logger=logger,
-                max_retries=max_retries
+                max_retries=max_retries,
+                overwrite=True
             )
-            log_message(f"{file_prefix} successfully saved to GCS at {cloud_path} and locally at {local_path}.")
+            log_message(f"{file_prefix} successfully saved (ovewritten={result.get("ovewritten")}) to GCS at {result.get("gcs_path")} and locally at {result.get("local_path")}.")
         except Exception as e:
             log_error(f"Failed at export_notices_to_gcs_file for {file_prefix} for file {file_name} to bucket {bucket_name}: {type(e).__name__} - {str(e)}")
 
-        return cloud_path, local_path
+        return result
 
     def import_notices_from_json(self, json_or_file, logger=None):
         def log_message(message):
@@ -353,17 +361,17 @@ class NoticesManager:
 
         if remove:
             if level_code >= self.ERROR_CODE_START_VALUE:
-                self._error_count -= 1
+                self._errors_count -= 1
             elif level_code >= self.WARNING_CODE_START_VALUE:
-                self._warning_count -= 1
+                self._warnings_count -= 1
             elif level_code >= self.SUCCESS_CODE_START_VALUE:
-                self._success_count -= 1
+                self._successes_count -= 1
             self._level_counts[level_name] -= 1
         else:
             if level_code >= self.ERROR_CODE_START_VALUE:
-                self._error_count += 1
+                self._errors_count += 1
             elif level_code >= self.WARNING_CODE_START_VALUE:
-                self._warning_count += 1
+                self._warnings_count += 1
             elif level_code == self.SUCCESS_CODE_START_VALUE:
-                self._success_count += 1
+                self._successes_count += 1
             self._level_counts[level_name] += 1

{ipulse_shared_core_ftredge-2.51 → ipulse_shared_core_ftredge-2.52}/src/ipulse_shared_core_ftredge/utils_gcp.py

@@ -122,47 +122,32 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
 
 
 
-def write_data_to_gcs(bucket_name, storage_client, data, file_name=None, 
-                      save_locally=False, local_path=None, logger=None, max_retries=3):
+def write_json_to_gcs(bucket_name, storage_client, data, file_name=None, 
+                      save_locally=False, local_path=None, logger=None, max_retries=3, overwrite=True):
     """Saves data to Google Cloud Storage and optionally locally.
 
     This function attempts to upload data to GCS. If the upload fails after
     retries and `save_locally` is True or `local_path` is provided, it attempts
     to save the data locally.
 
-    Args:
-        bucket_name (str): Name of the GCS bucket.
-        storage_client (google.cloud.storage.Client): GCS client object.
-        data (list, dict, or str): Data to be saved.
-        file_name (str, optional): File name for GCS and local. Defaults to None.
-        save_locally (bool, optional): Save locally if GCS fails. Defaults to False.
-        local_path (str, optional): Local directory to save. Defaults to None.
-        logger (logging.Logger, optional): Logger for messages. Defaults to None.
-        max_retries (int, optional): Number of GCS upload retries. Defaults to 3.
-
     Returns:
-        tuple: A tuple containing the GCS path (or None if upload failed) and
-               the local path (or None if not saved locally).
-
-    Raises:
-        ValueError: If data is not a list, dict, or str.
-        Exception: If GCS upload fails after retries and local saving fails or
-                   is not requested. If GCS upload fails after retries and
-                   local saving is requested but unsuccessful.
+        dict: A dictionary containing the GCS path (or None if upload failed),
+              the local path (or None if not saved locally), and a boolean indicating if the file was overwritten.
     """
 
     def log_message(message):
         if logger:
             logger.info(message)
 
-    def log_error(message, exc_info=False):
+    def log_error(message,exc_info=False):
         if logger:
             logger.error(message, exc_info=exc_info)
 
     attempts = 0
     success = False
-    cloud_path = None
+    gcs_path = None
     local_path_final = None
+    overwritten = False
     gcs_upload_exception = None  # Store potential GCS exception
 
     if isinstance(data, (list, dict)):
@@ -176,9 +161,17 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
         try:
             bucket = storage_client.bucket(bucket_name)
             blob = bucket.blob(file_name)
+
+            # Check if the file exists and if we should overwrite it
+            if blob.exists():
+                if not overwrite:
+                    raise FileExistsError(f"File {file_name} already exists in bucket {bucket_name} and overwrite is set to False.")
+                else:
+                    overwritten = True
+
             blob.upload_from_string(data_str, content_type='application/json')
-            cloud_path = f"{bucket_name}/{file_name}"
-            log_message(f"Successfully saved file to GCS {cloud_path}.")
+            gcs_path = f"gs://{bucket_name}/{file_name}"
+            log_message(f"Successfully saved file to GCS {gcs_path}.")
             success = True
         except Exception as e:
             gcs_upload_exception = e
@@ -186,7 +179,7 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
             if attempts < max_retries:
                 time.sleep(2 ** attempts)
             else:
-                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts :{e}")
+                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts: {e}")
 
     if not success and (save_locally or local_path):
         try:
@@ -198,14 +191,16 @@ def write_data_to_gcs(bucket_name, storage_client, data, file_name=None,
                 f.write(data_str)
             log_message(f"Saved {file_name} locally at {local_path_final}.")
         except Exception as local_e:
-            log_error(f"Failed to write {file_name} locally: {local_e}",exc_info=True)
-
-        # If GCS upload failed, raise a single exception here
+            log_error(f"Failed to write {file_name} locally: {local_e}", exc_info=True)
 
     if gcs_upload_exception is not None:
-        raise gcs_upload_exception # Propagate without nesting
+        raise gcs_upload_exception  # Propagate without nesting
 
-    return cloud_path, local_path_final
+    return {
+        "gcs_path": gcs_path,
+        "local_path": local_path_final,
+        "overwritten": overwritten
+    }
 
 
 def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_info_verbose=True):

{ipulse_shared_core_ftredge-2.51 → ipulse_shared_core_ftredge-2.52/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.51
+Version: 2.52
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar