ipulse-shared-core-ftredge 2.56__py3-none-any.whl → 2.57__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -1,13 +1,18 @@
 from .models import (Organisation, UserAuth, UserProfile,
                      UserStatus, UserProfileUpdate, pulse_enums)
-from .utils_gcp import (setup_gcp_logger_and_error_report, 
-                        read_csv_from_gcs, read_json_from_gcs, 
-                        write_csv_to_gcs, write_json_to_gcs)
+
+
+from .utils_gcp import (setup_gcp_logger_and_error_report,
+                        read_csv_from_gcs, read_json_from_gcs,
+                        write_csv_to_gcs,write_json_to_gcs)
+from .utils_custom_logs import (ContextLog)
+from .utils_pipelinemon import ( Pipelinemon)
+from .utils_gcp_for_pipelines import (write_json_to_gcs_in_pipeline )
+
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
                                           check_format_against_schema_template)
-from .utils_common import (ContextLog,  Pipelinemon)
 
-from .enums import (TargetLogs, LogLevel, Unit, Frequency,
+from .enums import (TargetLogs, LogStatus, LogLevel, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,

ipulse_shared_core_ftredge/enums/__init__.py

@@ -4,6 +4,7 @@
 # pylint: disable=missing-class-docstring
 
 from .enums_common_utils import (LogLevel,
+                                 LogStatus,
                                 SystemsImpacted,
                                 TargetLogs,
                                 Unit,

ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -42,9 +42,15 @@ class LogLevel(Enum):
     DEBUG = 10  # Detailed debug information (for development/troubleshooting)
 
     INFO = 100
-    INFO_PERSISTNACE_COMPLETE= 101
-    INFO_UPDATE_COMPLETE = 102
-    INFO_DELETE_COMPLETE = 103
+    INFO_REMOTE_PERSISTNACE_COMPLETE= 101
+    INFO_REMOTE_UPDATE_COMPLETE = 102
+    INFO_REMOTE_DELETE_COMPLETE = 103
+
+    INFO_REMOTE_BULK_PERSISTNACE_COMPLETE= 111
+    INFO_REMOTE_BULK_UPDATE_COMPLETE = 112
+    INFO_REMOTE_BULK_DELETE_COMPLETE = 113
+
+    INFO_LOCAL_PERSISTNACE_COMPLETE = 121
 
     SUCCESS = 201
     SUCCESS_WITH_NOTICES = 211

ipulse_shared_core_ftredge/models/user_profile.py

@@ -33,9 +33,9 @@ class UserProfile(BaseModel):
     provider_id: str   #User can Read only
     
     username: Optional[str] = None #User can Read and Edit
-    dob: Optional[date] = None #User can Read and Edit 
-    first_name: Optional[str] = None #User can Read and Edit 
-    last_name: Optional[str] = None #User can Read and Edit 
+    dob: Optional[date] = None #User can Read and Edit
+    first_name: Optional[str] = None #User can Read and Edit
+    last_name: Optional[str] = None #User can Read and Edit
     mobile: Optional[str] = None #User can Read and Edit
     class Config:
         extra = "forbid"

ipulse_shared_core_ftredge/utils_custom_logs.py

@@ -0,0 +1,201 @@
+
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+import traceback
+import json
+from datetime import datetime, timezone
+from typing import List
+from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel, LogStatus
+
+
+class ContextLog:
+
+    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
+                context: str = None, description: str = None,
+                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                log_status: LogStatus = LogStatus.OPEN, subject: str = None, systems_impacted: List[str] = None,
+                ):
+
+        if e is not None:
+            e_type = type(e).__name__ if e_type is None else e_type
+            e_message = str(e) if e_message is None else e_message
+            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+        elif e_traceback is None and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+        self.level = level
+        self.subject = subject
+        self.description = description
+        self._base_context = base_context
+        self._context = context
+        self._systems_impacted = systems_impacted if systems_impacted else []
+        self.collector_id = collector_id
+        self.exception_type = e_type
+        self.exception_message = e_message
+        self.exception_traceback = e_traceback
+        self.log_status = log_status
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+
+    @property
+    def base_context(self):
+        return self._base_context
+
+    @base_context.setter
+    def base_context(self, value):
+        self._base_context = value
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = value
+
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def remove_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is not None:
+            self._systems_impacted.remove(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
+    def _format_traceback(self, e_traceback, e_message, max_field_len:int, max_traceback_lines:int):
+        if not e_traceback or e_traceback == 'None\n':
+            return None
+
+        traceback_lines = e_traceback.splitlines()
+
+            # Check if the traceback is within the limits
+        if len(traceback_lines) <= max_traceback_lines and len(e_traceback) <= max_field_len:
+            return e_traceback
+
+        # Remove lines that are part of the exception message if they are present in traceback
+        message_lines = e_message.splitlines() if e_message else []
+        if message_lines:
+            for message_line in message_lines:
+                if message_line in traceback_lines:
+                    traceback_lines.remove(message_line)
+
+        # Filter out lines from third-party libraries (like site-packages)
+        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
+
+        # If filtering results in too few lines, revert to original traceback
+        if len(filtered_lines) < 2:
+            filtered_lines = traceback_lines
+
+        # Combine standalone bracket lines with previous or next lines
+        combined_lines = []
+        for line in filtered_lines:
+            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
+                combined_lines[-1] += " " + line.strip()
+            else:
+                combined_lines.append(line)
+
+            # Ensure the number of lines doesn't exceed MAX_TRACEBACK_LINES
+        if len(combined_lines) > max_traceback_lines:
+            keep_lines_start = min(max_traceback_lines // 2, len(combined_lines))
+            keep_lines_end = min(max_traceback_lines // 2, len(combined_lines) - keep_lines_start)
+            combined_lines = (
+                combined_lines[:keep_lines_start] +
+                ['... (truncated) ...'] +
+                combined_lines[-keep_lines_end:]
+            )
+
+        formatted_traceback = '\n'.join(combined_lines)
+
+        # Ensure the total length doesn't exceed MAX_TRACEBACK_LENGTH
+        if len(formatted_traceback) > max_field_len:
+            truncated_length = max_field_len - len('... (truncated) ...')
+            half_truncated_length = truncated_length // 2
+            formatted_traceback = (
+                formatted_traceback[:half_truncated_length] +
+                '\n... (truncated) ...\n' +
+                formatted_traceback[-half_truncated_length:]
+            )
+        return formatted_traceback
+
+    def to_dict(self, max_field_len:int =10000, size_limit:float=256 * 1024 * 0.80,max_traceback_lines:int = 30):
+        size_limit = int(size_limit)  # Ensure size_limit is an integer
+
+        # Unified list of all fields
+        systems_impacted_str = f"{len(self.systems_impacted)} system(s): " + " ,,, ".join(self.systems_impacted) if self.systems_impacted else None
+        fields = [
+            ("log_status", str(self.log_status.name)),
+            ("level_code", self.level.value),
+            ("level_name", str(self.level.name)),
+            ("base_context", str(self.base_context)),
+            ("timestamp", str(self.timestamp)),
+            ("collector_id", str(self.collector_id)),
+            ("systems_impacted", systems_impacted_str),
+            ("context", str(self.context)),  # special sizing rules apply to it
+            ("subject", str(self.subject)),
+            ("description", str(self.description)),
+            ("exception_type", str(self.exception_type)),
+            ("exception_message", str(self.exception_message)),
+            ("exception_traceback", str(self._format_traceback(self.exception_traceback,self.exception_message, max_field_len, max_traceback_lines)))
+        ]
+
+        # Function to calculate the byte size of a JSON-encoded field
+        def field_size(key, value):
+            return len(json.dumps({key: value}).encode('utf-8'))
+
+        # Function to truncate a value based on its type
+        # Function to truncate a value based on its type
+        def truncate_value(value, max_size):
+            if isinstance(value, str):
+                half_size = max_size // 2
+                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
+            return value
+
+         # Ensure no field exceeds max_field_len
+        for i, (key, value) in enumerate(fields):
+            if isinstance(value, str) and len(value) > max_field_len:
+                fields[i] = (key, truncate_value(value, max_field_len))
+
+        # Ensure total size of the dict doesn't exceed size_limit
+        total_size = sum(field_size(key, value) for key, value in fields)
+        log_dict = {}
+        truncated = False
+
+        if total_size > size_limit:
+            truncated = True
+            remaining_size = size_limit
+            remaining_fields = len(fields)
+
+            for key, value in fields:
+                if remaining_fields > 0:
+                    max_size_per_field = remaining_size // remaining_fields
+                else:
+                    max_size_per_field = 0
+
+                field_sz = field_size(key, value)
+                if field_sz > max_size_per_field:
+                    value = truncate_value(value, max_size_per_field)
+                    field_sz = field_size(key, value)
+
+                log_dict[key] = value
+                remaining_size -= field_sz
+                remaining_fields -= 1
+        else:
+            log_dict = dict(fields)
+
+        log_dict['trunc'] = truncated
+
+        return log_dict

ipulse_shared_core_ftredge/utils_gcp.py

@@ -14,7 +14,6 @@ import traceback
 from google.cloud import error_reporting, logging as cloud_logging
 from google.api_core.exceptions import NotFound
 
-
 ############################################################################
 ##################### SETTING UP LOGGER ##########################
 
@@ -124,15 +123,22 @@ def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
 
 
 
-def write_json_to_gcs(bucket_name, storage_client, data, file_name,
-                      save_locally=False, local_path=None, logger=None, max_retries=2,
-                      overwrite_if_exists=False, increment_if_exists=False):
+def write_json_to_gcs( storage_client, data, bucket_name, file_name,
+                      file_exists_if_starts_with_prefix=None, overwrite_if_exists=False, increment_if_exists=False,
+                      save_locally=False, local_path=None,  max_retries=2, max_deletable_files=1, logger=None):
     """Saves data to Google Cloud Storage and optionally locally.
-
-    This function attempts to upload data to GCS. If the upload fails after
-    retries and `save_locally` is True or `local_path` is provided, it attempts
-    to save the data locally.
-    It also tries to handle file name conflicts by overwriting or incrementing. If both are provided as Ture, an exception will be raised.
+    
+    This function attempts to upload data to GCS. 
+    - If the upload fails after retries and `save_locally` is True or `local_path` is provided, it attempts to save the data locally.
+    - It handles file name conflicts based on these rules:
+        - If `overwrite_if_exists` is True: 
+            - If `file_exists_if_contains_substr` is provided, ANY existing file containing the substring is deleted, and the new file is saved with the provided `file_name`.
+            - If `file_exists_if_contains_substr` is None, and a file with the exact `file_name` exists, it's overwritten.
+        - If `increment_if_exists` is True:
+            - If `file_exists_if_contains_substr` is provided, a new file with an incremented version is created ONLY if a file with the EXACT `file_name` exists.
+            - If `file_exists_if_contains_substr` is None, a new file with an incremented version is created if a file with the exact `file_name` exists. 
+            
+        -If both overwrite_if_exists and increment_if_exists are provided as Ture, an exception will be raised.
     """
 
     def log_message(message):
@@ -147,100 +153,141 @@ def write_json_to_gcs(bucket_name, storage_client, data, file_name,
         if logger:
             logger.warning(message)
 
-    attempts = 0
-    success = False
-    gcs_path = None
-    local_path_final = None
-    gcs_file_overwritten = False
-    gcs_file_already_exists = False
-    gcs_file_saved_with_increment = False
-    gcs_upload_exception = None  # Store potential GCS exception
-
-    # Check for conflicting options
+    # Input validation
     if overwrite_if_exists and increment_if_exists:
-        raise ValueError("When writing JSON to GCS, both overwrite and increment_if_exists cannot be True at the same time.")
+        raise ValueError("Both 'overwrite_if_exists' and 'increment_if_exists' cannot be True simultaneously.")
+    if not isinstance(data, (list, dict, str)):
+        raise ValueError("Unsupported data type. Data must be a list, dict, or str.")
+    if max_deletable_files > 10:
+        raise ValueError("max_deletable_files should be less than 10 for safety. For more use another method.")
 
+    # Prepare data
     if isinstance(data, (list, dict)):
         data_str = json.dumps(data, indent=2)
-    elif isinstance(data, str):
-        data_str = data
     else:
-        raise ValueError("Unsupported data type. It should be a list, dict, or str.")
+        data_str = data
 
     bucket = storage_client.bucket(bucket_name)
     base_file_name, ext = os.path.splitext(file_name)
     increment = 0
+    attempts = 0
+    success = False
 
-    while attempts < max_retries and not success:
-        try:
-            if increment_if_exists:
-                while bucket.blob(file_name).exists():
-                    gcs_file_already_exists = True
-                    increment += 1
-                    file_name = f"{base_file_name}_{increment}{ext}"
-                    gcs_file_saved_with_increment = True
-                    log_warning(f"File {file_name} already exists in bucket {bucket_name}. Writing with increment: {increment_if_exists}")
+    # GCS-related metadata
+    gcs_path = None
+    gcs_file_overwritten = False
+    gcs_file_already_exists = False
+    gcs_file_saved_with_increment = False
+    gcs_file_exists_checked_on_name = file_name
+    gcs_deleted_files=[]
+
+    # GCS upload exception
+    gcs_upload_exception = None
+
+     # Local file path
+    local_path_final = None
+
+    try:
+        # --- Overwrite Logic ---
+        if overwrite_if_exists:
+            if file_exists_if_starts_with_prefix:
+                gcs_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+                blobs_to_delete = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+                if len(blobs_to_delete) > max_deletable_files:
+                    raise Exception(f"Error: Attempt to delete {len(blobs_to_delete)} matched files, but limit is {max_deletable_files}.")
+                if blobs_to_delete:
+                    log_message(f"Deleting files containing '{file_exists_if_starts_with_prefix}' for overwrite.")
+                    for blob in blobs_to_delete:
+                        blob.delete()
+                        gcs_deleted_files.append(blob.name)
+                        log_message(f"Deleted: gs://{bucket_name}/{blob.name}")
+                    gcs_file_overwritten = True
             else:
                 blob = bucket.blob(file_name)
-
-                # Check if the file exists
                 if blob.exists():
                     gcs_file_already_exists = True
                     gcs_path = f"gs://{bucket_name}/{file_name}"
-                    log_message(f"File {file_name} already exists in bucket {bucket_name}. Overwriting: {overwrite_if_exists}")
-                    if not overwrite_if_exists:
-                        log_warning(f"File {file_name} already exists and overwrite is set to False. Skipping save to GCS.")
-                        break
+                    log_message(f"File '{file_name}' already exists. Overwriting.")
+                    blob.delete()  # Delete the existing blob
+                    gcs_deleted_files.append(blob.name)
+                    gcs_file_overwritten = True
+
+        # --- Increment Logic ---
+        elif increment_if_exists:
+            gcs_file_exists_checked_on_name = file_name  # We only increment if the exact name exists
+            while bucket.blob(file_name).exists():
+                gcs_file_already_exists = True
+                increment += 1
+                file_name = f"{base_file_name}_v{increment}{ext}"
+                gcs_file_saved_with_increment = True
+                log_warning(f"File already exists. Using incremented name: {file_name}")
+
+        # --- GCS Upload ---
+        if overwrite_if_exists or increment_if_exists:  # Only upload if either overwrite or increment is True
+            while attempts < max_retries and not success:
+                try:
+                    blob = bucket.blob(file_name)  # Use the potentially updated file_name
+                    blob.upload_from_string(data_str, content_type='application/json')
+                    gcs_path = f"gs://{bucket_name}/{file_name}"
+                    log_message(f"Successfully saved file to GCS: {gcs_path}")
+                    success = True
+                except Exception as e:
+                    gcs_upload_exception=e
+                    attempts += 1
+                    if attempts < max_retries:
+                        log_warning(f"Attempt {attempts} to upload to GCS failed. Retrying...")
+                        time.sleep(2 ** attempts)
                     else:
-                        gcs_file_overwritten = True
-
-            blob.upload_from_string(data_str, content_type='application/json')
-            gcs_path = f"gs://{bucket_name}/{file_name}"
-            log_message(f"Successfully saved file to GCS {gcs_path}.")
-            success = True
-        except Exception as e:
-            gcs_upload_exception = e
-            attempts += 1
-            if attempts < max_retries:
-                time.sleep(2 ** attempts)
-            else:
-                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts: {e}")
+                        log_error(f"Failed to write '{file_name}' to GCS bucket '{bucket_name}' after {max_retries} attempts: {e}", exc_info=True)
+                        if save_locally or local_path:
+                            log_message(f"Attempting to save '{file_name}' locally due to GCS upload failure.")
+    except Exception as e:
+        log_error(f"Error during GCS operations: {e}", exc_info=True)
+        gcs_upload_exception = e
 
+    # --- Save Locally ---
+    write_out=False
     if not success or save_locally or local_path:
         try:
-            if not local_path:
-                local_path_final = os.path.join("/tmp", file_name)
-            else:
-                local_path_final = os.path.join(local_path, file_name)
+            local_path=local_path if local_path else "/tmp"
+            local_path_final = os.path.join(local_path, file_name)
 
             if os.path.exists(local_path_final):
                 if increment_if_exists:
                     increment = 0
                     while os.path.exists(local_path_final):
                         increment += 1
-                        local_path_final = os.path.join(local_path, f"{base_file_name}_{increment}{ext}")
-                elif not overwrite_if_exists:
-                    log_message(f"File {file_name} already exists locally at {local_path_final} and overwrite is set to False. Skipping save.")
-                    success = True
-                else:
+                        local_path_final = os.path.join(local_path, f"{base_file_name}_v{increment}{ext}")
+                    log_warning(f"Local file already exists. Using incremented name: {local_path_final}")
+                    write_out=True
+                elif overwrite_if_exists:
+                    write_out=True
                     log_message(f"File {file_name} already exists locally at {local_path_final}. Overwriting: {overwrite_if_exists}")
+                else:
+                    log_message(f"File {file_name} already exists locally at {local_path_final} and overwrite is set to False. Skipping save.")
+                    write_out=False
+            else:
+                write_out=True
 
-            if not success:
+            if write_out:
                 with open(local_path_final, 'w', encoding='utf-8') as f:
                     f.write(data_str)
-                log_message(f"Saved {file_name} locally at {local_path_final}. Overwritten: {overwrite_if_exists}")
-                success = True
+                    log_message(f"Saved {file_name} locally at {local_path_final}. Overwritten: {overwrite_if_exists}")
+
         except Exception as local_e:
             log_error(f"Failed to write {file_name} locally: {local_e}", exc_info=True)
 
     if gcs_upload_exception is not None:
         raise gcs_upload_exception  # Propagate without nesting
 
+    # --- Return Metadata ---
     return {
-        "gcs_path": gcs_path,
-        "local_path": local_path_final,
+        "gcs_path": gcs_path if success else None,  # Only set gcs_path if upload succeeded
+        "local_path": local_path_final if write_out else None,  # Only set local_path if saved locally
         "gcs_file_already_exists": gcs_file_already_exists,
+        "gcs_file_exists_checked_on_name":gcs_file_exists_checked_on_name ,
         "gcs_file_overwritten": gcs_file_overwritten,
+        "gcs_deleted_file_names": ",,,".join(gcs_deleted_files) if gcs_deleted_files else None,
         "gcs_file_saved_with_increment": gcs_file_saved_with_increment
     }
 

ipulse_shared_core_ftredge/utils_gcp_for_pipelines.py

@@ -0,0 +1,201 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+import json
+import os
+import time
+from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel
+from .utils_custom_logs import ContextLog
+from ipulse_shared_core_ftredge.utils_pipelinemon import Pipelinemon
+
+############################################################################
+##################### SETTING UP LOGGER ##########################
+
+####DEPCREACATED: THIS APPROACH WAS GOOD, BUT ERRORS WERE NOT REPORTED TO ERROR REPORTING
+# logging.basicConfig(level=logging.INFO)
+# logging_client = google.cloud.logging.Client()
+# logging_client.setup_logging()
+###################################
+
+
+##### THIS APPROACH IS USED NOW ########
+ENV = os.getenv('ENV', 'LOCAL').strip("'")
+
+
+def write_json_to_gcs_in_pipeline( pipelinemon:Pipelinemon, storage_client, data, bucket_name, file_name,
+                      file_exists_if_starts_with_prefix:str=None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
+                      save_locally:bool=False, local_path=None,  max_retries:int=2, max_deletable_files:int=1):
+    """Saves data to Google Cloud Storage and optionally locally.
+    
+    This function attempts to upload data to GCS. 
+    - If the upload fails after retries and `save_locally` is True or `local_path` is provided, it attempts to save the data locally.
+    - It handles file name conflicts based on these rules:
+        - If `overwrite_if_exists` is True: 
+            - If `file_exists_if_contains_substr` is provided, ANY existing file containing the substring is deleted, and the new file is saved with the provided `file_name`.
+            - If `file_exists_if_contains_substr` is None, and a file with the exact `file_name` exists, it's overwritten.
+        - If `increment_if_exists` is True:
+            - If `file_exists_if_contains_substr` is provided, a new file with an incremented version is created ONLY if a file with the EXACT `file_name` exists.
+            - If `file_exists_if_contains_substr` is None, a new file with an incremented version is created if a file with the exact `file_name` exists. 
+            
+        -If both overwrite_if_exists and increment_if_exists are provided as Ture, an exception will be raised.
+    """
+
+    with pipelinemon.context("write_json_to_gcs_in_pipeline"):
+        # GCS upload exception
+        gcs_upload_error = False
+        # Input validation
+        if overwrite_if_exists and increment_if_exists:
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM, subject="Param validation", description="Both 'overwrite_if_exists' and 'increment_if_exists' cannot be True simultaneously."))
+            gcs_upload_error=True
+        if not isinstance(data, (list, dict, str)):
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM,subject="Data validation", description="Unsupported data type. Data must be a list, dict, or str."))
+            gcs_upload_error=True
+        if max_deletable_files > 10:
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM,subject="max_deletable_files", description="max_deletable_files should be less than 10 for safety. For more use another method."))
+            gcs_upload_error=True
+
+        # Prepare data
+        if isinstance(data, (list, dict)):
+            data_str = json.dumps(data, indent=2)
+        else:
+            data_str = data
+
+        bucket = storage_client.bucket(bucket_name)
+        base_file_name, ext = os.path.splitext(file_name)
+        increment = 0
+        attempts = 0
+        success = False
+
+        # GCS-related metadata
+        gcs_path = None
+        gcs_file_overwritten = False
+        gcs_file_already_exists = False
+        gcs_file_saved_with_increment = False
+        gcs_file_exists_checked_on_name = file_name
+        gcs_deleted_files=[]
+
+        # Local file path
+        local_path_final = None
+
+        try:
+            upload_allowed = True
+            # --- Overwrite Logic ---
+            if overwrite_if_exists:
+                with pipelinemon.context("overwriting"):
+                    if file_exists_if_starts_with_prefix:
+                        gcs_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+                        blobs_to_delete = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+                        if len(blobs_to_delete) > max_deletable_files:
+                            pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
+                            #### Ensuring to quit the operation if too many files are found
+                            raise Exception(f"Error: Attempt to delete {len(blobs_to_delete)} matched files, but limit is {max_deletable_files}.")
+                        if blobs_to_delete:
+                            gcs_file_already_exists = True
+                            pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
+                            for blob in blobs_to_delete:
+                                gcs_path_del = f"gs://{bucket_name}/{blob.name}"
+                                pipelinemon.add_system_impacted(f"delete: gcs_bucket_file: {gcs_path_del}")
+                                blob.delete()
+                                pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= gcs_path_del, description="file deleted from GCS as part of overwrite, matched with prefix"))
+                                gcs_deleted_files.append(blob.name)
+                            gcs_file_overwritten = True
+                    else:
+                        blob = bucket.blob(file_name)
+                        if blob.exists():
+                            gcs_file_already_exists = True
+                            pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}"))
+                            gcs_path_del = f"gs://{bucket_name}/{file_name}"
+                            pipelinemon.add_system_impacted(f"delete: gcs_bucket_file: {gcs_path_del}")
+                            blob.delete()  # Delete the existing blob
+                            pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= gcs_path_del, description="file deleted from GCS as part of overwrite, matched with exact name"))
+                            gcs_deleted_files.append(blob.name)
+                            gcs_file_overwritten = True
+
+            # --- Increment Logic ---
+            elif increment_if_exists:
+                with pipelinemon.context("incrementing"):
+                    gcs_file_exists_checked_on_name = file_name  # We only increment if the exact name exists
+                    while bucket.blob(file_name).exists():
+                        gcs_file_already_exists = True
+                        increment += 1
+                        file_name = f"{base_file_name}_v{increment}{ext}"
+                        gcs_file_saved_with_increment = True
+                    if increment>0:
+                        pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"File saved with incremented version in {bucket_name}"))
+
+             # --- Check for Conflicts (Including Prefix) ---
+            else:
+                if file_exists_if_starts_with_prefix:
+                    blobs_matched = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+                    if blobs_matched:
+                        upload_allowed = False
+                        pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_matched)} existing files in bucket {bucket_name}."))
+                    elif bucket.blob(file_name).exists():
+                        pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}."))
+                        upload_allowed = False
+
+            # --- GCS Upload ---
+            if overwrite_if_exists or increment_if_exists or upload_allowed:
+                with pipelinemon.context("uploading"):
+                    while attempts < max_retries and not success:
+                        try:
+                            gcs_path = f"gs://{bucket_name}/{file_name}"
+                            blob = bucket.blob(file_name)  # Use the potentially updated file_name
+                            pipelinemon.add_system_impacted(f"upload: gcs_bucket_file: {gcs_path}")
+                            blob.upload_from_string(data_str, content_type='application/json')
+                            pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_PERSISTNACE_COMPLETE, subject= gcs_path, description="file uploaded to GCS"))
+                            success = True
+                        except Exception as e:
+                            attempts += 1
+                            if attempts < max_retries:
+                                time.sleep(2 ** attempts)
+                            else:
+                                pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e))
+                                gcs_upload_error = True
+
+        except Exception as e:
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e))
+            gcs_upload_error = True
+
+        # --- Save Locally ---
+        ###### Not logging the local save operation in pipeline, as it is not a critical operation
+        write_out=False
+        if not success or gcs_upload_error or save_locally or local_path:
+            try:
+                local_path=local_path if local_path else "/tmp"
+                local_path_final = os.path.join(local_path, file_name)
+
+                if os.path.exists(local_path_final):
+                    if increment_if_exists:
+                        increment = 0
+                        while os.path.exists(local_path_final):
+                            increment += 1
+                            local_path_final = os.path.join(local_path, f"{base_file_name}_v{increment}{ext}")
+                        write_out=True
+                    elif overwrite_if_exists:
+                        write_out=True
+                    else:
+                        write_out=False
+                else:
+                    write_out=True
+
+                if write_out:
+                    with open(local_path_final, 'w', encoding='utf-8') as f:
+                        f.write(data_str)
+
+            except Exception as local_e:
+                pipelinemon.add_log(ContextLog(LogLevel.WARNING_FIX_RECOMMENDED, e=local_e, description="Failed to write to local file"))
+
+        # --- Return Metadata ---
+        return {
+            "gcs_path": gcs_path if success else None,  # Only set gcs_path if upload succeeded
+            "local_path": local_path_final if write_out else None,  # Only set local_path if saved locally
+            "gcs_file_already_exists": gcs_file_already_exists,
+            "gcs_file_exists_checked_on_name":gcs_file_exists_checked_on_name ,
+            "gcs_file_overwritten": gcs_file_overwritten,
+            "gcs_deleted_file_names": ",,,".join(gcs_deleted_files) if gcs_deleted_files else None,
+            "gcs_file_saved_with_increment": gcs_file_saved_with_increment
+        }

ipulse_shared_core_ftredge/{utils_common.py → utils_pipelinemon.py}

@@ -4,208 +4,17 @@
 # pylint: disable=line-too-long
 # pylint: disable=missing-class-docstring
 # pylint: disable=broad-exception-caught
-import traceback
 import json
 import uuid
 from datetime import datetime, timezone
 from contextlib import contextmanager
 from typing import List
 from google.cloud import logging as cloudlogging
-from ipulse_shared_core_ftredge.enums.enums_common_utils import TargetLogs, LogLevel, LogStatus
+from ipulse_shared_core_ftredge.enums.enums_common_utils import TargetLogs, LogLevel
 from ipulse_shared_core_ftredge.utils_gcp import write_json_to_gcs
+from ipulse_shared_core_ftredge.utils_custom_logs import ContextLog
 
 
-# ["data_import","data_quality", "data_processing","data_general","data_persistance","metadata_quality", "metadata_processing", "metadata_persistance","metadata_general"]
-
-class ContextLog:
-    MAX_FIELD_LINES = 26  # Define the maximum number of traceback lines to include
-    MAX_FIELD_LENGTH=10000
-
-    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
-                context: str = None, description: str = None,
-                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
-                log_status: LogStatus = LogStatus.OPEN, subject: str = None, systems_impacted: List[str] = None
-                ):
-        if e is not None:
-            e_type = type(e).__name__ if e_type is None else e_type
-            e_message = str(e) if e_message is None else e_message
-            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
-        elif e_traceback is None and (e_type or e_message):
-            e_traceback = traceback.format_exc()
-
-        self.level = level
-        self.subject = subject
-        self.description = description
-        self._base_context = base_context
-        self._context = context
-        self._systems_impacted = systems_impacted if systems_impacted else []
-        self.collector_id = collector_id
-        self.exception_type = e_type
-        self.exception_message = e_message
-        self.exception_traceback = e_traceback
-        self.log_status = log_status
-        self.timestamp = datetime.now(timezone.utc).isoformat()
-
-    @property
-    def base_context(self):
-        return self._base_context
-
-    @base_context.setter
-    def base_context(self, value):
-        self._base_context = value
-
-    @property
-    def context(self):
-        return self._context
-
-    @context.setter
-    def context(self, value):
-        self._context = value
-
-    @property
-    def systems_impacted(self):
-        return self._systems_impacted
-
-    @systems_impacted.setter
-    def systems_impacted(self, list_of_si: List[str]):
-        self._systems_impacted = list_of_si
-
-    def add_system_impacted(self, system_impacted: str):
-        if self._systems_impacted is None:
-            self._systems_impacted = []
-        self._systems_impacted.append(system_impacted)
-
-    def remove_system_impacted(self, system_impacted: str):
-        if self._systems_impacted is not None:
-            self._systems_impacted.remove(system_impacted)
-
-    def clear_systems_impacted(self):
-        self._systems_impacted = []
-
-    def _format_traceback(self, e_traceback, e_message):
-        if not e_traceback or e_traceback == 'None\n':
-            return None
-
-        traceback_lines = e_traceback.splitlines()
-
-            # Check if the traceback is within the limits
-        if len(traceback_lines) <= self.MAX_FIELD_LINES and len(e_traceback) <= self.MAX_FIELD_LENGTH:
-            return e_traceback
-
-        # Remove lines that are part of the exception message if they are present in traceback
-        message_lines = e_message.splitlines() if e_message else []
-        if message_lines:
-            for message_line in message_lines:
-                if message_line in traceback_lines:
-                    traceback_lines.remove(message_line)
-
-        # Filter out lines from third-party libraries (like site-packages)
-        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
-
-        # If filtering results in too few lines, revert to original traceback
-        if len(filtered_lines) < 2:
-            filtered_lines = traceback_lines
-
-        # Combine standalone bracket lines with previous or next lines
-        combined_lines = []
-        for line in filtered_lines:
-            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
-                combined_lines[-1] += " " + line.strip()
-            else:
-                combined_lines.append(line)
-
-            # Ensure the number of lines doesn't exceed MAX_TRACEBACK_LINES
-        if len(combined_lines) > self.MAX_FIELD_LINES:
-            keep_lines_start = min(self.MAX_FIELD_LINES // 2, len(combined_lines))
-            keep_lines_end = min(self.MAX_FIELD_LINES // 2, len(combined_lines) - keep_lines_start)
-            combined_lines = (
-                combined_lines[:keep_lines_start] +
-                ['... (truncated) ...'] +
-                combined_lines[-keep_lines_end:]
-            )
-
-        formatted_traceback = '\n'.join(combined_lines)
-
-        # Ensure the total length doesn't exceed MAX_TRACEBACK_LENGTH
-        if len(formatted_traceback) > self.MAX_FIELD_LENGTH:
-            truncated_length = self.MAX_FIELD_LENGTH - len('... (truncated) ...')
-            half_truncated_length = truncated_length // 2
-            formatted_traceback = (
-                formatted_traceback[:half_truncated_length] +
-                '\n... (truncated) ...\n' +
-                formatted_traceback[-half_truncated_length:]
-            )
-        return formatted_traceback
-
-    def to_dict(self, max_field_len:int =10000, size_limit:float=256 * 1024 * 0.80):
-        size_limit = int(size_limit)  # Ensure size_limit is an integer
-
-        # Unified list of all fields
-        systems_impacted_str = f"{len(self.systems_impacted)} system(s): " + " ,,, ".join(self.systems_impacted) if self.systems_impacted else None
-        fields = [
-            ("log_status", str(self.log_status.name)),
-            ("level_code", self.level.value),
-            ("level_name", str(self.level.name)),
-            ("base_context", str(self.base_context)),
-            ("timestamp", str(self.timestamp)),
-            ("collector_id", str(self.collector_id)),
-            ("systems_impacted", systems_impacted_str),
-            ("context", str(self.context)),  # special sizing rules apply to it
-            ("subject", str(self.subject)),
-            ("description", str(self.description)),
-            ("exception_type", str(self.exception_type)),
-            ("exception_message", str(self.exception_message)),
-            ("exception_traceback", str(self._format_traceback(self.exception_traceback,self.exception_message)))
-        ]
-
-        # Function to calculate the byte size of a JSON-encoded field
-        def field_size(key, value):
-            return len(json.dumps({key: value}).encode('utf-8'))
-
-        # Function to truncate a value based on its type
-        # Function to truncate a value based on its type
-        def truncate_value(value, max_size):
-            if isinstance(value, str):
-                half_size = max_size // 2
-                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
-            return value
-
-         # Ensure no field exceeds max_field_len
-        for i, (key, value) in enumerate(fields):
-            if isinstance(value, str) and len(value) > max_field_len:
-                fields[i] = (key, truncate_value(value, max_field_len))
-
-        # Ensure total size of the dict doesn't exceed size_limit
-        total_size = sum(field_size(key, value) for key, value in fields)
-        log_dict = {}
-        truncated = False
-
-        if total_size > size_limit:
-            truncated = True
-            remaining_size = size_limit
-            remaining_fields = len(fields)
-
-            for key, value in fields:
-                if remaining_fields > 0:
-                    max_size_per_field = remaining_size // remaining_fields
-                else:
-                    max_size_per_field = 0
-
-                field_sz = field_size(key, value)
-                if field_sz > max_size_per_field:
-                    value = truncate_value(value, max_size_per_field)
-                    field_sz = field_size(key, value)
-
-                log_dict[key] = value
-                remaining_size -= field_sz
-                remaining_fields -= 1
-        else:
-            log_dict = dict(fields)
-
-        log_dict['trunc'] = truncated
-
-        return log_dict
-
 class Pipelinemon:
     ERROR_START_CODE = LogLevel.ERROR.value
     WARNING_START_CODE = LogLevel.WARNING.value
@@ -213,7 +22,7 @@ class Pipelinemon:
     SUCCESS_START_CODE = LogLevel.SUCCESS.value
     INFO_START_CODE = LogLevel.INFO.value
 
-    def __init__(self, base_context: str, target_logs: TargetLogs = TargetLogs.MIXED, logger_name=None, max_log_field_size:int =10000, max_log_dict_size:float=256 * 1024 * 0.80):
+    def __init__(self, base_context: str, target_logs: TargetLogs = TargetLogs.MIXED, logger_name=None, max_log_field_size:int =10000, max_log_dict_size:float=256 * 1024 * 0.80, max_log_traceback_lines:int = 30):
         self._id = str(uuid.uuid4())
         self._logs = []
         self._early_stop = False
@@ -223,13 +32,14 @@ class Pipelinemon:
         self._successes_count = 0
         self._infos_count = 0
         self._systems_impacted = []
-        self._level_counts = {level.name: 0 for level in LogLevel}
+        self._by_level_counts = {level.name: 0 for level in LogLevel}
         self._base_context = base_context
         self._context_stack = []
         self._target_logs = target_logs.value
         self._logger = self._initialize_logger(logger_name)
         self._max_log_field_size = max_log_field_size
         self._max_log_dict_size = max_log_dict_size
+        self._max_log_traceback_lines = max_log_traceback_lines
 
     def _initialize_logger(self, logger_name):
         if logger_name:
@@ -280,6 +90,14 @@ class Pipelinemon:
     def clear_systems_impacted(self):
         self._systems_impacted = []
 
+    @property
+    def max_log_dict_size(self):
+        return self._max_log_dict_size
+
+    @max_log_dict_size.setter
+    def max_log_dict_size(self, value):
+        self._max_log_dict_size = value
+
     @property
     def max_log_field_size(self):
         return self._max_log_field_size
@@ -289,12 +107,12 @@ class Pipelinemon:
         self._max_log_field_size = value
 
     @property
-    def max_log_dict_size(self):
-        return self._max_log_dict_size
+    def max_log_traceback_lines(self):
+        return self._max_log_traceback_lines
 
-    @max_log_dict_size.setter
-    def max_log_dict_size(self, value):
-        self._max_log_dict_size = value
+    @max_log_traceback_lines.setter
+    def max_log_traceback_lines(self, value):
+        self._max_log_traceback_lines = value
 
     @property
     def early_stop(self):
@@ -313,7 +131,7 @@ class Pipelinemon:
         self._early_stop = False
 
 
-    def add_log(self, log: ContextLog, ):
+    def add_log(self, log: ContextLog ):
         if (self._target_logs == TargetLogs.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
            (self._target_logs == TargetLogs.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
             raise ValueError(f"Invalid log level {log.level.name} for Pipelinemon target logs setup: {self._target_logs}")
@@ -321,7 +139,7 @@ class Pipelinemon:
         log.context = self.current_context
         log.collector_id = self.id
         log.systems_impacted = self.systems_impacted
-        log_dict = log.to_dict(max_field_len=self.max_log_field_size, size_limit=self.max_log_dict_size)
+        log_dict = log.to_dict(max_field_len=self.max_log_field_size, size_limit=self.max_log_dict_size, max_traceback_lines=self.max_log_traceback_lines)
         self._logs.append(log_dict)
         self._update_counts(log_dict)
 
@@ -346,7 +164,7 @@ class Pipelinemon:
         self._notices_count = 0
         self._successes_count = 0
         self._infos_count = 0
-        self._level_counts = {level.name: 0 for level in LogLevel}
+        self._by_level_counts = {level.name: 0 for level in LogLevel}
 
     def clear_logs(self):
         self._logs = []
@@ -397,7 +215,7 @@ class Pipelinemon:
         return len(self._logs)
 
     def count_logs_by_level(self, level: LogLevel):
-        return self._level_counts.get(level.name, 0)
+        return self._by_level_counts.get(level.name, 0)
 
     def _count_logs(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
         return sum(
@@ -528,7 +346,7 @@ class Pipelinemon:
                 self._successes_count -= 1
             elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
                 self._infos_count -= 1
-            self._level_counts[level_name] -= 1
+            self._by_level_counts[level_name] -= 1
         else:
             if level_code >= self.ERROR_START_CODE:
                 self._errors_count += 1
@@ -540,4 +358,5 @@ class Pipelinemon:
                 self._successes_count += 1
             elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
                 self._infos_count += 1
-            self._level_counts[level_name] += 1
+            self._by_level_counts[level_name] += 1
+

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -6,7 +6,7 @@
 import datetime
 from google.cloud import bigquery
 from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel
-from ipulse_shared_core_ftredge.utils_common import ContextLog
+from ipulse_shared_core_ftredge.utils_custom_logs import ContextLog
 
 
 def create_bigquery_schema_from_json(json_schema):
@@ -59,16 +59,14 @@ def check_format_against_schema_template(data_to_check, schema, dt_ts_to_str=Tru
                     checked_data[field_name] = value
 
             elif mode == "REQUIRED":
-                warning = ContextLog(level=LogLevel.WARNING,
+                warnings_or_error.append(ContextLog(level=LogLevel.WARNING,
                                               subject=field_name,
-                                              description=f"Required field '{field_name}' is missing in the updates.")
-                warnings_or_error.append(warning)
+                                              description=f"Required field '{field_name}' is missing in the updates."))
 
     except Exception as e:
-        error_log = ContextLog(level=LogLevel.ERROR_EXCEPTION_REDO,
+        warnings_or_error.append(ContextLog(level=LogLevel.ERROR_EXCEPTION,
                                subject=data_to_check,
-                               description=f"An error occurred during update check: {str(e)}")
-        warnings_or_error.append(error_log)
+                               description=f"An error occurred during update check: {str(e)}"))
 
     return checked_data, warnings_or_error
 
@@ -144,11 +142,11 @@ def handle_type_conversion(field_type, field_name, value):
         try:
             return float(value), None
         except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED, 
+            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
                                                 subject=field_name,
                                                 description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
     if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED, 
+        return bool(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
                                                 subject=field_name,
                                                 description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
 

{ipulse_shared_core_ftredge-2.56.dist-info → ipulse_shared_core_ftredge-2.57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.56
+Version: 2.57
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.56.dist-info → ipulse_shared_core_ftredge-2.57.dist-info}/RECORD

@@ -1,9 +1,11 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=gIwh_6xjoG7UIuTKNkq0iGvCuxsbyXNrTyV3vB8YdGc,869
-ipulse_shared_core_ftredge/utils_common.py,sha256=UOaEiTE9moJCoSGFnD6aPI_w1UujovWcYOH6ifJxPxg,22533
-ipulse_shared_core_ftredge/utils_gcp.py,sha256=z1Rdgi1wP0768a9vnp9qmj_UtoLdmFOqYMcnrL1v1pw,10999
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=AwGl9J-XQc_aO_VKWhR_TuA1ML8nWxHuzHtxBH8yfwE,7499
-ipulse_shared_core_ftredge/enums/__init__.py,sha256=pdi9OIDyb8pfTFbeBqnhQ_ZO-gnYfmLKwHZBYOMnkqE,888
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=UQG-dd302r5VFhV2kwDMXCg20nsE2ht4XBEZx9rzX1o,6724
+ipulse_shared_core_ftredge/__init__.py,sha256=QHWbeWYQgs314BKxQDzG7_P2mdnryC1Zqc3xvw8OfnM,987
+ipulse_shared_core_ftredge/utils_custom_logs.py,sha256=OJ6STEL7GER-s3x03cKcsYtZ4jKhndPOAJXT1oiAY2M,8183
+ipulse_shared_core_ftredge/utils_gcp.py,sha256=xI0iOxSjGQ75ECFlmPirOB8Cz8qjqYNabfRLEg0cE8o,14142
+ipulse_shared_core_ftredge/utils_gcp_for_pipelines.py,sha256=rO7KD8JUmit0NwKoepKmBX8MlcQ8GzDhUfAyp6OWAw0,11816
+ipulse_shared_core_ftredge/utils_pipelinemon.py,sha256=KRDJW0fqF6sfqsxL8YKH378qpXnOzpZ3C53VAkMoT10,15011
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=vi8hBU95_N-Znfs-FClBNCFMKvJYBOdBwZm2pgBZ7BQ,7433
+ipulse_shared_core_ftredge/enums/__init__.py,sha256=MRmvcrFhbGZbww759KCledFe6BpOArmnCQ18mb_F8Fg,932
+ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=VdOipu5YNVO4TcQMXy7UkCjSnBfUzutEkgfFFsnsp7k,6922
 ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=7w3Jjmw84Wq22Bb5Qs09Z82Bdf-j8nhRiQJfw60_g80,1903
 ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=W1TkSLu3ryLf_aif2VcKsFznWz0igeMUR_buoGEG6w8,1406
 ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
@@ -13,13 +15,11 @@ ipulse_shared_core_ftredge/models/organisation.py,sha256=4f1ATEWh5WT-CDJBLEZUhUw
 ipulse_shared_core_ftredge/models/pulse_enums.py,sha256=YJhtvoX6Dk3_SyJUD8vVDSRIzWy5n0I0AOwe19fmDT8,4851
 ipulse_shared_core_ftredge/models/resource_catalog_item.py,sha256=PxeRvI8fe8KOiHr6NW2Jz_yocyLId09PW8QyTZxjHAA,9809
 ipulse_shared_core_ftredge/models/user_auth.py,sha256=35HNN7ZW4ZELCqaJrAtoSsVLFAZ1KL2S_VmuzbcEMm4,119
-ipulse_shared_core_ftredge/models/user_profile.py,sha256=3-HHB3wK2s4hsP9d5Xp2JYrsGe8iFzz4VaoygDIO_-s,1930
+ipulse_shared_core_ftredge/models/user_profile.py,sha256=D3BB9D6XEv7IVZgsURgf0hWmUZW5rms3uiBXS0ZGLeE,1927
 ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFPhX2XlqEqlKLBQ4AkvPHXEuZbFMY,1712
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
-ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.56.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.56.dist-info/METADATA,sha256=WdSqfe6RTQDh4M9EBSA33UaYrmMud26ZYm-P-LRj5OQ,561
-ipulse_shared_core_ftredge-2.56.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-ipulse_shared_core_ftredge-2.56.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.56.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.57.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.57.dist-info/METADATA,sha256=OFCR8U5m0OwnbR9hyIqSwOWW40QmwDfz1WqnWSVnBxg,561
+ipulse_shared_core_ftredge-2.57.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+ipulse_shared_core_ftredge-2.57.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.57.dist-info/RECORD,,

{ipulse_shared_core_ftredge-2.56.dist-info → ipulse_shared_core_ftredge-2.57.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ipulse_shared_core_ftredge/tests/test.py

@@ -1,17 +0,0 @@
-from ipulse_shared_core_ftredge import UserProfile,  UserAuth
-import datetime
-import logging
-logging.basicConfig(level=logging.INFO)
-ex=UserProfile(uid="uid",
-                organizations_uids={"20231220retailcustomer_coreorgn"},
-                email="email@gmail.com",
-                creat_date= datetime.datetime.now(datetime.UTC),
-                creat_by_user='creat_by_user',
-                updt_date=datetime.datetime.now(datetime.UTC),
-                updt_by_user="subscriber_cf_persistUserAuthToUserProfile",
-                approved=True,
-                provider_id='provider_id',
-                username='username')
-
-
-logging.info(ex.model_dump(exclude_unset=True))