ipulse-shared-core-ftredge 2.6__py3-none-any.whl → 2.6.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils/utils_cloud_gcp_with_collectors.py

@@ -0,0 +1,169 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+
+import json
+import os
+import time
+from typing import Optional
+from google.cloud.storage import Client as GCSClient
+from ipulse_shared_core_ftredge.enums.enums_logs import LogLevel
+from .utils_collector_pipelinemon import Pipelinemon
+from .logs.context_log import ContextLog
+
+
+############################################################################
+##################### SETTING UP LOGGER ##########################
+
+
+
+
+def write_json_to_gcs_with_pipelinemon_extended( pipelinemon:Pipelinemon, storage_client:GCSClient, data:dict | list | str, bucket_name: str, file_name: str,
+                      file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
+                      max_retries:int=2, max_deletable_files:int=1):
+    """Saves data to Google Cloud Storage and optionally locally.
+    
+    This function attempts to upload data to GCS. 
+    - If the upload fails after retries and `save_locally` is True or `local_path` is provided, it attempts to save the data locally.
+    - It handles file name conflicts based on these rules:
+        - If `overwrite_if_exists` is True: 
+            - If `file_exists_if_contains_substr` is provided, ANY existing file containing the substring is deleted, and the new file is saved with the provided `file_name`.
+            - If `file_exists_if_contains_substr` is None, and a file with the exact `file_name` exists, it's overwritten.
+        - If `increment_if_exists` is True:
+            - If `file_exists_if_contains_substr` is provided, a new file with an incremented version is created ONLY if a file with the EXACT `file_name` exists.
+            - If `file_exists_if_contains_substr` is None, a new file with an incremented version is created if a file with the exact `file_name` exists. 
+            
+        -If both overwrite_if_exists and increment_if_exists are provided as Ture, an exception will be raised.
+    """
+
+    cloud_storage_ref="GCP_GCS"
+
+    with pipelinemon.context(f"write_json_to_{cloud_storage_ref}_with_pipelinemon"):
+        cloud_storage_upload_error = None
+        # Input validation
+        if overwrite_if_exists and increment_if_exists:
+            err_msg="Both 'overwrite_if_exists' and 'increment_if_exists' cannot be True simultaneously."
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM, subject="Param validation", description=err_msg))
+            return {"cloud_storage_upload_error": err_msg}
+        if max_deletable_files > 10:
+            err_msg="max_deletable_files should be less than 10 for safety. For more use another method."
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM,subject="max_deletable_files", description=err_msg))
+            return {"cloud_storage_upload_error": err_msg}
+
+        # Prepare data
+        if isinstance(data, (list, dict)):
+            data_str = json.dumps(data, indent=2)
+        else:
+            data_str = data
+
+        bucket = storage_client.bucket(bucket_name)
+        base_file_name, ext = os.path.splitext(file_name)
+        increment = 0
+        attempts = 0
+        success = False
+
+        # GCS-related metadata
+        cloud_storage_path = None
+        cloud_storage_file_overwritten = False
+        cloud_storage_file_already_exists = False
+        cloud_storage_file_saved_with_increment = False
+        cloud_storage_file_exists_checked_on_name = file_name
+        cloud_storage_deleted_files=[]
+
+        try:
+            upload_allowed = True
+            # --- Overwrite Logic ---
+            if overwrite_if_exists:
+                with pipelinemon.context("overwriting"):
+                    if file_exists_if_starts_with_prefix:
+                        cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+                        blobs_to_delete = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+                        if len(blobs_to_delete) > max_deletable_files:
+                            err_msg=f"Error: Attempt to delete {len(blobs_to_delete)} matched files, but limit is {max_deletable_files}."
+                            pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
+                            pipelinemon.add_log(ContextLog(LogLevel.ERROR_CUSTOM, subject="Too many files", description=err_msg))
+                            #### Ensuring to quit the operation if too many files are found, it will be catched below
+                            return {"cloud_storage_upload_error": err_msg}
+                        if blobs_to_delete:
+                            cloud_storage_file_already_exists = True
+                            pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_to_delete)} files in bucket {bucket_name}"))
+                            for blob in blobs_to_delete:
+                                cloud_storage_path_del = f"gs://{bucket_name}/{blob.name}"
+                                pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_del}")
+                                blob.delete()
+                                pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= cloud_storage_path_del, description=f"file deleted from {cloud_storage_ref} as part of overwrite, matched with prefix"))
+                                cloud_storage_deleted_files.append(cloud_storage_path_del)
+                            cloud_storage_file_overwritten = True
+                    elif bucket.blob(file_name).exists():
+                        cloud_storage_file_already_exists = True
+                        pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}"))
+                        cloud_storage_path_del = f"gs://{bucket_name}/{file_name}"
+                        pipelinemon.add_system_impacted(f"delete: {cloud_storage_ref}_bucket_file: {cloud_storage_path_del}")
+                        blob.delete()  # Delete the existing blob
+                        pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_DELETE_COMPLETE, subject= cloud_storage_path_del, description=f"file deleted from {cloud_storage_ref} as part of overwrite, matched with exact name"))
+                        cloud_storage_deleted_files.append(cloud_storage_path_del)
+                        cloud_storage_file_overwritten = True
+            # --- Increment Logic ---
+            elif increment_if_exists:
+                with pipelinemon.context("incrementing"):
+                    cloud_storage_file_exists_checked_on_name = file_name  # We only increment if the exact name exists
+                    while bucket.blob(file_name).exists():
+                        cloud_storage_file_already_exists = True
+                        increment += 1
+                        file_name = f"{base_file_name}_v{increment}{ext}"
+                        cloud_storage_file_saved_with_increment = True
+                    if increment>0:
+                        cloud_storage_path = f"gs://{bucket_name}/{file_name}"
+                        pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Attempting to save file with incremented version in {bucket_name}"))
+             # --- Check for Conflicts (Including Prefix) ---
+            else:
+                if file_exists_if_starts_with_prefix:
+                    blobs_matched = list(bucket.list_blobs(prefix=file_exists_if_starts_with_prefix))
+                    cloud_storage_file_exists_checked_on_name = file_exists_if_starts_with_prefix
+                    if blobs_matched:
+                        upload_allowed = False
+                        cloud_storage_file_already_exists = True
+                        pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_exists_if_starts_with_prefix, description=f"Prefix matched with {len(blobs_matched)} existing files in bucket {bucket_name}."))
+                elif bucket.blob(file_name).exists():
+                    pipelinemon.add_log(ContextLog(LogLevel.NOTICE_ALREADY_EXISTS, subject=file_name, description=f"Exact name matched with existing file in bucket {bucket_name}."))
+                    upload_allowed = False
+                    cloud_storage_file_already_exists = True
+
+            # --- GCS Upload ---
+            cloud_storage_path = f"gs://{bucket_name}/{file_name}"
+            if overwrite_if_exists or increment_if_exists or upload_allowed:
+                with pipelinemon.context("uploading"):
+                    while attempts < max_retries and not success:
+                        try:
+                            blob = bucket.blob(file_name)  # Use the potentially updated file_name
+                            pipelinemon.add_system_impacted(f"upload: {cloud_storage_ref}_bucket_file: {cloud_storage_path}")
+                            blob.upload_from_string(data_str, content_type='application/json')
+                            pipelinemon.add_log(ContextLog(LogLevel.INFO_REMOTE_PERSISTNACE_COMPLETE, subject= cloud_storage_path, description=f"file uploaded to {cloud_storage_ref}"))
+                            success = True
+                        except Exception as e:
+                            attempts += 1
+                            if attempts < max_retries:
+                                time.sleep(2 ** attempts)
+                            else:
+                                err_msg=f"Error uploading file to {cloud_storage_ref} bucket {bucket_name} with name {file_name} : {type(e).__name__}-{str(e)}"
+                                pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e, description=err_msg))
+                                return {"cloud_storage_upload_error": err_msg}
+
+        except Exception as e:
+            pipelinemon.add_log(ContextLog(LogLevel.ERROR_EXCEPTION, e=e))
+            return {"cloud_storage_upload_error": f"Exception in GCS upload {type(e).__name__}-{str(e)}"}
+    # --- Return Metadata ---
+    return {
+        "cloud_storage_path": cloud_storage_path if ((success or not upload_allowed) and not cloud_storage_upload_error ) else None,
+        "cloud_storage_file_already_exists": cloud_storage_file_already_exists,
+        "cloud_storage_file_exists_checked_on_name":cloud_storage_file_exists_checked_on_name ,
+        "cloud_storage_file_overwritten": cloud_storage_file_overwritten,
+        "cloud_storage_deleted_file_names": ",,,".join(cloud_storage_deleted_files) if cloud_storage_deleted_files else None,
+        "cloud_storage_file_saved_with_increment": cloud_storage_file_saved_with_increment,
+        "cloud_storage_upload_error": cloud_storage_upload_error
+    }
+

ipulse_shared_core_ftredge/utils/utils_cloud_with_collectors.py

@@ -0,0 +1,26 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+from typing import Optional
+from ipulse_shared_core_ftredge.enums.enums_cloud import CloudProvider
+from .utils_collector_pipelinemon import Pipelinemon
+from .utils_cloud_gcp_with_collectors import write_json_to_gcs_with_pipelinemon_extended
+
+
+def write_json_to_cloud_storage_with_pipelinemon_extended(cloud_provider:CloudProvider, pipelinemon:Pipelinemon, storage_client, data:dict | list | str, bucket_name: str, file_name: str,
+                      file_exists_if_starts_with_prefix:Optional[str] =None, overwrite_if_exists:bool=False, increment_if_exists:bool=False,
+                      max_retries:int=2, max_deletable_files:int=1):
+
+
+    if cloud_provider == CloudProvider.GCP:
+        return write_json_to_gcs_with_pipelinemon_extended(pipelinemon=pipelinemon, storage_client=storage_client, data=data, bucket_name=bucket_name, file_name=file_name,
+                                                  file_exists_if_starts_with_prefix=file_exists_if_starts_with_prefix,
+                                                  overwrite_if_exists=overwrite_if_exists, increment_if_exists=increment_if_exists,
+                                                  max_retries=max_retries,
+                                                  max_deletable_files=max_deletable_files)
+
+    raise ValueError(f"Unsupported cloud provider: {cloud_provider}. Supported cloud providers: {CloudProvider.GCP.value}")

ipulse_shared_core_ftredge/utils/utils_collector_pipelinemon.py

@@ -0,0 +1,356 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+import json
+import uuid
+from datetime import datetime, timezone
+from contextlib import contextmanager
+from typing import List
+from ipulse_shared_core_ftredge.enums.enums_logs import TargetLogs, LogLevel
+from ipulse_shared_core_ftredge.utils.logs.context_log import ContextLog
+############################################################################
+##### PIPINEMON Collector for Logs and Statuses of running pipelines #######
+class Pipelinemon:
+    ERROR_START_CODE = LogLevel.ERROR.value
+    WARNING_START_CODE = LogLevel.WARNING.value
+    NOTICE_START_CODE = LogLevel.NOTICE.value
+    SUCCESS_START_CODE = LogLevel.SUCCESS.value
+    INFO_START_CODE = LogLevel.INFO.value
+
+    def __init__(self,  base_context: str, logger, target_logs: TargetLogs = TargetLogs.MIXED,
+                 max_log_field_size:int =10000, max_log_dict_size:float=256 * 1024 * 0.80, max_log_traceback_lines:int = 30):
+
+        self._id = str(uuid.uuid4())
+        self._logs = []
+        self._early_stop = False
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._notices_count = 0
+        self._successes_count = 0
+        self._infos_count = 0
+        self._systems_impacted = []
+        self._by_level_counts = {level.name: 0 for level in LogLevel}
+        self._base_context = base_context
+        self._context_stack = []
+        self._target_logs = target_logs.value
+        self._logger = logger
+        self._max_log_field_size = max_log_field_size
+        self._max_log_dict_size = max_log_dict_size
+        self._max_log_traceback_lines = max_log_traceback_lines
+
+
+    @contextmanager
+    def context(self, context):
+        self.push_context(context)
+        try:
+            yield
+        finally:
+            self.pop_context()
+
+    def push_context(self, context):
+        self._context_stack.append(context)
+
+    def pop_context(self):
+        if self._context_stack:
+            self._context_stack.pop()
+
+    @property
+    def current_context(self):
+        return " >> ".join(self._context_stack)
+
+    @property
+    def base_context(self):
+        return self._base_context
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
+    @property
+    def max_log_dict_size(self):
+        return self._max_log_dict_size
+
+    @max_log_dict_size.setter
+    def max_log_dict_size(self, value):
+        self._max_log_dict_size = value
+
+    @property
+    def max_log_field_size(self):
+        return self._max_log_field_size
+
+    @max_log_field_size.setter
+    def max_log_field_size(self, value):
+        self._max_log_field_size = value
+
+    @property
+    def max_log_traceback_lines(self):
+        return self._max_log_traceback_lines
+
+    @max_log_traceback_lines.setter
+    def max_log_traceback_lines(self, value):
+        self._max_log_traceback_lines = value
+
+    @property
+    def early_stop(self):
+        return self._early_stop
+
+    def set_early_stop(self, max_errors_tolerance:int=0, max_warnings_tolerance:int=0, create_error_log=True, pop_context=False):
+        self._early_stop = True
+        if create_error_log:
+            if pop_context:
+                self.pop_context()
+            if max_errors_tolerance > 0:
+                self.add_log(ContextLog(level=LogLevel.ERROR_PIPELINE_THRESHOLD_REACHED,
+                                        subject="EARLY_STOP",
+                                        description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+            elif max_warnings_tolerance > 0:
+                self.add_log(ContextLog(level=LogLevel.ERROR_PIPELINE_THRESHOLD_REACHED,
+                                        subject="EARLY_STOP",
+                                        description=f"Total MAX_WARNINGS_TOLERANCE of {max_warnings_tolerance} has been reached."))
+            else:
+                self.add_log(ContextLog(level=LogLevel.ERROR_PIPELINE_THRESHOLD_REACHED,
+                                        subject="EARLY_STOP",
+                                        description="Early stop has been triggered."))
+
+    def reset_early_stop(self):
+        self._early_stop = False
+
+
+    def add_log(self, log: ContextLog ):
+        if (self._target_logs == TargetLogs.SUCCESSES and log.level >=self.NOTICE_START_CODE) or \
+           (self._target_logs == TargetLogs.WARNINGS_AND_ERRORS and log.level.value < self.WARNING_START_CODE):
+            raise ValueError(f"Invalid log level {log.level.name} for Pipelinemon target logs setup: {self._target_logs}")
+        log.base_context = self.base_context
+        log.context = self.current_context if self.current_context else "root"
+        log.collector_id = self.id
+        log.systems_impacted = self.systems_impacted
+        log_dict = log.to_dict(max_field_len=self.max_log_field_size, size_limit=self.max_log_dict_size, max_traceback_lines=self.max_log_traceback_lines)
+        self._logs.append(log_dict)
+        self._update_counts(log_dict)
+
+        if self._logger:
+            # We specifically want to avoid having an ERROR log level for this structured Pipelinemon reporting, to ensure Errors are alerting on Critical Application Services.
+            # A single ERROR log level is usually added at the end of the entire pipeline
+            if log.level.value >= self.WARNING_START_CODE:
+                self._logger.warning(log_dict)
+            else:
+                self._logger.info(log_dict)
+
+    def add_logs(self, logs: List[ContextLog]):
+        for log in logs:
+            self.add_log(log)
+
+    def clear_logs_and_counts(self):
+        self._logs = []
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._notices_count = 0
+        self._successes_count = 0
+        self._infos_count = 0
+        self._by_level_counts = {level.name: 0 for level in LogLevel}
+
+    def clear_logs(self):
+        self._logs = []
+
+    def get_all_logs(self,in_json_format=False):
+        if in_json_format:
+            return json.dumps(self._logs)
+        return self._logs
+
+    def get_logs_for_level(self, level: LogLevel):
+        return [log for log in self._logs if log["level_code"] == level.value]
+
+    def get_logs_by_str_in_context(self, context_substring: str):
+        return [
+            log for log in self._logs
+            if context_substring in log["context"]
+        ]
+
+    def contains_errors(self):
+        return self._errors_count > 0
+
+    def count_errors(self):
+        return self._errors_count
+
+    def contains_warnings_or_errors(self):
+        return self._warnings_count > 0 or self._errors_count > 0
+
+    def count_warnings_and_errors(self):
+        return self._warnings_count + self._errors_count
+
+    def count_warnings(self):
+        return self._warnings_count
+
+    def count_notices(self):
+        return self._notices_count
+
+    def count_successes(self):
+        return self._successes_count
+
+    def count_successes_with_notice(self):
+        return self.count_logs_by_level(LogLevel.SUCCESS_WITH_NOTICES)
+
+    def count_successes_no_notice(self):
+        return self.count_logs_by_level(LogLevel.SUCCESS)
+
+    def count_infos(self):
+        return self._infos_count
+
+    def count_all_logs(self):
+        return len(self._logs)
+
+    def count_logs_by_level(self, level: LogLevel):
+        return self._by_level_counts.get(level.name, 0)
+
+    def _count_logs(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
+        return sum(
+            1 for log in self._logs
+            if (log["context"] == context_substring if exact_match else context_substring in log["context"]) and
+               (level_code_min is None or log["level_code"] >= level_code_min) and
+               (level_code_max is None or log["level_code"] <= level_code_max)
+        )
+
+    def count_logs_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True)
+
+    def count_logs_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context)
+
+    def count_logs_by_level_for_current_context(self, level: LogLevel):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
+
+    def count_logs_by_level_for_current_and_nested_contexts(self, level: LogLevel):
+        return self._count_logs(self.current_context, level_code_min=level.value, level_code_max=level.value)
+
+    def count_errors_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.ERROR_START_CODE)
+
+    def count_errors_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.ERROR_START_CODE)
+
+    def count_warnings_and_errors_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE)
+
+    def count_warnings_and_errors_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE)
+
+    def count_warnings_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+    def count_warnings_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+    def count_notices_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
+
+    def count_notices_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
+
+    def count_successes_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
+
+    def count_successes_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
+
+    def count_infos_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.INFO_START_CODE, level_code_max=self.SUCCESS_START_CODE-1)
+
+    def count_infos_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.INFO_START_CODE, level_code_max=self.SUCCESS_START_CODE-1)
+    
+    def _update_counts(self, log, remove=False):
+        level_code = log["level_code"]
+        level_name = log["level_name"]
+
+        if remove:
+            if level_code >= self.ERROR_START_CODE:
+                self._errors_count -= 1
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
+                self._warnings_count -= 1
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count -= 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
+                self._successes_count -= 1
+            elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
+                self._infos_count -= 1
+            self._by_level_counts[level_name] -= 1
+        else:
+            if level_code >= self.ERROR_START_CODE:
+                self._errors_count += 1
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
+                self._warnings_count += 1
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count += 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
+                self._successes_count += 1
+            elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
+                self._infos_count += 1
+            self._by_level_counts[level_name] += 1
+
+    def generate_file_name(self, file_prefix=None, include_base_context=True):
+        if not file_prefix:
+            file_prefix = self._target_logs
+        timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+        if include_base_context:
+            file_name = f"{file_prefix}_{timestamp}_{self.base_context}_len{len(self._logs)}.json"
+        else:
+            file_name = f"{file_prefix}_{timestamp}_len{len(self._logs)}.json"
+
+        return file_name
+
+    def import_logs_from_json(self, json_or_file, logger=None):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_warning(message, exc_info=False):
+            if logger:
+                logger.warning(message, exc_info=exc_info)
+
+        try:
+            if isinstance(json_or_file, str):  # Load from string
+                imported_logs = json.loads(json_or_file)
+            elif hasattr(json_or_file, 'read'):  # Load from file-like object
+                imported_logs = json.load(json_or_file)
+            self.add_logs(imported_logs)
+            log_message("Successfully imported logs from json.")
+        except Exception as e:
+            log_warning(f"Failed to import logs from json: {type(e).__name__} - {str(e)}", exc_info=True)
+
+
+    def generate_final_log_message(self, subjectref: str, total_subjs: int) -> str:
+        return f"""
+            Pipeline for {self.base_context} with pipelinemon.id: {self.id},
+            SUMMARY: Early_Stop= {self.early_stop} ;
+            A)SUCCESSES: {self.count_successes()}/{total_subjs} {subjectref}(s) ; out of which SUCCESSES_WITH_NOTICES: {self.count_successes_with_notice()}/{total_subjs} {subjectref}(s) ;
+            B)NOTICES: {self.count_notices()} ;
+            C)WARNINGS: {self.count_warnings()} ;
+            D)ERRORS: {self.count_errors()} ;
+            E)INFOS: {self.count_infos()} ;
+        """
+
+    def log_final_message(self, subjectref: str, total_subjs: int, generallogger):
+        final_log_message = self.generate_final_log_message(subjectref=subjectref, total_subjs=total_subjs)
+        if self.count_warnings_and_errors() > 0:
+            generallogger.error(final_log_message)
+        else:
+            generallogger.info(final_log_message)