ipulse-shared-data-eng-ftredge 2.1.1__tar.gz

ipulse_shared_data_eng_ftredge-2.1.1/LICENCE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Future Edge Group
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ipulse_shared_data_eng_ftredge-2.1.1/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.1
+Name: ipulse_shared_data_eng_ftredge
+Version: 2.1.1
+Summary: Shared Data Engineering functions for the Pulse platform project. Using AI for financial advisory and investment management.
+Home-page: https://github.com/TheFutureEdge/ipulse_shared_data_eng
+Author: Russlan Ramdowar
+License-File: LICENCE
+Requires-Dist: python-dateutil~=2.8
+Requires-Dist: pytest~=7.1
+Requires-Dist: ipulse_shared_core_ftredge>=1.1.1
+Requires-Dist: ipulse_shared_enums>=1.1.1
+Requires-Dist: google-cloud-bigquery~=3.24.0
+Requires-Dist: google-cloud-storage~=1.41.1
+Requires-Dist: google-cloud-pubsub~=2.7.0
+Requires-Dist: google-cloud-secret-manager~=2.7.0

ipulse_shared_data_eng_ftredge-2.1.1/README.md

@@ -0,0 +1,21 @@
+# ipulse_shared_data_eng
+Shared Data Engineering Code for ipulse platform, especially for Oracle module
+
+
+### Collectors i.e. Pipelinemon
+
+Collectors are smart Objects which are added to long running functions or pipelines for which we want to collect an overall number of successes, notices, warnings or errors. 
+
+We can wait until the full pipeline is finished in order to write off a single Summary file from a Collector, or we can attach to it a logger, which will be reporting major status along the journey, which is often times better. Because if a function crashes midway through , all logs will be lost, and it would be hard to investigate if anythign has bee persisted and has to be rolled back. THis will require a lot of manual effort to recollect.
+
+Pipelinemon , short of Pipeline Monitoring system is a type of very powerful Collector which Russlan created specifically for Pulse Data Engineering pipelines.
+
+Pipelinemon writes all observation logs to Google CLoud Logging, and you have to setup a Log Sink (Router) which will send the Pipelinemon's observation logs to BigQuery. 
+
+Great thing about Pipelinemin is its "context" keeping feature.
+
+
+
+### Utils : Schema Checkers , Cloud Utils ( save file to cloud storage etc. for GCP, AWS, Azure etc.) , local files utils etc.
+
+

ipulse_shared_data_eng_ftredge-2.1.1/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

ipulse_shared_data_eng_ftredge-2.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ipulse_shared_data_eng_ftredge-2.1.1/setup.py

@@ -0,0 +1,24 @@
+# pylint: disable=import-error
+from setuptools import setup, find_packages
+
+setup(
+    name='ipulse_shared_data_eng_ftredge',
+    version='2.1.1',
+    package_dir={'': 'src'},  # Specify the source directory
+    packages=find_packages(where='src'),  # Look for packages in 'src'
+    install_requires=[
+        # List your dependencies here
+        'python-dateutil~=2.8',
+        'pytest~=7.1',
+        'ipulse_shared_core_ftredge>=1.1.1',
+        'ipulse_shared_enums>=1.1.1',
+        'google-cloud-bigquery~=3.24.0',
+        'google-cloud-storage~=1.41.1',
+        'google-cloud-pubsub~=2.7.0',
+        'google-cloud-secret-manager~=2.7.0'
+        
+    ],
+    author='Russlan Ramdowar',
+    description='Shared Data Engineering functions for the Pulse platform project. Using AI for financial advisory and investment management.',
+    url='https://github.com/TheFutureEdge/ipulse_shared_data_eng'
+)

ipulse_shared_data_eng_ftredge-2.1.1/src/ipulse_shared_data_eng_ftredge/__init__.py

@@ -0,0 +1,10 @@
+from .collectors import ContextLog, Pipelinemon
+from .utils import (check_format_against_schema_template,
+                    write_json_to_cloud_storage_extended,
+                    read_json_from_cloud_storage,
+                    write_json_to_gcs_extended,
+                    read_json_from_gcs,
+                    save_json_locally_extended,
+                    prepare_full_file_path
+        )
+

ipulse_shared_data_eng_ftredge-2.1.1/src/ipulse_shared_data_eng_ftredge/collectors/__init__.py

@@ -0,0 +1,2 @@
+from .context_log import ContextLog
+from .pipelinemon import Pipelinemon

ipulse_shared_data_eng_ftredge-2.1.1/src/ipulse_shared_data_eng_ftredge/collectors/context_log.py

@@ -0,0 +1,210 @@
+
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=unused-variable
+import traceback
+import json
+from datetime import datetime, timezone
+from typing import List
+from ipulse_shared_enums import Status, LogLevel
+
+############################################################################
+##################### SETTING UP custom LOGGING format= DICT ##########################
+### Cloud  Agnostic, can be used with any cloud provider , jsut use to_dict() method to get the log in dict format
+class ContextLog:
+
+    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
+                context: str = None, description: str = None,
+                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                log_status: Status = Status.OPEN, subject: str = None, systems_impacted: List[str] = None,
+                ):
+
+        if e is not None:
+            e_type = type(e).__name__ if e_type is None else e_type
+            e_message = str(e) if e_message is None else e_message
+            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+        elif not e_traceback and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+        self.level = level
+        self.subject = subject
+        self.description = description
+        self._base_context = base_context
+        self._context = context
+        self._systems_impacted = systems_impacted if systems_impacted else []
+        self.collector_id = collector_id
+        self.exception_type = e_type
+        self.exception_message = e_message
+        self.exception_traceback = e_traceback
+        self.log_status = log_status
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+
+    @property
+    def base_context(self):
+        return self._base_context
+
+    @base_context.setter
+    def base_context(self, value):
+        self._base_context = value
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = value
+
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def remove_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is not None:
+            self._systems_impacted.remove(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
+    def _format_traceback(self, e_traceback, e_message, max_field_len:int, max_traceback_lines:int):
+        if not e_traceback or e_traceback == 'None\n':
+            return None
+
+        traceback_lines = e_traceback.splitlines()
+
+            # Check if the traceback is within the limits
+        if len(traceback_lines) <= max_traceback_lines and len(e_traceback) <= max_field_len:
+            return e_traceback
+
+        # Remove lines that are part of the exception message if they are present in traceback
+        message_lines = e_message.splitlines() if e_message else []
+        if message_lines:
+            for message_line in message_lines:
+                if message_line in traceback_lines:
+                    traceback_lines.remove(message_line)
+
+        # Filter out lines from third-party libraries (like site-packages)
+        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
+
+        # If filtering results in too few lines, revert to original traceback
+        if len(filtered_lines) < 2:
+            filtered_lines = traceback_lines
+
+        # Combine standalone bracket lines with previous or next lines
+        combined_lines = []
+        for line in filtered_lines:
+            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
+                combined_lines[-1] += " " + line.strip()
+            else:
+                combined_lines.append(line)
+
+            # Ensure the number of lines doesn't exceed MAX_TRACEBACK_LINES
+        if len(combined_lines) > max_traceback_lines:
+            keep_lines_start = min(max_traceback_lines // 2, len(combined_lines))
+            keep_lines_end = min(max_traceback_lines // 2, len(combined_lines) - keep_lines_start)
+            combined_lines = (
+                combined_lines[:keep_lines_start] +
+                ['... (truncated) ...'] +
+                combined_lines[-keep_lines_end:]
+            )
+
+        formatted_traceback = '\n'.join(combined_lines)
+
+        # Ensure the total length doesn't exceed MAX_TRACEBACK_LENGTH
+        if len(formatted_traceback) > max_field_len:
+            truncated_length = max_field_len - len('... (truncated) ...')
+            half_truncated_length = truncated_length // 2
+            formatted_traceback = (
+                formatted_traceback[:half_truncated_length] +
+                '\n... (truncated) ...\n' +
+                formatted_traceback[-half_truncated_length:]
+            )
+        return formatted_traceback
+
+    def to_dict(self, max_field_len:int =10000, size_limit:float=256 * 1024 * 0.80,max_traceback_lines:int = 30):
+        size_limit = int(size_limit)  # Ensure size_limit is an integer
+
+        # Unified list of all fields
+        systems_impacted_str = f"{len(self.systems_impacted)} system(s): " + " ,,, ".join(self.systems_impacted) if self.systems_impacted else None
+        fields = [
+            ("log_status", str(self.log_status.name)),
+            ("level_code", self.level.value),
+            ("level_name", str(self.level.name)),
+            ("base_context", str(self.base_context)),
+            ("timestamp", str(self.timestamp)),
+            ("collector_id", str(self.collector_id)),
+            ("systems_impacted", systems_impacted_str),
+            ("context", str(self.context)),  # special sizing rules apply to it
+            ("subject", str(self.subject)),
+            ("description", str(self.description)),
+            ("exception_type", str(self.exception_type)),
+            ("exception_message", str(self.exception_message)),
+            ("exception_traceback", str(self._format_traceback(self.exception_traceback,self.exception_message, max_field_len, max_traceback_lines)))
+        ]
+
+        # Function to calculate the byte size of a JSON-encoded field
+        def field_size(key, value):
+            return len(json.dumps({key: value}).encode('utf-8'))
+
+        # Function to truncate a value based on its type
+        # Function to truncate a value based on its type
+        def truncate_value(value, max_size):
+            if isinstance(value, str):
+                half_size = max_size // 2
+                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
+            return value
+
+         # Ensure no field exceeds max_field_len
+        for i, (key, value) in enumerate(fields):
+            if isinstance(value, str) and len(value) > max_field_len:
+                fields[i] = (key, truncate_value(value, max_field_len))
+
+        # Ensure total size of the dict doesn't exceed size_limit
+        total_size = sum(field_size(key, value) for key, value in fields)
+        log_dict = {}
+        truncated = False
+
+        if total_size > size_limit:
+            truncated = True
+            remaining_size = size_limit
+            remaining_fields = len(fields)
+
+            for key, value in fields:
+                if remaining_fields > 0:
+                    max_size_per_field = remaining_size // remaining_fields
+                else:
+                    max_size_per_field = 0
+
+                field_sz = field_size(key, value)
+                if field_sz > max_size_per_field:
+                    value = truncate_value(value, max_size_per_field)
+                    field_sz = field_size(key, value)
+
+                log_dict[key] = value
+                remaining_size -= field_sz
+                remaining_fields -= 1
+        else:
+            log_dict = dict(fields)
+
+        log_dict['trunc'] = truncated
+
+        return log_dict
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), indent=4)
+
+    def __repr__(self):
+        return self.__str__()

ipulse_shared_data_eng_ftredge-2.1.1/src/ipulse_shared_data_eng_ftredge/collectors/pipelinemon.py

@@ -0,0 +1,354 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+import json
+import uuid
+from datetime import datetime, timezone
+from contextlib import contextmanager
+from typing import List
+from ipulse_shared_enums import LogLevel
+from .context_log import  ContextLog
+############################################################################
+##### PIPINEMON Collector for Logs and Statuses of running pipelines #######
+class Pipelinemon:
+    ERROR_START_CODE = LogLevel.ERROR.value
+    WARNING_START_CODE = LogLevel.WARNING.value
+    NOTICE_START_CODE = LogLevel.NOTICE.value
+    SUCCESS_START_CODE = LogLevel.SUCCESS.value
+    INFO_START_CODE = LogLevel.INFO.value
+
+    def __init__(self,  base_context: str, logger,
+                 max_log_field_size:int =10000,
+                 max_log_dict_size:float=256 * 1024 * 0.80,
+                 max_log_traceback_lines:int = 30):
+
+        self._id = str(uuid.uuid4())
+        self._logs = []
+        self._early_stop = False
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._notices_count = 0
+        self._successes_count = 0
+        self._infos_count = 0
+        self._systems_impacted = []
+        self._by_level_counts = {level.name: 0 for level in LogLevel}
+        self._base_context = base_context
+        self._context_stack = []
+        self._logger = logger
+        self._max_log_field_size = max_log_field_size
+        self._max_log_dict_size = max_log_dict_size
+        self._max_log_traceback_lines = max_log_traceback_lines
+
+
+    @contextmanager
+    def context(self, context):
+        self.push_context(context)
+        try:
+            yield
+        finally:
+            self.pop_context()
+
+    def push_context(self, context):
+        self._context_stack.append(context)
+
+    def pop_context(self):
+        if self._context_stack:
+            self._context_stack.pop()
+
+    @property
+    def current_context(self):
+        return " >> ".join(self._context_stack)
+
+    @property
+    def base_context(self):
+        return self._base_context
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
+    @property
+    def max_log_dict_size(self):
+        return self._max_log_dict_size
+
+    @max_log_dict_size.setter
+    def max_log_dict_size(self, value):
+        self._max_log_dict_size = value
+
+    @property
+    def max_log_field_size(self):
+        return self._max_log_field_size
+
+    @max_log_field_size.setter
+    def max_log_field_size(self, value):
+        self._max_log_field_size = value
+
+    @property
+    def max_log_traceback_lines(self):
+        return self._max_log_traceback_lines
+
+    @max_log_traceback_lines.setter
+    def max_log_traceback_lines(self, value):
+        self._max_log_traceback_lines = value
+
+    @property
+    def early_stop(self):
+        return self._early_stop
+
+    def set_early_stop(self, max_errors_tolerance:int=0, max_warnings_tolerance:int=0, create_error_log=True, pop_context=False):
+        self._early_stop = True
+        if create_error_log:
+            if pop_context:
+                self.pop_context()
+            if max_errors_tolerance > 0:
+                self.add_log(ContextLog(level=LogLevel.ERROR_PIPELINE_THRESHOLD_REACHED,
+                                        subject="EARLY_STOP",
+                                        description=f"Total MAX_ERRORS_TOLERANCE of {max_errors_tolerance} has been reached."))
+            elif max_warnings_tolerance > 0:
+                self.add_log(ContextLog(level=LogLevel.ERROR_PIPELINE_THRESHOLD_REACHED,
+                                        subject="EARLY_STOP",
+                                        description=f"Total MAX_WARNINGS_TOLERANCE of {max_warnings_tolerance} has been reached."))
+            else:
+                self.add_log(ContextLog(level=LogLevel.ERROR_PIPELINE_THRESHOLD_REACHED,
+                                        subject="EARLY_STOP",
+                                        description="Early stop has been triggered."))
+
+    def reset_early_stop(self):
+        self._early_stop = False
+
+
+    def add_log(self, log: ContextLog ):
+        log.base_context = self.base_context
+        log.context = self.current_context if self.current_context else "root"
+        log.collector_id = self.id
+        log.systems_impacted = self.systems_impacted
+        log_dict = log.to_dict(max_field_len=self.max_log_field_size, size_limit=self.max_log_dict_size, max_traceback_lines=self.max_log_traceback_lines)
+        self._logs.append(log_dict)
+        self._update_counts(log_dict)
+
+        if self._logger:
+            # We specifically want to avoid having an ERROR log level for this structured Pipelinemon reporting, to ensure Errors are alerting on Critical Application Services.
+            # A single ERROR log level is usually added at the end of the entire pipeline
+            if log.level.value >= self.WARNING_START_CODE:
+                self._logger.warning(log_dict)
+            else:
+                self._logger.info(log_dict)
+
+    def add_logs(self, logs: List[ContextLog]):
+        for log in logs:
+            self.add_log(log)
+
+    def clear_logs_and_counts(self):
+        self._logs = []
+        self._errors_count = 0
+        self._warnings_count = 0
+        self._notices_count = 0
+        self._successes_count = 0
+        self._infos_count = 0
+        self._by_level_counts = {level.name: 0 for level in LogLevel}
+
+    def clear_logs(self):
+        self._logs = []
+
+    def get_all_logs(self,in_json_format=False):
+        if in_json_format:
+            return json.dumps(self._logs)
+        return self._logs
+
+    def get_logs_for_level(self, level: LogLevel):
+        return [log for log in self._logs if log["level_code"] == level.value]
+
+    def get_logs_by_str_in_context(self, context_substring: str):
+        return [
+            log for log in self._logs
+            if context_substring in log["context"]
+        ]
+
+    def contains_errors(self):
+        return self._errors_count > 0
+
+    def count_errors(self):
+        return self._errors_count
+
+    def contains_warnings_or_errors(self):
+        return self._warnings_count > 0 or self._errors_count > 0
+
+    def count_warnings_and_errors(self):
+        return self._warnings_count + self._errors_count
+
+    def count_warnings(self):
+        return self._warnings_count
+
+    def count_notices(self):
+        return self._notices_count
+
+    def count_successes(self):
+        return self._successes_count
+
+    def count_successes_with_notice(self):
+        return self.count_logs_by_level(LogLevel.SUCCESS_WITH_NOTICES)
+
+    def count_successes_no_notice(self):
+        return self.count_logs_by_level(LogLevel.SUCCESS)
+
+    def count_infos(self):
+        return self._infos_count
+
+    def count_all_logs(self):
+        return len(self._logs)
+
+    def count_logs_by_level(self, level: LogLevel):
+        return self._by_level_counts.get(level.name, 0)
+
+    def _count_logs(self, context_substring: str, exact_match=False, level_code_min=None, level_code_max=None):
+        return sum(
+            1 for log in self._logs
+            if (log["context"] == context_substring if exact_match else context_substring in log["context"]) and
+               (level_code_min is None or log["level_code"] >= level_code_min) and
+               (level_code_max is None or log["level_code"] <= level_code_max)
+        )
+
+    def count_logs_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True)
+
+    def count_logs_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context)
+
+    def count_logs_by_level_for_current_context(self, level: LogLevel):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=level.value, level_code_max=level.value)
+
+    def count_logs_by_level_for_current_and_nested_contexts(self, level: LogLevel):
+        return self._count_logs(self.current_context, level_code_min=level.value, level_code_max=level.value)
+
+    def count_errors_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.ERROR_START_CODE)
+
+    def count_errors_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.ERROR_START_CODE)
+
+    def count_warnings_and_errors_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE)
+
+    def count_warnings_and_errors_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE)
+
+    def count_warnings_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+    def count_warnings_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.WARNING_START_CODE, level_code_max=self.ERROR_START_CODE - 1)
+
+    def count_notices_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
+
+    def count_notices_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.NOTICE_START_CODE, level_code_max=self.WARNING_START_CODE-1)
+
+    def count_successes_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
+
+    def count_successes_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.SUCCESS_START_CODE, level_code_max=self.NOTICE_START_CODE-1)
+
+    def count_infos_for_current_context(self):
+        return self._count_logs(self.current_context, exact_match=True, level_code_min=self.INFO_START_CODE, level_code_max=self.SUCCESS_START_CODE-1)
+
+    def count_infos_for_current_and_nested_contexts(self):
+        return self._count_logs(self.current_context, level_code_min=self.INFO_START_CODE, level_code_max=self.SUCCESS_START_CODE-1)
+    
+    def _update_counts(self, log, remove=False):
+        level_code = log["level_code"]
+        level_name = log["level_name"]
+
+        if remove:
+            if level_code >= self.ERROR_START_CODE:
+                self._errors_count -= 1
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
+                self._warnings_count -= 1
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count -= 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
+                self._successes_count -= 1
+            elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
+                self._infos_count -= 1
+            self._by_level_counts[level_name] -= 1
+        else:
+            if level_code >= self.ERROR_START_CODE:
+                self._errors_count += 1
+            elif self.WARNING_START_CODE <= level_code < self.ERROR_START_CODE:
+                self._warnings_count += 1
+            elif self.NOTICE_START_CODE <= level_code < self.WARNING_START_CODE:
+                self._notices_count += 1
+            elif self.SUCCESS_START_CODE <= level_code < self.NOTICE_START_CODE:
+                self._successes_count += 1
+            elif self.INFO_START_CODE <= level_code < self.SUCCESS_START_CODE:
+                self._infos_count += 1
+            self._by_level_counts[level_name] += 1
+
+    def generate_file_name(self, file_prefix=None, include_base_context=True):
+        timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+        if not file_prefix:
+            file_prefix = "pipelinelogs"
+        if include_base_context:
+            file_name = f"{file_prefix}_{timestamp}_{self.base_context}_len{len(self._logs)}.json"
+        else:
+            file_name = f"{file_prefix}_{timestamp}_len{len(self._logs)}.json"
+
+        return file_name
+
+    def import_logs_from_json(self, json_or_file, logger=None):
+        def log_message(message):
+            if logger:
+                logger.info(message)
+
+        def log_warning(message, exc_info=False):
+            if logger:
+                logger.warning(message, exc_info=exc_info)
+
+        try:
+            if isinstance(json_or_file, str):  # Load from string
+                imported_logs = json.loads(json_or_file)
+            elif hasattr(json_or_file, 'read'):  # Load from file-like object
+                imported_logs = json.load(json_or_file)
+            self.add_logs(imported_logs)
+            log_message("Successfully imported logs from json.")
+        except Exception as e:
+            log_warning(f"Failed to import logs from json: {type(e).__name__} - {str(e)}", exc_info=True)
+
+
+    def generate_final_log_message(self, subjectref: str, total_subjs: int) -> str:
+        return f"""
+            Pipeline for {self.base_context} with pipelinemon.id: {self.id},
+            SUMMARY: Early_Stop= {self.early_stop} ;
+            A)SUCCESSES: {self.count_successes()}/{total_subjs} {subjectref}(s) ; out of which SUCCESSES_WITH_NOTICES: {self.count_successes_with_notice()}/{total_subjs} {subjectref}(s) ;
+            B)NOTICES: {self.count_notices()} ;
+            C)WARNINGS: {self.count_warnings()} ;
+            D)ERRORS: {self.count_errors()} ;
+            E)INFOS: {self.count_infos()} ;
+        """
+
+    def log_final_message(self, subjectref: str, total_subjs: int, generallogger):
+        final_log_message = self.generate_final_log_message(subjectref=subjectref, total_subjs=total_subjs)
+        if self.count_warnings_and_errors() > 0:
+            generallogger.error(final_log_message)
+        else:
+            generallogger.info(final_log_message)

ipulse_shared_data_eng_ftredge-2.1.1/src/ipulse_shared_data_eng_ftredge/utils/__init__.py

@@ -0,0 +1,8 @@
+from .utils_check_data_schema import check_format_against_schema_template
+from .utils_cloud import (write_json_to_cloud_storage_extended,
+                          read_json_from_cloud_storage)
+from .utils_cloud_gcp import (write_json_to_gcs_extended,
+                              read_json_from_gcs
+        )
+from .utils_local_files import (save_json_locally_extended,
+                                prepare_full_file_path)