ipulse-shared-core-ftredge 2.7.1__py3-none-any.whl → 2.8.1__py3-none-any.whl

ipulse_shared_core_ftredge/enums/pulse_enums.py

@@ -1,182 +0,0 @@
-resource_classifications = {
-    "*",
-    "childs_based", # Meaning need to look into child fields to determine classifications
-    
-    "public", #Anyone Can Access ex: synthetic data 
-    "authuser_open", # Any Authenticated Can Access ex: prices of gold, bitcoin etc.
-    # "authuser_subscription",
-    "authuser_confidential", ## Only User Owner Can Access and Specific Admin
-    "authuser_limitedacl" , ## Has to be in the ACL
-    "authuser_owner"
-    "internal_open", ## Any Internal employees only Can Access ex: public reports, emails etc. 
-    "internal_sensitive", ## Many Internal employees Can Access IF meet special condition ex: internal financials summary reports , web and app analytics, list of admin users etc.
-    "internal_confidential", ## Few Internal employees Can Access. ex: internal user data, key financials, salaries and bonuses etc
-    "internal_limitedacl", ## Has to employee usertype and in the ACL
-    "internal_owner"
-}
-
-
-resource_domain = {
-    "*",
-    ############### GYM #########
-    "gym_domain",
-    "gym_data_domain",
-    "gym_ai_domain",
-    ############## ORACLE #########
-    "oracle_domain",
-    "oracle_historic_prices_domain",
-    "oracle_ai_domain",
-    "oracle_assests_historic_info_domain",
-    "oracle_historic_econometrics_domain",
-    "oracle_news_historic_domain",
-    "oracle_calendar_domain",
-    "oracle_modelinfo_domain",
-    "oracle_modelmetrics_domain",
-    "oracle_modelpredictions_domain",
-    ######### ORGANISATIONS #########
-    "organisation_domain",
-    ################### USER #########
-    "user_domain",
-    "user_management_domain",
-    "user_portfolio_domain",
-    "user_groups_and_roles_domain",
-    ############### BUSINESS #########
-    "business_domain",
-    ############### ANALYTICS #########
-    "analytics_domain",
-    "system_domain"
-}
-
-resource_types =  {
-    "db", "sql_db", "nosql_db", "dynamodb",
-    "big_query", "big_query_project", "big_query_table", "big_query_column", 
-    "big_query_row", "big_query_cell",
-    "firestore", "firestore_project", "firestore_collection", 
-    "firestore_document","firestore_document_with_timeseries" "firestore_document_field",
-    "pandas_dataframe", "spark_dataframe",
-    "s3_bucket", "storage_bucket",
-    "folder", "file", "json_file", "csv_file", "pdf_file", 
-    "unstructured_file", "image", "video", "audio", "text",
-    "api", "report", "dashboard", "webpage", "website", "web"
-}
-
-organisation_relations = {
-    "*",
-    "retail_customer",
-    "corporate_customer",
-    "parent",
-    "sister",
-    "self",
-    "partner",
-    "supplier",
-    "sponsor",
-    "investor",
-    "regulator",
-    "other"
-    }
-
-organisation_industries  = {
-    "*",
-    "data",
-    "government",
-    "media",
-    "academic",
-    "commercial",
-    "fund",
-    "finance",
-    "advisory",
-    "hedgefund",
-    "bank",
-    "vc",
-    "pe",
-    "construction",
-    "healthcare",
-    "technology",
-    "consulting",
-    "retail",
-    "non_profit",
-    "individual",
-    "freelancer",
-    "other"
-}
-
-licences_types={
-    "*",
-    ######################################### OPEN or FULL Rights
-    "public",
-    "open",
-    "open_no_tandc",
-    "full_rights",
-    "full_rights_for_sale",
-    "commercial_licence_perpetual",
-    "customer_private_tac",
-    ######################################### SPECIAL CONDITIONS
-    "open_with_tandc",
-    "on_special_request",
-    "commercial_licence_limited_time",
-    "customer_owned_for_sale",
-     ######################################### Not for Commercial Use
-    "full_rights_not_for_sale",
-    "internal_only",
-    "academic_licence",
-    "not_for_commercial_use",
-    "customer_private"
-    ######################################### Unknown
-    "commercial_licence_not_purchased",
-    "web_scrapped",
-    "unknown"
-}
-
-
-actions ={"GET",
-          "POST",
-          "DELETE",
-          "PUT",
-          "create",
-          "batch_create",
-          "read", 
-          "batch_read",
-          "edit",
-          "batch_edit",
-          "add",
-          "batch_add",
-          "remove",
-          "batch_remove",
-          "delete",
-          "batch_delete", 
-          "rename" ,
-          "batch_rename",
-          "move",
-          "batch_move",
-          "download",
-          "upload",
-          "share"
-                }
-
-resource_readable_by={
-    "*",
-    "all",
-    "authenticated",
-    "restircted",
-    "owner",
-    "selected_by_owner",
-    "admin",
-    "selected_by_admin",
-    "super_admin",
-    "super_admin_selected",
-    "system"
-}
-
-resource_updatable_by={
-     "*",
-    "all",
-    "authenticated",
-    "restircted",
-    "owner",
-    "selected_by_owner",
-    "admin",
-    "selected_by_admin",
-    "super_admin",
-    "super_admin_selected",
-    "system"
-}

ipulse_shared_core_ftredge/utils/logs/__init__.py

@@ -1,2 +0,0 @@
-from .context_log import ContextLog
-from .get_logger import get_logger

ipulse_shared_core_ftredge/utils/logs/context_log.py

@@ -1,210 +0,0 @@
-
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=logging-fstring-interpolation
-# pylint: disable=line-too-long
-# pylint: disable=missing-class-docstring
-# pylint: disable=broad-exception-caught
-# pylint: disable=unused-variable
-import traceback
-import json
-from datetime import datetime, timezone
-from typing import List
-from ipulse_shared_core_ftredge import Status, LogLevel
-
-############################################################################
-##################### SETTING UP custom LOGGING format= DICT ##########################
-### Cloud  Agnostic, can be used with any cloud provider , jsut use to_dict() method to get the log in dict format
-class ContextLog:
-
-    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
-                context: str = None, description: str = None,
-                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
-                log_status: Status = Status.OPEN, subject: str = None, systems_impacted: List[str] = None,
-                ):
-
-        if e is not None:
-            e_type = type(e).__name__ if e_type is None else e_type
-            e_message = str(e) if e_message is None else e_message
-            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
-        elif (e_traceback is None or e_traceback== "") and (e_type or e_message):
-            e_traceback = traceback.format_exc()
-
-        self.level = level
-        self.subject = subject
-        self.description = description
-        self._base_context = base_context
-        self._context = context
-        self._systems_impacted = systems_impacted if systems_impacted else []
-        self.collector_id = collector_id
-        self.exception_type = e_type
-        self.exception_message = e_message
-        self.exception_traceback = e_traceback
-        self.log_status = log_status
-        self.timestamp = datetime.now(timezone.utc).isoformat()
-
-    @property
-    def base_context(self):
-        return self._base_context
-
-    @base_context.setter
-    def base_context(self, value):
-        self._base_context = value
-
-    @property
-    def context(self):
-        return self._context
-
-    @context.setter
-    def context(self, value):
-        self._context = value
-
-    @property
-    def systems_impacted(self):
-        return self._systems_impacted
-
-    @systems_impacted.setter
-    def systems_impacted(self, list_of_si: List[str]):
-        self._systems_impacted = list_of_si
-
-    def add_system_impacted(self, system_impacted: str):
-        if self._systems_impacted is None:
-            self._systems_impacted = []
-        self._systems_impacted.append(system_impacted)
-
-    def remove_system_impacted(self, system_impacted: str):
-        if self._systems_impacted is not None:
-            self._systems_impacted.remove(system_impacted)
-
-    def clear_systems_impacted(self):
-        self._systems_impacted = []
-
-    def _format_traceback(self, e_traceback, e_message, max_field_len:int, max_traceback_lines:int):
-        if not e_traceback or e_traceback == 'None\n':
-            return None
-
-        traceback_lines = e_traceback.splitlines()
-
-            # Check if the traceback is within the limits
-        if len(traceback_lines) <= max_traceback_lines and len(e_traceback) <= max_field_len:
-            return e_traceback
-
-        # Remove lines that are part of the exception message if they are present in traceback
-        message_lines = e_message.splitlines() if e_message else []
-        if message_lines:
-            for message_line in message_lines:
-                if message_line in traceback_lines:
-                    traceback_lines.remove(message_line)
-
-        # Filter out lines from third-party libraries (like site-packages)
-        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
-
-        # If filtering results in too few lines, revert to original traceback
-        if len(filtered_lines) < 2:
-            filtered_lines = traceback_lines
-
-        # Combine standalone bracket lines with previous or next lines
-        combined_lines = []
-        for line in filtered_lines:
-            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
-                combined_lines[-1] += " " + line.strip()
-            else:
-                combined_lines.append(line)
-
-            # Ensure the number of lines doesn't exceed MAX_TRACEBACK_LINES
-        if len(combined_lines) > max_traceback_lines:
-            keep_lines_start = min(max_traceback_lines // 2, len(combined_lines))
-            keep_lines_end = min(max_traceback_lines // 2, len(combined_lines) - keep_lines_start)
-            combined_lines = (
-                combined_lines[:keep_lines_start] +
-                ['... (truncated) ...'] +
-                combined_lines[-keep_lines_end:]
-            )
-
-        formatted_traceback = '\n'.join(combined_lines)
-
-        # Ensure the total length doesn't exceed MAX_TRACEBACK_LENGTH
-        if len(formatted_traceback) > max_field_len:
-            truncated_length = max_field_len - len('... (truncated) ...')
-            half_truncated_length = truncated_length // 2
-            formatted_traceback = (
-                formatted_traceback[:half_truncated_length] +
-                '\n... (truncated) ...\n' +
-                formatted_traceback[-half_truncated_length:]
-            )
-        return formatted_traceback
-
-    def to_dict(self, max_field_len:int =10000, size_limit:float=256 * 1024 * 0.80,max_traceback_lines:int = 30):
-        size_limit = int(size_limit)  # Ensure size_limit is an integer
-
-        # Unified list of all fields
-        systems_impacted_str = f"{len(self.systems_impacted)} system(s): " + " ,,, ".join(self.systems_impacted) if self.systems_impacted else None
-        fields = [
-            ("log_status", str(self.log_status.name)),
-            ("level_code", self.level.value),
-            ("level_name", str(self.level.name)),
-            ("base_context", str(self.base_context)),
-            ("timestamp", str(self.timestamp)),
-            ("collector_id", str(self.collector_id)),
-            ("systems_impacted", systems_impacted_str),
-            ("context", str(self.context)),  # special sizing rules apply to it
-            ("subject", str(self.subject)),
-            ("description", str(self.description)),
-            ("exception_type", str(self.exception_type)),
-            ("exception_message", str(self.exception_message)),
-            ("exception_traceback", str(self._format_traceback(self.exception_traceback,self.exception_message, max_field_len, max_traceback_lines)))
-        ]
-
-        # Function to calculate the byte size of a JSON-encoded field
-        def field_size(key, value):
-            return len(json.dumps({key: value}).encode('utf-8'))
-
-        # Function to truncate a value based on its type
-        # Function to truncate a value based on its type
-        def truncate_value(value, max_size):
-            if isinstance(value, str):
-                half_size = max_size // 2
-                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
-            return value
-
-         # Ensure no field exceeds max_field_len
-        for i, (key, value) in enumerate(fields):
-            if isinstance(value, str) and len(value) > max_field_len:
-                fields[i] = (key, truncate_value(value, max_field_len))
-
-        # Ensure total size of the dict doesn't exceed size_limit
-        total_size = sum(field_size(key, value) for key, value in fields)
-        log_dict = {}
-        truncated = False
-
-        if total_size > size_limit:
-            truncated = True
-            remaining_size = size_limit
-            remaining_fields = len(fields)
-
-            for key, value in fields:
-                if remaining_fields > 0:
-                    max_size_per_field = remaining_size // remaining_fields
-                else:
-                    max_size_per_field = 0
-
-                field_sz = field_size(key, value)
-                if field_sz > max_size_per_field:
-                    value = truncate_value(value, max_size_per_field)
-                    field_sz = field_size(key, value)
-
-                log_dict[key] = value
-                remaining_size -= field_sz
-                remaining_fields -= 1
-        else:
-            log_dict = dict(fields)
-
-        log_dict['trunc'] = truncated
-
-        return log_dict
-
-    def __str__(self):
-        return json.dumps(self.to_dict(), indent=4)
-
-    def __repr__(self):
-        return self.__str__()

ipulse_shared_core_ftredge/utils/logs/get_logger.py

@@ -1,103 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=broad-exception-caught
-# pylint: disable=line-too-long
-# pylint: disable=unused-variable
-# pylint: disable=broad-exception-raised
-import logging
-import os
-import json
-import traceback
-from typing import List, Union
-from ipulse_shared_core_ftredge.enums import LoggingHandlers
-from ipulse_shared_core_ftredge.utils.utils_cloud_gcp import add_gcp_cloud_logging, add_gcp_error_reporting
-
-###################################################################################################
-##################################################################################################
-##################################### SETTING UP LOGGER ##########################################
-
-class CloudLogFormatter(logging.Formatter):
-    """Formats log records as structured JSON."""
-
-    def format(self, record):
-        log_entry = {
-            'message': record.msg,
-            'timestamp': self.formatTime(record, self.datefmt),
-            'name': record.name,
-            'severity': record.levelname,
-            'pathname': record.pathname,
-            'lineno': record.lineno,
-        }
-        if record.exc_info:
-            log_entry['exception_traceback'] = ''.join(traceback.format_exception(*record.exc_info))
-        if isinstance(record.msg, dict):
-            log_entry.update(record.msg)
-        return json.dumps(log_entry)
-
-
-class LocalLogFormatter(logging.Formatter):
-    """Formats log records for local output to the console."""
-
-    def format(self, record):  # Make sure you have the 'record' argument here!
-        path_parts = record.pathname.split(os.sep)
-
-        # Get the last two parts of the path if they exist
-        if len(path_parts) >= 2:
-            short_path = os.path.join(path_parts[-2], path_parts[-1])
-        else:
-            short_path = record.pathname
-        
-        # Format log messages differently based on the log level
-        if record.levelno == logging.INFO:
-            log_message = f"[INFO] {self.formatTime(record, self.datefmt)} :: {record.msg}"
-        elif record.levelno == logging.DEBUG:
-            log_message = f"[DEBUG] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
-        elif record.levelno == logging.ERROR:
-            log_message = f"[ERROR] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
-            if record.exc_info:
-                log_message += "\n" + ''.join(traceback.format_exception(*record.exc_info))
-        else:
-            log_message = f"[{record.levelname}] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
-
-
-        return log_message
-
-
-def get_logger( logger_name:str ,level=logging.INFO, logging_handler_providers: Union[LoggingHandlers, List[LoggingHandlers]] = LoggingHandlers.NONE):
-
-    """Creates and configures a logger with the specified handlers."""
-
-    logger = logging.getLogger(logger_name)
-    logger.setLevel(level)
-    cloud_formatter = CloudLogFormatter()
-
-    # Ensure logging_handler_providers is a list for consistent processing
-    if not isinstance(logging_handler_providers, list):
-        logging_handler_providers = [logging_handler_providers]
-
-    supported_remote_handlers = [
-        LoggingHandlers.GCP_CLOUD_LOGGING,
-        LoggingHandlers.GCP_ERROR_REPORTING,
-        LoggingHandlers.LOCAL_STREAM,
-        LoggingHandlers.NONE, # If NONE is considered a remote handler
-    ]
-
-    # Remote handlers
-
-    for handler_provider in logging_handler_providers:
-        if handler_provider in supported_remote_handlers:
-            if handler_provider == LoggingHandlers.GCP_CLOUD_LOGGING:
-                add_gcp_cloud_logging(logger, cloud_formatter)
-            elif handler_provider == LoggingHandlers.GCP_ERROR_REPORTING:
-                add_gcp_error_reporting(logger)
-            elif handler_provider == LoggingHandlers.LOCAL_STREAM:  # Handle local stream
-                local_handler = logging.StreamHandler()
-                local_handler.setFormatter(LocalLogFormatter())
-                logger.addHandler(local_handler)
-        else:
-            raise ValueError(
-            f"Unsupported logging provider: {handler_provider}. "
-            f"Supported providers: {[h.value for h in supported_remote_handlers]}"
-            )
-    return logger

ipulse_shared_core_ftredge/utils/utils_cloud.py

@@ -1,53 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=broad-exception-caught
-# pylint: disable=line-too-long
-# pylint: disable=unused-variable
-# pylint: disable=broad-exception-caught
-from ipulse_shared_core_ftredge.enums import CloudProvider, DataSourceType, DuplicationHandling, MatchConditionType
-from .utils_collector_pipelinemon import Pipelinemon
-from .utils_cloud_gcp import (write_json_to_gcs_extended,
-                              read_json_from_gcs)
-
-#######################################################################################################################
-#######################################################################################################################
-#################################################     cloud IO functions      ########################################
-
-# Define the central function that routes to the relevant cloud-specific function
-def write_json_to_cloud_storage_extended(cloud_storage:CloudProvider | DataSourceType, storage_client, data:dict | list | str, bucket_name: str, file_name: str,
-                      duplication_handling:DuplicationHandling,  duplication_match_condition_type: MatchConditionType, duplication_match_condition: str = "",
-                       max_retries:int=2, max_matched_deletable_files:int=1,
-                      pipelinemon: Pipelinemon = None, logger=None, print_out=False, raise_e=False):
-
-
-    supported_cloud_storage_values = [CloudProvider.GCP, DataSourceType.GCS]
-
-    if cloud_storage in [CloudProvider.GCP, DataSourceType.GCS]:
-        return write_json_to_gcs_extended(
-            pipelinemon=pipelinemon,
-            storage_client=storage_client,
-            data=data,
-            bucket_name=bucket_name,
-            file_name=file_name,
-            duplication_handling_enum=duplication_handling,
-            duplication_match_condition_type_enum=duplication_match_condition_type,
-            duplication_match_condition=duplication_match_condition,
-            max_retries=max_retries,
-            max_deletable_files=max_matched_deletable_files,
-            logger=logger,
-            print_out=print_out,
-            raise_e=raise_e
-        )
-
-    raise ValueError(f"Unsupported cloud storage : {cloud_storage}. Supported cloud storage values: {supported_cloud_storage_values}")
-
-
-def read_json_from_cloud_storage(cloud_storage:CloudProvider | DataSourceType , storage_client, bucket_name:str, file_name:str, logger=None, print_out:bool=False):
-
-    supported_cloud_storage_values = [CloudProvider.GCP, DataSourceType.GCS]
-
-    if cloud_storage in [CloudProvider.GCP, DataSourceType.GCS]:
-        return read_json_from_gcs(storage_client=storage_client, bucket_name=bucket_name, file_name=file_name, logger=logger, print_out=print_out)
-
-    raise ValueError(f"Unsupported cloud storage: {cloud_storage}. Supported cloud storage values: {supported_cloud_storage_values}")