ipulse-shared-core-ftredge 2.6__py3-none-any.whl → 2.7.1__py3-none-any.whl

ipulse_shared_core_ftredge/models/user_profile_update.py

@@ -1,18 +1,36 @@
-
 from typing import Optional, Set
-from pydantic import BaseModel
-from datetime import datetime , date
+from pydantic import BaseModel, Field, EmailStr
+from datetime import date, datetime
+
+
+CLASS_ORIGIN_AUTHOR="Russlan Ramdowar;russlan@ftredge.com"
+CLASS_ORGIN_DATE=datetime(2024, 3, 15, 20, 15)
+
+CLASS_VERSION = 2.01
+CLASS_REVISION_AUTHOR="Russlan Ramdowar;russlan@ftredge.com"
+CLASS_REVISION_DATE=datetime(2024, 3, 15, 20, 15)
+LAST_MODIFICATION="Created , with all fields Optional"
 
 class UserProfileUpdate(BaseModel):
-    email:  Optional[str] = None
-    username: Optional[str] = None
-    aliases: Optional[Set[str]] = None
-    first_name: Optional[str] = None
-    last_name: Optional[str] = None
-    mobile: Optional[str] = None
-    dob: Optional[date] = None 
-    updt_date: Optional[datetime] = None 
-    updt_by_user: Optional[str] = None
+    schema_version: Optional[float] = Field(None, description="Version of this Class == version of DB Schema")
+    email: Optional[EmailStr] = Field(None, description="Propagated from Firebase Auth")
+    organizations_uids: Optional[Set[str]] = Field(None, description="Depends on Subscription Plan, Regularly Updated")
+    creat_date: Optional[datetime] = Field(None, description="Creation date")
+    creat_by_user: Optional[str] = Field(None, description="Created by user")
+    updt_date: Optional[datetime] = Field(None, description="Update date")
+    updt_by_user: Optional[str] = Field(None, description="Updated by user")
+    aliases: Optional[Set[str]] = Field(None, description="User aliases")
+    provider_id: Optional[str] = Field(None, description="Provider ID")
+
+    username: Optional[str] = Field(None, description="Username")
+    dob: Optional[date] = Field(None, description="Date of Birth")
+    first_name: Optional[str] = Field(None, description="First Name")
+    last_name: Optional[str] = Field(None, description="Last Name")
+    mobile: Optional[str] = Field(None, description="Mobile Number")
     
-    def model_dump(self, **kwargs):
-            return super().model_dump(exclude_none=True, **kwargs)
+    # def model_dump(self, **kwargs):
+    #         return super().model_dump(exclude_none=True, **kwargs)
+        
+        
+        
+        

ipulse_shared_core_ftredge/models/user_status.py

@@ -1,36 +1,46 @@
 from pydantic import BaseModel, Field
+
 from datetime import datetime
 from dateutil.relativedelta import relativedelta
-from typing import Set, Optional
-# import uuid
-from . import pulse_enums as enums
+from typing import Set, Optional, Dict, List
+
+
 
 
 CLASS_ORIGIN_AUTHOR="Russlan Ramdowar;russlan@ftredge.com"
 CLASS_ORGIN_DATE=datetime(2024, 2, 12, 20, 5)
 
-CLASS_VERSION = 2.1
+SCHEMA_VERSION = 2.3
 CLASS_REVISION_AUTHOR="Russlan Ramdowar;russlan@ftredge.com"
 CLASS_REVISION_DATE=datetime(2024, 2, 13, 20, 15)
+LAST_MODIFICATION="Changed default IAM_GROUPS"
 
 DOMAIN="user"
 OBJ_REF = "usrsttus"
+
+DEFAULT_IAM_GROUPS={"pulseroot":["full_open_read"]}
+DEFAULT_SUBSCRIPTION_PLAN="subscription_free"
+DEFAULT_SUBSCRIPTION_STATUS="active"
+DEFAULT_SUBSCRIPTION_INSIGHT_CREDITS=10
+DEFAULT_EXTRA_INSIGHT_CREDITS=0
+
+############################################ !!!!! ALWAYS UPDATE SCHEMA VERSION , IF SCHEMA IS BEING MODIFIED !!! ############################################
 class UserStatus(BaseModel):
-    schema_version: float = Field(default=CLASS_VERSION, description="Version of this Class == version of DB Schema") #User can Read only
+    schema_version: float = Field(default=SCHEMA_VERSION, description="Version of this Class == version of DB Schema") #User can Read only
     # uid: str = Field(frozen=True, description="Generated by Firebase Auth") #User can Read only
     # puid:str = Field(default_factory=lambda: f"{DOMAIN}{OBJ_REF}{datetime.utcnow().strftime('%Y%m%d%H%M')}{uuid.uuid4().hex[:8]}".lower(),
     #                   frozen=True, 
     #                   description="Generated Automatically by default_factory") #User can Read only
     
-    iam_groups: Set[str] = Field(default_factory=lambda:{"pulseroot__authuser_open"}, description="User's Groups, with a default one for all authenticated Pulse users") #User can Read only
-    sbscrptn_plan: str=Field(default_factory=lambda:"subscription_free", description="Subscription Plan ") #User can Read only
-    sbscrptn_status: str=Field(default_factory=lambda:"active", description="Subscription Status") #User can Read only
+    iam_groups: Dict[str, List[str]] = Field(default_factory=lambda:DEFAULT_IAM_GROUPS, description="User's Groups, with a default one for all authenticated Pulse users") #User can Read only
+    sbscrptn_plan: str=Field(default_factory=lambda:DEFAULT_SUBSCRIPTION_PLAN, description="Subscription Plan ") #User can Read only
+    sbscrptn_status: str=Field(default_factory=lambda:DEFAULT_SUBSCRIPTION_STATUS, description="Subscription Status") #User can Read only
     sbscrptn_start_date: datetime=Field(default_factory=lambda:datetime.utcnow(), description="Subscription Start Date") #User can Read only
     sbscrptn_end_date: datetime=Field(default_factory=lambda:datetime.utcnow()+relativedelta(years=1) , description="Subscription End Date") #User can Read only
-    sbscrptn_insight_credits: int= Field(default_factory=lambda:7, description="Depends on Subscription Plan, Set Amount udated at Regular Intervals or at Regular Time") #User can Read only
+    sbscrptn_insight_credits: int= Field(default_factory=lambda:DEFAULT_SUBSCRIPTION_INSIGHT_CREDITS, description="Depends on Subscription Plan, Set Amount udated at Regular Intervals or at Regular Time") #User can Read only
     sbscrptn_ins_crdts_updtd_since_datetime: datetime=Field(default_factory=lambda:datetime.utcnow(), description="Subscription Start Date") #User can Read only #User can Read only #User can Read only
-    extra_insigth_credits: int= Field(default_factory=lambda:0, description="If user purchased extra Insigth Credits they shouldn't Expire") #User can Read only
-    payment_ref_uids: Optional[Set[str]] = None #User can Read only
+    extra_insight_credits: int= Field(default_factory=lambda:DEFAULT_EXTRA_INSIGHT_CREDITS, description="If user purchased extra Insigth Credits they shouldn't Expire") #User can Read only
+    payment_refs_uids: Optional[Set[str]] = None #User can Read only
     
     creat_date: datetime #User can Read only
     creat_by_user: str #User can Read only / Ideally shouldn't be able to see even

ipulse_shared_core_ftredge/utils/__init__.py

@@ -0,0 +1,23 @@
+# pylint: disable=missing-module-docstring
+
+from .logs import (ContextLog, get_logger)
+from .utils_common import (save_json_locally_extended,
+                           log_error,
+                           log_warning,
+                           log_info,
+                           prepare_full_file_path)
+
+from .utils_collector_pipelinemon import ( Pipelinemon)
+
+from .utils_cloud_gcp import (add_gcp_cloud_logging,
+                              add_gcp_error_reporting,
+                              create_bigquery_schema_from_json,
+                              read_csv_from_gcs, read_json_from_gcs,
+                                write_csv_to_gcs,write_json_to_gcs_extended)
+
+
+from .utils_cloud import (write_json_to_cloud_storage_extended,
+                          read_json_from_cloud_storage)
+
+
+from .utils_templates_and_schemas import (check_format_against_schema_template)

ipulse_shared_core_ftredge/utils/logs/__init__.py

@@ -0,0 +1,2 @@
+from .context_log import ContextLog
+from .get_logger import get_logger

ipulse_shared_core_ftredge/{models → utils/logs}/audit_log_firestore.py

@@ -9,4 +9,4 @@ class AuditLogFirestore(BaseModel):
     field_name: str
     old_value: str
     new_value: str
-    timestamp: datetime.utcnow()
+    timestamp: datetime

ipulse_shared_core_ftredge/utils/logs/context_log.py

@@ -0,0 +1,210 @@
+
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=unused-variable
+import traceback
+import json
+from datetime import datetime, timezone
+from typing import List
+from ipulse_shared_core_ftredge import Status, LogLevel
+
+############################################################################
+##################### SETTING UP custom LOGGING format= DICT ##########################
+### Cloud  Agnostic, can be used with any cloud provider , jsut use to_dict() method to get the log in dict format
+class ContextLog:
+
+    def __init__(self, level: LogLevel, base_context: str = None, collector_id: str = None,
+                context: str = None, description: str = None,
+                e: Exception = None, e_type: str = None, e_message: str = None, e_traceback: str = None,
+                log_status: Status = Status.OPEN, subject: str = None, systems_impacted: List[str] = None,
+                ):
+
+        if e is not None:
+            e_type = type(e).__name__ if e_type is None else e_type
+            e_message = str(e) if e_message is None else e_message
+            e_traceback = traceback.format_exc() if e_traceback is None else e_traceback
+        elif (e_traceback is None or e_traceback== "") and (e_type or e_message):
+            e_traceback = traceback.format_exc()
+
+        self.level = level
+        self.subject = subject
+        self.description = description
+        self._base_context = base_context
+        self._context = context
+        self._systems_impacted = systems_impacted if systems_impacted else []
+        self.collector_id = collector_id
+        self.exception_type = e_type
+        self.exception_message = e_message
+        self.exception_traceback = e_traceback
+        self.log_status = log_status
+        self.timestamp = datetime.now(timezone.utc).isoformat()
+
+    @property
+    def base_context(self):
+        return self._base_context
+
+    @base_context.setter
+    def base_context(self, value):
+        self._base_context = value
+
+    @property
+    def context(self):
+        return self._context
+
+    @context.setter
+    def context(self, value):
+        self._context = value
+
+    @property
+    def systems_impacted(self):
+        return self._systems_impacted
+
+    @systems_impacted.setter
+    def systems_impacted(self, list_of_si: List[str]):
+        self._systems_impacted = list_of_si
+
+    def add_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is None:
+            self._systems_impacted = []
+        self._systems_impacted.append(system_impacted)
+
+    def remove_system_impacted(self, system_impacted: str):
+        if self._systems_impacted is not None:
+            self._systems_impacted.remove(system_impacted)
+
+    def clear_systems_impacted(self):
+        self._systems_impacted = []
+
+    def _format_traceback(self, e_traceback, e_message, max_field_len:int, max_traceback_lines:int):
+        if not e_traceback or e_traceback == 'None\n':
+            return None
+
+        traceback_lines = e_traceback.splitlines()
+
+            # Check if the traceback is within the limits
+        if len(traceback_lines) <= max_traceback_lines and len(e_traceback) <= max_field_len:
+            return e_traceback
+
+        # Remove lines that are part of the exception message if they are present in traceback
+        message_lines = e_message.splitlines() if e_message else []
+        if message_lines:
+            for message_line in message_lines:
+                if message_line in traceback_lines:
+                    traceback_lines.remove(message_line)
+
+        # Filter out lines from third-party libraries (like site-packages)
+        filtered_lines = [line for line in traceback_lines if "site-packages" not in line]
+
+        # If filtering results in too few lines, revert to original traceback
+        if len(filtered_lines) < 2:
+            filtered_lines = traceback_lines
+
+        # Combine standalone bracket lines with previous or next lines
+        combined_lines = []
+        for line in filtered_lines:
+            if line.strip() in {"(", ")", "{", "}", "[", "]"} and combined_lines:
+                combined_lines[-1] += " " + line.strip()
+            else:
+                combined_lines.append(line)
+
+            # Ensure the number of lines doesn't exceed MAX_TRACEBACK_LINES
+        if len(combined_lines) > max_traceback_lines:
+            keep_lines_start = min(max_traceback_lines // 2, len(combined_lines))
+            keep_lines_end = min(max_traceback_lines // 2, len(combined_lines) - keep_lines_start)
+            combined_lines = (
+                combined_lines[:keep_lines_start] +
+                ['... (truncated) ...'] +
+                combined_lines[-keep_lines_end:]
+            )
+
+        formatted_traceback = '\n'.join(combined_lines)
+
+        # Ensure the total length doesn't exceed MAX_TRACEBACK_LENGTH
+        if len(formatted_traceback) > max_field_len:
+            truncated_length = max_field_len - len('... (truncated) ...')
+            half_truncated_length = truncated_length // 2
+            formatted_traceback = (
+                formatted_traceback[:half_truncated_length] +
+                '\n... (truncated) ...\n' +
+                formatted_traceback[-half_truncated_length:]
+            )
+        return formatted_traceback
+
+    def to_dict(self, max_field_len:int =10000, size_limit:float=256 * 1024 * 0.80,max_traceback_lines:int = 30):
+        size_limit = int(size_limit)  # Ensure size_limit is an integer
+
+        # Unified list of all fields
+        systems_impacted_str = f"{len(self.systems_impacted)} system(s): " + " ,,, ".join(self.systems_impacted) if self.systems_impacted else None
+        fields = [
+            ("log_status", str(self.log_status.name)),
+            ("level_code", self.level.value),
+            ("level_name", str(self.level.name)),
+            ("base_context", str(self.base_context)),
+            ("timestamp", str(self.timestamp)),
+            ("collector_id", str(self.collector_id)),
+            ("systems_impacted", systems_impacted_str),
+            ("context", str(self.context)),  # special sizing rules apply to it
+            ("subject", str(self.subject)),
+            ("description", str(self.description)),
+            ("exception_type", str(self.exception_type)),
+            ("exception_message", str(self.exception_message)),
+            ("exception_traceback", str(self._format_traceback(self.exception_traceback,self.exception_message, max_field_len, max_traceback_lines)))
+        ]
+
+        # Function to calculate the byte size of a JSON-encoded field
+        def field_size(key, value):
+            return len(json.dumps({key: value}).encode('utf-8'))
+
+        # Function to truncate a value based on its type
+        # Function to truncate a value based on its type
+        def truncate_value(value, max_size):
+            if isinstance(value, str):
+                half_size = max_size // 2
+                return value[:half_size] + '...' + value[-(max_size - half_size - 3):]
+            return value
+
+         # Ensure no field exceeds max_field_len
+        for i, (key, value) in enumerate(fields):
+            if isinstance(value, str) and len(value) > max_field_len:
+                fields[i] = (key, truncate_value(value, max_field_len))
+
+        # Ensure total size of the dict doesn't exceed size_limit
+        total_size = sum(field_size(key, value) for key, value in fields)
+        log_dict = {}
+        truncated = False
+
+        if total_size > size_limit:
+            truncated = True
+            remaining_size = size_limit
+            remaining_fields = len(fields)
+
+            for key, value in fields:
+                if remaining_fields > 0:
+                    max_size_per_field = remaining_size // remaining_fields
+                else:
+                    max_size_per_field = 0
+
+                field_sz = field_size(key, value)
+                if field_sz > max_size_per_field:
+                    value = truncate_value(value, max_size_per_field)
+                    field_sz = field_size(key, value)
+
+                log_dict[key] = value
+                remaining_size -= field_sz
+                remaining_fields -= 1
+        else:
+            log_dict = dict(fields)
+
+        log_dict['trunc'] = truncated
+
+        return log_dict
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), indent=4)
+
+    def __repr__(self):
+        return self.__str__()

ipulse_shared_core_ftredge/utils/logs/get_logger.py

@@ -0,0 +1,103 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-raised
+import logging
+import os
+import json
+import traceback
+from typing import List, Union
+from ipulse_shared_core_ftredge.enums import LoggingHandlers
+from ipulse_shared_core_ftredge.utils.utils_cloud_gcp import add_gcp_cloud_logging, add_gcp_error_reporting
+
+###################################################################################################
+##################################################################################################
+##################################### SETTING UP LOGGER ##########################################
+
+class CloudLogFormatter(logging.Formatter):
+    """Formats log records as structured JSON."""
+
+    def format(self, record):
+        log_entry = {
+            'message': record.msg,
+            'timestamp': self.formatTime(record, self.datefmt),
+            'name': record.name,
+            'severity': record.levelname,
+            'pathname': record.pathname,
+            'lineno': record.lineno,
+        }
+        if record.exc_info:
+            log_entry['exception_traceback'] = ''.join(traceback.format_exception(*record.exc_info))
+        if isinstance(record.msg, dict):
+            log_entry.update(record.msg)
+        return json.dumps(log_entry)
+
+
+class LocalLogFormatter(logging.Formatter):
+    """Formats log records for local output to the console."""
+
+    def format(self, record):  # Make sure you have the 'record' argument here!
+        path_parts = record.pathname.split(os.sep)
+
+        # Get the last two parts of the path if they exist
+        if len(path_parts) >= 2:
+            short_path = os.path.join(path_parts[-2], path_parts[-1])
+        else:
+            short_path = record.pathname
+        
+        # Format log messages differently based on the log level
+        if record.levelno == logging.INFO:
+            log_message = f"[INFO] {self.formatTime(record, self.datefmt)} :: {record.msg}"
+        elif record.levelno == logging.DEBUG:
+            log_message = f"[DEBUG] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+        elif record.levelno == logging.ERROR:
+            log_message = f"[ERROR] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+            if record.exc_info:
+                log_message += "\n" + ''.join(traceback.format_exception(*record.exc_info))
+        else:
+            log_message = f"[{record.levelname}] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+
+
+        return log_message
+
+
+def get_logger( logger_name:str ,level=logging.INFO, logging_handler_providers: Union[LoggingHandlers, List[LoggingHandlers]] = LoggingHandlers.NONE):
+
+    """Creates and configures a logger with the specified handlers."""
+
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(level)
+    cloud_formatter = CloudLogFormatter()
+
+    # Ensure logging_handler_providers is a list for consistent processing
+    if not isinstance(logging_handler_providers, list):
+        logging_handler_providers = [logging_handler_providers]
+
+    supported_remote_handlers = [
+        LoggingHandlers.GCP_CLOUD_LOGGING,
+        LoggingHandlers.GCP_ERROR_REPORTING,
+        LoggingHandlers.LOCAL_STREAM,
+        LoggingHandlers.NONE, # If NONE is considered a remote handler
+    ]
+
+    # Remote handlers
+
+    for handler_provider in logging_handler_providers:
+        if handler_provider in supported_remote_handlers:
+            if handler_provider == LoggingHandlers.GCP_CLOUD_LOGGING:
+                add_gcp_cloud_logging(logger, cloud_formatter)
+            elif handler_provider == LoggingHandlers.GCP_ERROR_REPORTING:
+                add_gcp_error_reporting(logger)
+            elif handler_provider == LoggingHandlers.LOCAL_STREAM:  # Handle local stream
+                local_handler = logging.StreamHandler()
+                local_handler.setFormatter(LocalLogFormatter())
+                logger.addHandler(local_handler)
+        else:
+            raise ValueError(
+            f"Unsupported logging provider: {handler_provider}. "
+            f"Supported providers: {[h.value for h in supported_remote_handlers]}"
+            )
+    return logger

ipulse_shared_core_ftredge/utils/utils_cloud.py

@@ -0,0 +1,53 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+# pylint: disable=line-too-long
+# pylint: disable=unused-variable
+# pylint: disable=broad-exception-caught
+from ipulse_shared_core_ftredge.enums import CloudProvider, DataSourceType, DuplicationHandling, MatchConditionType
+from .utils_collector_pipelinemon import Pipelinemon
+from .utils_cloud_gcp import (write_json_to_gcs_extended,
+                              read_json_from_gcs)
+
+#######################################################################################################################
+#######################################################################################################################
+#################################################     cloud IO functions      ########################################
+
+# Define the central function that routes to the relevant cloud-specific function
+def write_json_to_cloud_storage_extended(cloud_storage:CloudProvider | DataSourceType, storage_client, data:dict | list | str, bucket_name: str, file_name: str,
+                      duplication_handling:DuplicationHandling,  duplication_match_condition_type: MatchConditionType, duplication_match_condition: str = "",
+                       max_retries:int=2, max_matched_deletable_files:int=1,
+                      pipelinemon: Pipelinemon = None, logger=None, print_out=False, raise_e=False):
+
+
+    supported_cloud_storage_values = [CloudProvider.GCP, DataSourceType.GCS]
+
+    if cloud_storage in [CloudProvider.GCP, DataSourceType.GCS]:
+        return write_json_to_gcs_extended(
+            pipelinemon=pipelinemon,
+            storage_client=storage_client,
+            data=data,
+            bucket_name=bucket_name,
+            file_name=file_name,
+            duplication_handling_enum=duplication_handling,
+            duplication_match_condition_type_enum=duplication_match_condition_type,
+            duplication_match_condition=duplication_match_condition,
+            max_retries=max_retries,
+            max_deletable_files=max_matched_deletable_files,
+            logger=logger,
+            print_out=print_out,
+            raise_e=raise_e
+        )
+
+    raise ValueError(f"Unsupported cloud storage : {cloud_storage}. Supported cloud storage values: {supported_cloud_storage_values}")
+
+
+def read_json_from_cloud_storage(cloud_storage:CloudProvider | DataSourceType , storage_client, bucket_name:str, file_name:str, logger=None, print_out:bool=False):
+
+    supported_cloud_storage_values = [CloudProvider.GCP, DataSourceType.GCS]
+
+    if cloud_storage in [CloudProvider.GCP, DataSourceType.GCS]:
+        return read_json_from_gcs(storage_client=storage_client, bucket_name=bucket_name, file_name=file_name, logger=logger, print_out=print_out)
+
+    raise ValueError(f"Unsupported cloud storage: {cloud_storage}. Supported cloud storage values: {supported_cloud_storage_values}")