ipulse-shared-core-ftredge 2.7.1__tar.gz → 2.8.1__tar.gz

{ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.7.1
+Version: 2.8.1
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar
@@ -10,4 +10,3 @@ Requires-Dist: python-dateutil~=2.8
 Requires-Dist: pytest~=7.1
 Requires-Dist: google-cloud-logging~=3.10.0
 Requires-Dist: google-cloud-error-reporting~=1.11.0
-Requires-Dist: google-cloud-bigquery~=3.24.0

ipulse_shared_core_ftredge-2.8.1/README.md

@@ -0,0 +1,7 @@
+# ipulse_shared_core
+Shared Models like User, Organisation etc. Also includes shared enum_sets
+
+
+### Enums 
+
+Contains majority of all Enums used in Pulse

{ipulse_shared_core_ftredge-2.7.1 → ipulse_shared_core_ftredge-2.8.1}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.7.1',
+    version='2.8.1',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[
@@ -12,8 +12,7 @@ setup(
         'python-dateutil~=2.8',
         'pytest~=7.1',
         'google-cloud-logging~=3.10.0',
-        'google-cloud-error-reporting~=1.11.0',
-        'google-cloud-bigquery~=3.24.0'
+        'google-cloud-error-reporting~=1.11.0'
     ],
     author='Russlan Ramdowar',
     description='Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.',

{ipulse_shared_core_ftredge-2.7.1 → ipulse_shared_core_ftredge-2.8.1}/src/ipulse_shared_core_ftredge/__init__.py

@@ -9,15 +9,10 @@ from .enums import (TargetLogs,LogLevel,  Status, Unit, Frequency,
                     DataSourceType,PipelineTriggerType,DataOperationType,
                     MatchConditionType, DuplicationHandling, DuplicationHandlingStatus,
                     CodingLanguage, ExecutionLocation, ExecutionComputeType,
-                    CloudProvider,LoggingHandlers)
-from .utils import (get_logger,
-                    save_json_locally_extended,
-                    write_json_to_cloud_storage_extended,
-                    write_json_to_gcs_extended,
-                    write_csv_to_gcs,
-                    read_json_from_cloud_storage,
-                    read_csv_from_gcs,
-                    read_json_from_gcs,
-                    check_format_against_schema_template,
-                    create_bigquery_schema_from_json,
-                    Pipelinemon, ContextLog)
+                    CloudProvider,LoggingHandler)
+from .utils import (list_as_strings)
+
+from .logging import (get_logger,
+                      log_error,
+                      log_warning,
+                      log_info)

ipulse_shared_core_ftredge-2.8.1/src/ipulse_shared_core_ftredge/logging/__init__.py

@@ -0,0 +1 @@
+from .utils_logging import get_logger, log_error, log_warning, log_info

ipulse_shared_core_ftredge-2.8.1/src/ipulse_shared_core_ftredge/logging/logging_handlers_and_formatters.py

@@ -0,0 +1,144 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+
+import logging
+import traceback
+import json
+import os
+from google.cloud import error_reporting
+from google.cloud import logging as cloud_logging
+
+##########################################################################################################################
+####################################    Custom logging FORMATTERS    ##################################################### 
+##########################################################################################################################
+
+
+class CloudLogFormatter(logging.Formatter):
+    """Formats log records as structured JSON."""
+
+    def format(self, record):
+        log_entry = {
+            'message': record.msg,
+            'timestamp': self.formatTime(record, self.datefmt),
+            'name': record.name,
+            'severity': record.levelname,
+            'pathname': record.pathname,
+            'lineno': record.lineno,
+        }
+        if record.exc_info:
+            log_entry['exception_traceback'] = ''.join(traceback.format_exception(*record.exc_info))
+        if isinstance(record.msg, dict):
+            log_entry.update(record.msg)
+        return json.dumps(log_entry)
+
+
+class LocalLogFormatter(logging.Formatter):
+    """Formats log records for local output to the console."""
+
+    def format(self, record):  # Make sure you have the 'record' argument here!
+        path_parts = record.pathname.split(os.sep)
+
+        # Get the last two parts of the path if they exist
+        if len(path_parts) >= 2:
+            short_path = os.path.join(path_parts[-2], path_parts[-1])
+        else:
+            short_path = record.pathname
+        
+        # Format log messages differently based on the log level
+        if record.levelno == logging.INFO:
+            log_message = f"[INFO] {self.formatTime(record, self.datefmt)} :: {record.msg}"
+        elif record.levelno == logging.DEBUG:
+            log_message = f"[DEBUG] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+        elif record.levelno == logging.ERROR:
+            log_message = f"[ERROR] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+            if record.exc_info:
+                log_message += "\n" + ''.join(traceback.format_exception(*record.exc_info))
+        else:
+            log_message = f"[{record.levelname}] {self.formatTime(record, self.datefmt)} :: {record.msg} :: {short_path} :: lineno {record.lineno} :: {record.name}"
+
+
+        return log_message
+
+#############################################################################################################################################
+######################################## Logging handlers for Google Cloud ########################################
+#############################################################################################################################################
+
+class CustomGCPLoggingHandler(cloud_logging.handlers.CloudLoggingHandler):
+    """Custom handler for Google Cloud Logging with a dynamic logName."""
+    def __init__(self, client, name, resource=None, labels=None):
+        super().__init__(client=client, name=name, resource=resource, labels=labels)
+        self.client = client  # Ensure client is consistently used
+
+    def emit(self, record):
+        try:
+            # 1. Create the basic log entry dictionary
+            log_entry = {
+                'message': record.msg,
+                'severity': record.levelname,
+                'name': record.name,
+                'pathname': record.filename,
+                'lineno': record.lineno,
+            }
+            if record.exc_info:
+                log_entry['exception_traceback'] = ''.join(
+                    traceback.format_exception(*record.exc_info)
+                )
+
+            # 2. Apply the formatter to the 'message' field if it's a dictionary
+            if isinstance(record.msg, dict):
+                formatted_message = self.formatter.format(record)
+                try:
+                    log_entry['message'] = json.loads(formatted_message)
+                except json.JSONDecodeError:
+                    log_entry['message'] = formatted_message
+            else:
+                log_entry['message'] = record.msg
+
+            # 3. Set the custom logName
+            log_entry['logName'] = f"projects/{self.client.project}/logs/{record.name}"
+
+            # 4. Send to Google Cloud Logging
+            super().emit(record)
+        except Exception as e:
+            self.handleError(record)
+
+class CustomGCPErrorReportingHandler(logging.Handler):
+    def __init__(self, client=None, level=logging.ERROR):
+        super().__init__(level)
+        self.error_client = error_reporting.Client() if client is None else client
+        self.propagate = True
+
+    def emit(self, record):
+        try:
+            if record.levelno >= logging.ERROR:
+                log_struct = {
+                'message': self.format(record),
+                'severity': record.levelname,
+                'pathname': getattr(record, 'pathname', None),
+                'lineno': getattr(record, 'lineno', None)
+            }
+            if record.exc_info:
+                log_struct['exception'] = ''.join(
+                    traceback.format_exception(*record.exc_info)
+                )
+            self.error_client.report(str(log_struct))
+        except Exception as e:
+            self.handleError(record)
+
+
+def add_gcp_cloud_logging(logger, formatter, client=None):
+    """Sets up Google Cloud Logging for the logger."""
+    client = client or cloud_logging.Client()
+    handler = CustomGCPLoggingHandler(client, logger.name)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+def add_gcp_error_reporting(logger, client=None):
+    """Sets up Google Cloud Error Reporting for the logger."""
+    client = client or error_reporting.Client()
+    handler = CustomGCPErrorReportingHandler(client=client)
+    logger.addHandler(handler)

ipulse_shared_core_ftredge-2.8.1/src/ipulse_shared_core_ftredge/logging/utils_logging.py

@@ -0,0 +1,72 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+import logging
+from typing import List, Union
+from ipulse_shared_core_ftredge.enums import LoggingHandler
+from ipulse_shared_core_ftredge.logging.logging_handlers_and_formatters import (CloudLogFormatter,
+                                                                                      LocalLogFormatter,
+                                                                                      add_gcp_cloud_logging,
+                                                                                      add_gcp_error_reporting)
+
+
+def get_logger( logger_name:str ,level=logging.INFO, logging_handler_providers: Union[LoggingHandler, List[LoggingHandler]] = LoggingHandler.NONE):
+
+    """Creates and configures a logger with the specified handlers."""
+
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(level)
+    cloud_formatter = CloudLogFormatter()
+
+    # Ensure logging_handler_providers is a list for consistent processing
+    if not isinstance(logging_handler_providers, list):
+        logging_handler_providers = [logging_handler_providers]
+
+    supported_remote_handlers = [
+        LoggingHandler.GCP_CLOUD_LOGGING,
+        LoggingHandler.GCP_ERROR_REPORTING,
+        LoggingHandler.LOCAL_STREAM,
+        LoggingHandler.NONE, # If NONE is considered a remote handler
+    ]
+
+    # Remote handlers
+
+    for handler_provider in logging_handler_providers:
+        if handler_provider in supported_remote_handlers:
+            if handler_provider == LoggingHandler.GCP_CLOUD_LOGGING:
+                add_gcp_cloud_logging(logger, cloud_formatter)
+            elif handler_provider == LoggingHandler.GCP_ERROR_REPORTING:
+                add_gcp_error_reporting(logger)
+            elif handler_provider == LoggingHandler.LOCAL_STREAM:  # Handle local stream
+                local_handler = logging.StreamHandler()
+                local_handler.setFormatter(LocalLogFormatter())
+                logger.addHandler(local_handler)
+        else:
+            raise ValueError(
+            f"Unsupported logging provider: {handler_provider}. "
+            f"Supported providers: {[h.value for h in supported_remote_handlers]}"
+            )
+    return logger
+
+
+def log_error(msg,logger=None , print_out=False, exc_info=False):
+    if print_out:
+        print(msg)
+    if logger:
+        logger.error(msg, exc_info=exc_info)
+
+def log_warning(msg, logger=None, print_out=False):
+    if print_out:
+        print(msg)
+    if logger:
+        logger.warning(msg)
+
+
+def log_info(msg, logger=None, print_out=False):
+    if print_out:
+        print(msg)
+    if logger:
+        logger.info(msg)

ipulse_shared_core_ftredge-2.8.1/src/ipulse_shared_core_ftredge/utils/__init__.py

@@ -0,0 +1,3 @@
+# pylint: disable=missing-module-docstring
+
+from .utils_common import (list_as_strings)

ipulse_shared_core_ftredge-2.8.1/src/ipulse_shared_core_ftredge/utils/utils_common.py

@@ -0,0 +1,10 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+# pylint: disable=missing-class-docstring
+# pylint: disable=broad-exception-caught
+
+def list_as_strings(*enums):
+    """Converts a list of Enum members to their string values."""
+    return [str(enum) for enum in enums]

{ipulse_shared_core_ftredge-2.7.1 → ipulse_shared_core_ftredge-2.8.1/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.7.1
+Version: 2.8.1
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar
@@ -10,4 +10,3 @@ Requires-Dist: python-dateutil~=2.8
 Requires-Dist: pytest~=7.1
 Requires-Dist: google-cloud-logging~=3.10.0
 Requires-Dist: google-cloud-error-reporting~=1.11.0
-Requires-Dist: google-cloud-bigquery~=3.24.0

ipulse_shared_core_ftredge-2.8.1/src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt

@@ -0,0 +1,24 @@
+LICENCE
+README.md
+pyproject.toml
+setup.py
+src/ipulse_shared_core_ftredge/__init__.py
+src/ipulse_shared_core_ftredge.egg-info/PKG-INFO
+src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt
+src/ipulse_shared_core_ftredge.egg-info/dependency_links.txt
+src/ipulse_shared_core_ftredge.egg-info/requires.txt
+src/ipulse_shared_core_ftredge.egg-info/top_level.txt
+src/ipulse_shared_core_ftredge/logging/__init__.py
+src/ipulse_shared_core_ftredge/logging/audit_log_firestore.py
+src/ipulse_shared_core_ftredge/logging/logging_handlers_and_formatters.py
+src/ipulse_shared_core_ftredge/logging/utils_logging.py
+src/ipulse_shared_core_ftredge/models/__init__.py
+src/ipulse_shared_core_ftredge/models/organisation.py
+src/ipulse_shared_core_ftredge/models/resource_catalog_item.py
+src/ipulse_shared_core_ftredge/models/user_auth.py
+src/ipulse_shared_core_ftredge/models/user_profile.py
+src/ipulse_shared_core_ftredge/models/user_profile_update.py
+src/ipulse_shared_core_ftredge/models/user_status.py
+src/ipulse_shared_core_ftredge/utils/__init__.py
+src/ipulse_shared_core_ftredge/utils/utils_common.py
+tests/test_utils_gcp.py

{ipulse_shared_core_ftredge-2.7.1 → ipulse_shared_core_ftredge-2.8.1}/src/ipulse_shared_core_ftredge.egg-info/requires.txt

@@ -3,4 +3,3 @@ python-dateutil~=2.8
 pytest~=7.1
 google-cloud-logging~=3.10.0
 google-cloud-error-reporting~=1.11.0
-google-cloud-bigquery~=3.24.0

ipulse_shared_core_ftredge-2.7.1/README.md

@@ -1,21 +0,0 @@
-# ipulse_shared_core
-Shared Models like User, Organisation etc. Also includes shared enum_sets
-
-
-### Enums 
-
-Contains majority of all Enums used in Pulse
-
-
-
-### Collectors i.e. Pipelinemon
-
-Collectors are smart Objects which are added to long running functions or pipelines for which we want to collect an overall number of successes, notices, warnings or errors. 
-
-We can wait until the full pipeline is finished in order to write off a single Summary file from a Collector, or we can attach to it a logger, which will be reporting major status along the journey, which is often times better. Because if a function crashes midway through , all logs will be lost, and it would be hard to investigate if anythign has bee persisted and has to be rolled back. THis will require a lot of manual effort to recollect.
-
-Pipelinemon , short of Pipeline Monitoring system is a type of very powerful Collector which Russlan created specifically for Pulse Data Engineering pipelines.
-
-Pipelinemon writes all observation logs to Google CLoud Logging, and you have to setup a Log Sink (Router) which will send the Pipelinemon's observation logs to BigQuery. 
-
-Great thing about Pipelinemin is its "context" keeping feature.

ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge/enums/__init__.py

@@ -1,37 +0,0 @@
-
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-
-from .enums_common_utils import (Status,
-                                Unit,
-                                Frequency)
-
-
-from .enums_modules import(Module,
-                           Domain)
-
-
-from .enums_module_fincore import (FinCoreCategory,
-                                    FincCoreSubCategory,
-                                    FinCoreRecordsCategory,
-                                    FinancialExchangeOrPublisher)
-
-from .enums_logging import (TargetLogs,
-                            LogLevel,
-                            LoggingHandlers)
-
-from .enums_data_eng import (DataPrimaryCategory,
-                            DataState,
-                            DatasetScope,
-                            DataSourceType,
-                            PipelineTriggerType,
-                            DataOperationType,
-                            MatchConditionType,
-                            DuplicationHandling,
-                            DuplicationHandlingStatus,
-                            CodingLanguage,
-                            ExecutionLocation,
-                            ExecutionComputeType)
-
-from .enums_solution_providers import (CloudProvider)

ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -1,107 +0,0 @@
-
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=line-too-long
-
-from enum import Enum
-
-
-class Status(Enum):
-    OPEN = "open"
-    ACKNOWLEDGED = "acknowledged"
-    ESCALATED = "escalated"
-    IN_PROGRESS = "in_progress"
-    IN_REVIEW = "in_review"
-    RESOLVED = "resolved"
-    IGNORED = "ignored"
-    CANCELLED = "cancelled"
-    CLOSED = "closed"
-
-    def __str__(self):
-        return self.value
-
-### Exception during full exection, partially saved
-# Exception during ensemble pipeline; modifications collected in local object , nothing persisted
-# Exception during ensemble pipeline; modifications persisted , metadata failed
-# Exception during ensemble pipeline; modifications persisted , metadata persisted
-# Exception during ensemble pipeline; modifications persisted , metadata persisted
-
-
-class Unit(Enum):
-    MIX="MIX"
-    # Currency and Financial Values
-    USD = "USD"  # United States Dollar
-    EUR = "EUR"  # Euro
-    JPY = "JPY"  # Japanese Yen
-    GBP = "GBP"  # British Pound Sterling
-    AUD = "AUD"  # Australian Dollar
-    CAD = "CAD"  # Canadian Dollar
-    CHF = "CHF"  # Swiss Franc
-    CNY = "CNY"  # Chinese Yuan Renminbi
-    SEK = "SEK"  # Swedish Krona
-    NZD = "NZD"  # New Zealand Dollar
-    MXN = "MXN"  # Mexican Peso
-    SGD = "SGD"  # Singapore Dollar
-    HKD = "HKD"  # Hong Kong Dollar
-    NOK = "NOK"  # Norwegian Krone
-    KRW = "KRW"  # South Korean Won
-    RUB = "RUB"  # Russian Ruble
-    INR = "INR"  # Indian Rupee
-    BRL = "BRL"  # Brazilian Real
-    ZAR = "ZAR"  # South African Rand
-    CURRENCY = "currency"    # General currency, when specific currency is not needed
-
-    # Stock Market and Investments
-    SHARES = "shares"        # Number of shares
-    PERCENT = "prcnt"      # Percentage, used for rates and ratios
-    BPS = "bps"              # Basis points, often used for interest rates and financial ratios
-
-    # Volume and Quantitative Measurements
-    VOLUME = "volume"        # Trading volume in units
-    MILLIONS = "mills"    # Millions, used for large quantities or sums
-    BILLIONS = "bills"    # Billions, used for very large quantities or sums
-
-    # Commodity Specific Units
-    BARRELS = "barrels"      # Barrels, specifically for oil and similar liquids
-    TONNES = "tonnes"        # Tonnes, for bulk materials like metals or grains
-    TROY_OUNCES = "troy_oz" # Troy ounces, specifically for precious metals
-
-    # Real Estate and Physical Properties
-    SQUARE_FEET = "sq_ft"    # Square feet, for area measurement in real estate
-    METER_SQUARE = "m2"      # Square meters, for area measurement in real estate
-    ACRES = "acres"          # Acres, used for measuring large plots of land
-
-    # Miscellaneous and Other Measures
-    UNITS = "units"          # Generic units, applicable when other specific units are not suitable
-    COUNT = "count"          # Count, used for tallying items or events
-    INDEX_POINTS = "index_pnts"  # Index points, used in measuring indices like stock market indices
-    RATIO = "ratio"          # Ratio, for various financial ratios
-
-    def __str__(self):
-        return self.value
-
-class Frequency(Enum):
-    ONE_MIN = "1min"
-    FIVE_MIN="5min"
-    FIFTEEN_MIN="15min"
-    THIRTY_MIN = "30min"
-    ONE_H = "1h"
-    TWO_H = "2h"
-    SIX_H = "6h"
-    TWELVE_H = "12h"
-    FOUR_H = "4h"
-    EOD="eod"
-    ONE_D = "1d"
-    TWO_D = "2d"
-    THREE_D = "3d"
-    ONE_W = "1w"
-    ONE_M = "1m"
-    TWO_M="2m"
-    THREE_M="3m"
-    SIX_M="6m"
-    ONE_Y="1y"
-    THREE_Y="3y"
-
-    def __str__(self):
-        return self.value