ipulse-shared-core-ftredge 2.26__py3-none-any.whl → 2.28__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -1,5 +1,6 @@
 from .models import Organisation, UserAuth, UserProfile, UserStatus, UserProfileUpdate, pulse_enums
 from .utils_gcp import setup_gcp_logger_and_error_report, read_csv_from_gcs, read_json_from_gcs, write_csv_to_gcs, write_json_to_gcs
-from .utils_templates_and_schemas import create_bigquery_schema_from_json,update_check_with_schema_template, update_check_with_dict_template
+from .utils_templates_and_schemas import create_bigquery_schema_from_json,update_check_with_schema_template
+from .enums_common import NoticeSeverity
 
  

ipulse_shared_core_ftredge/enums_common.py

@@ -0,0 +1,27 @@
+from enum import Enum
+
+class NoticeSeverity(Enum):
+    """
+    Standardized logging levels for data engineering pipelines,
+    designed for easy analysis and identification of manual 
+    intervention needs.
+    """
+    DEBUG = 100  # Detailed debug information (for development/troubleshooting)
+    INFO = 200   # Normal pipeline execution information 
+    NOTICE = 300  # Events requiring attention, but not necessarily errors
+
+     # Warnings indicate potential issues that might require attention:
+    WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
+    WARNING_ACTION_RECOMMENDED = 402 # Action recommended to prevent potential future issues
+    WARNING_ACTION_REQUIRED = 403  # Action required, pipeline can likely continue
+
+    # Errors indicate a problem that disrupts normal pipeline execution:
+    ERROR_TRANSIENT_RETRY = 501 # Temporary error, automatic retry likely to succeed
+    ERROR_DATA_ISSUE_ISOLATED = 502 # Error likely caused by data issues, manual intervention likely needed
+    ERROR_DATA_ISSUE_WITH_DEPENDENCIES = 503 # Error likely in code/configuration, investigation required
+    ERROR_CONFIG_OR_CODE_ISSUE = 504 # Error likely in code/configuration, investigation required
+    ERROR_UNKNOWN_EXCEPTION = 505
+
+    # Critical errors indicate severe failures requiring immediate attention:
+    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure), requires immediate action
+    CRITICAL_PIPELINE_FAILURE = 602 # Complete pipeline failure, requires investigation and potential rollback

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -1,5 +1,11 @@
-import datetime 
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+
+import datetime
 from google.cloud import bigquery
+from . import NoticeSeverity
 
 def create_bigquery_schema_from_json(json_schema):
     schema = []
@@ -10,67 +16,66 @@ def create_bigquery_schema_from_json(json_schema):
             schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
     return schema
 
-def update_check_with_dict_template(template, updates, logger):
-    template_dict = template.copy()
-    filtered_updates = {k: v for k, v in updates.items() if k in template}
-    template_dict.update(filtered_updates)
-    # Log warnings for any fields in the updates that are not in the template
-    extra_fields = set(updates.keys()) - set(template.keys())
-    if extra_fields:
-        logger.warning(f"Fields in the updates extra to the template config: {extra_fields}. You probably forgot to update the Template Config. Do so now.")
-    return template_dict
 
 def update_check_with_schema_template(updates, schema, logger, dt_ts_to_str=True, check_max_length=True):
+    
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
     valid_updates = {}
-    schema_dict = {field["name"]: field for field in schema}
     
-    for field_name, value in updates.items():
-        if field_name in schema_dict:
-            field_type = schema_dict[field_name]["type"]
+    # Process updates to conform to the schema
+    for field in schema:
+        field_name = field["name"]
+        field_type = field["type"]
+        mode = field["mode"]
+        
+        if field_name in updates:
+            value = updates[field_name]
             
+            # Handle date and timestamp formatting
             if dt_ts_to_str:
                 if field_type == "DATE":
                     value = handle_date_fields(field_name, value, logger)
                 elif field_type == "TIMESTAMP":
                     value = handle_timestamp_fields(field_name, value, logger)
             
-            if check_max_length and "max_length" in schema_dict[field_name]:
-                value = check_and_truncate_length(field_name, value, schema_dict[field_name]["max_length"], logger)
+            # Check and handle max length restriction
+            if check_max_length and "max_length" in field:
+                value = check_and_truncate_length(field_name, value, field["max_length"], logger)
             
-            # Set the value in the template dictionary based on the field type
-            if field_type == "STRING" and not isinstance(value, str):
-                logger.warning(f"Field {field_name} expected to be a string but got {type(value).__name__}.")
-                value = str(value)
-            elif field_type == "INT64" and not isinstance(value, int):
-                logger.warning(f"Field {field_name} expected to be an int but got {type(value).__name__}.")
-                try:
-                    value = int(value)
-                except ValueError:
-                    logger.warning(f"Cannot convert value {value} of field {field_name} to int.")
-                    value = None
-            elif field_type == "FLOAT64" and not isinstance(value, float):
-                logger.warning(f"Field {field_name} expected to be a float but got {type(value).__name__}.")
-                try:
-                    value = float(value)
-                except ValueError:
-                    logger.warning(f"Cannot convert value {value} of field {field_name} to float.")
-                    value = None
-            elif field_type == "BOOL" and not isinstance(value, bool):
-                logger.warning(f"Field {field_name} expected to be a bool but got {type(value).__name__}.")
-                value = bool(value)
-
-            valid_updates[field_name] = value
-        else:
-            logger.warning(f"Field {field_name} not in schema template, it will be ignored.You probably forgot to update the Schema Config. Do so now. "
-            "And assess the manual work required to fix missed data.")
-
-        # Check for missing REQUIRED fields
-    for field in schema:
-        if field["mode"] == "REQUIRED" and field["name"] not in valid_updates:
-            logger.warning(f"Required field '{field['name']}' is missing in the updates.")
+            # Validate and convert types
+            if field_type == "STRING":
+                if not isinstance(value, str):
+                    logger.warning(f"Field {field_name} expected to be a string but got {type(value).__name__}.")
+                    value = str(value)
+            elif field_type == "INT64":
+                if not isinstance(value, int):
+                    logger.warning(f"Field {field_name} expected to be an int but got {type(value).__name__}.")
+                    try:
+                        value = int(value)
+                    except ValueError:
+                        logger.warning(f"Cannot convert value {value} of field {field_name} to int.")
+                        continue
+            elif field_type == "FLOAT64":
+                if not isinstance(value, float):
+                    logger.warning(f"Field {field_name} expected to be a float but got {type(value).__name__}.")
+                    try:
+                        value = float(value)
+                    except ValueError:
+                        logger.warning(f"Cannot convert value {value} of field {field_name} to float.")
+                        continue
+            elif field_type == "BOOL":
+                if not isinstance(value, bool):
+                    logger.warning(f"Field {field_name} expected to be a bool but got {type(value).__name__}.")
+                    value = bool(value)
+
+            # Only add to the dictionary if value is not None or the field is required
+            if value is not None or mode == "REQUIRED":
+                valid_updates[field_name] = value
+
+        elif mode == "REQUIRED":
+            logger.warning(f"Required field '{field_name}' is missing in the updates.")
     
-    return  valid_updates
+    return valid_updates
 
 
 def check_updates_formatting(updates, schema, logger, dt_ts_to_str, check_max_length):

{ipulse_shared_core_ftredge-2.26.dist-info → ipulse_shared_core_ftredge-2.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.26
+Version: 2.28
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.26.dist-info → ipulse_shared_core_ftredge-2.28.dist-info}/RECORD

@@ -1,6 +1,7 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=OzKQDJcK-xsfy4j1ldBhoMCsdRHq0vPUsyVUOiTCROY,376
+ipulse_shared_core_ftredge/__init__.py,sha256=gJQzYYV1NyehTS6ZPtzaGv6-ihRXaxaew7dBNxM4V1g,384
+ipulse_shared_core_ftredge/enums_common.py,sha256=abs_PFWpd79ZikFEWbwidPN4bKjcDq5qFsvYgupLQMo,1601
 ipulse_shared_core_ftredge/utils_gcp.py,sha256=E8TvZ05fTjNr-VQXxSZNCiqT9PwPhtqeKOifIGhb2sg,6289
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=iRUA_0IFrKHIiqnaRirNDrhlz9Jf-IdXISNqsHgwlHI,6227
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=rYk547qbiRx7Q5a5pcb29lkwSArjS_F9Hf1JkX05mBU,5861
 ipulse_shared_core_ftredge/models/__init__.py,sha256=gE22Gzhil0RYQa7YLtdtT44_AsWqklcDfRtgLAQc1dI,200
 ipulse_shared_core_ftredge/models/audit_log_firestore.py,sha256=5AwO6NHuOncq65n400eqM8QPrS2EGGaP3Z_6l2rxdBE,261
 ipulse_shared_core_ftredge/models/organisation.py,sha256=4f1ATEWh5WT-CDJBLEZUhUwyl3V06ogRkteZAqW_nko,2953
@@ -12,8 +13,8 @@ ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFP
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
 ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.26.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.26.dist-info/METADATA,sha256=Cch9ylDzsq944x_iIaECNM9d0Vengx9PCPsYgmE5FEk,561
-ipulse_shared_core_ftredge-2.26.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-ipulse_shared_core_ftredge-2.26.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.26.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.28.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.28.dist-info/METADATA,sha256=YqkikZJUwwzn1QwAU2f1Sk5rJPG30TxsSTF4yLQp-iU,561
+ipulse_shared_core_ftredge-2.28.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+ipulse_shared_core_ftredge-2.28.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.28.dist-info/RECORD,,