ipulse-shared-core-ftredge 2.33__py3-none-any.whl → 2.36__py3-none-any.whl

ipulse_shared_core_ftredge/__init__.py

@@ -4,8 +4,9 @@ from .utils_gcp import (setup_gcp_logger_and_error_report,
                         read_csv_from_gcs, read_json_from_gcs, 
                         write_csv_to_gcs, write_json_to_gcs)
 from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
-                                          update_check_with_schema_template)
-from .enums import (NoticeSeverity,Unit, Frequency,
+                                          update_check_with_schema_template,
+                                          create_data_check_notice)
+from .enums import (NoticeSeverity, Unit, Frequency,
                     Module, SubModule, BaseDataCategory,
                     FinCoreCategory, FincCoreSubCategory,
                     FinCoreRecordsCategory, ExchangeOrPublisher,

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -5,7 +5,15 @@
 
 import datetime
 from google.cloud import bigquery
-from . import NoticeSeverity
+from ipulse_shared_core_ftredge.enums.enums_common_utils import NoticeSeverity
+
+def create_data_check_notice(severity, field_name, message):
+    return {
+        "severity_code": severity.value,
+        "severity_name": severity.name,
+        "subject": field_name,
+        "message": message
+    }
 
 def create_bigquery_schema_from_json(json_schema):
     schema = []
@@ -17,124 +25,123 @@ def create_bigquery_schema_from_json(json_schema):
     return schema
 
 
-def update_check_with_schema_template(updates, schema, logger, dt_ts_to_str=True, check_max_length=True):
-    
+def update_check_with_schema_template(updates, schema, dt_ts_to_str=True, check_max_length=True):
+
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
     valid_updates = {}
-    
+    notices=[] ### THIS IS TO AVOID LOGGING A WARNING RANDOMLY , INSTEAD GROUPPING FOR A GIVEN RUN
+
     # Process updates to conform to the schema
     for field in schema:
         field_name = field["name"]
         field_type = field["type"]
         mode = field["mode"]
-        
+
         if field_name in updates:
             value = updates[field_name]
-            
+
             # Handle date and timestamp formatting
             if dt_ts_to_str:
                 if field_type == "DATE":
-                    value = handle_date_fields(field_name, value, logger)
+                    value, notice = handle_date_fields(field_name, value)
                 elif field_type == "TIMESTAMP":
-                    value = handle_timestamp_fields(field_name, value, logger)
-            
+                    value,notice = handle_timestamp_fields(field_name, value)
+                if notice:
+                    notices.append(notice)
             # Check and handle max length restriction
             if check_max_length and "max_length" in field:
-                value = check_and_truncate_length(field_name, value, field["max_length"], logger)
-            
+                value,notice = check_and_truncate_length(field_name, value, field["max_length"])
+                if notice:
+                    notices.append(notice)
             # Validate and convert types
-            if field_type == "STRING":
-                if not isinstance(value, str):
-                    logger.warning(f"Field {field_name} expected to be a string but got {type(value).__name__}.")
-                    value = str(value)
-            elif field_type == "INT64":
-                if not isinstance(value, int):
-                    logger.warning(f"Field {field_name} expected to be an int but got {type(value).__name__}.")
-                    try:
-                        value = int(value)
-                    except ValueError:
-                        logger.warning(f"Cannot convert value {value} of field {field_name} to int.")
-                        continue
-            elif field_type == "FLOAT64":
-                if not isinstance(value, float):
-                    logger.warning(f"Field {field_name} expected to be a float but got {type(value).__name__}.")
-                    try:
-                        value = float(value)
-                    except ValueError:
-                        logger.warning(f"Cannot convert value {value} of field {field_name} to float.")
-                        continue
-            elif field_type == "BOOL":
-                if not isinstance(value, bool):
-                    logger.warning(f"Field {field_name} expected to be a bool but got {type(value).__name__}.")
-                    value = bool(value)
+            if field_type in ["STRING", "INT64", "FLOAT64", "BOOL"]:
+                value, notice = handle_type_conversion(field_type, field_name, value )
+                if notice:
+                    notices.append(notice)
 
             # Only add to the dictionary if value is not None or the field is required
             if value is not None or mode == "REQUIRED":
                 valid_updates[field_name] = value
 
         elif mode == "REQUIRED":
-            logger.warning(f"Required field '{field_name}' is missing in the updates.")
-    
-    return valid_updates
-
-
-def check_updates_formatting(updates, schema, logger, dt_ts_to_str, check_max_length):
-    """Processes updates to ensure they match the schema, handling dates, timestamps, and lengths."""
-    for field in schema:
-        field_name = field["name"]
-        field_type = field["type"]
-
-        if field_name in updates:
-            value = updates[field_name]
+            notice=create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
+                             field_name,
+                             f"Required field '{field_name}' is missing in the updates.")
 
-            if dt_ts_to_str:
-                if field_type == "DATE":
-                    updates[field_name] = handle_date_fields(field_name, value, logger)
-                elif field_type == "TIMESTAMP":
-                    updates[field_name] = handle_timestamp_fields(field_name, value, logger)
-
-            if check_max_length and "max_length" in field:
-                updates[field_name] = check_and_truncate_length(field_name, value, field["max_length"], logger)
-
-    return updates
+            notices.append(notice)
+      
+    return valid_updates, notices
 
 
-def handle_date_fields(field_name, value, logger):
-    """Handles date fields, ensuring they are in the correct format."""
+def handle_date_fields(field_name, value):
+    """Handles date fields, ensuring they are in the correct format.
+        Return is a tuple of the formatted date and a notice."""
     if isinstance(value, datetime.date):
-        return value.strftime("%Y-%m-%d")
+        return value.strftime("%Y-%m-%d"), None
     elif isinstance(value, str):
         try:
             datetime.datetime.strptime(value, "%Y-%m-%d")
             return value
         except ValueError:
-            logger.warning(f"Invalid date format for field {field_name}, expected YYYY-MM-DD")
-            return None
+            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
+                             field_name,
+                             f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
     else:
-        logger.warning(f"Invalid date format for field {field_name}")
-        return None
-    
+        return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
+                             field_name,
+                             f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
+
 
-def handle_timestamp_fields(field_name, value, logger):
+def handle_timestamp_fields(field_name, value):
     """Handles timestamp fields, ensuring they are in the correct format."""
     if isinstance(value, datetime.datetime):
-        return value.isoformat()
+        return value.isoformat(), None
     elif isinstance(value, str):
         try:
             datetime.datetime.fromisoformat(value)
             return value
         except ValueError:
-            logger.warning(f"Invalid timestamp format for field {field_name}, expected ISO format")
-            return None
+            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED, 
+                             field_name,
+                             f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
     else:
-        logger.warning(f"Invalid timestamp format for field {field_name}")
-        return None
+        return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED, 
+                             field_name,
+                             f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
 
-def check_and_truncate_length(field_name, value, max_length, logger):
+
+def check_and_truncate_length(field_name, value, max_length):
     """Checks and truncates the length of string fields if they exceed the max length."""
     if isinstance(value, str) and len(value) > max_length:
-        logger.warning(f"Field {field_name} exceeds max length of {max_length}. Truncating.")
-        return value[:max_length]
-    return value
+        return value[:max_length], create_data_check_notice(NoticeSeverity.WARNING_ACTION_RECOMMENDED, 
+                             field_name,
+                             f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
+   
+    return value, None
+
 
 
+def handle_type_conversion(field_type, field_name, value):
+    if field_type == "STRING" and not isinstance(value, str):
+        return str(value), create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED, 
+                             field_name,
+                             f"Expected STRING but got {value} of type {type(value).__name__}.")
+
+    if field_type == "INT64" and not isinstance(value, int):
+        try:
+            return int(value), None
+        except ValueError:
+            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED,
+                                                   field_name,
+                                                   f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
+    if field_type == "FLOAT64" and not isinstance(value, float):
+        try:
+            return float(value), None
+        except ValueError:
+            return None, create_data_check_notice(NoticeSeverity.WARNING_ACTION_REQUIRED, 
+                                                  field_name,
+                                                  f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
+    if field_type == "BOOL" and not isinstance(value, bool):
+        return bool(value), None
+    
+    return value, None

{ipulse_shared_core_ftredge-2.33.dist-info → ipulse_shared_core_ftredge-2.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.33
+Version: 2.36
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.33.dist-info → ipulse_shared_core_ftredge-2.36.dist-info}/RECORD

@@ -1,6 +1,6 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=Z8qv6nL-cXEGTHl3zTnL9vxGNZ94uCfJBgsSV7HAkcw,826
+ipulse_shared_core_ftredge/__init__.py,sha256=snLFrVzGmYy6TmXM6-D7Yq4dkcJDRP3g9DeuJQDFPMc,895
 ipulse_shared_core_ftredge/utils_gcp.py,sha256=E8TvZ05fTjNr-VQXxSZNCiqT9PwPhtqeKOifIGhb2sg,6289
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=rYk547qbiRx7Q5a5pcb29lkwSArjS_F9Hf1JkX05mBU,5861
+ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=UxRZnTOmV0VfjtrA-0YrTcKgJOMpa6kFiuLSdSUD_XA,6513
 ipulse_shared_core_ftredge/enums/__init__.py,sha256=PjxJiUConI2TuaG_Ushe2BaFVjBDw1rbq1E9Vt9nXvE,801
 ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=ukChcm8R2QwlK4NAfM291mWryCAZvfg7bMpdScQQfok,4360
 ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=A8_uiGGacd-_AZP09Zft-DX3rF8aVqqFzFak8s4MsfY,1801
@@ -17,8 +17,8 @@ ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFP
 ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
 ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.33.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.33.dist-info/METADATA,sha256=nBXAyVU5_otXscH8TQ8DmeIGVkoR01wu8c6LnB6bBec,561
-ipulse_shared_core_ftredge-2.33.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-ipulse_shared_core_ftredge-2.33.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.33.dist-info/RECORD,,
+ipulse_shared_core_ftredge-2.36.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
+ipulse_shared_core_ftredge-2.36.dist-info/METADATA,sha256=vYm8146CUjDdbz9XOZGikiH2wfd29iIthI6zq608ShI,561
+ipulse_shared_core_ftredge-2.36.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+ipulse_shared_core_ftredge-2.36.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
+ipulse_shared_core_ftredge-2.36.dist-info/RECORD,,