ipulse-shared-core-ftredge 2.23__tar.gz → 2.26__tar.gz

{ipulse_shared_core_ftredge-2.23/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.23
+Version: 2.26
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar
@@ -10,3 +10,4 @@ Requires-Dist: python-dateutil~=2.8
 Requires-Dist: pytest~=7.1
 Requires-Dist: google-cloud-logging~=3.10.0
 Requires-Dist: google-cloud-error-reporting~=1.11.0
+Requires-Dist: google-cloud-bigquery~=3.24.0

{ipulse_shared_core_ftredge-2.23 → ipulse_shared_core_ftredge-2.26}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.23',
+    version='2.26',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[
@@ -11,7 +11,8 @@ setup(
         'python-dateutil~=2.8',
         'pytest~=7.1',
         'google-cloud-logging~=3.10.0',
-        'google-cloud-error-reporting~=1.11.0'
+        'google-cloud-error-reporting~=1.11.0',
+        'google-cloud-bigquery~=3.24.0'
     ],
     author='Russlan Ramdowar',
     description='Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.',

ipulse_shared_core_ftredge-2.26/src/ipulse_shared_core_ftredge/__init__.py

@@ -0,0 +1,5 @@
+from .models import Organisation, UserAuth, UserProfile, UserStatus, UserProfileUpdate, pulse_enums
+from .utils_gcp import setup_gcp_logger_and_error_report, read_csv_from_gcs, read_json_from_gcs, write_csv_to_gcs, write_json_to_gcs
+from .utils_templates_and_schemas import create_bigquery_schema_from_json,update_check_with_schema_template, update_check_with_dict_template
+
+ 

ipulse_shared_core_ftredge-2.23/src/ipulse_shared_core_ftredge/gcp_utils.py → ipulse_shared_core_ftredge-2.26/src/ipulse_shared_core_ftredge/utils_gcp.py

@@ -77,7 +77,7 @@ def setup_gcp_logger_and_error_report(logger_name):
 
 
 ############################################################################
-##################### GOOGLE  CLOUD STORAGE UTILS ##########################
+##################### GOOGLE CLOUD STORAGE UTILS ##########################
 
 def read_json_from_gcs(bucket_name, file_name, stor_client, logger):
     """ Helper function to read a JSON file from Google Cloud Storage """

ipulse_shared_core_ftredge-2.26/src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -0,0 +1,135 @@
+import datetime 
+from google.cloud import bigquery
+
+def create_bigquery_schema_from_json(json_schema):
+    schema = []
+    for field in json_schema:
+        if "max_length" in field:
+            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"], max_length=field["max_length"]))
+        else:
+            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
+    return schema
+
+def update_check_with_dict_template(template, updates, logger):
+    template_dict = template.copy()
+    filtered_updates = {k: v for k, v in updates.items() if k in template}
+    template_dict.update(filtered_updates)
+    # Log warnings for any fields in the updates that are not in the template
+    extra_fields = set(updates.keys()) - set(template.keys())
+    if extra_fields:
+        logger.warning(f"Fields in the updates extra to the template config: {extra_fields}. You probably forgot to update the Template Config. Do so now.")
+    return template_dict
+
+def update_check_with_schema_template(updates, schema, logger, dt_ts_to_str=True, check_max_length=True):
+    """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
+    valid_updates = {}
+    schema_dict = {field["name"]: field for field in schema}
+    
+    for field_name, value in updates.items():
+        if field_name in schema_dict:
+            field_type = schema_dict[field_name]["type"]
+            
+            if dt_ts_to_str:
+                if field_type == "DATE":
+                    value = handle_date_fields(field_name, value, logger)
+                elif field_type == "TIMESTAMP":
+                    value = handle_timestamp_fields(field_name, value, logger)
+            
+            if check_max_length and "max_length" in schema_dict[field_name]:
+                value = check_and_truncate_length(field_name, value, schema_dict[field_name]["max_length"], logger)
+            
+            # Set the value in the template dictionary based on the field type
+            if field_type == "STRING" and not isinstance(value, str):
+                logger.warning(f"Field {field_name} expected to be a string but got {type(value).__name__}.")
+                value = str(value)
+            elif field_type == "INT64" and not isinstance(value, int):
+                logger.warning(f"Field {field_name} expected to be an int but got {type(value).__name__}.")
+                try:
+                    value = int(value)
+                except ValueError:
+                    logger.warning(f"Cannot convert value {value} of field {field_name} to int.")
+                    value = None
+            elif field_type == "FLOAT64" and not isinstance(value, float):
+                logger.warning(f"Field {field_name} expected to be a float but got {type(value).__name__}.")
+                try:
+                    value = float(value)
+                except ValueError:
+                    logger.warning(f"Cannot convert value {value} of field {field_name} to float.")
+                    value = None
+            elif field_type == "BOOL" and not isinstance(value, bool):
+                logger.warning(f"Field {field_name} expected to be a bool but got {type(value).__name__}.")
+                value = bool(value)
+
+            valid_updates[field_name] = value
+        else:
+            logger.warning(f"Field {field_name} not in schema template, it will be ignored.You probably forgot to update the Schema Config. Do so now. "
+            "And assess the manual work required to fix missed data.")
+
+        # Check for missing REQUIRED fields
+    for field in schema:
+        if field["mode"] == "REQUIRED" and field["name"] not in valid_updates:
+            logger.warning(f"Required field '{field['name']}' is missing in the updates.")
+    
+    return  valid_updates
+
+
+def check_updates_formatting(updates, schema, logger, dt_ts_to_str, check_max_length):
+    """Processes updates to ensure they match the schema, handling dates, timestamps, and lengths."""
+    for field in schema:
+        field_name = field["name"]
+        field_type = field["type"]
+
+        if field_name in updates:
+            value = updates[field_name]
+
+            if dt_ts_to_str:
+                if field_type == "DATE":
+                    updates[field_name] = handle_date_fields(field_name, value, logger)
+                elif field_type == "TIMESTAMP":
+                    updates[field_name] = handle_timestamp_fields(field_name, value, logger)
+
+            if check_max_length and "max_length" in field:
+                updates[field_name] = check_and_truncate_length(field_name, value, field["max_length"], logger)
+
+    return updates
+
+
+def handle_date_fields(field_name, value, logger):
+    """Handles date fields, ensuring they are in the correct format."""
+    if isinstance(value, datetime.date):
+        return value.strftime("%Y-%m-%d")
+    elif isinstance(value, str):
+        try:
+            datetime.datetime.strptime(value, "%Y-%m-%d")
+            return value
+        except ValueError:
+            logger.warning(f"Invalid date format for field {field_name}, expected YYYY-MM-DD")
+            return None
+    else:
+        logger.warning(f"Invalid date format for field {field_name}")
+        return None
+    
+
+def handle_timestamp_fields(field_name, value, logger):
+    """Handles timestamp fields, ensuring they are in the correct format."""
+    if isinstance(value, datetime.datetime):
+        return value.isoformat()
+    elif isinstance(value, str):
+        try:
+            datetime.datetime.fromisoformat(value)
+            return value
+        except ValueError:
+            logger.warning(f"Invalid timestamp format for field {field_name}, expected ISO format")
+            return None
+    else:
+        logger.warning(f"Invalid timestamp format for field {field_name}")
+        return None
+
+def check_and_truncate_length(field_name, value, max_length, logger):
+    """Checks and truncates the length of string fields if they exceed the max length."""
+    if isinstance(value, str) and len(value) > max_length:
+        logger.warning(f"Field {field_name} exceeds max length of {max_length}. Truncating.")
+        return value[:max_length]
+    return value
+
+

{ipulse_shared_core_ftredge-2.23 → ipulse_shared_core_ftredge-2.26/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.23
+Version: 2.26
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar
@@ -10,3 +10,4 @@ Requires-Dist: python-dateutil~=2.8
 Requires-Dist: pytest~=7.1
 Requires-Dist: google-cloud-logging~=3.10.0
 Requires-Dist: google-cloud-error-reporting~=1.11.0
+Requires-Dist: google-cloud-bigquery~=3.24.0

{ipulse_shared_core_ftredge-2.23 → ipulse_shared_core_ftredge-2.26}/src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt

@@ -3,8 +3,8 @@ README.md
 pyproject.toml
 setup.py
 src/ipulse_shared_core_ftredge/__init__.py
-src/ipulse_shared_core_ftredge/gcp_utils.py
-src/ipulse_shared_core_ftredge/template_config_utils.py
+src/ipulse_shared_core_ftredge/utils_gcp.py
+src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py
 src/ipulse_shared_core_ftredge.egg-info/PKG-INFO
 src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt
 src/ipulse_shared_core_ftredge.egg-info/dependency_links.txt

{ipulse_shared_core_ftredge-2.23 → ipulse_shared_core_ftredge-2.26}/src/ipulse_shared_core_ftredge.egg-info/requires.txt

@@ -3,3 +3,4 @@ python-dateutil~=2.8
 pytest~=7.1
 google-cloud-logging~=3.10.0
 google-cloud-error-reporting~=1.11.0
+google-cloud-bigquery~=3.24.0

ipulse_shared_core_ftredge-2.23/src/ipulse_shared_core_ftredge/__init__.py

@@ -1,5 +0,0 @@
-from .models import Organisation, UserAuth, UserProfile, UserStatus, UserProfileUpdate, pulse_enums
-from .gcp_utils import setup_gcp_logger_and_error_report, read_csv_from_gcs, read_json_from_gcs, write_csv_to_gcs, write_json_to_gcs
-from .template_config_utils import full_update_template, partial_update_template
-
- 

ipulse_shared_core_ftredge-2.23/src/ipulse_shared_core_ftredge/template_config_utils.py

@@ -1,32 +0,0 @@
-def full_update_template(template, updates, logger):
-    template_dict = template.copy()
-    filtered_updates = {k: v for k, v in updates.items() if k in template}
-    template_dict.update(filtered_updates)
-    
-    # Log warnings for any fields in the template that are not updated
-    missing_fields = set(template.keys()) - set(filtered_updates.keys())
-    if missing_fields:
-        logger.warning(f"Fields in the  template config but not updated: {missing_fields}")
-    # Log warnings for any fields in the updates that are not in the template
-    extra_fields = set(updates.keys()) - set(template.keys())
-    if extra_fields:
-        logger.warning(f"Fields in the updates extea to the template config: {extra_fields}. You probably forgot to update the Template Config. Do so now.")
-    if extra_fields:
-        raise ValueError(f"Updates contain fields not present in the template: {extra_fields}")
-    
-    return template_dict
-
-def partial_update_template(template, updates, logger):
-    template_dict = template.copy()
-    filtered_updates = {k: v for k, v in updates.items() if k in template}
-    template_dict.update(filtered_updates)
-
-    # Log warnings for any fields in the updates that are not in the template
-    extra_fields = set(updates.keys()) - set(template.keys())
-    if extra_fields:
-        logger.warning(f"Fields in the updates not in the template: {extra_fields}")
-    
-    if extra_fields:
-        raise ValueError(f"Updates contain fields not present in the template: {extra_fields}")
-    
-    return template_dict