ipulse-shared-core-ftredge 2.27__tar.gz → 2.29__tar.gz

{ipulse_shared_core_ftredge-2.27/src/ipulse_shared_core_ftredge.egg-info → ipulse_shared_core_ftredge-2.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.27
+Version: 2.29
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.27 → ipulse_shared_core_ftredge-2.29}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.27',
+    version='2.29',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[

ipulse_shared_core_ftredge-2.29/src/ipulse_shared_core_ftredge/__init__.py

@@ -0,0 +1,12 @@
+from .models import (Organisation, UserAuth, UserProfile,
+                     UserStatus, UserProfileUpdate, pulse_enums)
+from .utils_gcp import (setup_gcp_logger_and_error_report, 
+                        read_csv_from_gcs, read_json_from_gcs, 
+                        write_csv_to_gcs, write_json_to_gcs)
+from .utils_templates_and_schemas import (create_bigquery_schema_from_json,
+                                          update_check_with_schema_template)
+from .enums import (NoticeSeverity,
+                    SourcingPipelineType,
+                    SourcingTriggerType,
+                    DWEvents,
+                    DWEventTriggerType )

ipulse_shared_core_ftredge-2.29/src/ipulse_shared_core_ftredge/enums/__init__.py

@@ -0,0 +1,5 @@
+from .enums_common import NoticeSeverity
+from .enums_data_eng import (SourcingTriggerType,
+                             SourcingPipelineType,
+                             DWEvents,
+                             DWEventTriggerType)

ipulse_shared_core_ftredge-2.29/src/ipulse_shared_core_ftredge/enums/enums_common.py

@@ -0,0 +1,27 @@
+from enum import Enum
+
+class NoticeSeverity(Enum):
+    """
+    Standardized logging levels for data engineering pipelines,
+    designed for easy analysis and identification of manual 
+    intervention needs.
+    """
+    DEBUG = 100  # Detailed debug information (for development/troubleshooting)
+    INFO = 200   # Normal pipeline execution information 
+    NOTICE = 300  # Events requiring attention, but not necessarily errors
+
+     # Warnings indicate potential issues that might require attention:
+    WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
+    WARNING_ACTION_RECOMMENDED = 402 # Action recommended to prevent potential future issues
+    WARNING_ACTION_REQUIRED = 403  # Action required, pipeline can likely continue
+
+    # Errors indicate a problem that disrupts normal pipeline execution:
+    ERROR_TRANSIENT_RETRY = 501 # Temporary error, automatic retry likely to succeed
+    ERROR_DATA_ISSUE_ISOLATED = 502 # Error likely caused by data issues, manual intervention likely needed
+    ERROR_DATA_ISSUE_WITH_DEPENDENCIES = 503 # Error likely in code/configuration, investigation required
+    ERROR_CONFIG_OR_CODE_ISSUE = 504 # Error likely in code/configuration, investigation required
+    ERROR_UNKNOWN_EXCEPTION = 505
+
+    # Critical errors indicate severe failures requiring immediate attention:
+    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure), requires immediate action
+    CRITICAL_PIPELINE_FAILURE = 602 # Complete pipeline failure, requires investigation and potential rollback

ipulse_shared_core_ftredge-2.29/src/ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -0,0 +1,40 @@
+from enum import Enum, auto
+
+class SourcingTriggerType(Enum):
+    HISTORIC_MANUAL = auto()
+    LIVE_SCHEDULED = auto()
+    ADHOC_MANUAL = auto()
+    ADHOC_SCHEDULED = auto()
+    LIVE_MANUAL = auto()
+
+class SourcingPipelineType(Enum):
+    LOCAL_GET_API_TO_GCS_BUCKET_FILE = auto()
+    GCLOUD_GET_API_INMEMORY = auto()
+    LOCAL_GET_API_INMEMORY = auto()
+    GCLOUD_GET_API_TO_GCS_BUCKET_FILE = auto()
+    LOCAL_GET_API_TO_LOCAL_FILE = auto()
+    LOCAL_DOWNLOAD_WEB_FILE_TO_LOCAL = auto()
+    LOCAL_DOWNLOAD_WEB_FILE_UPLOAD_TO_GCS_BUCKET = auto()
+
+class DWEventTriggerType(Enum):
+    GCS_BUCKET_FILE_UPLOAD = auto()
+    INSIDE_SOURCING_FUNCTION = auto()
+    MANUAL_FROM_GCS_BUCKET_FILE = auto()
+    MANUAL_FROM_LOCAL_FILE = auto()
+    PUBSUBC_TOPIC = auto()
+
+class DWEvents(Enum):
+    INSERT_NOREPLACE_1A_NT = auto()
+    MERGE_NOREPLACE_NA_1T = auto()
+    MERGE_NOREPLACE_NA_NT = auto()
+    INSERT_NOREPLACE_1A_1T = auto()
+    MERGE_NOREPLACE_1A_NT = auto()
+    INSERT_REPLACE_1A_1T = auto()
+    INSERT_REPLACE_1A_NT = auto()
+    MERGE_REPL_NA_NT = auto()
+    MERGE_REPL_1A_NT = auto()
+    MERGE_REPL_NA_1T = auto()
+    DELETE_1A_1T = auto()
+    DELETE_1A_NT = auto()
+    DELETE_NA_1T = auto()
+    DELETE_NA_NT = auto()

{ipulse_shared_core_ftredge-2.27 → ipulse_shared_core_ftredge-2.29}/src/ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -1,5 +1,11 @@
-import datetime 
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=logging-fstring-interpolation
+# pylint: disable=line-too-long
+
+import datetime
 from google.cloud import bigquery
+from . import NoticeSeverity
 
 def create_bigquery_schema_from_json(json_schema):
     schema = []
@@ -10,24 +16,11 @@ def create_bigquery_schema_from_json(json_schema):
             schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
     return schema
 
-def update_check_with_dict_template(template, updates, logger):
-    template_dict = template.copy()
-    filtered_updates = {k: v for k, v in updates.items() if k in template}
-    template_dict.update(filtered_updates)
-    # Log warnings for any fields in the updates that are not in the template
-    extra_fields = set(updates.keys()) - set(template.keys())
-    if extra_fields:
-        logger.warning(f"Fields in the updates extra to the template config: {extra_fields}. You probably forgot to update the Template Config. Do so now.")
-    return template_dict
 
 def update_check_with_schema_template(updates, schema, logger, dt_ts_to_str=True, check_max_length=True):
-    """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
-    valid_updates = {}
-    schema_dict = {field["name"]: field for field in schema}
     
     """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
     valid_updates = {}
-    schema_dict = {field["name"]: field for field in schema}
     
     # Process updates to conform to the schema
     for field in schema:

{ipulse_shared_core_ftredge-2.27 → ipulse_shared_core_ftredge-2.29/src/ipulse_shared_core_ftredge.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipulse_shared_core_ftredge
-Version: 2.27
+Version: 2.29
 Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
 Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
 Author: Russlan Ramdowar

{ipulse_shared_core_ftredge-2.27 → ipulse_shared_core_ftredge-2.29}/src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt

@@ -10,6 +10,9 @@ src/ipulse_shared_core_ftredge.egg-info/SOURCES.txt
 src/ipulse_shared_core_ftredge.egg-info/dependency_links.txt
 src/ipulse_shared_core_ftredge.egg-info/requires.txt
 src/ipulse_shared_core_ftredge.egg-info/top_level.txt
+src/ipulse_shared_core_ftredge/enums/__init__.py
+src/ipulse_shared_core_ftredge/enums/enums_common.py
+src/ipulse_shared_core_ftredge/enums/enums_data_eng.py
 src/ipulse_shared_core_ftredge/models/__init__.py
 src/ipulse_shared_core_ftredge/models/audit_log_firestore.py
 src/ipulse_shared_core_ftredge/models/organisation.py

ipulse_shared_core_ftredge-2.27/src/ipulse_shared_core_ftredge/__init__.py

@@ -1,5 +0,0 @@
-from .models import Organisation, UserAuth, UserProfile, UserStatus, UserProfileUpdate, pulse_enums
-from .utils_gcp import setup_gcp_logger_and_error_report, read_csv_from_gcs, read_json_from_gcs, write_csv_to_gcs, write_json_to_gcs
-from .utils_templates_and_schemas import create_bigquery_schema_from_json,update_check_with_schema_template, update_check_with_dict_template
-
-