ipulse-shared-core-ftredge 2.56__py3-none-any.whl → 3.1.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils_gcp.py

@@ -1,267 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=broad-exception-caught
-# pylint: disable=line-too-long
-# pylint: disable=unused-variable
-import json
-import csv
-from io import StringIO
-import logging
-import os
-import time
-import traceback
-from google.cloud import error_reporting, logging as cloud_logging
-from google.api_core.exceptions import NotFound
-
-
-############################################################################
-##################### SETTING UP LOGGER ##########################
-
-####DEPCREACATED: THIS APPROACH WAS GOOD, BUT ERRORS WERE NOT REPORTED TO ERROR REPORTING
-# logging.basicConfig(level=logging.INFO)
-# logging_client = google.cloud.logging.Client()
-# logging_client.setup_logging()
-###################################
-
-
-##### THIS APPROACH IS USED NOW ########
-ENV = os.getenv('ENV', 'LOCAL').strip("'")
-
-def setup_gcp_logger_and_error_report(logger_name,level=logging.INFO, use_cloud_logging=True):
-    """Sets up a logger with Error Reporting and Cloud Logging handlers.
-
-    Args:
-        logger_name: The name of the logger.
-
-    Returns:
-        logging.Logger: The configured logger instance.
-    """
-
-    class ErrorReportingHandler(logging.Handler):
-        def __init__(self, level=logging.ERROR):
-            super().__init__(level)
-            self.error_client = error_reporting.Client()
-            self.propagate = True
-
-        def emit(self, record):
-            try:
-                if record.levelno >= logging.ERROR:
-                    message = self.format(record)
-                    if record.exc_info:
-                        message += "\n" + ''.join(traceback.format_exception(*record.exc_info))
-                    if hasattr(record, 'pathname') and hasattr(record, 'lineno'):
-                        message += f"\nFile: {record.pathname}, Line: {record.lineno}"
-                    self.error_client.report(message)
-            except Exception as e:
-                # Ensure no exceptions are raised during logging
-                self.handleError(record)
-
-    logger = logging.getLogger(logger_name)
-    logger.setLevel(level)
-
-    # Add a console handler for local development
-    if ENV == "LOCAL" or not use_cloud_logging:
-        formatter = logging.Formatter('%(levelname)s : %(name)s : %(asctime)s : %(message)s')
-        console_handler = logging.StreamHandler()
-        console_handler.setFormatter(formatter)
-        logger.addHandler(console_handler)
-
-    if use_cloud_logging:
-        # Create Error Reporting handler
-        error_reporting_handler = ErrorReportingHandler()
-
-        # Create Google Cloud Logging handler
-        cloud_logging_client = cloud_logging.Client()
-        cloud_logging_handler = cloud_logging_client.get_default_handler()
-
-        # Add handlers to the logger
-        logger.addHandler(error_reporting_handler)
-        logger.addHandler(cloud_logging_handler)
-    return logger
-############################################################################
-
-
-############################################################################
-##################### GOOGLE CLOUD STORAGE UTILS ##########################
-
-def read_json_from_gcs(bucket_name, file_name, stor_client, logger):
-    """ Helper function to read a JSON file from Google Cloud Storage """
-    try:
-        bucket = stor_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        data_string = blob.download_as_text()
-        data = json.loads(data_string)
-        return data
-    except NotFound:
-        logger.error(f"Error: The file {file_name} was not found in the bucket {bucket_name}.")
-        return None
-    except json.JSONDecodeError:
-        logger.error(f"Error: The file {file_name} could not be decoded as JSON.")
-        return None
-    except Exception as e:
-        logger.error(f"An unexpected error occurred: {e}", exc_info=True)
-        return None
-
-def read_csv_from_gcs(bucket_name, file_name, storage_client, logger):
-    """ Helper function to read a CSV file from Google Cloud Storage """
-    try:
-        bucket = storage_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        data_string = blob.download_as_text()
-        data_file = StringIO(data_string)
-        reader = csv.DictReader(data_file)
-        return list(reader)
-    except NotFound:
-        logger.error(f"Error: The file {file_name} was not found in the bucket {bucket_name}.")
-        return None
-    except csv.Error:
-        logger.error(f"Error: The file {file_name} could not be read as CSV.")
-        return None
-    except Exception as e:
-        logger.error(f"An unexpected error occurred: {e}", exc_info=True)
-        return None
-
-
-
-def write_json_to_gcs(bucket_name, storage_client, data, file_name,
-                      save_locally=False, local_path=None, logger=None, max_retries=2,
-                      overwrite_if_exists=False, increment_if_exists=False):
-    """Saves data to Google Cloud Storage and optionally locally.
-
-    This function attempts to upload data to GCS. If the upload fails after
-    retries and `save_locally` is True or `local_path` is provided, it attempts
-    to save the data locally.
-    It also tries to handle file name conflicts by overwriting or incrementing. If both are provided as Ture, an exception will be raised.
-    """
-
-    def log_message(message):
-        if logger:
-            logger.info(message)
-
-    def log_error(message, exc_info=False):
-        if logger:
-            logger.error(message, exc_info=exc_info)
-
-    def log_warning(message):
-        if logger:
-            logger.warning(message)
-
-    attempts = 0
-    success = False
-    gcs_path = None
-    local_path_final = None
-    gcs_file_overwritten = False
-    gcs_file_already_exists = False
-    gcs_file_saved_with_increment = False
-    gcs_upload_exception = None  # Store potential GCS exception
-
-    # Check for conflicting options
-    if overwrite_if_exists and increment_if_exists:
-        raise ValueError("When writing JSON to GCS, both overwrite and increment_if_exists cannot be True at the same time.")
-
-    if isinstance(data, (list, dict)):
-        data_str = json.dumps(data, indent=2)
-    elif isinstance(data, str):
-        data_str = data
-    else:
-        raise ValueError("Unsupported data type. It should be a list, dict, or str.")
-
-    bucket = storage_client.bucket(bucket_name)
-    base_file_name, ext = os.path.splitext(file_name)
-    increment = 0
-
-    while attempts < max_retries and not success:
-        try:
-            if increment_if_exists:
-                while bucket.blob(file_name).exists():
-                    gcs_file_already_exists = True
-                    increment += 1
-                    file_name = f"{base_file_name}_{increment}{ext}"
-                    gcs_file_saved_with_increment = True
-                    log_warning(f"File {file_name} already exists in bucket {bucket_name}. Writing with increment: {increment_if_exists}")
-            else:
-                blob = bucket.blob(file_name)
-
-                # Check if the file exists
-                if blob.exists():
-                    gcs_file_already_exists = True
-                    gcs_path = f"gs://{bucket_name}/{file_name}"
-                    log_message(f"File {file_name} already exists in bucket {bucket_name}. Overwriting: {overwrite_if_exists}")
-                    if not overwrite_if_exists:
-                        log_warning(f"File {file_name} already exists and overwrite is set to False. Skipping save to GCS.")
-                        break
-                    else:
-                        gcs_file_overwritten = True
-
-            blob.upload_from_string(data_str, content_type='application/json')
-            gcs_path = f"gs://{bucket_name}/{file_name}"
-            log_message(f"Successfully saved file to GCS {gcs_path}.")
-            success = True
-        except Exception as e:
-            gcs_upload_exception = e
-            attempts += 1
-            if attempts < max_retries:
-                time.sleep(2 ** attempts)
-            else:
-                log_error(f"Failed to write {file_name} to GCS bucket {bucket_name} after {max_retries} attempts: {e}")
-
-    if not success or save_locally or local_path:
-        try:
-            if not local_path:
-                local_path_final = os.path.join("/tmp", file_name)
-            else:
-                local_path_final = os.path.join(local_path, file_name)
-
-            if os.path.exists(local_path_final):
-                if increment_if_exists:
-                    increment = 0
-                    while os.path.exists(local_path_final):
-                        increment += 1
-                        local_path_final = os.path.join(local_path, f"{base_file_name}_{increment}{ext}")
-                elif not overwrite_if_exists:
-                    log_message(f"File {file_name} already exists locally at {local_path_final} and overwrite is set to False. Skipping save.")
-                    success = True
-                else:
-                    log_message(f"File {file_name} already exists locally at {local_path_final}. Overwriting: {overwrite_if_exists}")
-
-            if not success:
-                with open(local_path_final, 'w', encoding='utf-8') as f:
-                    f.write(data_str)
-                log_message(f"Saved {file_name} locally at {local_path_final}. Overwritten: {overwrite_if_exists}")
-                success = True
-        except Exception as local_e:
-            log_error(f"Failed to write {file_name} locally: {local_e}", exc_info=True)
-
-    if gcs_upload_exception is not None:
-        raise gcs_upload_exception  # Propagate without nesting
-
-    return {
-        "gcs_path": gcs_path,
-        "local_path": local_path_final,
-        "gcs_file_already_exists": gcs_file_already_exists,
-        "gcs_file_overwritten": gcs_file_overwritten,
-        "gcs_file_saved_with_increment": gcs_file_saved_with_increment
-    }
-
-
-def write_csv_to_gcs(bucket_name, file_name, data, storage_client, logger,log_info_verbose=True):
-    """ Helper function to write a CSV file to Google Cloud Storage """
-    try:
-        bucket = storage_client.bucket(bucket_name)
-        blob = bucket.blob(file_name)
-        data_file = StringIO()
-        if data and isinstance(data, list) and isinstance(data[0], dict):
-            fieldnames = data[0].keys()
-            writer = csv.DictWriter(data_file, fieldnames=fieldnames)
-            writer.writeheader()
-            writer.writerows(data)
-        else:
-            raise ValueError("Data should be a list of dictionaries")
-        blob.upload_from_string(data_file.getvalue(), content_type='text/csv')
-        if log_info_verbose:
-            logger.info(f"Successfully wrote CSV to {file_name} in bucket {bucket_name}.")
-    except ValueError as e:
-        logger.error(f"ValueError: {e}")
-    except Exception as e:
-        logger.error(f"An unexpected error occurred while writing CSV to GCS: {e}", exc_info=True)

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -1,155 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=logging-fstring-interpolation
-# pylint: disable=line-too-long
-
-import datetime
-from google.cloud import bigquery
-from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel
-from ipulse_shared_core_ftredge.utils_common import ContextLog
-
-
-def create_bigquery_schema_from_json(json_schema):
-    schema = []
-    for field in json_schema:
-        if "max_length" in field:
-            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"], max_length=field["max_length"]))
-        else:
-            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
-    return schema
-
-
-def check_format_against_schema_template(data_to_check, schema, dt_ts_to_str=True, check_max_length=True):
-    """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
-    checked_data = {}
-    warnings_or_error = []  # Group warnings and errors for a given run
-
-    try:
-        # Process updates to conform to the schema
-        for field in schema:
-            field_name = field["name"]
-            field_type = field["type"]
-            mode = field["mode"]
-
-            # Initialize notice to None at the start of each field processing
-            warning = None
-
-            if field_name in data_to_check:
-                value = data_to_check[field_name]
-
-                # Handle date and timestamp formatting
-                if field_type == "DATE":
-                    value, warning = handle_date_fields(field_name, value, dt_ts_to_str)
-                elif field_type == "TIMESTAMP":
-                    value, warning = handle_timestamp_fields(field_name, value, dt_ts_to_str)
-                elif field_type in ["STRING", "INT64", "FLOAT64", "BOOL"]:
-                    value, warning = handle_type_conversion(field_type, field_name, value)
-
-                if warning:
-                    warnings_or_error.append(warning)
-
-                # Check and handle max length restriction
-                if check_max_length and "max_length" in field:
-                    value, warning = check_and_truncate_length(field_name, value, field["max_length"])
-                    if warning:
-                        warnings_or_error.append(warning)
-
-                # Only add to the dictionary if value is not None or the field is required
-                if value is not None or mode == "REQUIRED":
-                    checked_data[field_name] = value
-
-            elif mode == "REQUIRED":
-                warning = ContextLog(level=LogLevel.WARNING,
-                                              subject=field_name,
-                                              description=f"Required field '{field_name}' is missing in the updates.")
-                warnings_or_error.append(warning)
-
-    except Exception as e:
-        error_log = ContextLog(level=LogLevel.ERROR_EXCEPTION_REDO,
-                               subject=data_to_check,
-                               description=f"An error occurred during update check: {str(e)}")
-        warnings_or_error.append(error_log)
-
-    return checked_data, warnings_or_error
-
-def handle_date_fields(field_name, value, dt_ts_to_str):
-    """Handles date fields, ensuring they are in the correct format and optionally converts them to string."""
-    if isinstance(value, datetime.date):
-        if dt_ts_to_str:
-            return value.strftime("%Y-%m-%d"), None
-        return value, None
-    elif isinstance(value, str):
-        try:
-            parsed_date = datetime.datetime.strptime(value, "%Y-%m-%d").date()
-            if dt_ts_to_str:
-                return value, None
-            return parsed_date, None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                                  subject=field_name,
-                                                   description=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
-    else:
-        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                              subject=field_name,
-                                              description= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
-
-
-def handle_timestamp_fields(field_name, value, dt_ts_to_str):
-    """Handles timestamp fields, ensuring they are in the correct format and optionally converts them to ISO format string."""
-    if isinstance(value, datetime.datetime):
-        if dt_ts_to_str:
-            return value.isoformat(), None
-        return value, None
-    elif isinstance(value, str):
-        try:
-            parsed_datetime = datetime.datetime.fromisoformat(value)
-            if dt_ts_to_str:
-                return value, None
-            return parsed_datetime, None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                                  subject=field_name,
-                                                  description= f"Expected ISO format TIMESTAMP but got {value}.")
-    else:
-        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                              subject=field_name,
-                                              description= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
-
-
-def check_and_truncate_length(field_name, value, max_length):
-    """Checks and truncates the length of string fields if they exceed the max length."""
-    if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], ContextLog(level=LogLevel.WARNING_FIX_RECOMMENDED,
-                            subject= field_name,
-                             description= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
-
-    return value, None
-
-
-
-def handle_type_conversion(field_type, field_name, value):
-    if field_type == "STRING" and not isinstance(value, str):
-        return str(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
-                             subject=field_name,
-                             description= f"Expected STRING but got {value} of type {type(value).__name__}.")
-
-    if field_type == "INT64" and not isinstance(value, int):
-        try:
-            return int(value), None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                                subject= field_name,
-                                                description=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
-    if field_type == "FLOAT64" and not isinstance(value, float):
-        try:
-            return float(value), None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED, 
-                                                subject=field_name,
-                                                description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
-    if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED, 
-                                                subject=field_name,
-                                                description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
-
-    return value, None

ipulse_shared_core_ftredge-2.56.dist-info/RECORD

@@ -1,25 +0,0 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=gIwh_6xjoG7UIuTKNkq0iGvCuxsbyXNrTyV3vB8YdGc,869
-ipulse_shared_core_ftredge/utils_common.py,sha256=UOaEiTE9moJCoSGFnD6aPI_w1UujovWcYOH6ifJxPxg,22533
-ipulse_shared_core_ftredge/utils_gcp.py,sha256=z1Rdgi1wP0768a9vnp9qmj_UtoLdmFOqYMcnrL1v1pw,10999
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=AwGl9J-XQc_aO_VKWhR_TuA1ML8nWxHuzHtxBH8yfwE,7499
-ipulse_shared_core_ftredge/enums/__init__.py,sha256=pdi9OIDyb8pfTFbeBqnhQ_ZO-gnYfmLKwHZBYOMnkqE,888
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=UQG-dd302r5VFhV2kwDMXCg20nsE2ht4XBEZx9rzX1o,6724
-ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=7w3Jjmw84Wq22Bb5Qs09Z82Bdf-j8nhRiQJfw60_g80,1903
-ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=W1TkSLu3ryLf_aif2VcKsFznWz0igeMUR_buoGEG6w8,1406
-ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
-ipulse_shared_core_ftredge/models/__init__.py,sha256=gE22Gzhil0RYQa7YLtdtT44_AsWqklcDfRtgLAQc1dI,200
-ipulse_shared_core_ftredge/models/audit_log_firestore.py,sha256=5AwO6NHuOncq65n400eqM8QPrS2EGGaP3Z_6l2rxdBE,261
-ipulse_shared_core_ftredge/models/organisation.py,sha256=4f1ATEWh5WT-CDJBLEZUhUwyl3V06ogRkteZAqW_nko,2953
-ipulse_shared_core_ftredge/models/pulse_enums.py,sha256=YJhtvoX6Dk3_SyJUD8vVDSRIzWy5n0I0AOwe19fmDT8,4851
-ipulse_shared_core_ftredge/models/resource_catalog_item.py,sha256=PxeRvI8fe8KOiHr6NW2Jz_yocyLId09PW8QyTZxjHAA,9809
-ipulse_shared_core_ftredge/models/user_auth.py,sha256=35HNN7ZW4ZELCqaJrAtoSsVLFAZ1KL2S_VmuzbcEMm4,119
-ipulse_shared_core_ftredge/models/user_profile.py,sha256=3-HHB3wK2s4hsP9d5Xp2JYrsGe8iFzz4VaoygDIO_-s,1930
-ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFPhX2XlqEqlKLBQ4AkvPHXEuZbFMY,1712
-ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
-ipulse_shared_core_ftredge/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ipulse_shared_core_ftredge/tests/test.py,sha256=0lS8HP5Quo_BqNoscU40qOH9aJRaa1Pfam5VUBmdld8,682
-ipulse_shared_core_ftredge-2.56.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.56.dist-info/METADATA,sha256=WdSqfe6RTQDh4M9EBSA33UaYrmMud26ZYm-P-LRj5OQ,561
-ipulse_shared_core_ftredge-2.56.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-ipulse_shared_core_ftredge-2.56.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.56.dist-info/RECORD,,