ipulse-shared-core-ftredge 2.57__py3-none-any.whl → 3.2.1__py3-none-any.whl

ipulse_shared_core_ftredge/utils_templates_and_schemas.py

@@ -1,153 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=logging-fstring-interpolation
-# pylint: disable=line-too-long
-
-import datetime
-from google.cloud import bigquery
-from ipulse_shared_core_ftredge.enums.enums_common_utils import LogLevel
-from ipulse_shared_core_ftredge.utils_custom_logs import ContextLog
-
-
-def create_bigquery_schema_from_json(json_schema):
-    schema = []
-    for field in json_schema:
-        if "max_length" in field:
-            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"], max_length=field["max_length"]))
-        else:
-            schema.append(bigquery.SchemaField(field["name"], field["type"], mode=field["mode"]))
-    return schema
-
-
-def check_format_against_schema_template(data_to_check, schema, dt_ts_to_str=True, check_max_length=True):
-    """Ensure Update dict corresponds to the config schema, ensuring proper formats and lengths."""
-    checked_data = {}
-    warnings_or_error = []  # Group warnings and errors for a given run
-
-    try:
-        # Process updates to conform to the schema
-        for field in schema:
-            field_name = field["name"]
-            field_type = field["type"]
-            mode = field["mode"]
-
-            # Initialize notice to None at the start of each field processing
-            warning = None
-
-            if field_name in data_to_check:
-                value = data_to_check[field_name]
-
-                # Handle date and timestamp formatting
-                if field_type == "DATE":
-                    value, warning = handle_date_fields(field_name, value, dt_ts_to_str)
-                elif field_type == "TIMESTAMP":
-                    value, warning = handle_timestamp_fields(field_name, value, dt_ts_to_str)
-                elif field_type in ["STRING", "INT64", "FLOAT64", "BOOL"]:
-                    value, warning = handle_type_conversion(field_type, field_name, value)
-
-                if warning:
-                    warnings_or_error.append(warning)
-
-                # Check and handle max length restriction
-                if check_max_length and "max_length" in field:
-                    value, warning = check_and_truncate_length(field_name, value, field["max_length"])
-                    if warning:
-                        warnings_or_error.append(warning)
-
-                # Only add to the dictionary if value is not None or the field is required
-                if value is not None or mode == "REQUIRED":
-                    checked_data[field_name] = value
-
-            elif mode == "REQUIRED":
-                warnings_or_error.append(ContextLog(level=LogLevel.WARNING,
-                                              subject=field_name,
-                                              description=f"Required field '{field_name}' is missing in the updates."))
-
-    except Exception as e:
-        warnings_or_error.append(ContextLog(level=LogLevel.ERROR_EXCEPTION,
-                               subject=data_to_check,
-                               description=f"An error occurred during update check: {str(e)}"))
-
-    return checked_data, warnings_or_error
-
-def handle_date_fields(field_name, value, dt_ts_to_str):
-    """Handles date fields, ensuring they are in the correct format and optionally converts them to string."""
-    if isinstance(value, datetime.date):
-        if dt_ts_to_str:
-            return value.strftime("%Y-%m-%d"), None
-        return value, None
-    elif isinstance(value, str):
-        try:
-            parsed_date = datetime.datetime.strptime(value, "%Y-%m-%d").date()
-            if dt_ts_to_str:
-                return value, None
-            return parsed_date, None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                                  subject=field_name,
-                                                   description=f"Expected a DATE in YYYY-MM-DD format but got {value}.")
-    else:
-        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                              subject=field_name,
-                                              description= f"Expected a DATE or YYYY-MM-DD str format but got {value} of type {type(value).__name__}.")
-
-
-def handle_timestamp_fields(field_name, value, dt_ts_to_str):
-    """Handles timestamp fields, ensuring they are in the correct format and optionally converts them to ISO format string."""
-    if isinstance(value, datetime.datetime):
-        if dt_ts_to_str:
-            return value.isoformat(), None
-        return value, None
-    elif isinstance(value, str):
-        try:
-            parsed_datetime = datetime.datetime.fromisoformat(value)
-            if dt_ts_to_str:
-                return value, None
-            return parsed_datetime, None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                                  subject=field_name,
-                                                  description= f"Expected ISO format TIMESTAMP but got {value}.")
-    else:
-        return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                              subject=field_name,
-                                              description= f"Expected ISO format TIMESTAMP but got {value} of type {type(value).__name__}.")
-
-
-def check_and_truncate_length(field_name, value, max_length):
-    """Checks and truncates the length of string fields if they exceed the max length."""
-    if isinstance(value, str) and len(value) > max_length:
-        return value[:max_length], ContextLog(level=LogLevel.WARNING_FIX_RECOMMENDED,
-                            subject= field_name,
-                             description= f"Field exceeds max length: {len(value)}/{max_length}. Truncating.")
-
-    return value, None
-
-
-
-def handle_type_conversion(field_type, field_name, value):
-    if field_type == "STRING" and not isinstance(value, str):
-        return str(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
-                             subject=field_name,
-                             description= f"Expected STRING but got {value} of type {type(value).__name__}.")
-
-    if field_type == "INT64" and not isinstance(value, int):
-        try:
-            return int(value), None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                                subject= field_name,
-                                                description=f"Expected INTEGER, but got {value} of type {type(value).__name__}.")
-    if field_type == "FLOAT64" and not isinstance(value, float):
-        try:
-            return float(value), None
-        except ValueError:
-            return None, ContextLog(level=LogLevel.WARNING_FIX_REQUIRED,
-                                                subject=field_name,
-                                                description=f"Expected FLOAT, but got  {value} of type {type(value).__name__}.")
-    if field_type == "BOOL" and not isinstance(value, bool):
-        return bool(value), ContextLog(level=LogLevel.WARNING_REVIEW_RECOMMENDED,
-                                                subject=field_name,
-                                                description=f"Expected BOOL, but got  {value}. Converting as {bool(value)}.")
-
-    return value, None

ipulse_shared_core_ftredge-2.57.dist-info/RECORD

@@ -1,25 +0,0 @@
-ipulse_shared_core_ftredge/__init__.py,sha256=QHWbeWYQgs314BKxQDzG7_P2mdnryC1Zqc3xvw8OfnM,987
-ipulse_shared_core_ftredge/utils_custom_logs.py,sha256=OJ6STEL7GER-s3x03cKcsYtZ4jKhndPOAJXT1oiAY2M,8183
-ipulse_shared_core_ftredge/utils_gcp.py,sha256=xI0iOxSjGQ75ECFlmPirOB8Cz8qjqYNabfRLEg0cE8o,14142
-ipulse_shared_core_ftredge/utils_gcp_for_pipelines.py,sha256=rO7KD8JUmit0NwKoepKmBX8MlcQ8GzDhUfAyp6OWAw0,11816
-ipulse_shared_core_ftredge/utils_pipelinemon.py,sha256=KRDJW0fqF6sfqsxL8YKH378qpXnOzpZ3C53VAkMoT10,15011
-ipulse_shared_core_ftredge/utils_templates_and_schemas.py,sha256=vi8hBU95_N-Znfs-FClBNCFMKvJYBOdBwZm2pgBZ7BQ,7433
-ipulse_shared_core_ftredge/enums/__init__.py,sha256=MRmvcrFhbGZbww759KCledFe6BpOArmnCQ18mb_F8Fg,932
-ipulse_shared_core_ftredge/enums/enums_common_utils.py,sha256=VdOipu5YNVO4TcQMXy7UkCjSnBfUzutEkgfFFsnsp7k,6922
-ipulse_shared_core_ftredge/enums/enums_data_eng.py,sha256=7w3Jjmw84Wq22Bb5Qs09Z82Bdf-j8nhRiQJfw60_g80,1903
-ipulse_shared_core_ftredge/enums/enums_module_fincore.py,sha256=W1TkSLu3ryLf_aif2VcKsFznWz0igeMUR_buoGEG6w8,1406
-ipulse_shared_core_ftredge/enums/enums_modules.py,sha256=AyXUoNmR75DZLaEHi3snV6LngR25LeZRqzrLDaAupbY,1244
-ipulse_shared_core_ftredge/models/__init__.py,sha256=gE22Gzhil0RYQa7YLtdtT44_AsWqklcDfRtgLAQc1dI,200
-ipulse_shared_core_ftredge/models/audit_log_firestore.py,sha256=5AwO6NHuOncq65n400eqM8QPrS2EGGaP3Z_6l2rxdBE,261
-ipulse_shared_core_ftredge/models/organisation.py,sha256=4f1ATEWh5WT-CDJBLEZUhUwyl3V06ogRkteZAqW_nko,2953
-ipulse_shared_core_ftredge/models/pulse_enums.py,sha256=YJhtvoX6Dk3_SyJUD8vVDSRIzWy5n0I0AOwe19fmDT8,4851
-ipulse_shared_core_ftredge/models/resource_catalog_item.py,sha256=PxeRvI8fe8KOiHr6NW2Jz_yocyLId09PW8QyTZxjHAA,9809
-ipulse_shared_core_ftredge/models/user_auth.py,sha256=35HNN7ZW4ZELCqaJrAtoSsVLFAZ1KL2S_VmuzbcEMm4,119
-ipulse_shared_core_ftredge/models/user_profile.py,sha256=D3BB9D6XEv7IVZgsURgf0hWmUZW5rms3uiBXS0ZGLeE,1927
-ipulse_shared_core_ftredge/models/user_profile_update.py,sha256=oKK0XsQDKkgDvjFPhX2XlqEqlKLBQ4AkvPHXEuZbFMY,1712
-ipulse_shared_core_ftredge/models/user_status.py,sha256=8TyRd8tBK9_xb0MPKbI5pn9-lX7ovKbeiuWYYPtIOiw,3202
-ipulse_shared_core_ftredge-2.57.dist-info/LICENCE,sha256=YBtYAXNqCCOo9Mr2hfkbSPAM9CeAr2j1VZBSwQTrNwE,1060
-ipulse_shared_core_ftredge-2.57.dist-info/METADATA,sha256=OFCR8U5m0OwnbR9hyIqSwOWW40QmwDfz1WqnWSVnBxg,561
-ipulse_shared_core_ftredge-2.57.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-ipulse_shared_core_ftredge-2.57.dist-info/top_level.txt,sha256=8sgYrptpexkA_6_HyGvho26cVFH9kmtGvaK8tHbsGHk,27
-ipulse_shared_core_ftredge-2.57.dist-info/RECORD,,