ipulse-shared-core-ftredge 2.6__tar.gz → 2.7.1__tar.gz

ipulse_shared_core_ftredge-2.7.1/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: ipulse_shared_core_ftredge
+Version: 2.7.1
+Summary: Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.
+Home-page: https://github.com/TheFutureEdge/ipulse_shared_core
+Author: Russlan Ramdowar
+License-File: LICENCE
+Requires-Dist: pydantic[email]~=2.5
+Requires-Dist: python-dateutil~=2.8
+Requires-Dist: pytest~=7.1
+Requires-Dist: google-cloud-logging~=3.10.0
+Requires-Dist: google-cloud-error-reporting~=1.11.0
+Requires-Dist: google-cloud-bigquery~=3.24.0

ipulse_shared_core_ftredge-2.7.1/README.md

@@ -0,0 +1,21 @@
+# ipulse_shared_core
+Shared Models like User, Organisation etc. Also includes shared enum_sets
+
+
+### Enums 
+
+Contains majority of all Enums used in Pulse
+
+
+
+### Collectors i.e. Pipelinemon
+
+Collectors are smart Objects which are added to long running functions or pipelines for which we want to collect an overall number of successes, notices, warnings or errors. 
+
+We can wait until the full pipeline is finished in order to write off a single Summary file from a Collector, or we can attach to it a logger, which will be reporting major status along the journey, which is often times better. Because if a function crashes midway through , all logs will be lost, and it would be hard to investigate if anythign has bee persisted and has to be rolled back. THis will require a lot of manual effort to recollect.
+
+Pipelinemon , short of Pipeline Monitoring system is a type of very powerful Collector which Russlan created specifically for Pulse Data Engineering pipelines.
+
+Pipelinemon writes all observation logs to Google CLoud Logging, and you have to setup a Log Sink (Router) which will send the Pipelinemon's observation logs to BigQuery. 
+
+Great thing about Pipelinemin is its "context" keeping feature.

{ipulse_shared_core_ftredge-2.6 → ipulse_shared_core_ftredge-2.7.1}/setup.py

@@ -1,8 +1,9 @@
+# pylint: disable=import-error
 from setuptools import setup, find_packages
 
 setup(
     name='ipulse_shared_core_ftredge',
-    version='2.6',
+    version='2.7.1',
     package_dir={'': 'src'},  # Specify the source directory
     packages=find_packages(where='src'),  # Look for packages in 'src'
     install_requires=[
@@ -10,8 +11,11 @@ setup(
         'pydantic[email]~=2.5',
         'python-dateutil~=2.8',
         'pytest~=7.1',
+        'google-cloud-logging~=3.10.0',
+        'google-cloud-error-reporting~=1.11.0',
+        'google-cloud-bigquery~=3.24.0'
     ],
     author='Russlan Ramdowar',
-    description='Shared models for the Pulse platform project. Using AI for financial advisory and investment management.',
+    description='Shared Core models and Logger util for the Pulse platform project. Using AI for financial advisory and investment management.',
     url='https://github.com/TheFutureEdge/ipulse_shared_core',
 )

ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge/__init__.py

@@ -0,0 +1,23 @@
+# pylint: disable=missing-module-docstring
+from .models import ( UserAuth, UserProfile,
+                     UserStatus, UserProfileUpdate)
+
+from .enums import (TargetLogs,LogLevel,  Status, Unit, Frequency,
+                    Module, Domain, FinCoreCategory, FincCoreSubCategory,
+                    FinCoreRecordsCategory, FinancialExchangeOrPublisher,
+                    DataPrimaryCategory, DataState, DatasetScope,
+                    DataSourceType,PipelineTriggerType,DataOperationType,
+                    MatchConditionType, DuplicationHandling, DuplicationHandlingStatus,
+                    CodingLanguage, ExecutionLocation, ExecutionComputeType,
+                    CloudProvider,LoggingHandlers)
+from .utils import (get_logger,
+                    save_json_locally_extended,
+                    write_json_to_cloud_storage_extended,
+                    write_json_to_gcs_extended,
+                    write_csv_to_gcs,
+                    read_json_from_cloud_storage,
+                    read_csv_from_gcs,
+                    read_json_from_gcs,
+                    check_format_against_schema_template,
+                    create_bigquery_schema_from_json,
+                    Pipelinemon, ContextLog)

ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge/enums/__init__.py

@@ -0,0 +1,37 @@
+
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+
+from .enums_common_utils import (Status,
+                                Unit,
+                                Frequency)
+
+
+from .enums_modules import(Module,
+                           Domain)
+
+
+from .enums_module_fincore import (FinCoreCategory,
+                                    FincCoreSubCategory,
+                                    FinCoreRecordsCategory,
+                                    FinancialExchangeOrPublisher)
+
+from .enums_logging import (TargetLogs,
+                            LogLevel,
+                            LoggingHandlers)
+
+from .enums_data_eng import (DataPrimaryCategory,
+                            DataState,
+                            DatasetScope,
+                            DataSourceType,
+                            PipelineTriggerType,
+                            DataOperationType,
+                            MatchConditionType,
+                            DuplicationHandling,
+                            DuplicationHandlingStatus,
+                            CodingLanguage,
+                            ExecutionLocation,
+                            ExecutionComputeType)
+
+from .enums_solution_providers import (CloudProvider)

ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge/enums/enums_common_utils.py

@@ -0,0 +1,107 @@
+
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=line-too-long
+
+from enum import Enum
+
+
+class Status(Enum):
+    OPEN = "open"
+    ACKNOWLEDGED = "acknowledged"
+    ESCALATED = "escalated"
+    IN_PROGRESS = "in_progress"
+    IN_REVIEW = "in_review"
+    RESOLVED = "resolved"
+    IGNORED = "ignored"
+    CANCELLED = "cancelled"
+    CLOSED = "closed"
+
+    def __str__(self):
+        return self.value
+
+### Exception during full exection, partially saved
+# Exception during ensemble pipeline; modifications collected in local object , nothing persisted
+# Exception during ensemble pipeline; modifications persisted , metadata failed
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
+# Exception during ensemble pipeline; modifications persisted , metadata persisted
+
+
+class Unit(Enum):
+    MIX="MIX"
+    # Currency and Financial Values
+    USD = "USD"  # United States Dollar
+    EUR = "EUR"  # Euro
+    JPY = "JPY"  # Japanese Yen
+    GBP = "GBP"  # British Pound Sterling
+    AUD = "AUD"  # Australian Dollar
+    CAD = "CAD"  # Canadian Dollar
+    CHF = "CHF"  # Swiss Franc
+    CNY = "CNY"  # Chinese Yuan Renminbi
+    SEK = "SEK"  # Swedish Krona
+    NZD = "NZD"  # New Zealand Dollar
+    MXN = "MXN"  # Mexican Peso
+    SGD = "SGD"  # Singapore Dollar
+    HKD = "HKD"  # Hong Kong Dollar
+    NOK = "NOK"  # Norwegian Krone
+    KRW = "KRW"  # South Korean Won
+    RUB = "RUB"  # Russian Ruble
+    INR = "INR"  # Indian Rupee
+    BRL = "BRL"  # Brazilian Real
+    ZAR = "ZAR"  # South African Rand
+    CURRENCY = "currency"    # General currency, when specific currency is not needed
+
+    # Stock Market and Investments
+    SHARES = "shares"        # Number of shares
+    PERCENT = "prcnt"      # Percentage, used for rates and ratios
+    BPS = "bps"              # Basis points, often used for interest rates and financial ratios
+
+    # Volume and Quantitative Measurements
+    VOLUME = "volume"        # Trading volume in units
+    MILLIONS = "mills"    # Millions, used for large quantities or sums
+    BILLIONS = "bills"    # Billions, used for very large quantities or sums
+
+    # Commodity Specific Units
+    BARRELS = "barrels"      # Barrels, specifically for oil and similar liquids
+    TONNES = "tonnes"        # Tonnes, for bulk materials like metals or grains
+    TROY_OUNCES = "troy_oz" # Troy ounces, specifically for precious metals
+
+    # Real Estate and Physical Properties
+    SQUARE_FEET = "sq_ft"    # Square feet, for area measurement in real estate
+    METER_SQUARE = "m2"      # Square meters, for area measurement in real estate
+    ACRES = "acres"          # Acres, used for measuring large plots of land
+
+    # Miscellaneous and Other Measures
+    UNITS = "units"          # Generic units, applicable when other specific units are not suitable
+    COUNT = "count"          # Count, used for tallying items or events
+    INDEX_POINTS = "index_pnts"  # Index points, used in measuring indices like stock market indices
+    RATIO = "ratio"          # Ratio, for various financial ratios
+
+    def __str__(self):
+        return self.value
+
+class Frequency(Enum):
+    ONE_MIN = "1min"
+    FIVE_MIN="5min"
+    FIFTEEN_MIN="15min"
+    THIRTY_MIN = "30min"
+    ONE_H = "1h"
+    TWO_H = "2h"
+    SIX_H = "6h"
+    TWELVE_H = "12h"
+    FOUR_H = "4h"
+    EOD="eod"
+    ONE_D = "1d"
+    TWO_D = "2d"
+    THREE_D = "3d"
+    ONE_W = "1w"
+    ONE_M = "1m"
+    TWO_M="2m"
+    THREE_M="3m"
+    SIX_M="6m"
+    ONE_Y="1y"
+    THREE_Y="3y"
+
+    def __str__(self):
+        return self.value

ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -0,0 +1,313 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+from enum import Enum
+
+
+class DataPrimaryCategory(Enum):
+    SIMULATION="simulation" # Simulation data, based on models and simulations
+    HISTORIC = "historic" # Historical data, usually accurate and complete
+    REALTIME="realtime" # Real-time data, not always certain, can have error
+    ANALYTICS="analytics" # Analytical data and modelling, derived from historical and prediction data. Normally shall be making Human readable sense. vs. Features
+    FEATURES="features" # Feature data, used for training models
+    PREDICTIVE="predictive" # Predictive data, based on models and simulations
+
+    def __str__(self):
+        return self.value
+
+class DataState(Enum):
+    RAW = "raw"
+    FORMATTED= "formatted"
+    CLEANED = "cleaned"
+    PROCESSED = "processed"
+    SIMULATED = "simulated"
+    ANALYZED = "analyzed"
+    VALIDATED = "validated"
+    INVALID = "invalid"
+
+    def __str__(self):
+        return self.value
+
+class DatasetScope(Enum):
+    FULL = "full_dataset"
+    INCREMENTAL = "incremental_dataset"
+    PARTIAL = "partial_dataset"
+    UNKNOWN = "unknown_dataset"
+
+    def __str__(self):
+        return self.value
+
+
+class DataSourceType(Enum):
+    # --- General ---
+    API = "api"
+    RPC = "rpc"
+    GRPC = "grpc"
+    WEBSITE = "website"
+    # --SQL Databases--
+    ORACLE = "oracle"
+    POSTGRESQL = "postgresql"
+    SQLSERVER = "sqlserver"
+    MYSQL = "mysql"
+    BIGQUERY = "bigquery"
+    SNOWFLAKE = "snowflake"
+    REDSHIFT = "redshift"
+    ATHENA = "athena"
+    # --NOSQL Databases--
+    MONGODB = "mongodb"
+    REDIS = "redis"
+    CASSANDRA = "cassandra"
+    NEO4J = "neo4j"
+    FIRESTORE = "firestore"
+    DYNAMODB = "dynamodb"
+    # --NEWSQL Databases--
+    COCKROACHDB = "cockroachdb"
+    SPANNER = "spanner"
+    # --- Messaging ---
+    MESSAGING_KAFKA = "messaging_kafka"
+    MESSAGING_SQS = "messaging_sqs"
+    MESSAGING_PUBSUB = "messaging_pubsub"
+    # --- Real-time Communication ---
+    REALTIME_WEBSOCKET = "websocket"
+     # --- Notifications ---
+    NOTIFICATION_WEBHOOK = "webhook"
+    # --- Storage ---
+    LOCAL_STORAGE = "local_storage"
+    INMEMORY = "inmemory"
+    GCS = "gcs"
+    S3 = "s3"
+    AZURE_BLOB = "azure_blob"
+    HDFS = "hdfs"
+    # --- Files ---
+    FILE = "file"
+    FILE_CSV = "file_csv"
+    FILE_EXCEL = "file_excel"
+    FILE_JSON = "file_json"
+    FILE_PARQUET = "file_parquet"
+    FILE_ORC = "file_orc"
+    FILE_AVRO = "file_avro"
+    FILE_TEXT = "file_text"
+    FILE_IMAGE = "file_image"
+    FILE_VIDEO = "file_video"
+    FILE_AUDIO = "file_audio"
+    FILE_PDF = "file_pdf"
+    FILE_WORD = "file_word"
+    FILE_POWERPOINT = "file_powerpoint"
+    FILE_HTML = "file_html"
+    FILE_MARKDOWN = "file_markdown"
+    FILE_XML = "file_xml"
+    FILE_YAML = "file_yaml"
+    FILE_TOML = "file_toml"
+    FILE_OTHER = "file_other"
+
+    def __str__(self):
+        return self.value
+
+class PipelineTriggerType(Enum):
+    MANUAL = "manual"
+    SCHEDULER = "scheduler"
+    SCHEDULER_MAIN = "scheduler_main"
+    SCHEDULER_FALLBACK = "scheduler_fallback"
+    SCHEDULER_RETRY = "scheduler_retry"
+    SCHEDULED_VERIFICATION = "scheduled_verification"
+    EVENT_GCS_UPLOAD= "event_gcs_upload"
+    EVENT_PUBSUB= "event_pubsub"
+    ANOTHER_PIPELINE = "another_pipeline"
+
+    def __str__(self):
+        return self.value
+
+
+class DataOperationType(Enum):
+    # --- Read operations ---
+    SOURCE="source" # For reading data from source
+    QUERY = "query" # For databases or systems that support queries
+    SCAN = "scan" # For reading all data sequentially (e.g., files)
+    READ= "read" # For general read operations
+    GET= "get" # For getting a single record
+    IMPORT = "import"
+    # --- Transform operations ---
+    TRANSFORM = "transform"
+    PREPROCESS = "preprocess"
+    ENRICH = "enrich"
+    JOIN = "join"
+    AGGREGATE = "aggregate"
+    FILTER = "filter"
+    SORT = "sort"
+    GROUP = "group"
+    # --- Write operations ---
+    POST= "post" # For creating new records
+    PUT= "put"
+    PATCH= "patch"
+    WRITE = "write"
+    WRITE_TO_FILE = "write_to_file"
+    APPEND = "append"
+    UPSERT = "upsert"
+    INSERT = "insert"
+    OVERWRITE = "overwrite"
+    INCREMENT = "increment"
+    UPDATE = "update"
+    DELETE = "delete"
+    EXPORT = "export"
+    COPY = "copy"
+    MERGE = "merge" ## For merging data, combines INSERT, UPDATE, DELETE operations
+    MERGE_UPSERT = "merge_upsert" ## For merging data, combines INSERT, UPDATE, DELETE operations
+    BIGQUERY_WRITE_APPEND = "bigquery_write_append" # For emptying table and writing data, specific to BIGQUERY
+    BIGQUERY_WRITE_TRUNCATE = "bigquery_write_truncate" #For writing data to empty table, fails if table not empty, specific to BIGQUERY
+    BIGQUERY_WRITE_EMPTY = "bigquery_write_empty" # For updating or inserting records
+    # --- Create operations ---
+    CREATE_TABLE = "create_table"
+    CREATE_DATABASE = "create_database"
+    CREATE_COLLECTION = "create_collection"
+    CREATE_INDEX = "create_index"
+    CREATE_SCHEMA = "create_schema"
+    CREATE_MODEL = "create_model"
+    CREATE_VIEW = "create_view"
+    # --- Alter operations ---
+    ALTER_TABLE = "alter_table"
+    ALTER_DATABASE = "alter_database"
+    ALTER_COLLECTION = "alter_collection"
+    ALTER_INDEX = "alter_index"
+    ALTER_SCHEMA = "alter_schema"
+    ALTER_MODEL = "alter_model"
+    ALTER_VIEW = "alter_view"
+    # --- Drop operations ---
+    DROP_TABLE = "drop_table"
+    DROP_DATABASE = "drop_database"
+    DROP_COLLECTION = "drop_collection"
+    DROP_INDEX = "drop_index"
+    DROP_SCHEMA = "drop_schema"
+    DROP_MODEL = "drop_model"
+    DROP_VIEW = "drop_view"
+    # --- Truncate operations ---
+    TRUNCATE_TABLE = "truncate_table"
+    TRUNCATE_COLLECTION = "truncate_collection"
+
+    def __str__(self):
+        return self.value
+class MatchConditionType(Enum):
+    EXACT = "exact"
+    PREFIX = "prefix"
+    SUFFIX = "suffix"
+    CONTAINS = "contains"
+    REGEX = "regex"
+    IN_RANGE = "in_range"
+    NOT_IN_RANGE = "not_in_range"
+    GREATER_THAN = "greater_than"
+    LESS_THAN = "less_than"
+    GREATER_THAN_OR_EQUAL = "greater_than_or_equal"
+    LESS_THAN_OR_EQUAL = "less_than_or_equal"
+    IN_LIST = "in_list"
+    NOT_IN_LIST = "not_in_list"
+    ON_FIELD_MATCH = "on_field_match"
+    ON_FIELD_EQUAL = "on_field_equal"
+    ON_FIELDS_EQUAL_TO = "on_fields_equal_to"
+    ON_FIELDS_COMBINATION = "on_fields_combination"
+    NOT_APPLICABLE = "not_applicable"
+
+    def __str__(self):
+        return self.value
+
+
+class DuplicationHandling(Enum):
+    RAISE_ERROR = "raise_error"
+    OVERWRITE = "overwrite"
+    INCREMENT = "increment"
+    SKIP = "skip"
+    SYSTEM_DEFAULT = "system_default"
+    ALLOW = "allow" ## applicable for databases allowing this operation i.e. BigQuery 
+    MERGE_DEFAULT = "merge_default"
+    MERGE_PRESERVE_SOURCE_ON_DUPLICATES = "merge_preserve_source_on_dups"
+    MERGE_PRESERVE_TARGET_ON_DUPLICATES = "merge_preserve_target_on_dups"
+    MERGE_PRESERVE_BOTH_ON_DUPLICATES = "merge_preserve_both_on_dups"
+    MERGE_RAISE_ERROR_ON_DUPLICATES = "merge_raise_error_on_dups"
+    MERGE_CUSTOM = "merge_custom"
+
+    def __str__(self):
+        return self.value
+
+
+class DuplicationHandlingStatus(Enum):
+    ALLOWED = "allowed"
+    RAISED_ERROR = "raised_error"
+    SYSTEM_DEFAULT = "system_default"
+    OVERWRITTEN = "overwritten"
+    SKIPPED = "skipped"
+    INCREMENTED = "incremented"
+    OPERATION_CANCELLED = "operation_cancelled"
+    MERGED = "merged"
+    MERGED_PRESERVED_SOURCE = "merged_preserved_source"
+    MERGED_PRESERVED_TARGET = "merged_preserved_target"
+    MERGED_PRESERVED_BOTH = "merged_preserved_both"
+    MERGED_RAISED_ERROR = "merged_raised_error"
+    MERGED_CUSTOM = "merged_custom"
+    NO_DUPLICATES = "no_duplicates"
+    UNKNOWN = "unknown"
+    UNEXPECTED_ERROR= "unexpected_error"
+    CONDITIONAL_ERROR = "conditional_error"
+    NOT_APPLICABLE = "not_applicable"
+
+    def __str__(self):
+        return self.value
+
+class CodingLanguage(Enum):
+    PYTHON = "python"
+    NODEJS = "nodejs"
+    JAVA = "java"
+    JAVASCRIPT = "javascript"
+    TYPESCRIPT = "typescript"
+    REACTJS = "reactjs"
+
+    def __str__(self):
+        return self.value
+
+
+class ExecutionLocation(Enum):
+# Add local execution environments
+    LOCAL_SCRIPT = "local_script"
+    LOCAL_JUPYTER_NOTEBOOK = "local_jupyter_notebook"
+    LOCAL_SERVER = "local_server"
+    LOCAL_DOCKER = "local_docker"  # Add local Docker environment
+    LOCAL_KUBERNETES = "local_kubernetes"  # Add local Kubernetes environment
+
+    LOCAL_GCP_CLOUD_FUNCTION = "local_gcp_cloud_function"
+    LOCAL_GCP_CLOUD_RUN = "local_gcp_cloud_run"
+
+# Add GCP execution environments
+    CLOUD_GCP_JUPYTER_NOTEBOOK = "cloud_gcp_jupyter_notebook"
+    CLOUD_GCP_CLOUD_FUNCTION = "cloud_gcp_cloud_function"
+    CLOUD_GCP_CLOUD_RUN = "cloud_gcp_cloud_run"
+    CLOUD_GCP_COMPUTE_ENGINE = "cloud_gcp_compute_engine"
+    CLOUD_GCP_DATAPROC = "cloud_gcp_dataproc"
+    CLOUD_GCP_DATAFLOW = "cloud_gcp_dataflow"
+    CLOUD_GCP_BIGQUERY = "cloud_gcp_bigquery"
+# Add AWS execution environments
+    CLOUD_AWS_LAMBDA = "cloud_aws_lambda"
+    CLOUD_AWS_EC2 = "cloud_aws_ec2"
+    CLOUD_AWS_EMR = "cloud_aws_emr"
+    CLOUD_AWS_GLUE = "cloud_aws_glue"
+    CLOUD_AWS_ATHENA = "cloud_aws_athena"
+    CLOUD_AWS_REDSHIFT = "cloud_aws_redshift"
+# Add Azure execution environments
+    CLOUD_AZURE_FUNCTIONS = "cloud_azure_functions"
+    CLOUD_AZURE_VIRTUAL_MACHINES = "cloud_azure_virtual_machines"
+    CLOUD_AZURE_SYNAPSE_ANALYTICS = "cloud_azure_synapse_analytics"
+    CLOUD_AZURE_DATA_FACTORY = "cloud_azure_data_factory"
+
+    def __str__(self):
+        return self.value
+
+class ExecutionComputeType(Enum):
+
+    CPU_INTEL = "cpu_intel"
+    CPU_AMD = "cpu_amd"
+    CPU_ARM = "cpu_arm"
+    GPU_NVIDIA = "gpu_nvidia"
+    GPU_AMD = "gpu_amd"
+    GPU_INTEL = "gpu_intel"
+    TPU_GOOGLE = "tpu_google"
+    TPU_INTEL = "tpu_intel"
+    TPU_AMD = "tpu_amd"
+
+    def __str__(self):
+        return self.value

ipulse_shared_core_ftredge-2.7.1/src/ipulse_shared_core_ftredge/enums/enums_logging.py

@@ -0,0 +1,108 @@
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-function-docstring
+# pylint: disable=missing-class-docstring
+from enum import Enum
+
+
+class LoggingHandlers(Enum):
+    NONE = "none"  # No remote handler
+    LOCAL_STREAM = "local_stream"  # Local stream handler
+    GCP_CLOUD_LOGGING = "gcp_cloud_logging"
+    GCP_ERROR_REPORTING = "gcp_error_reporting"
+    GCP_FIREBASE = "gcp_firebase"
+    AWS_CLOUD_WATCH = "aws_cloud_watch"
+    AZURE_MONITOR = "azure_monitor"
+    AZURE_APPLICATION_INSIGHTS = "azure_application_insights"
+    IBM_LOG_ANALYTICS = "ibm_log_analytics"
+    ALIBABA_LOG_SERVICE = "alibaba_log_service"
+    LOGGLY = "loggly"
+    DATADOG = "datadog"
+    NEW_RELIC = "new_relic"
+    SENTRY = "sentry"
+    SUMOLOGIC = "sumologic"
+    # --- Other ---
+    SYSLOG = "syslog" # For system logs
+    CUSTOM = "custom" # For a user-defined remote handler
+    OTHER = "other"
+
+    def __str__(self):
+        return self.value
+
+class TargetLogs(Enum):
+    MIXED="mixed_logs"
+    SUCCESSES = "success_logs"
+    NOTICES = "notice_logs"
+    SUCCESSES_AND_NOTICES = "succs_n_notc_logs"
+    WARNINGS = "warning_logs"
+    WARNINGS_AND_ERRORS = "warn_n_err_logs"
+    ERRORS = "error_logs"
+
+    def __str__(self):
+        return self.value
+
+class LogLevel(Enum):
+    """
+    Standardized notice levels for data engineering pipelines,
+    designed for easy analysis and identification of manual 
+    intervention needs.
+    """
+    DEBUG = 10  # Detailed debug information (for development/troubleshooting)
+
+    INFO = 100
+    INFO_REMOTE_PERSISTNACE_COMPLETE= 101
+    INFO_REMOTE_UPDATE_COMPLETE = 102
+    INFO_REMOTE_DELETE_COMPLETE = 103
+
+    INFO_REMOTE_BULK_PERSISTNACE_COMPLETE= 111
+    INFO_REMOTE_BULK_UPDATE_COMPLETE = 112
+    INFO_REMOTE_BULK_DELETE_COMPLETE = 113
+
+    INFO_LOCAL_PERSISTNACE_COMPLETE = 121
+
+    SUCCESS = 201
+    SUCCESS_WITH_NOTICES = 211
+    SUCCESS_WITH_WARNINGS = 212
+
+    NOTICE = 300  # Maybe same file or data already fully or partially exists
+    NOTICE_ALREADY_EXISTS = 301 # Data already exists, no action required
+    NOTICE_PARTIAL_EXISTS = 302 # Partial data exists, no action required
+    NOTICE_ACTION_CANCELLED = 303 # Data processing cancelled, no action required
+
+     # Warnings indicate potential issues that might require attention:
+    WARNING = 400 # General warning, no immediate action required
+    # WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
+    WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
+    WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
+    WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
+
+    ERROR = 500 # General error, no immediate action required
+
+    ERROR_EXCEPTION = 501
+    ERROR_CUSTOM = 502 # Temporary error, automatic retry likely to succeed
+
+    ERROR_OPERATION_PARTIALLY_FAILED = 511 # Partial or full failure, manual intervention required
+    ERROR_OPERATION_FAILED = 512 # Operation failed, manual intervention required
+    ERORR_OPERATION_WITH_WARNINGS = 513 # Partial or full failure, manual intervention required
+    ERORR_OPERATION_WITH_ERRORS = 514 # Partial or full failure, manual intervention required
+    ERORR_OPERATION_WITH_WARNINGS_OR_ERRORS = 515 # Partial or full failure, manual intervention required
+
+    ERROR_PERSISTANCE_FAILED = 522 # Data persistance failed, manual intervention required
+    ERROR_UPDATE_FAILED = 523 # Data update failed, manual intervention required
+    ERROR_DELETE_FAILED = 524 # Data deletion failed, manual intervention required
+    ERROR_PERSISTANCE_WITH_ERRORS = 525 # Data persistance failed, manual intervention required
+    ERROR_UPDATE_WITH_ERRORS = 526 # Data update failed, manual intervention required
+    ERROR_DELETE_WITH_ERRORS = 527 # Data deletion failed, manual intervention required
+
+    ERROR_THRESHOLD_REACHED = 551
+    ERROR_PIPELINE_THRESHOLD_REACHED = 552 # Error due to threshold reached, no immediate action required
+    ERROR_SUBTHRESHOLD_REACHED = 553 # Error due to threshold reached, no immediate action required
+    ERROR_DATA_QUALITY_THRESHOLD_REACHED = 554 # Error due to threshold reached, no immediate action required
+    ERROR_METADATA_QUALITY_THRESHOLD_REACHED = 555 # Error due to threshold reached, no immediate action required
+    # Critical errors indicate severe failures requiring immediate attention:
+    CRITICAL=600 # General critical error, requires immediate action
+    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure, stackoverflow ), requires immediate action
+
+    UNKNOWN=1001 # Unknown error, should not be used in normal operation
+
+    def __str__(self):
+        return self.value