ipulse-shared-core-ftredge 2.7.1__py3-none-any.whl → 2.8.1__py3-none-any.whl

ipulse_shared_core_ftredge/enums/enums_data_eng.py

@@ -1,313 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-from enum import Enum
-
-
-class DataPrimaryCategory(Enum):
-    SIMULATION="simulation" # Simulation data, based on models and simulations
-    HISTORIC = "historic" # Historical data, usually accurate and complete
-    REALTIME="realtime" # Real-time data, not always certain, can have error
-    ANALYTICS="analytics" # Analytical data and modelling, derived from historical and prediction data. Normally shall be making Human readable sense. vs. Features
-    FEATURES="features" # Feature data, used for training models
-    PREDICTIVE="predictive" # Predictive data, based on models and simulations
-
-    def __str__(self):
-        return self.value
-
-class DataState(Enum):
-    RAW = "raw"
-    FORMATTED= "formatted"
-    CLEANED = "cleaned"
-    PROCESSED = "processed"
-    SIMULATED = "simulated"
-    ANALYZED = "analyzed"
-    VALIDATED = "validated"
-    INVALID = "invalid"
-
-    def __str__(self):
-        return self.value
-
-class DatasetScope(Enum):
-    FULL = "full_dataset"
-    INCREMENTAL = "incremental_dataset"
-    PARTIAL = "partial_dataset"
-    UNKNOWN = "unknown_dataset"
-
-    def __str__(self):
-        return self.value
-
-
-class DataSourceType(Enum):
-    # --- General ---
-    API = "api"
-    RPC = "rpc"
-    GRPC = "grpc"
-    WEBSITE = "website"
-    # --SQL Databases--
-    ORACLE = "oracle"
-    POSTGRESQL = "postgresql"
-    SQLSERVER = "sqlserver"
-    MYSQL = "mysql"
-    BIGQUERY = "bigquery"
-    SNOWFLAKE = "snowflake"
-    REDSHIFT = "redshift"
-    ATHENA = "athena"
-    # --NOSQL Databases--
-    MONGODB = "mongodb"
-    REDIS = "redis"
-    CASSANDRA = "cassandra"
-    NEO4J = "neo4j"
-    FIRESTORE = "firestore"
-    DYNAMODB = "dynamodb"
-    # --NEWSQL Databases--
-    COCKROACHDB = "cockroachdb"
-    SPANNER = "spanner"
-    # --- Messaging ---
-    MESSAGING_KAFKA = "messaging_kafka"
-    MESSAGING_SQS = "messaging_sqs"
-    MESSAGING_PUBSUB = "messaging_pubsub"
-    # --- Real-time Communication ---
-    REALTIME_WEBSOCKET = "websocket"
-     # --- Notifications ---
-    NOTIFICATION_WEBHOOK = "webhook"
-    # --- Storage ---
-    LOCAL_STORAGE = "local_storage"
-    INMEMORY = "inmemory"
-    GCS = "gcs"
-    S3 = "s3"
-    AZURE_BLOB = "azure_blob"
-    HDFS = "hdfs"
-    # --- Files ---
-    FILE = "file"
-    FILE_CSV = "file_csv"
-    FILE_EXCEL = "file_excel"
-    FILE_JSON = "file_json"
-    FILE_PARQUET = "file_parquet"
-    FILE_ORC = "file_orc"
-    FILE_AVRO = "file_avro"
-    FILE_TEXT = "file_text"
-    FILE_IMAGE = "file_image"
-    FILE_VIDEO = "file_video"
-    FILE_AUDIO = "file_audio"
-    FILE_PDF = "file_pdf"
-    FILE_WORD = "file_word"
-    FILE_POWERPOINT = "file_powerpoint"
-    FILE_HTML = "file_html"
-    FILE_MARKDOWN = "file_markdown"
-    FILE_XML = "file_xml"
-    FILE_YAML = "file_yaml"
-    FILE_TOML = "file_toml"
-    FILE_OTHER = "file_other"
-
-    def __str__(self):
-        return self.value
-
-class PipelineTriggerType(Enum):
-    MANUAL = "manual"
-    SCHEDULER = "scheduler"
-    SCHEDULER_MAIN = "scheduler_main"
-    SCHEDULER_FALLBACK = "scheduler_fallback"
-    SCHEDULER_RETRY = "scheduler_retry"
-    SCHEDULED_VERIFICATION = "scheduled_verification"
-    EVENT_GCS_UPLOAD= "event_gcs_upload"
-    EVENT_PUBSUB= "event_pubsub"
-    ANOTHER_PIPELINE = "another_pipeline"
-
-    def __str__(self):
-        return self.value
-
-
-class DataOperationType(Enum):
-    # --- Read operations ---
-    SOURCE="source" # For reading data from source
-    QUERY = "query" # For databases or systems that support queries
-    SCAN = "scan" # For reading all data sequentially (e.g., files)
-    READ= "read" # For general read operations
-    GET= "get" # For getting a single record
-    IMPORT = "import"
-    # --- Transform operations ---
-    TRANSFORM = "transform"
-    PREPROCESS = "preprocess"
-    ENRICH = "enrich"
-    JOIN = "join"
-    AGGREGATE = "aggregate"
-    FILTER = "filter"
-    SORT = "sort"
-    GROUP = "group"
-    # --- Write operations ---
-    POST= "post" # For creating new records
-    PUT= "put"
-    PATCH= "patch"
-    WRITE = "write"
-    WRITE_TO_FILE = "write_to_file"
-    APPEND = "append"
-    UPSERT = "upsert"
-    INSERT = "insert"
-    OVERWRITE = "overwrite"
-    INCREMENT = "increment"
-    UPDATE = "update"
-    DELETE = "delete"
-    EXPORT = "export"
-    COPY = "copy"
-    MERGE = "merge" ## For merging data, combines INSERT, UPDATE, DELETE operations
-    MERGE_UPSERT = "merge_upsert" ## For merging data, combines INSERT, UPDATE, DELETE operations
-    BIGQUERY_WRITE_APPEND = "bigquery_write_append" # For emptying table and writing data, specific to BIGQUERY
-    BIGQUERY_WRITE_TRUNCATE = "bigquery_write_truncate" #For writing data to empty table, fails if table not empty, specific to BIGQUERY
-    BIGQUERY_WRITE_EMPTY = "bigquery_write_empty" # For updating or inserting records
-    # --- Create operations ---
-    CREATE_TABLE = "create_table"
-    CREATE_DATABASE = "create_database"
-    CREATE_COLLECTION = "create_collection"
-    CREATE_INDEX = "create_index"
-    CREATE_SCHEMA = "create_schema"
-    CREATE_MODEL = "create_model"
-    CREATE_VIEW = "create_view"
-    # --- Alter operations ---
-    ALTER_TABLE = "alter_table"
-    ALTER_DATABASE = "alter_database"
-    ALTER_COLLECTION = "alter_collection"
-    ALTER_INDEX = "alter_index"
-    ALTER_SCHEMA = "alter_schema"
-    ALTER_MODEL = "alter_model"
-    ALTER_VIEW = "alter_view"
-    # --- Drop operations ---
-    DROP_TABLE = "drop_table"
-    DROP_DATABASE = "drop_database"
-    DROP_COLLECTION = "drop_collection"
-    DROP_INDEX = "drop_index"
-    DROP_SCHEMA = "drop_schema"
-    DROP_MODEL = "drop_model"
-    DROP_VIEW = "drop_view"
-    # --- Truncate operations ---
-    TRUNCATE_TABLE = "truncate_table"
-    TRUNCATE_COLLECTION = "truncate_collection"
-
-    def __str__(self):
-        return self.value
-class MatchConditionType(Enum):
-    EXACT = "exact"
-    PREFIX = "prefix"
-    SUFFIX = "suffix"
-    CONTAINS = "contains"
-    REGEX = "regex"
-    IN_RANGE = "in_range"
-    NOT_IN_RANGE = "not_in_range"
-    GREATER_THAN = "greater_than"
-    LESS_THAN = "less_than"
-    GREATER_THAN_OR_EQUAL = "greater_than_or_equal"
-    LESS_THAN_OR_EQUAL = "less_than_or_equal"
-    IN_LIST = "in_list"
-    NOT_IN_LIST = "not_in_list"
-    ON_FIELD_MATCH = "on_field_match"
-    ON_FIELD_EQUAL = "on_field_equal"
-    ON_FIELDS_EQUAL_TO = "on_fields_equal_to"
-    ON_FIELDS_COMBINATION = "on_fields_combination"
-    NOT_APPLICABLE = "not_applicable"
-
-    def __str__(self):
-        return self.value
-
-
-class DuplicationHandling(Enum):
-    RAISE_ERROR = "raise_error"
-    OVERWRITE = "overwrite"
-    INCREMENT = "increment"
-    SKIP = "skip"
-    SYSTEM_DEFAULT = "system_default"
-    ALLOW = "allow" ## applicable for databases allowing this operation i.e. BigQuery 
-    MERGE_DEFAULT = "merge_default"
-    MERGE_PRESERVE_SOURCE_ON_DUPLICATES = "merge_preserve_source_on_dups"
-    MERGE_PRESERVE_TARGET_ON_DUPLICATES = "merge_preserve_target_on_dups"
-    MERGE_PRESERVE_BOTH_ON_DUPLICATES = "merge_preserve_both_on_dups"
-    MERGE_RAISE_ERROR_ON_DUPLICATES = "merge_raise_error_on_dups"
-    MERGE_CUSTOM = "merge_custom"
-
-    def __str__(self):
-        return self.value
-
-
-class DuplicationHandlingStatus(Enum):
-    ALLOWED = "allowed"
-    RAISED_ERROR = "raised_error"
-    SYSTEM_DEFAULT = "system_default"
-    OVERWRITTEN = "overwritten"
-    SKIPPED = "skipped"
-    INCREMENTED = "incremented"
-    OPERATION_CANCELLED = "operation_cancelled"
-    MERGED = "merged"
-    MERGED_PRESERVED_SOURCE = "merged_preserved_source"
-    MERGED_PRESERVED_TARGET = "merged_preserved_target"
-    MERGED_PRESERVED_BOTH = "merged_preserved_both"
-    MERGED_RAISED_ERROR = "merged_raised_error"
-    MERGED_CUSTOM = "merged_custom"
-    NO_DUPLICATES = "no_duplicates"
-    UNKNOWN = "unknown"
-    UNEXPECTED_ERROR= "unexpected_error"
-    CONDITIONAL_ERROR = "conditional_error"
-    NOT_APPLICABLE = "not_applicable"
-
-    def __str__(self):
-        return self.value
-
-class CodingLanguage(Enum):
-    PYTHON = "python"
-    NODEJS = "nodejs"
-    JAVA = "java"
-    JAVASCRIPT = "javascript"
-    TYPESCRIPT = "typescript"
-    REACTJS = "reactjs"
-
-    def __str__(self):
-        return self.value
-
-
-class ExecutionLocation(Enum):
-# Add local execution environments
-    LOCAL_SCRIPT = "local_script"
-    LOCAL_JUPYTER_NOTEBOOK = "local_jupyter_notebook"
-    LOCAL_SERVER = "local_server"
-    LOCAL_DOCKER = "local_docker"  # Add local Docker environment
-    LOCAL_KUBERNETES = "local_kubernetes"  # Add local Kubernetes environment
-
-    LOCAL_GCP_CLOUD_FUNCTION = "local_gcp_cloud_function"
-    LOCAL_GCP_CLOUD_RUN = "local_gcp_cloud_run"
-
-# Add GCP execution environments
-    CLOUD_GCP_JUPYTER_NOTEBOOK = "cloud_gcp_jupyter_notebook"
-    CLOUD_GCP_CLOUD_FUNCTION = "cloud_gcp_cloud_function"
-    CLOUD_GCP_CLOUD_RUN = "cloud_gcp_cloud_run"
-    CLOUD_GCP_COMPUTE_ENGINE = "cloud_gcp_compute_engine"
-    CLOUD_GCP_DATAPROC = "cloud_gcp_dataproc"
-    CLOUD_GCP_DATAFLOW = "cloud_gcp_dataflow"
-    CLOUD_GCP_BIGQUERY = "cloud_gcp_bigquery"
-# Add AWS execution environments
-    CLOUD_AWS_LAMBDA = "cloud_aws_lambda"
-    CLOUD_AWS_EC2 = "cloud_aws_ec2"
-    CLOUD_AWS_EMR = "cloud_aws_emr"
-    CLOUD_AWS_GLUE = "cloud_aws_glue"
-    CLOUD_AWS_ATHENA = "cloud_aws_athena"
-    CLOUD_AWS_REDSHIFT = "cloud_aws_redshift"
-# Add Azure execution environments
-    CLOUD_AZURE_FUNCTIONS = "cloud_azure_functions"
-    CLOUD_AZURE_VIRTUAL_MACHINES = "cloud_azure_virtual_machines"
-    CLOUD_AZURE_SYNAPSE_ANALYTICS = "cloud_azure_synapse_analytics"
-    CLOUD_AZURE_DATA_FACTORY = "cloud_azure_data_factory"
-
-    def __str__(self):
-        return self.value
-
-class ExecutionComputeType(Enum):
-
-    CPU_INTEL = "cpu_intel"
-    CPU_AMD = "cpu_amd"
-    CPU_ARM = "cpu_arm"
-    GPU_NVIDIA = "gpu_nvidia"
-    GPU_AMD = "gpu_amd"
-    GPU_INTEL = "gpu_intel"
-    TPU_GOOGLE = "tpu_google"
-    TPU_INTEL = "tpu_intel"
-    TPU_AMD = "tpu_amd"
-
-    def __str__(self):
-        return self.value

ipulse_shared_core_ftredge/enums/enums_logging.py

@@ -1,108 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-from enum import Enum
-
-
-class LoggingHandlers(Enum):
-    NONE = "none"  # No remote handler
-    LOCAL_STREAM = "local_stream"  # Local stream handler
-    GCP_CLOUD_LOGGING = "gcp_cloud_logging"
-    GCP_ERROR_REPORTING = "gcp_error_reporting"
-    GCP_FIREBASE = "gcp_firebase"
-    AWS_CLOUD_WATCH = "aws_cloud_watch"
-    AZURE_MONITOR = "azure_monitor"
-    AZURE_APPLICATION_INSIGHTS = "azure_application_insights"
-    IBM_LOG_ANALYTICS = "ibm_log_analytics"
-    ALIBABA_LOG_SERVICE = "alibaba_log_service"
-    LOGGLY = "loggly"
-    DATADOG = "datadog"
-    NEW_RELIC = "new_relic"
-    SENTRY = "sentry"
-    SUMOLOGIC = "sumologic"
-    # --- Other ---
-    SYSLOG = "syslog" # For system logs
-    CUSTOM = "custom" # For a user-defined remote handler
-    OTHER = "other"
-
-    def __str__(self):
-        return self.value
-
-class TargetLogs(Enum):
-    MIXED="mixed_logs"
-    SUCCESSES = "success_logs"
-    NOTICES = "notice_logs"
-    SUCCESSES_AND_NOTICES = "succs_n_notc_logs"
-    WARNINGS = "warning_logs"
-    WARNINGS_AND_ERRORS = "warn_n_err_logs"
-    ERRORS = "error_logs"
-
-    def __str__(self):
-        return self.value
-
-class LogLevel(Enum):
-    """
-    Standardized notice levels for data engineering pipelines,
-    designed for easy analysis and identification of manual 
-    intervention needs.
-    """
-    DEBUG = 10  # Detailed debug information (for development/troubleshooting)
-
-    INFO = 100
-    INFO_REMOTE_PERSISTNACE_COMPLETE= 101
-    INFO_REMOTE_UPDATE_COMPLETE = 102
-    INFO_REMOTE_DELETE_COMPLETE = 103
-
-    INFO_REMOTE_BULK_PERSISTNACE_COMPLETE= 111
-    INFO_REMOTE_BULK_UPDATE_COMPLETE = 112
-    INFO_REMOTE_BULK_DELETE_COMPLETE = 113
-
-    INFO_LOCAL_PERSISTNACE_COMPLETE = 121
-
-    SUCCESS = 201
-    SUCCESS_WITH_NOTICES = 211
-    SUCCESS_WITH_WARNINGS = 212
-
-    NOTICE = 300  # Maybe same file or data already fully or partially exists
-    NOTICE_ALREADY_EXISTS = 301 # Data already exists, no action required
-    NOTICE_PARTIAL_EXISTS = 302 # Partial data exists, no action required
-    NOTICE_ACTION_CANCELLED = 303 # Data processing cancelled, no action required
-
-     # Warnings indicate potential issues that might require attention:
-    WARNING = 400 # General warning, no immediate action required
-    # WARNING_NO_ACTION = 401 # Minor issue or Unexpected Behavior, no immediate action required (can be logged frequently)
-    WARNING_REVIEW_RECOMMENDED = 402 # Action recommended to prevent potential future issues
-    WARNING_FIX_RECOMMENDED = 403 # Action recommended to prevent potential future issues
-    WARNING_FIX_REQUIRED = 404  # Action required, pipeline can likely continue
-
-    ERROR = 500 # General error, no immediate action required
-
-    ERROR_EXCEPTION = 501
-    ERROR_CUSTOM = 502 # Temporary error, automatic retry likely to succeed
-
-    ERROR_OPERATION_PARTIALLY_FAILED = 511 # Partial or full failure, manual intervention required
-    ERROR_OPERATION_FAILED = 512 # Operation failed, manual intervention required
-    ERORR_OPERATION_WITH_WARNINGS = 513 # Partial or full failure, manual intervention required
-    ERORR_OPERATION_WITH_ERRORS = 514 # Partial or full failure, manual intervention required
-    ERORR_OPERATION_WITH_WARNINGS_OR_ERRORS = 515 # Partial or full failure, manual intervention required
-
-    ERROR_PERSISTANCE_FAILED = 522 # Data persistance failed, manual intervention required
-    ERROR_UPDATE_FAILED = 523 # Data update failed, manual intervention required
-    ERROR_DELETE_FAILED = 524 # Data deletion failed, manual intervention required
-    ERROR_PERSISTANCE_WITH_ERRORS = 525 # Data persistance failed, manual intervention required
-    ERROR_UPDATE_WITH_ERRORS = 526 # Data update failed, manual intervention required
-    ERROR_DELETE_WITH_ERRORS = 527 # Data deletion failed, manual intervention required
-
-    ERROR_THRESHOLD_REACHED = 551
-    ERROR_PIPELINE_THRESHOLD_REACHED = 552 # Error due to threshold reached, no immediate action required
-    ERROR_SUBTHRESHOLD_REACHED = 553 # Error due to threshold reached, no immediate action required
-    ERROR_DATA_QUALITY_THRESHOLD_REACHED = 554 # Error due to threshold reached, no immediate action required
-    ERROR_METADATA_QUALITY_THRESHOLD_REACHED = 555 # Error due to threshold reached, no immediate action required
-    # Critical errors indicate severe failures requiring immediate attention:
-    CRITICAL=600 # General critical error, requires immediate action
-    CRITICAL_SYSTEM_FAILURE = 601 # System-level failure (e.g., infrastructure, stackoverflow ), requires immediate action
-
-    UNKNOWN=1001 # Unknown error, should not be used in normal operation
-
-    def __str__(self):
-        return self.value

ipulse_shared_core_ftredge/enums/enums_module_fincore.py

@@ -1,72 +0,0 @@
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-from enum import Enum
-
-
-class FinCoreCategory(Enum):
-    MARKET="market" # Market prices data
-    CORPORATE="corp" # Corporate data such as financial statements and earnings, similar to fundamental data
-    FUNDAMENTAL="fundam"
-    ECONOMY="economy"
-    NEWS="news"
-    SENTIMENT="sntmnt"
-    SOCIAL="social"
-    POLITICS="poltcs"
-    OTHER="other"
-
-    def __str__(self):
-        return self.value
-
-    
-
-class FincCoreSubCategory(Enum):
-    STOCKS = "stocks"
-    BONDS = "bonds"
-    COMMODITIES = "cmmdt"
-    CURRENCIES = "crrncy"
-    CRYPTOCURRENCIES = "crypto"
-    REAL_ESTATE = "realest"
-    EQUITY_INDICES = "eqindx"
-    OPTIONS = "options"
-    FUTURES = "futures"
-    ETF = "etf"
-    ECONOMIC_INDICATORS = "ecoind"
-    FUNDAMENTALS = "fundam"
-    OTHER = "othr"
-
-    def __str__(self):
-        return self.value
-
-class FinCoreRecordsCategory(Enum):
-    PRICE="pric"
-    SPOT= "spot"
-    OHLCVA="ohlcva"
-    OHLCV="ohlcv"
-    OPEN="open"
-    HIGH="high"
-    LOW="low"
-    CLOSE="close"
-    VOLUME="volume"
-    ADJC="adjc"
-    FUNDAMENTAL="fundam" # treat this differently
-    EARNINGS="earnings"
-    CASH_FLOW="cashflw"
-    BALANCE_SHEET="blnce_sht"
-    INTERNAL_TRANSACTIONS="internaltrans"
-    INDICATORS="indic"
-    ARTICLE="article"
-    INSTA_POST="isntapost"
-    TWEET="tweet"
-    OTHER="othr"
-
-    def __str__(self):
-        return self.value
-
-class FinancialExchangeOrPublisher(Enum):
-    CC="cc"
-    US="us"
-    NASDAQ="nasdaq"
-
-    def __str__(self):
-        return self.value

ipulse_shared_core_ftredge/enums/enums_modules.py

@@ -1,31 +0,0 @@
-
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=line-too-long
-from enum import Enum
-
-class Module(Enum):
-    CORE="core"
-    ORACLE="oracle"
-    PORTFOLIO="portfolio"
-    RISK="risk"
-    RESEARCH="research"
-    TRADING="trading"
-    SIMULATION="simulation"
-
-    def __str__(self):
-        return self.value
-
-class Domain(Enum):
-    FINCORE="fincore"
-    GYMCORE="gymcore"
-    HEALTHCORE="healthcore"
-    ENVICORE="envicore"
-    SPORTSCORE="sportscore"
-    POLICORE="policore"
-    CUSTOM="custom"
-
-    def __str__(self):
-        return self.value
-

ipulse_shared_core_ftredge/enums/enums_solution_providers.py

@@ -1,24 +0,0 @@
-
-# pylint: disable=missing-module-docstring
-# pylint: disable=missing-function-docstring
-# pylint: disable=missing-class-docstring
-# pylint: disable=line-too-long
-
-from enum import Enum
-
-class CloudProvider(Enum):
-    GCP = "cloud_gcp"
-    AWS = "cloud_aws"
-    AZURE = "cloud_azure"
-    IBM = "cloud_ibm"
-    ALIBABA = "cloud_alibaba"
-    NO_CLOUD = "no_cloud"
-    CLOUD_AGNOSTIC = "cloud_agnostic"
-    OTHER = "other"
-    UNKNWON = "unknown"
-
-    def __str__(self):
-        return self.value
-
-
-