logryn 1.0.0b1__tar.gz

logryn-1.0.0b1/Logryn/__init__.py

@@ -0,0 +1,37 @@
+"""
+Advanced High-Availability Logging Library
+==========================================
+A robust, asynchronous, and thread-safe logging framework designed for 
+long-running pipelines. It supports RAM caching, Database batch writing, 
+and graceful shutdowns under heavy concurrent loads.
+
+Core Components:
+- MainLog: The central async logging orchestrator.
+- project_log_decorator: External wrapper for pipeline functions.
+- ThreadConfigManager: Thread identity and configuration management.
+- setup_global_error_handler: OS-level signal and crash handling.
+"""
+
+__version__ = "1.0.0b1"
+__author__ = "Elad Segev"
+
+# 1. Core Logger & Decorators
+from .main_logger import (
+    MainLog, 
+    project_log_decorator
+)
+
+# 2. Thread Management
+from .thread_config_manager import ThreadConfigManager
+
+# 3. System Stability & Crash Handling
+from .logger_e import setup_global_error_handler
+
+# 4. Explicit Public API Definition
+__all__ = [
+    "MainLog",
+    "project_log_decorator",
+    "ThreadConfigManager",
+    "setup_global_error_handler",
+    "__version__"
+]

logryn-1.0.0b1/Logryn/db_core.py

@@ -0,0 +1,553 @@
+from sqlalchemy import create_engine, inspect, text
+from functools import lru_cache
+
+@lru_cache(maxsize=32)
+def _get_engine(connection_string: str):
+    """
+    Returns a cached SQLAlchemy database engine for the specified connection.
+
+    The engine is configured with connection health checks and connection
+    recycling to improve reliability for long-running database operations.
+
+    :param connection_string: Database connection string used to create the engine.
+    :type connection_string: str
+
+    :return: SQLAlchemy engine associated with the connection string.
+    :rtype: Engine
+    """
+    return create_engine(
+        connection_string,
+        pool_pre_ping=True,     
+        pool_recycle=3600,      
+    )
+
+
+def _set_connection_string(database_name:str|None, db_config:dict):
+    """
+    Builds a MariaDB connection string from the provided database configuration.
+
+    :param database_name: Name of the target database. If omitted, the connection
+                        string points to the MariaDB server without a database.
+    :type database_name: str | None
+    :param db_config: Dictionary containing the database connection parameters,
+                    including user, password, host, and port.
+    :type db_config: dict
+
+    :return: MariaDB SQLAlchemy connection string.
+    :rtype: str
+    """
+    user = db_config.get("user")
+    password = db_config.get("password")
+    host = db_config.get("host")
+    port = db_config.get("port")
+    if database_name:
+        return f"mariadb+mariadbconnector://{user}:{password}@{host}:{port}/{database_name}"
+    return f"mariadb+mariadbconnector://{user}:{password}@{host}:{port}/"
+
+
+def is_table_exists(table_name: str, database_name: str, db_config: dict) -> bool:
+    """
+    Checks whether a table exists in the specified database.
+
+    :param table_name: Name of the table to check.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: True if the table exists, otherwise False.
+    :rtype: bool
+    """
+    engine =  _get_engine(_set_connection_string(database_name, db_config))
+    inspector = inspect(engine)
+    return inspector.has_table(table_name)
+        
+
+def get_all_tables(database_name: str, db_config: dict) -> list:
+    """
+    Retrieves all table names from the specified database.
+
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: List of table names, or an empty list if the operation fails.
+    :rtype: list
+    """
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        inspector = inspect(engine)
+        
+        return inspector.get_table_names()
+        
+    except Exception as e:
+        print(f"Failed to fetch tables: {e}")
+        return []
+
+
+def is_db_exists(database_name: str, db_config: dict) -> bool:
+    """
+    Checks whether a database exists on the configured MariaDB server.
+
+    :param database_name: Name of the database to check.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: True if the database exists, otherwise False.
+    :rtype: bool
+    """
+
+    engine = _get_engine(_set_connection_string(None, db_config))
+    
+    with engine.connect() as conn:
+        query_check = text(f"SHOW DATABASES LIKE '{database_name}'")
+        result = conn.execute(query_check)
+        db_exists = result.fetchone() is not None
+
+    return db_exists
+
+
+def create_database_(database_name: str, db_config: dict) -> tuple:
+    """
+    Creates a database on the configured MariaDB server if it does not already exist.
+
+    :param database_name: Name of the database to create.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    try:
+
+        if is_db_exists(database_name=database_name, db_config=db_config):
+            return True, 'Existing database'
+            
+        try:
+
+            engine = _get_engine(_set_connection_string(None, db_config))
+            with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn_create:
+                conn_create.execute(text(f"CREATE DATABASE {database_name}"))
+            return True, 'Database created'
+                
+        except Exception as e_create:
+            msg = f"Failed to create database '{database_name}': {e_create}"
+            return False, msg
+                
+    except Exception as e_conn:
+        mag = f"Failed to connect to MariaDB server: {e_conn}"
+        return False, mag
+
+
+def create_table_(table_name: str, database_name: str, db_config: dict, create_query: str) -> tuple:
+    """
+    Creates a table in the specified database if it does not already exist.
+
+    :param table_name: Name of the table to create.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+    :param create_query: SQL statement used to create the table.
+    :type create_query: str
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    try:
+        exists_status = is_table_exists(table_name, database_name, db_config)
+
+    except Exception as e:
+        msg = f"Failed to connect or check table: {e}"
+        return False, msg
+
+        
+    if exists_status is True:
+        return True, 'Existing table'
+    
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            conn.execute(text(create_query))
+            
+        return True, 'Table created'
+        
+    except Exception as e:
+        mag = f"Failed to create table '{table_name}': {e}"
+        return False, mag
+
+
+def insert_row_(data_dict: dict, table_name: str, database_name: str, db_config: dict) -> tuple:
+    """
+    Inserts a row into the specified database table.
+
+    :param data_dict: Dictionary containing column names and their corresponding
+                    values for the row to insert.
+    :type data_dict: dict
+    :param table_name: Name of the target table.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Tuple containing a success flag and the inserted row ID or an error
+            message.
+    :rtype: tuple
+    """
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        columns = ", ".join(data_dict.keys())
+        placeholders = ", ".join([f":{k}" for k in data_dict.keys()])
+        query_string = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
+        
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            result = conn.execute(text(query_string), data_dict)
+            inserted_id = result.lastrowid
+            
+        return True, inserted_id
+        
+    except Exception as e:
+        msg = f"Failed to insert row into '{table_name}': {e}"
+        return False, msg
+
+
+def insert_rows_batch_(data_list: list[dict], table_name: str, database_name: str, db_config: dict) -> tuple:
+    """
+    Inserts multiple rows into the specified database table as a batch operation.
+
+    :param data_list: List of dictionaries containing column names and their
+                    corresponding values for each row.
+    :type data_list: list[dict]
+    :param table_name: Name of the target table.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    if not data_list:
+        return True, 'No data to insert'
+            
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        columns = ", ".join(data_list[0].keys())
+        placeholders = ", ".join([f":{k}" for k in data_list[0].keys()])
+        query_string = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
+        
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            conn.execute(text(query_string), data_list)
+            
+        return True, f'{len(data_list)} rows inserted successfully'
+        
+    except Exception as e:
+        msg = f"Failed to insert batch into '{table_name}': {e}"
+        return False, msg
+
+
+def drop_table_(table_name: str, database_name: str, db_config: dict) -> tuple:
+    """
+    Drops the specified table from the database if it exists.
+
+    :param table_name: Name of the table to drop.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            conn.execute(text(f"DROP TABLE IF EXISTS {table_name}"))
+            
+        return True, f"Table '{table_name}' dropped successfully"
+        
+    except Exception as e:
+        msg = f"Failed to drop table '{table_name}': {e}"
+        return False, msg
+
+
+def drop_database_(database_name: str, db_config: dict) -> tuple:
+    """
+    Drops the specified database if it exists.
+
+    :param database_name: Name of the database to drop.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    if not is_db_exists(database_name, db_config):
+        return False, f"The database '{database_name}' doesn't exsist "
+    
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            conn.execute(text(f"DROP DATABASE IF EXISTS {database_name}"))
+            
+        return True, f"Database '{database_name}' dropped successfully"
+        
+    except Exception as e:
+        msg = f"Failed to drop database '{database_name}': {e}"
+        return False, msg
+
+
+def alter_table_(table_name: str, database_name: str, db_config: dict, alter_query: str) -> tuple:
+    """
+    Alters the specified table using the provided SQL statement.
+
+    :param table_name: Name of the table to alter.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+    :param alter_query: SQL statement used to alter the table.
+    :type alter_query: str
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            conn.execute(text(alter_query))
+            
+        return True, f"Table '{table_name}' altered successfully"
+        
+    except Exception as e:
+        msg = f"Failed to alter table '{table_name}': {e}"
+        return False, msg
+
+
+def get_row_count_(table_name: str, database_name: str, db_config: dict) -> int:
+    """
+    Returns the number of rows in the specified database table.
+
+    :param table_name: Name of the table to count rows from.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Number of rows in the table, or 0 if the operation fails.
+    :rtype: int
+    """
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        with engine.connect() as conn:
+            query = text(f"SELECT COUNT(*) FROM {table_name}")
+            result = conn.execute(query)
+            count = result.scalar()
+            
+            return int(count) if count is not None else 0
+            
+    except Exception as e:
+        print(f"Failed to get row count for '{table_name}': {e}")
+        return 0
+
+
+def delete_table_(table_name: str, database_name: str, db_config: dict) -> tuple:
+    """
+    Deletes all rows from the specified database table.
+
+    :param table_name: Name of the table to clear.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            conn.execute(text(f"DELETE FROM {table_name}"))
+        return True, f"Table '{table_name}' cleared successfully"
+    except Exception as e:
+        return False, f"Failed to clear table '{table_name}': {e}"
+
+
+def upsert_rows_batch_(data_list: list[dict], table_name: str, database_name: str, db_config: dict) -> tuple:
+    """
+    Inserts multiple rows into the specified database table and updates existing
+    rows when a duplicate key is encountered.
+
+    :param data_list: List of dictionaries containing column names and their
+                    corresponding values for each row.
+    :type data_list: list[dict]
+    :param table_name: Name of the target table.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+
+    :return: Tuple containing a success flag and a status or error message.
+    :rtype: tuple
+    """
+    if not data_list:
+        return True, 'No data to upsert'
+    
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        
+        columns = ", ".join(data_list[0].keys())
+        placeholders = ", ".join([f":{k}" for k in data_list[0].keys()])
+        
+
+        update_cols = [k for k in data_list[0].keys() if k != 'station_id']
+        update_clause = ", ".join([f"{k} = VALUES({k})" for k in update_cols])
+        
+        query_string = f"""
+            INSERT INTO {table_name} ({columns}) 
+            VALUES ({placeholders})
+            ON DUPLICATE KEY UPDATE {update_clause}
+        """
+        
+        with engine.execution_options(isolation_level="AUTOCOMMIT").connect() as conn:
+            conn.execute(text(query_string), data_list)
+            
+        return True, f'{len(data_list)} rows upserted successfully'
+        
+    except Exception as e:
+        msg = f"Failed to upsert batch into '{table_name}': {e}"
+        return False, msg
+
+
+def select_(table_name: str, database_name: str, db_config: dict, where_dict: str | None = None, logical_op: str = "AND") -> tuple:
+    """
+    Retrieves rows from the specified database table with an optional filter.
+
+    :param table_name: Name of the table to query.
+    :type table_name: str
+    :param database_name: Name of the target database.
+    :type database_name: str
+    :param db_config: Dictionary containing the database connection parameters.
+    :type db_config: dict
+    :param where_dict: Optional filter definition used to build the SQL WHERE
+                    clause.
+    :type where_dict: str | None
+    :param logical_op: Logical operator used to combine multiple filter
+                    conditions.
+    :type logical_op: str
+
+    :return: Tuple containing a success flag and the retrieved rows or an error
+            message.
+    :rtype: tuple
+    """
+
+    query_str = f"SELECT * FROM {table_name}"
+
+    where_string, query_params = _build_where_clause(where_dict, logical_op)
+
+    
+    if where_string:
+        query_str += f" WHERE {where_string}"
+        
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        with engine.connect() as conn:
+            result = conn.execute(text(query_str), query_params)
+            rows = result.fetchall()
+            
+        return True, rows
+        
+    except Exception as e:
+        return False, str(e)
+
+
+def _build_where_clause(conditions: dict | None, logical_op: str = "AND") -> tuple[str, dict]:
+    """
+    Constructs a parameterized SQL WHERE clause from a dictionary of conditions.
+
+    The function converts each condition into a SQL comparison using named
+    placeholders and stores the corresponding values separately for safe query
+    execution. Conditions may use the default equality operator or explicitly
+    specify a supported SQL operator.
+
+    :param conditions: Dictionary mapping column names to values or
+                    ``(operator, value)`` tuples.
+    :type conditions: dict | None
+    :param logical_op: Logical operator used to combine multiple conditions.
+                    Supported values are ``"AND"`` and ``"OR"``. Defaults to
+                    ``"AND"``.
+    :type logical_op: str
+
+    :return: A tuple containing the generated WHERE clause string and a
+            dictionary of parameter values. Returns an empty clause and
+            parameter dictionary when no conditions are provided or the logical
+            operator is invalid.
+    :rtype: tuple[str, dict]
+
+    :raises ValueError: If a condition specifies an unsupported SQL operator.
+    """
+
+    if not conditions:
+        return "", {}
+    
+    if logical_op.lower not in ['and','or']:
+        return "", {}
+
+    ALLOWED_OPERATORS = {"=", "!=", ">", "<", ">=", "<=", "LIKE", "IN"}
+    
+    clauses = []
+    params = {}
+    
+    for i, (col_name, value) in enumerate(conditions.items()):
+        op = "="
+        val = value
+        
+        if isinstance(value, tuple) and len(value) == 2:
+            op, val = value
+            op = str(op).upper()
+            if op not in ALLOWED_OPERATORS:
+                raise ValueError(f"Security Alert: Unsupported SQL operator '{op}'")
+        
+       
+        placeholder = f"{col_name}_{i}"
+        
+        clauses.append(f"{col_name} {op} :{placeholder}")
+        
+        params[placeholder] = val
+        
+    where_string = f" {logical_op.upper()} ".join(clauses)
+    return where_string, params
+
+
+
+def get_table_columns_(table_name: str, database_name: str, db_config: dict) -> list:
+    try:
+        engine = _get_engine(_set_connection_string(database_name, db_config))
+        inspector = inspect(engine)
+        
+        columns = inspector.get_columns(table_name)
+        
+        return True, [col['name'] for col in columns]
+        
+    except Exception as e:
+        return False, f"Failed to fetch columns for '{table_name}': {e}"

logryn-1.0.0b1/Logryn/logger_e.py

@@ -0,0 +1,67 @@
+import sys
+import signal
+import threading
+import asyncio
+from datetime import datetime
+
+def setup_global_error_handler(logger_instance, loop: asyncio.AbstractEventLoop | None = None) -> None:
+    """
+    Configures global handlers for signals, uncaught exceptions, and thread
+    exceptions to ensure that process errors are logged and the logger is
+    properly flushed before shutdown.
+
+    :param logger_instance: Logger instance used to record errors and shutdown
+                            events.
+    :type logger_instance: Any
+    :param loop: Asyncio event loop used to schedule logger shutdown and signal
+                handling. Defaults to the current event loop.
+    :type loop: asyncio.AbstractEventLoop | None
+
+    :return: None.
+    :rtype: None
+    """
+    loop = loop or asyncio.get_event_loop()
+
+    def _flush_and_stop(details: str):
+        async def _do():
+            logger_instance.write(
+                event={'process': 'System Interruption', 'type': 'User/System Kill',
+                       'parameters': 'Signal received', 'thread_id': None},
+                scenario='error', details=details,
+            )
+            await logger_instance.shutdown()      # the piece that was never called
+            loop.stop()
+        asyncio.ensure_future(_do(), loop=loop)
+
+    def signal_handler():
+        _flush_and_stop("Signal received. Flushing queue before exit.")
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        try:
+            loop.add_signal_handler(sig, signal_handler)   # POSIX: safe, runs on the loop
+        except NotImplementedError:
+            signal.signal(sig, lambda s, f: signal_handler())  # Windows fallback
+
+    def exception_handler(exc_type, exc_value, exc_traceback):
+        if issubclass(exc_type, KeyboardInterrupt):
+            sys.__excepthook__(exc_type, exc_value, exc_traceback)
+            return
+        logger_instance.write(
+            event={'process': 'Fatal Crash', 'type': 'Unhandled Exception',
+                   'parameters': f"Type: {exc_type.__name__}", 'thread_id': None},
+            scenario='error', details=f"CRASH LOGIC: {exc_value}",
+            local_start_time=datetime.now(),
+        )
+        asyncio.ensure_future(logger_instance.shutdown(), loop=loop)
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
+
+    def thread_exception_handler(args: threading.ExceptHookArgs):
+        logger_instance.write(
+            event={'process': f"Thread Crash [{args.thread.name}]",
+                   'type': 'Unhandled Thread Exception',
+                   'parameters': f"Type: {args.exc_type.__name__}", 'thread_id': None},
+            scenario='error', details=str(args.exc_value),
+        )
+
+    sys.excepthook = exception_handler
+    threading.excepthook = thread_exception_handler   # <-- previously silent