limitsdb 0.5.0__py3-none-any.whl

limitsdb/db/ldb_engines.py

@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Literal
+
+from limitsdb.core.ldb_params_config import Config
+
+
+@dataclass(frozen=True)
+class ColumnDefinition:
+    """Generic column definition independent from a specific engine."""
+
+    name: str
+    data_type: str
+    id: int | None = None
+    length: int | None = None
+    precision: int | None = None
+    scale: int | None = None
+    nullable: bool = True
+    default: str | None = None
+
+
+IndexMap = dict[str, dict[str, Any]]
+ColumnMetadata = dict[str, ColumnDefinition]
+
+
+@dataclass(frozen=True)
+class IndexDefinition:
+    """Generic index definition."""
+
+    name: str
+    columns: tuple[str, ...]
+    unique: bool = False
+
+
+@dataclass(frozen=True)
+class TableDefinition:
+    """Generic table definition."""
+
+    owner: str
+    name: str
+    columns: tuple[ColumnDefinition, ...]
+    primary_key: tuple[str, ...] | None = None
+    indexes: tuple[IndexDefinition, ...] = ()
+
+
+@dataclass(frozen=True)
+class SequenceDefinition:
+    """Generic sequence definition."""
+
+    owner: str
+    name: str
+    start_with: int = 1
+    increment_by: int = 1
+    minvalue: int | None = None
+    maxvalue: int | None = None
+    cycle: bool = False
+    cache: int | None = 20
+
+
+@dataclass(frozen=True)
+class RoleDefinition:
+    """Generic role definition."""
+
+    name: str
+
+
+@dataclass(frozen=True)
+class UserDefinition:
+    """Generic user definition."""
+
+    name: str
+    password: str
+    default_tablespace: str | None = None
+    temporary_tablespace: str | None = None
+    roles: tuple[str, ...] = ()
+    roles_with_admin_option: tuple[str, ...] = ()
+    system_privileges: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class DatabaseLinkDefinition:
+    """Generic database link definition."""
+
+    name: str
+    username: str
+    password: str
+    dsn: str
+
+
+class DatabaseEngine(ABC):
+    """Static interface for database engine operations."""
+
+    @staticmethod
+    @abstractmethod
+    def get_connection(config: Config, *, admin: bool = False, env: Literal["SOURCE", "HISTORY"] | None = None) -> Any:
+        """Establishes a connection to the database.
+        Args:
+            config: Configuration object.
+            admin: If True, connects with admin credentials.
+        Returns:
+            An open database connection."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def close_connection(conn: Any) -> None:
+        """Closes the Oracle connection.
+        Args:
+            conn: Active Oracle connection."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_table_columns(conn: Any, owner: str, table_name: str) -> tuple[list[str], dict[str, ColumnDefinition]]:
+        """Retrieves column names and metadata for a given table.
+        Args:
+            conn: Active Oracle connection.
+            owner: Schema owner of the table.
+            table_name: Table name.
+        Returns:
+            Tuple containing a list of column names and a dict of column metadata."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_date_condition(date_expr: str, months_keep_src: int) -> str:
+        """Returns a date condition for the given date expression and months to keep.
+        Args:
+            date_expr: Date expression to evaluate.
+            months_keep_src: Months to keep.
+        Returns:
+            Date condition as string."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_fallback_expression(expression: str, fallback: str) -> str:
+        """Return an expression that uses ``fallback`` when ``expression`` is null."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_identifiers_from_expression(expression: str) -> set[str]:
+        """Returns a set of @prefixxed identifiers found in the given expression.
+        Args:
+            expression: The expression string to process.
+        Returns:
+            A set of identifier strings found in the expression.
+        """
+
+    @staticmethod
+    @abstractmethod
+    def rewrite_expression_identifiers(expression: str, replacements: Mapping[str, str], *, qualifier: str) -> str:
+        """Rewrite prefixed identifiers using engine-specific identifier rules.
+
+        Args:
+            expression: Expression containing identifiers prefixed with ``@``.
+            replacements: Mapping from normalized source identifiers to persisted identifiers.
+            qualifier: SQL qualifier to apply to each replacement.
+        Returns:
+            The expression with every mapped identifier rendered for the database engine.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_column_type(column: ColumnDefinition) -> str:
+        """Returns the database-specific column type definition for the given column.
+        Args:
+            column: The ColumnDefinition object.
+        Returns:
+            The database-specific column type as a string.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def generate_sql_block(config: Config, table_cnf: Mapping[str, Any], process_date: str) -> str:
+        """Generates a SQL/PL block to process a given table.
+        Args:
+            config: Configuration object.
+            table_cnf: Processed table configuration object.
+            process_date: Process date in 'YYYYMMDD' format.
+        Returns:
+            The PL/SQL or SQL block to execute."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_system_date(conn: Any) -> datetime:
+        """Retrieves the current system date from the database.
+        Args:
+            conn: Active database connection.
+        Returns:
+            Current system date."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def sql_block_run(conn: Any, plsql_code: str) -> None:
+        """Executes a PL/SQL or SQL block against the database.
+        Args:
+            conn: Active database connection.
+            plsql_code: The PL/SQL or SQL block to execute."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_status(conn: Any, process_date: str) -> list[dict[str, Any]]:
+        """Loads control status rows for a given process date.
+        Args:
+            conn: Active database connection.
+            process_date: The process date in 'YYYYMMDD' format.
+        Returns:
+            A list of control status rows."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_rows_processed(conn: Any, owner: str, table_name: str, process_date: str) -> int:
+        """Returns rows processed for a given table and process date from ldb_ctl.
+        Args:
+            conn: Active Oracle connection.
+            owner: Schema owner of the table.
+            table_name: Table name.
+            process_date: Target process date in 'YYYYMMDD'.
+        Returns:
+            Number of rows processed or 0 if none found or mismatched date."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def save_error_status(
+        conn: Any,
+        config: Config,
+        owner: str,
+        table_name: str,
+        process_date: str,
+        process_start: datetime,
+        message: str,
+        plsql_code: str,
+    ) -> None:
+        """Saves an error status entry in the control table.
+        Args:
+            conn: Active database connection.
+            config: Configuration object.
+            owner: Schema owner.
+            table_name: Table name.
+            process_date: Date in 'YYYYMMDD' format.
+            process_start: Start time of the process.
+            message: Error message.
+            plsql_code: PL/SQL or SQL block executed."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def all_status_tend(conn: Any, tables_config: dict[tuple[str, str], Any], process_date: str) -> bool:
+        """Checks if all referenced tables have status "Status.TABLE_END" in the control table.
+        Args:
+            conn: Active database connection.
+            tables_config: Dict of (owner, table_name) keys representing configured tables.
+            process_date: Processing date in 'YYYYMMDD' format.
+        Returns:
+            True if all tables have status "Status.TABLE_END" for the given process date, False otherwise."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_primary_key_columns(
+        conn: Any, owner: str, table_name: str, table_cnf: Mapping[str, Any]
+    ) -> tuple[str, ...]:
+        """Returns the primary key columns for the specified table."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_roles(conn: Any, roles: Sequence[RoleDefinition]) -> list[str]:
+        """Ensure that the provided roles exist, returning newly created ones."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_users(conn: Any, users: Sequence[UserDefinition]) -> list[str]:
+        """Ensure that the provided users exist, returning newly created ones."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_tables(conn: Any, tables: Sequence[TableDefinition]) -> list[str]:
+        """Ensure that the provided tables exist, returning newly created ones."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_table_structure(conn: Any, table: TableDefinition, table_cnf: Mapping[str, Any]) -> None:
+        """Ensure that the provided table matches the expected structure."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_sequences(conn: Any, sequences: Sequence[SequenceDefinition]) -> list[str]:
+        """Ensure that the provided sequences exist, returning newly created ones."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_database_links(conn: Any, links: Sequence[DatabaseLinkDefinition]) -> list[str]:
+        """Ensure that the provided database links exist, returning newly created ones."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_table_privileges(
+        conn: Any, role: str, tables: Sequence[tuple[str, str]], privileges: Sequence[str]
+    ) -> list[str]:
+        """Ensure the role has the specified privileges on each table, returning new grants."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def ensure_supporting_objects(conn: Any, owner: str) -> list[str]:
+        """Ensure auxiliary PL/SQL objects required by LimitsDb exist in the schema."""
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_ldb_columns_expressions() -> tuple[str, str]:
+        """Returns the expressions for the LDB process date and insert date columns.
+        Returns:
+            A tuple containing the process date expression and insert date expression.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_identifier_str(identifier: str) -> str:
+        """Returns the identifier string required to query dictionary views.
+        Args:
+            indentifier: The identifier string to process.
+        Returns:
+            The identifier string without angle brackets.
+        """
+        pass

limitsdb/db/oracle/ldb_conf.sql

@@ -0,0 +1,151 @@
+DROP TABLE "LDB"."LDB_CONF";
+--------------------------------------------------------
+--  DDL for Sequence LDB_CNF_ID
+--------------------------------------------------------
+
+drop sequence "LDB"."LDB_CNF_ID";
+CREATE SEQUENCE "LDB"."LDB_CNF_ID";
+--------------------------------------------------------
+--  DDL for Table LDB_CONF
+--------------------------------------------------------
+
+CREATE TABLE "LDB"."LDB_CONF"
+(
+    "CNF_ID" NUMBER, 
+    "CNF_SOURCE_OWNER" VARCHAR2(50 BYTE), 
+    "CNF_HISTORY_OWNER" VARCHAR2(50 BYTE), 
+    "CNF_TABLE" VARCHAR2(50 BYTE), 
+    "CNF_RETAIN_MONTHS_SOURCE" NUMBER, 
+    "CNF_RETAIN_MONTHS_HISTORY" NUMBER, 
+    "CNF_EXEC_DAY" VARCHAR2(10 BYTE), 
+    "CNF_FRECUENCY" VARCHAR2(10 BYTE), 
+    "CNF_IS_ACTIVE" CHAR(1 BYTE), 
+    "CNF_PURGE_DATE_EXPR" VARCHAR2(100 BYTE), 
+    "CNF_ADDITIONAL_FILTER_EXPR" VARCHAR2(4000 BYTE),
+    "CNF_HISTORY_ADDTL_FILTER_EXPR" VARCHAR2(4000 BYTE),
+    "CNF_SOURCE_ORPHAN_PURGE" CHAR(1 BYTE), 
+    "CNF_ORPHAN_CHECK_COLUMN" VARCHAR2(4000 BYTE), 
+    "CNF_HAS_LOB_COLUMNS" CHAR(1 BYTE), 
+    "CNF_REFERENCING_TABLES" VARCHAR2(100 BYTE), 
+    "CNF_JOIN_EXPR" VARCHAR2(4000 BYTE), 
+    "CNF_HINT_EXPR" VARCHAR2(4000 BYTE), 
+    "CNF_LONG_COLUMNS" VARCHAR2(4000 BYTE)
+);
+
+--------------------------------------------------------
+--  DDL for Index CNF_CONF_PK
+--------------------------------------------------------
+
+CREATE UNIQUE INDEX "LDB"."CNF_CONF_PK" ON "LDB"."LDB_CONF" ("CNF_ID");
+
+--------------------------------------------------------
+--  DDL for Index CNF_CONF_I1
+--------------------------------------------------------
+
+CREATE INDEX "LDB"."CNF_CONF_I1" ON "LDB"."LDB_CONF" ("CNF_SOURCE_OWNER", "CNF_TABLE_NAME");
+
+--------------------------------------------------------
+--  Constraints for Table LDB_CONF
+--------------------------------------------------------
+
+ALTER TABLE "LDB"."LDB_CONF" ADD CONSTRAINT "CNF_CONF_PK" PRIMARY KEY ("CNF_ID") USING INDEX;
+
+--------------------------------------------------------
+--  DDL for Trigger LDB_CNF_ID_CR
+--------------------------------------------------------
+
+CREATE OR REPLACE EDITIONABLE TRIGGER "LDB"."LDB_CNF_CR"
+    before insert on "LDB_CONF"
+    for each row 
+begin  
+    if inserting then 
+        if :NEW."CNF_ID" is null then 
+            select LDB_CNF_ID.nextval into :NEW."CNF_ID" from dual;
+        end if; 
+    end if; 
+end;
+/
+ALTER TRIGGER "LDB"."LDB_CNF_CR" ENABLE;
+
+REM INSERTING into LDB.LDB_CONF
+SET DEFINE OFF;
+Insert into LDB.LDB_CONF (CNF_SOURCE_OWNER,CNF_HISTORY_OWNER,CNF_TABLE,CNF_RETAIN_MONTHS_SOURCE,CNF_RETAIN_MONTHS_HISTORY,CNF_EXEC_DAY,CNF_FRECUENCY,CNF_IS_ACTIVE,CNF_PURGE_DATE_EXPR,CNF_ADDITIONAL_FILTER_EXPR,CNF_HISTORY_ADDTL_FILTER_EXPR,CNF_SOURCE_ORPHAN_PURGE,CNF_ORPHAN_CHECK_COLUMN,CNF_HAS_LOB_COLUMNS,CNF_REFERENCING_TABLES,CNF_JOIN_EXPR,CNF_HINT_EXPR,CNF_LONG_COLUMNS) values ('GL','GLHST','INV_HEAD','3','11',null,'D','Y','INV_DSP_DATE','A.INV_STATUS=''D''',null,null,null,'N',null,null,'full(A)',null);
+Insert into LDB.LDB_CONF (CNF_SOURCE_OWNER,CNF_HISTORY_OWNER,CNF_TABLE,CNF_RETAIN_MONTHS_SOURCE,CNF_RETAIN_MONTHS_HISTORY,CNF_EXEC_DAY,CNF_FRECUENCY,CNF_IS_ACTIVE,CNF_PURGE_DATE_EXPR,CNF_ADDITIONAL_FILTER_EXPR,CNF_HISTORY_ADDTL_FILTER_EXPR,CNF_SOURCE_ORPHAN_PURGE,CNF_ORPHAN_CHECK_COLUMN,CNF_HAS_LOB_COLUMNS,CNF_REFERENCING_TABLES,CNF_JOIN_EXPR,CNF_HINT_EXPR,CNF_LONG_COLUMNS) values ('GL','GLHST','INV_HEAD','1','5',null,'D','Y','INV_DATE','A.INV_STATUS=''A''',null,null,null,'N',null,null,'full(A)',null);
+Insert into LDB.LDB_CONF (CNF_SOURCE_OWNER,CNF_HISTORY_OWNER,CNF_TABLE,CNF_RETAIN_MONTHS_SOURCE,CNF_RETAIN_MONTHS_HISTORY,CNF_EXEC_DAY,CNF_FRECUENCY,CNF_IS_ACTIVE,CNF_PURGE_DATE_EXPR,CNF_ADDITIONAL_FILTER_EXPR,CNF_HISTORY_ADDTL_FILTER_EXPR,CNF_SOURCE_ORPHAN_PURGE,CNF_ORPHAN_CHECK_COLUMN,CNF_HAS_LOB_COLUMNS,CNF_REFERENCING_TABLES,CNF_JOIN_EXPR,CNF_HINT_EXPR,CNF_LONG_COLUMNS) values ('GL','GLHST','INV_DET',null,null,null,null,null,null,null,null,'Y','INV_ID B','N','INV_HEAD','JOIN INV_HEAD ON B.INV_ID=A.INV_ID','full(A) full(B) use_hash(A, B)',null);
+Insert into LDB.LDB_CONF (CNF_SOURCE_OWNER,CNF_HISTORY_OWNER,CNF_TABLE,CNF_RETAIN_MONTHS_SOURCE,CNF_RETAIN_MONTHS_HISTORY,CNF_EXEC_DAY,CNF_FRECUENCY,CNF_IS_ACTIVE,CNF_PURGE_DATE_EXPR,CNF_ADDITIONAL_FILTER_EXPR,CNF_HISTORY_ADDTL_FILTER_EXPR,CNF_SOURCE_ORPHAN_PURGE,CNF_ORPHAN_CHECK_COLUMN,CNF_HAS_LOB_COLUMNS,CNF_REFERENCING_TABLES,CNF_JOIN_EXPR,CNF_HINT_EXPR,CNF_LONG_COLUMNS) values ('GL','GLHST','DSP_HEAD','3','11',null,'D','Y','DSP_DATE','A.DSP_STATUS=''D''',null,null,null,'N',null,null,'full(A)',null);
+Insert into LDB.LDB_CONF (CNF_SOURCE_OWNER,CNF_HISTORY_OWNER,CNF_TABLE,CNF_RETAIN_MONTHS_SOURCE,CNF_RETAIN_MONTHS_HISTORY,CNF_EXEC_DAY,CNF_FRECUENCY,CNF_IS_ACTIVE,CNF_PURGE_DATE_EXPR,CNF_ADDITIONAL_FILTER_EXPR,CNF_HISTORY_ADDTL_FILTER_EXPR,CNF_SOURCE_ORPHAN_PURGE,CNF_ORPHAN_CHECK_COLUMN,CNF_HAS_LOB_COLUMNS,CNF_REFERENCING_TABLES,CNF_JOIN_EXPR,CNF_HINT_EXPR,CNF_LONG_COLUMNS) values ('GL','GLHST','DSP_HEAD','1','5',null,'D','Y','DSP_DATE','A.DSP_STATUS=''A''',null,null,null,'N',null,null,'full(A)',null);
+Insert into LDB.LDB_CONF (CNF_SOURCE_OWNER,CNF_HISTORY_OWNER,CNF_TABLE,CNF_RETAIN_MONTHS_SOURCE,CNF_RETAIN_MONTHS_HISTORY,CNF_EXEC_DAY,CNF_FRECUENCY,CNF_IS_ACTIVE,CNF_PURGE_DATE_EXPR,CNF_ADDITIONAL_FILTER_EXPR,CNF_HISTORY_ADDTL_FILTER_EXPR,CNF_SOURCE_ORPHAN_PURGE,CNF_ORPHAN_CHECK_COLUMN,CNF_HAS_LOB_COLUMNS,CNF_REFERENCING_TABLES,CNF_JOIN_EXPR,CNF_HINT_EXPR,CNF_LONG_COLUMNS) values ('GL','GLHST','DSP_DET',null,null,null,null,null,null,null,null,'Y','B.DSP_ID, C.IND_ID','N','DSP_HEAD, INV_DET','JOIN DSP_HEAD B ON B.DSP_ID=A.DSP_ID JOIN INV_DET C ON C.IND_ID = A.IND_ID','full(A) full(B) full(C) use_hash(A, B, C)',null);
+
+commit;
+
+create or replace procedure check_save_status(p_owner in varchar2, p_table_name in varchar2, p_process_date in date, p_action in varchar2, p_status in varchar2, p_process_start in date,
+    p_chunk_start in date, p_process_end in date, p_message in varchar2, p_rows_processed in varchar2, p_plsql in varchar2, p_sqlcode in out number, p_out_message out varchar2) is
+    l_status varchar2(10);
+    l_process_start date;
+    l_process_date date;
+    l_rowid rowid;
+    in_use exception;
+    pragma exception_init(in_use, -54);
+begin
+    if nvl(p_sqlcode,0) not in (-20001, -20002, -20003) then
+        begin
+            select ctl_status, ctl_process_date, ctl_process_start, rowid into l_status, l_process_date, l_process_start, l_rowid
+            from ldb_ctl
+            where ctl_owner = p_owner and ctl_table_name = p_table_name
+            for update nowait;
+            if p_status = 'TSTART' and l_status != 'TEND' and (sysdate - l_process_start)*3600*24 < 5 then
+                p_sqlcode := -20002;
+                p_out_message := 'Table '||p_table_name||' is currently in process without lock';
+                return;
+            elsif p_status = 'TSTART' and l_status = 'TEND' and l_process_date = p_process_date then
+                rollback;
+                p_sqlcode := -20003;
+                p_out_message := 'Table '||p_table_name||' is already processed for date '||to_char(p_process_date,'YYYYMMDD');
+                return;
+            end if;
+            update ldb_ctl set ctl_process_date = p_process_date, ctl_action = p_action, ctl_status = p_status,
+                ctl_process_start = nvl(p_process_start, ctl_process_start), ctl_process_end = p_process_end,
+                ctl_rows_processed = decode(p_status, '{TABLE_START}', 0, ctl_rows_processed) + p_rows_processed, ctl_plsql = nvl(p_plsql, ctl_plsql)
+            where rowid = l_rowid;
+        exception
+        when in_use then
+            p_sqlcode := -20001;
+            p_out_message := 'Table '||p_table_name||' is currently in process';
+            return;
+        when no_data_found then
+            insert into ldb_ctl (ctl_owner, ctl_table_name, ctl_process_date, ctl_action, ctl_status,
+                ctl_process_start, ctl_process_end, ctl_rows_processed, ctl_plsql)
+            values (p_owner, p_table_name, p_process_date, p_action, p_status,
+                l_process_start, null, 0, p_plsql);
+        end;
+    end if;
+    insert into ldb_log (log_id, log_owner, log_table_name, log_process_date, log_action, log_status,
+        log_process_start, log_process_end, log_message, log_rows_processed, log_plsql)
+    values (ldb_log_id.nextval, p_owner, p_table_name, p_process_date, p_action, p_status,
+        nvl(p_chunk_start, p_process_start), p_process_end, p_message, p_rows_processed, p_plsql);
+end check_save_status;
+/
+
+CREATE OR REPLACE TYPE t_referencing_tables AS TABLE OF VARCHAR2(100);
+/
+create or replace procedure check_referencing_tables(p_referencing_tables t_referencing_tables, p_process_date date) is
+    l_ref_table varchar2(100);
+    l_ref_owner varchar2(100);
+    l_ref_table_name varchar2(100);
+    l_dummy number;
+begin
+    for i in 1..p_referencing_tables.count loop
+        l_ref_table := p_referencing_tables(i);
+        l_ref_owner := substr(l_ref_table,1,instr(l_ref_table,'.')-1);
+        l_ref_table_name := substr(l_ref_table,instr(l_ref_table,'.')+1);
+        begin
+            select 1 into l_dummy
+            from ldb_ctl
+            where ctl_owner = l_ref_owner and ctl_table_name = l_ref_table_name and ctl_process_date = p_process_date and ctl_status='TEND';
+        exception
+        when no_data_found then
+            raise_application_error(-20004,'Referencing table '||l_ref_table||' was not fully processed for date '||to_char(p_process_date,'YYYYMMDD'));
+        end;
+    end loop;
+end check_referencing_tables;
+/