pydpm_xl 0.1.39rc32__py3-none-any.whl → 0.2.0__py3-none-any.whl

py_dpm/dpm/queries/tables.py

@@ -0,0 +1,133 @@
+from typing import Optional
+
+from sqlalchemy import distinct
+from py_dpm.dpm.models import (
+    TableVersion,
+    ViewDatapoints,
+    ModuleVersion,
+    ModuleVersionComposition,
+)
+from py_dpm.dpm.queries.base import BaseQuery
+from py_dpm.dpm.queries.filters import filter_by_release, filter_by_date
+
+
+class TableQuery:
+    """
+    Queries related to data structure references (Tables, Rows, Columns, Sheets).
+    """
+
+    @staticmethod
+    def get_tables(
+        session,
+        release_id: Optional[int] = None,
+        date: Optional[str] = None,
+        release_code: Optional[str] = None,
+    ) -> BaseQuery:
+        """Get all available table codes."""
+
+        if sum(bool(x) for x in [release_id, date, release_code]) > 1:
+            raise ValueError(
+                "Specify a maximum of one of release_id, release_code or date."
+            )
+
+        q = session.query(TableVersion)
+
+        if date:
+            q = q.join(
+                ModuleVersionComposition,
+                TableVersion.tablevid == ModuleVersionComposition.tablevid,
+            ).join(
+                ModuleVersion,
+                ModuleVersionComposition.modulevid == ModuleVersion.modulevid,
+            )
+            q = filter_by_date(
+                q, date, ModuleVersion.fromreferencedate, ModuleVersion.toreferencedate
+            )
+        elif release_id:
+            q = filter_by_release(
+                q,
+                release_id=release_id,
+                start_col=TableVersion.startreleaseid,
+                end_col=TableVersion.endreleaseid,
+            )
+        elif release_code:
+            q = filter_by_release(
+                q,
+                release_code=release_code,
+                start_col=TableVersion.startreleaseid,
+                end_col=TableVersion.endreleaseid,
+            )
+
+        q = q.order_by(TableVersion.code)
+
+        return BaseQuery(session, q)
+
+    @staticmethod
+    def get_available_tables_from_datapoints(
+        session, release_id: Optional[int] = None
+    ) -> BaseQuery:
+        """Get available table codes from datapoints view."""
+        base_query = ViewDatapoints.create_view_query(session)
+        subq = base_query.subquery()
+
+        q = session.query(distinct(subq.c.table_code).label("table_code")).filter(
+            subq.c.table_code.isnot(None)
+        )
+
+        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = q.order_by(subq.c.table_code)
+
+        return BaseQuery(session, q)
+
+    @staticmethod
+    def get_available_rows(
+        session, table_code: str, release_id: Optional[int] = None
+    ) -> BaseQuery:
+        """Get available row codes for a table."""
+        base_query = ViewDatapoints.create_view_query(session)
+        subq = base_query.subquery()
+
+        q = session.query(distinct(subq.c.row_code).label("row_code")).filter(
+            subq.c.table_code == table_code, subq.c.row_code.isnot(None)
+        )
+
+        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = q.order_by(subq.c.row_code)
+
+        return BaseQuery(session, q)
+
+    @staticmethod
+    def get_available_columns(
+        session, table_code: str, release_id: Optional[int] = None
+    ) -> BaseQuery:
+        """Get available column codes for a table."""
+        base_query = ViewDatapoints.create_view_query(session)
+        subq = base_query.subquery()
+
+        q = session.query(distinct(subq.c.column_code).label("column_code")).filter(
+            subq.c.table_code == table_code, subq.c.column_code.isnot(None)
+        )
+
+        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = q.order_by(subq.c.column_code)
+
+        return BaseQuery(session, q)
+
+    @staticmethod
+    def get_available_sheets(
+        session, table_code: str, release_id: Optional[int] = None
+    ) -> BaseQuery:
+        """Get available sheet codes."""
+        base_query = ViewDatapoints.create_view_query(session)
+        subq = base_query.subquery()
+
+        q = session.query(distinct(subq.c.sheet_code).label("sheet_code")).filter(
+            subq.c.table_code == table_code,
+            subq.c.sheet_code.isnot(None),
+            subq.c.sheet_code != "",
+        )
+
+        q = filter_by_release(q, release_id, subq.c.start_release, subq.c.end_release)
+        q = q.order_by(subq.c.sheet_code)
+
+        return BaseQuery(session, q)

py_dpm/dpm/utils.py

@@ -0,0 +1,356 @@
+import os
+from urllib.parse import quote_plus
+
+from dotenv import load_dotenv
+from sqlalchemy import create_engine
+from sqlalchemy.engine import URL
+from sqlalchemy.orm import close_all_sessions, sessionmaker
+from contextlib import contextmanager
+from rich.console import Console
+
+console = Console()
+
+# Try to load .env from parent directories
+current_dir = os.path.dirname(os.path.abspath(__file__))
+while current_dir != os.path.dirname(current_dir):  # Stop at root
+    env_path = os.path.join(current_dir, ".env")
+    if os.path.exists(env_path):
+        load_dotenv(env_path)
+        break
+    current_dir = os.path.dirname(current_dir)
+
+# SQLite configuration
+sqlite_db_path = os.getenv("SQLITE_DB_PATH", "database.db")
+
+# Unified RDBMS configuration (preferred)
+#
+# These environment variables provide a single, backend-agnostic way to
+# configure a server-based database for pyDPM. `PYDPM_RDBMS` selects the
+# backend and the remaining variables provide connection details:
+#   - PYDPM_RDBMS: "postgres" or "sqlserver"
+#   - PYDPM_DB_HOST: hostname or IP
+#   - PYDPM_DB_PORT: port number (optional; defaults per backend)
+#   - PYDPM_DB_NAME: database name
+#   - PYDPM_DB_USER: username
+#   - PYDPM_DB_PASSWORD: password
+pydpm_rdbms = os.getenv("PYDPM_RDBMS", "").strip().lower()
+db_host = os.getenv("PYDPM_DB_HOST", None)
+db_port = os.getenv("PYDPM_DB_PORT", None)
+db_name = os.getenv("PYDPM_DB_NAME", None)
+db_user = os.getenv("PYDPM_DB_USER", None)
+db_password = os.getenv("PYDPM_DB_PASSWORD", None)
+
+if pydpm_rdbms == "postgres" and not db_port:
+    db_port = "5432"
+elif pydpm_rdbms == "sqlserver" and not db_port:
+    db_port = "1433"
+
+
+
+# PostgreSQL configuration
+postgres_host = os.getenv("POSTGRES_HOST", None)
+postgres_port = os.getenv("POSTGRES_PORT", "5432")
+postgres_db = os.getenv("POSTGRES_DB", None)
+postgres_user = os.getenv("POSTGRES_USER", None)
+postgres_pass = os.getenv("POSTGRES_PASS", None)
+
+# Legacy SQL Server configuration (kept for backward compatibility)
+server = os.getenv("DATABASE_SERVER", None)
+username = os.getenv("DATABASE_USER", None)
+password = os.getenv("DATABASE_PASS", None)
+database_name = os.getenv("DATABASE_NAME", None)
+
+# Determine database type
+use_postgres = os.getenv("USE_POSTGRES", "false").lower() == "true"
+use_sqlite = os.getenv("USE_SQLITE", "true").lower() == "true" and not use_postgres
+
+if use_postgres and not (
+    postgres_host and postgres_user and postgres_pass and postgres_db
+):
+    console.print(f"Warning: PostgreSQL credentials not provided", style="bold yellow")
+elif not use_sqlite and not use_postgres and not (server and username and password):
+    console.print(f"Warning: Database credentials not provided", style="bold yellow")
+elif not use_sqlite and not use_postgres:
+    # Handling special characters in password for SQL Server
+    password = password.replace("}", "}}")
+    for x in "%&.@#/\\=;":
+        if x in password:
+            password = "{" + password + "}"
+            break
+
+engine = None
+connection = None
+sessionMakerObject = None
+_current_engine_url = None
+
+
+def create_engine_from_url(connection_url):
+    """
+    Create SQLAlchemy engine from a connection URL with appropriate pooling parameters.
+
+    Detects database type from URL scheme and applies pooling parameters conditionally:
+    - SQLite: Only pool_pre_ping=True (no connection pooling)
+    - PostgreSQL/MySQL/others: Full connection pooling parameters
+
+    Also initializes the global sessionMakerObject for use by get_session().
+
+    Args:
+        connection_url (str): SQLAlchemy connection URL (e.g., 'sqlite:///path.db', 'postgresql://user:pass@host/db')
+
+    Returns:
+        sqlalchemy.engine.Engine: Configured database engine
+
+    Examples:
+        >>> engine = create_engine_from_url('sqlite:///database.db')
+        >>> engine = create_engine_from_url('postgresql://user:pass@localhost/mydb')
+    """
+    global engine, sessionMakerObject, _current_engine_url
+
+    # Detect database type from URL scheme
+    is_sqlite = connection_url.startswith("sqlite://")
+
+    # For SQLite URLs, always create a fresh engine to avoid
+    # surprising cross-test or cross-call state sharing, especially
+    # for in-memory databases. For server-based databases, reuse the
+    # engine when the URL has not changed.
+    if not is_sqlite and engine is not None and _current_engine_url == str(
+        connection_url
+    ):
+        return engine
+
+    if is_sqlite:
+        # SQLite doesn't support connection pooling
+        engine = create_engine(connection_url, pool_pre_ping=True)
+    else:
+        # Server-based databases (PostgreSQL, MySQL, etc.) with connection pooling
+        engine = create_engine(
+            connection_url,
+            pool_size=20,
+            max_overflow=10,
+            pool_recycle=180,
+            pool_pre_ping=True,
+        )
+
+    # Initialize global sessionMakerObject
+    if sessionMakerObject is not None:
+        close_all_sessions()
+    sessionMakerObject = sessionmaker(bind=engine)
+    _current_engine_url = str(connection_url)
+
+    return engine
+
+
+def create_engine_object(url):
+    global engine, _current_engine_url
+
+    # Convert URL to string for type detection if needed
+    url_str = str(url)
+
+    # Detect database type from URL scheme (not from environment variables)
+    is_sqlite = url_str.startswith("sqlite://")
+
+    # Only reuse engines for non-SQLite URLs. SQLite (especially in-memory)
+    # should create independent engines to avoid leaking state between calls.
+    if not is_sqlite and engine is not None and _current_engine_url == url_str:
+        return engine
+
+    if is_sqlite:
+        engine = create_engine(url, pool_pre_ping=True)
+    else:
+        # Server-based databases (PostgreSQL, MySQL, SQL Server, etc.) with connection pooling
+        engine = create_engine(
+            url, pool_size=20, max_overflow=10, pool_recycle=180, pool_pre_ping=True
+        )
+
+    global sessionMakerObject
+    if sessionMakerObject is not None:
+        close_all_sessions()
+    sessionMakerObject = sessionmaker(bind=engine)
+    _current_engine_url = url_str
+    return engine
+
+
+def get_engine(owner=None, database_path=None, connection_url=None):
+    """
+    Get database engine based on configuration or explicit parameters.
+
+    Priority order:
+    1. Explicit connection_url parameter (for PostgreSQL or other databases)
+    2. Explicit database_path parameter (for SQLite)
+    3. Environment variable USE_POSTGRES (from .env)
+    4. Environment variable USE_SQLITE (from .env)
+
+    Args:
+        owner: Owner for SQL Server databases (EBA/EIOPA) - legacy support
+        database_path: Explicit SQLite database path
+        connection_url: Explicit SQLAlchemy connection URL (e.g., for PostgreSQL)
+
+    Returns:
+        SQLAlchemy Engine
+    """
+    # Priority 1: If explicit connection URL is provided, use it directly
+    if connection_url:
+        return create_engine_from_url(connection_url)
+
+    # Priority 2: If explicit database_path is provided, use SQLite with that path
+    if database_path:
+        connection_url = f"sqlite:///{database_path}"
+        return create_engine_object(connection_url)
+
+    # Priority 3: Check unified PYDPM_RDBMS configuration
+    if pydpm_rdbms in ("postgres", "sqlserver"):
+        if not (db_host and db_name and db_user and db_password):
+            console.print(
+                "Warning: PYDPM_RDBMS is set but PYDPM_DB_* variables are incomplete; "
+                "falling back to legacy configuration",
+                style="bold yellow",
+            )
+        else:
+            if pydpm_rdbms == "postgres":
+                # PostgreSQL via unified PYDPM_* configuration
+                port = db_port or "5432"
+                connection_url = (
+                    f"postgresql://{db_user}:{db_password}@{db_host}:{port}/{db_name}"
+                )
+                return create_engine_object(connection_url)
+            else:
+                # SQL Server via unified PYDPM_* configuration
+                port = db_port or "1433"
+                server_with_port = f"{db_host},{port}" if port else db_host
+
+                # Handling special characters in password for SQL Server
+                sqlserver_password = db_password.replace("}", "}}")
+                for x in "%&.@#/\\=;":
+                    if x in sqlserver_password:
+                        sqlserver_password = "{" + sqlserver_password + "}"
+                        break
+
+                if os.name == "nt":
+                    driver = "{SQL Server}"
+                else:
+                    driver = os.getenv(
+                        "SQL_DRIVER", "{ODBC Driver 18 for SQL Server}"
+                    )
+
+                connection_string = (
+                    f"DRIVER={driver}",
+                    f"SERVER={server_with_port}",
+                    f"DATABASE={db_name}",
+                    f"UID={db_user}",
+                    f"PWD={sqlserver_password}",
+                    "TrustServerCertificate=yes",
+                )
+                connection_url = URL.create(
+                    "mssql+pyodbc",
+                    query={"odbc_connect": quote_plus(';'.join(connection_string))},
+                )
+                return create_engine_object(connection_url)
+
+    # Priority 4: Check legacy PostgreSQL configuration
+    if use_postgres:
+        if not (postgres_host and postgres_user and postgres_pass and postgres_db):
+            console.print(
+                "Warning: USE_POSTGRES is true but PostgreSQL credentials are incomplete; "
+                "falling back to SQLite or SQL Server defaults",
+                style="bold yellow",
+            )
+        else:
+            connection_url = (
+                f"postgresql://{postgres_user}:{postgres_pass}@"
+                f"{postgres_host}:{postgres_port}/{postgres_db}"
+            )
+            return create_engine_object(connection_url)
+
+    # Priority 5: Check environment variable USE_SQLITE
+    if use_sqlite:
+        # For SQLite, create the database path if it doesn't exist
+        db_dir = os.path.dirname(sqlite_db_path)
+        if db_dir and not os.path.exists(db_dir):
+            os.makedirs(db_dir)
+
+        # If owner is specified, append it to the filename
+        if owner:
+            base_name = os.path.splitext(sqlite_db_path)[0]
+            extension = os.path.splitext(sqlite_db_path)[1] or ".db"
+            db_path = f"{base_name}_{owner}{extension}"
+        else:
+            db_path = sqlite_db_path
+
+        connection_url = f"sqlite:///{db_path}"
+        return create_engine_object(connection_url)
+
+    # Priority 6: Legacy SQL Server logic
+    if owner is None:
+        raise Exception("Cannot generate engine. No owner used.")
+
+    if owner not in ("EBA", "EIOPA"):
+        raise Exception("Invalid owner, must be EBA or EIOPA")
+
+    if database_name is None:
+        database = "DPM_" + owner
+    else:
+        database = database_name
+
+    if os.name == "nt":
+        driver = "{SQL Server}"
+    else:
+        driver = os.getenv("SQL_DRIVER", "{ODBC Driver 18 for SQL Server}")
+
+    # Handling special characters in password for legacy SQL Server configuration
+    sqlserver_password = password.replace("}", "}}") if password else ""
+    for x in "%&.@#/\\=;":
+        if x in sqlserver_password:
+            sqlserver_password = "{" + sqlserver_password + "}"
+            break
+
+    connection_string = (
+        f"DRIVER={driver}",
+        f"SERVER={server}",
+        f"DATABASE={database}",
+        f"UID={username}",
+        f"PWD={sqlserver_password}",
+        "TrustServerCertificate=yes",
+    )
+    connection_string = ";".join(connection_string)
+    connection_url = URL.create(
+        "mssql+pyodbc", query={"odbc_connect": quote_plus(connection_string)}
+    )
+    return create_engine_object(connection_url)
+
+
+def get_connection(owner=None):
+    global engine
+    if engine is None:
+        engine = get_engine(owner)
+    connection = engine.connect()
+    return connection
+
+
+def get_session():
+    global sessionMakerObject
+    """Returns as session on the connection string"""
+    if sessionMakerObject is None:
+        raise Exception("Not found Session Maker")
+    session = sessionMakerObject()
+    return session
+
+
+@contextmanager
+def session_scope():
+    """
+    Provide a transactional scope around a series of operations.
+
+    This helper is intended for short-lived, one-off operations where
+    explicit session closing is desired. It does not commit automatically,
+    leaving transaction control to the caller, but always closes the
+    session in a finally block.
+    """
+    session = get_session()
+    try:
+        yield session
+    finally:
+        try:
+            session.close()
+        except Exception:
+            # Best-effort close; suppress errors on shutdown paths
+            pass

py_dpm/dpm_xl/__init__.py

@@ -0,0 +1,8 @@
+"""
+DPM-XL Processing Engine
+
+DPM-XL expression parsing, validation, and analysis.
+Includes grammar, AST, operators, types, and validation.
+"""
+
+__all__ = []

py_dpm/dpm_xl/ast/__init__.py

@@ -0,0 +1,14 @@
+"""
+DPM-XL Abstract Syntax Tree
+
+AST construction, manipulation, and analysis for DPM-XL expressions.
+"""
+
+from py_dpm.dpm_xl.ast.nodes import *
+from py_dpm.dpm_xl.ast.constructor import *
+from py_dpm.dpm_xl.ast.visitor import *
+from py_dpm.dpm_xl.ast.template import *
+
+__all__ = [
+    # Re-export will be handled by import *
+]

py_dpm/{AST/ASTConstructor.py → dpm_xl/ast/constructor.py}

@@ -10,12 +10,12 @@ import re
 
 from antlr4.tree.Tree import TerminalNodeImpl
 
-from py_dpm.AST.ASTObjects import *
-from py_dpm.Exceptions import exceptions
-from py_dpm.Exceptions.exceptions import SemanticError
-from py_dpm.Utils.tokens import TABLE_GROUP_PREFIX
-from py_dpm.grammar.dist.dpm_xlParser import dpm_xlParser
-from py_dpm.grammar.dist.dpm_xlParserVisitor import dpm_xlParserVisitor
+from py_dpm.dpm_xl.ast.nodes import *
+from py_dpm.exceptions import exceptions
+from py_dpm.exceptions.exceptions import SemanticError
+from py_dpm.dpm_xl.utils.tokens import TABLE_GROUP_PREFIX
+from py_dpm.dpm_xl.grammar.generated.dpm_xlParser import dpm_xlParser
+from py_dpm.dpm_xl.grammar.generated.dpm_xlParserVisitor import dpm_xlParserVisitor
 
 
 class ASTVisitor(dpm_xlParserVisitor):