pydpm_xl 0.1.39rc32__py3-none-any.whl → 0.2.1__py3-none-any.whl

py_dpm/{ValidationsGeneration/VariantsProcessor.py → dpm_xl/validation/variants.py}

@@ -3,20 +3,33 @@ import re
 
 import pandas as pd
 
-from py_dpm.AST.ASTObjects import VarID, WithExpression
-from py_dpm.AST.ASTTemplate import ASTTemplate
-from py_dpm.Exceptions import exceptions
-from py_dpm.models import ModuleVersionComposition, TableGroup, TableGroupComposition, TableVersion, ViewDatapoints
-from py_dpm.Utils.tokens import EXPRESSION, STATUS, STATUS_CORRECT, STATUS_INCOMPLETE, STATUS_INCORRECT, VALIDATION_CODE
-from py_dpm.data_handlers import filter_all_data
-
-cell_components = ['table', 'rows', 'cols', 'sheets']
-TABLE_ID = 'TableID'
-MODULE_VID = 'ModuleVID'
-TABLE_CODE = 'Code'
-GROUP = 'group'
-MODULE_VERSION_ID = 'module_version_id'
-MODULE_CODE = 'module_code'
+from py_dpm.dpm_xl.ast.nodes import VarID, WithExpression
+from py_dpm.dpm_xl.ast.template import ASTTemplate
+from py_dpm.exceptions import exceptions
+from py_dpm.dpm.models import (
+    ModuleVersionComposition,
+    TableGroup,
+    TableGroupComposition,
+    TableVersion,
+    ViewDatapoints,
+)
+from py_dpm.dpm_xl.utils.tokens import (
+    EXPRESSION,
+    STATUS,
+    STATUS_CORRECT,
+    STATUS_INCOMPLETE,
+    STATUS_INCORRECT,
+    VALIDATION_CODE,
+)
+from py_dpm.dpm_xl.utils.data_handlers import filter_all_data
+
+cell_components = ["table", "rows", "cols", "sheets"]
+TABLE_ID = "TableID"
+MODULE_VID = "ModuleVID"
+TABLE_CODE = "Code"
+GROUP = "group"
+MODULE_VERSION_ID = "module_version_id"
+MODULE_CODE = "module_code"
 
 
 class VariantsProcessorChecker(ASTTemplate):
@@ -73,11 +86,15 @@ class VariantsProcessor(ASTTemplate):
         :return: True if table has cells, False otherwise.
         """
         if table_code not in self.table_data:
-            datapoints = ViewDatapoints.get_table_data(session=self.session, table=table_code, release_id=self.release_id)
+            datapoints = ViewDatapoints.get_table_data(
+                session=self.session, table=table_code, release_id=self.release_id
+            )
             self.table_data[table_code] = datapoints
         else:
             datapoints = self.table_data[table_code]
-        data = filter_all_data(data=datapoints, table_code=table_code, rows=rows, cols=cols, sheets=sheets)
+        data = filter_all_data(
+            data=datapoints, table_code=table_code, rows=rows, cols=cols, sheets=sheets
+        )
         return not data.empty
 
     def check_table_from_module(self, node, table_module_id, is_abstract):
@@ -89,13 +106,17 @@ class VariantsProcessor(ASTTemplate):
         :return: table version if table version has cells, None otherwise.
         """
 
-        table_versions = TableVersion.get_tables_versions_of_table_group_compositions(session=self.session,
-                                                                                      table_id=table_module_id,
-                                                                                      is_abstract=is_abstract,
-                                                                                      release_id=self.release_id)
+        table_versions = TableVersion.get_tables_versions_of_table_group_compositions(
+            session=self.session,
+            table_id=table_module_id,
+            is_abstract=is_abstract,
+            release_id=self.release_id,
+        )
 
         for table_version in table_versions:
-            table_with_cells = self.check_if_table_has_cells(table_version.Code, node.rows, node.cols, node.sheets)
+            table_with_cells = self.check_if_table_has_cells(
+                table_version.Code, node.rows, node.cols, node.sheets
+            )
             if table_with_cells:
                 return table_version
             else:
@@ -109,10 +130,10 @@ class VariantsProcessor(ASTTemplate):
         :param group_code: Group code.
         :param table_code: Table code.
         """
-        groups = re.search("(" + re.escape(group_code) + '[^.$]' + ")", self.expression)
+        groups = re.search("(" + re.escape(group_code) + "[^.$]" + ")", self.expression)
         if f"g{group_code}" in self.expression and groups:
             group = groups.group(0)
-            suffix = table_code[-(len(table_code)-len(group_code)):]
+            suffix = table_code[-(len(table_code) - len(group_code)) :]
             if group:
                 if not self.children_suffix:
                     self.children_suffix = suffix
@@ -120,7 +141,11 @@ class VariantsProcessor(ASTTemplate):
                     if self.children_suffix != suffix:
                         self.equal_children_suffix = False
                         return
-                self.expression = re.sub(re.escape(f"g{group_code}") + '[^.$]', f"t{table_code}" + group[-1], self.expression)
+                self.expression = re.sub(
+                    re.escape(f"g{group_code}") + "[^.$]",
+                    f"t{table_code}" + group[-1],
+                    self.expression,
+                )
 
     def generate_child_expressions(self):
         """
@@ -135,25 +160,44 @@ class VariantsProcessor(ASTTemplate):
             df_tables.drop_duplicates(inplace=True)
 
             table_ids = df_tables[TABLE_ID].tolist()
-            modules_df = ModuleVersionComposition.get_modules_from_table_ids(session=self.session, table_ids=table_ids,
-                                                                             release_id=self.release_id)
+            modules_df = ModuleVersionComposition.get_modules_from_table_ids(
+                session=self.session, table_ids=table_ids, release_id=self.release_id
+            )
 
             data = pd.merge(df_tables, modules_df, on=[TABLE_ID])
 
             expression = copy.deepcopy(self.expression)
             for module, modules_tables in data.groupby(MODULE_VID):
-                modules_tables.apply(lambda x: self.generate_child_expression(x[GROUP], x[TABLE_CODE]), axis=1)
+                modules_tables.apply(
+                    lambda x: self.generate_child_expression(x[GROUP], x[TABLE_CODE]),
+                    axis=1,
+                )
                 module_code = modules_tables[MODULE_CODE].unique().tolist()[0]
                 if len(modules_tables) < len(self.table_groups_compositions):
                     final_expressions_with_errors.append(
-                        {EXPRESSION: self.expression, MODULE_VERSION_ID: module, MODULE_CODE: module_code})
+                        {
+                            EXPRESSION: self.expression,
+                            MODULE_VERSION_ID: module,
+                            MODULE_CODE: module_code,
+                        }
+                    )
                 else:
                     if self.equal_children_suffix:
                         final_expressions.append(
-                            {EXPRESSION: self.expression, MODULE_VERSION_ID: module, MODULE_CODE: module_code})
+                            {
+                                EXPRESSION: self.expression,
+                                MODULE_VERSION_ID: module,
+                                MODULE_CODE: module_code,
+                            }
+                        )
                     else:
                         final_expressions_with_errors.append(
-                            {EXPRESSION: self.expression, MODULE_VERSION_ID: module, MODULE_CODE: module_code})
+                            {
+                                EXPRESSION: self.expression,
+                                MODULE_VERSION_ID: module,
+                                MODULE_CODE: module_code,
+                            }
+                        )
                         self.equal_children_suffix = True
                 self.expression = copy.deepcopy(expression)
                 self.children_suffix = None
@@ -170,13 +214,17 @@ class VariantsProcessor(ASTTemplate):
         :return: Validation with code, expression and status
         """
 
-        validation_code = self.generate_child_validation_code() if status == STATUS_CORRECT else None
-
-        validation = {VALIDATION_CODE: validation_code,
-                      EXPRESSION: expression_info[EXPRESSION],
-                      STATUS: status,
-                      MODULE_VERSION_ID: expression_info[MODULE_VERSION_ID],
-                      MODULE_CODE: expression_info[MODULE_CODE]}
+        validation_code = (
+            self.generate_child_validation_code() if status == STATUS_CORRECT else None
+        )
+
+        validation = {
+            VALIDATION_CODE: validation_code,
+            EXPRESSION: expression_info[EXPRESSION],
+            STATUS: status,
+            MODULE_VERSION_ID: expression_info[MODULE_VERSION_ID],
+            MODULE_CODE: expression_info[MODULE_CODE],
+        }
         return validation
 
     def create_validation_new_format(self, expressions_dict):
@@ -190,23 +238,49 @@ class VariantsProcessor(ASTTemplate):
         correct_expressions = expressions_dict[STATUS_CORRECT]
         incomplete_expressions = expressions_dict[STATUS_INCOMPLETE]
         incorrect_expressions = expressions_dict[STATUS_INCORRECT]
-        unique_correct_expressions = pd.DataFrame.from_records(correct_expressions)['expression'].unique().tolist() if correct_expressions else []
-        unique_incomplete_expressions = pd.DataFrame.from_records(incomplete_expressions)['expression'].unique().tolist() if incomplete_expressions else []
-        unique_incorrect_expressions = pd.DataFrame.from_records(incorrect_expressions)['expression'].unique().tolist() if incorrect_expressions else []
+        unique_correct_expressions = (
+            pd.DataFrame.from_records(correct_expressions)["expression"]
+            .unique()
+            .tolist()
+            if correct_expressions
+            else []
+        )
+        unique_incomplete_expressions = (
+            pd.DataFrame.from_records(incomplete_expressions)["expression"]
+            .unique()
+            .tolist()
+            if incomplete_expressions
+            else []
+        )
+        unique_incorrect_expressions = (
+            pd.DataFrame.from_records(incorrect_expressions)["expression"]
+            .unique()
+            .tolist()
+            if incorrect_expressions
+            else []
+        )
         validations = []
         if correct_expressions:
-            v = self._aux_create_validation_new_format(correct_expressions, unique_correct_expressions, STATUS_CORRECT)
+            v = self._aux_create_validation_new_format(
+                correct_expressions, unique_correct_expressions, STATUS_CORRECT
+            )
             validations.extend(v)
         if incomplete_expressions:
-            v = self._aux_create_validation_new_format(incomplete_expressions, unique_incomplete_expressions, STATUS_INCOMPLETE)
+            v = self._aux_create_validation_new_format(
+                incomplete_expressions, unique_incomplete_expressions, STATUS_INCOMPLETE
+            )
             validations.extend(v)
         if incorrect_expressions:
-            v = self._aux_create_validation_new_format(incorrect_expressions, unique_incorrect_expressions, STATUS_INCORRECT)
+            v = self._aux_create_validation_new_format(
+                incorrect_expressions, unique_incorrect_expressions, STATUS_INCORRECT
+            )
             validations.extend(v)
 
         return validations
 
-    def _aux_create_validation_new_format(self, expressions, unique_expressions, status):
+    def _aux_create_validation_new_format(
+        self, expressions, unique_expressions, status
+    ):
         validations = []
         for expr in unique_expressions:
             aux = {}
@@ -216,12 +290,13 @@ class VariantsProcessor(ASTTemplate):
                 aux[VALIDATION_CODE] = self.generate_child_validation_code()
             else:
                 aux[VALIDATION_CODE] = None
-            aux['scopes'] = []
+            aux["scopes"] = []
             for elto in expressions:
                 if elto[EXPRESSION] == aux[EXPRESSION]:
-                    aux['scopes'].append(
-                        {"module_versions_ids": [elto[MODULE_VERSION_ID]],
-                        "module_code": elto[MODULE_CODE]
+                    aux["scopes"].append(
+                        {
+                            "module_versions_ids": [elto[MODULE_VERSION_ID]],
+                            "module_code": elto[MODULE_CODE],
                         }
                     )
             validations.append(aux)
@@ -244,22 +319,41 @@ class VariantsProcessor(ASTTemplate):
     def visit_VarID(self, node: VarID):
         if self.partial_selection:
             for attribute in cell_components:
-                if not getattr(node, attribute, False) and hasattr(self.partial_selection, attribute):
+                if not getattr(node, attribute, False) and hasattr(
+                    self.partial_selection, attribute
+                ):
                     setattr(node, attribute, getattr(self.partial_selection, attribute))
 
         if node.is_table_group or self.partial_selection.is_table_group:
             if node.table:
                 if node.table not in self.table_groups_compositions:
-                    group: TableGroup = TableGroup.get_group_from_code(session=self.session, group_code=node.table)
+                    group: TableGroup = TableGroup.get_group_from_code(
+                        session=self.session, group_code=node.table
+                    )
                     if not group:
                         raise exceptions.SemanticError("1-6", table_group=node.table)
-                    table_groups_compositions = TableGroupComposition.get_from_parent_table_code(code=node.table, session=self.session)
-                    self.table_groups_compositions[node.table] = table_groups_compositions
+                    table_groups_compositions = (
+                        TableGroupComposition.get_from_parent_table_code(
+                            code=node.table, session=self.session
+                        )
+                    )
+                    self.table_groups_compositions[node.table] = (
+                        table_groups_compositions
+                    )
                 else:
-                    table_groups_compositions = self.table_groups_compositions[node.table]
+                    table_groups_compositions = self.table_groups_compositions[
+                        node.table
+                    ]
 
                 for table_id, is_abstract in table_groups_compositions:
-                    table_version = self.check_table_from_module(node, table_id, is_abstract)
+                    table_version = self.check_table_from_module(
+                        node, table_id, is_abstract
+                    )
                     if table_version:
                         self.group_tables.append(
-                            {GROUP: node.table, TABLE_CODE: table_version.Code, TABLE_ID: table_version.TableID})
+                            {
+                                GROUP: node.table,
+                                TABLE_CODE: table_version.Code,
+                                TABLE_ID: table_version.TableID,
+                            }
+                        )

py_dpm/exceptions/__init__.py

@@ -0,0 +1,23 @@
+"""
+PyDPM Exceptions Module
+
+Custom exceptions for DPM processing.
+"""
+
+from py_dpm.exceptions.exceptions import (
+    DrrException,
+    SyntaxError,
+    SemanticError,
+    DataTypeError,
+    ScriptingError,
+)
+from py_dpm.exceptions.messages import centralised_messages
+
+__all__ = [
+    "DrrException",
+    "SyntaxError",
+    "SemanticError",
+    "DataTypeError",
+    "ScriptingError",
+    "centralised_messages",
+]

py_dpm/{Exceptions → exceptions}/exceptions.py

@@ -1,5 +1,4 @@
-from py_dpm.Exceptions.messages import centralised_messages
-from py_dpm.Utils.operands_mapping import LabelHandler, get_type_from_label
+from py_dpm.exceptions.messages import centralised_messages
 
 """
 Exceptions management.
@@ -34,6 +33,9 @@ class SyntaxError(DrrException):
 
 
 def gather_expression(operand):
+    # Lazy import to avoid circular dependency
+    from py_dpm.dpm_xl.utils.operands_mapping import LabelHandler
+
     operands_labels = LabelHandler().operands_labels
 
     expression = operand
@@ -52,6 +54,9 @@ class SemanticError(DrrException):
     """
 
     def __init__(self, code, **kwargs):
+        # Lazy import to avoid circular dependency
+        from py_dpm.dpm_xl.utils.operands_mapping import LabelHandler, get_type_from_label
+
         operands_labels = LabelHandler().operands_labels
         message = centralised_messages[code].format(**kwargs)
         for operand in reversed(operands_labels):

pydpm_xl-0.2.1.dist-info/METADATA

@@ -0,0 +1,278 @@
+Metadata-Version: 2.4
+Name: pydpm_xl
+Version: 0.2.1
+Summary: Python library for DPM-XL data processing and analysis
+Author-email: "MeaningfulData S.L." <info@meaningfuldata.eu>
+License: GPL-3.0-or-later
+Keywords: dpm,dpm-xl,data-processing,migration,analysis
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rich<13.8.0,>=13.7.1
+Requires-Dist: sqlalchemy<1.5.0,>=1.4.50
+Requires-Dist: pandas>=2.1.4
+Requires-Dist: antlr4-python3-runtime<4.9.3,>=4.9.2
+Requires-Dist: pyodbc<5.2.0,>=5.1.0
+Requires-Dist: click<8.2.0,>=8.1.6
+Requires-Dist: python-dotenv<1.1.0,>=1.0.1
+Requires-Dist: psycopg2-binary<3.0.0,>=2.9.0
+Provides-Extra: test
+Requires-Dist: pytest<8.0.0,>=7.4.0; extra == "test"
+Requires-Dist: pytest-cov<5.0.0,>=4.1.0; extra == "test"
+Dynamic: license-file
+
+# pyDPM
+
+A Python library for processing DPM (Data Point Model) expressions and working with regulatory reporting data.
+
+## Overview
+
+pyDPM provides two main areas of functionality:
+
+- **DPM-XL Processing**: Parse, validate, and generate ASTs for DPM-XL expressions
+- **DPM Utilities**: Work with DPM databases, explore data dictionaries, and manage operation scopes
+
+## Installation
+
+### Using Poetry (Recommended)
+
+```bash
+poetry install
+```
+
+### Using pip
+
+```bash
+pip install .
+```
+
+## Architecture
+
+```
+py_dpm/
+├── api/              # Public APIs
+│   ├── dpm_xl/      # DPM-XL expression APIs
+│   └── dpm/         # General DPM APIs
+├── dpm_xl/          # DPM-XL Processing Engine
+│   ├── grammar/     # ANTLR grammar
+│   ├── ast/         # AST generation
+│   ├── operators/   # Expression operators
+│   ├── types/       # Type system
+│   ├── validation/  # Validation logic
+│   └── utils/       # DPM-XL utilities
+├── dpm/             # General DPM Core
+│   ├── db/          # Database models & views
+│   ├── scopes/      # Operation scopes
+│   └── explorer/    # Data dictionary explorer
+├── cli/             # Command-line interface
+├── exceptions/      # Custom exceptions
+└── utils/           # Shared utilities
+```
+
+## Database Configuration
+
+pyDPM supports multiple database backends. It selects the connection method based on the following hierarchy of preference:
+
+1.  **Explicit Argument**: Passing a connection URL or path directly in Python code overrides all configuration.
+2.  **Unified RDBMS Configuration**: If `PYDPM_RDBMS` and the `PYDPM_DB_*` variables are set, it connects to the configured server database.
+3.  **Legacy PostgreSQL**: If `USE_POSTGRES=true` in `.env`, it connects to the configured Postgres server.
+4.  **SQLite**: If `USE_SQLITE=true` (default), it connects to a local SQLite file.
+5.  **SQL Server**: Legacy fallback if no other option is selected.
+
+### Environment Variables (.env)
+
+Configure your database connection in the `.env` file:
+
+```ini
+# --- Option 1: SQLite (Default) ---
+USE_SQLITE=true
+SQLITE_DB_PATH=database.db
+
+# --- Option 2: Unified server database (recommended) ---
+# PYDPM_RDBMS=postgres        # or "sqlserver"
+# PYDPM_DB_HOST=localhost
+# PYDPM_DB_PORT=5432          # defaults: 5432 for postgres, 1433 for sqlserver
+# PYDPM_DB_NAME=dpm_db
+# PYDPM_DB_USER=myuser
+# PYDPM_DB_PASSWORD=mypassword
+
+# --- Option 3: Legacy PostgreSQL (backward compatible) ---
+# USE_POSTGRES=true
+# POSTGRES_HOST=localhost
+# POSTGRES_PORT=5432
+# POSTGRES_DB=dpm_db
+# POSTGRES_USER=myuser
+# POSTGRES_PASS=mypassword
+```
+
+## Usage
+
+### Command Line Interface
+
+#### Migrate Access Database
+```bash
+poetry run pydpm migrate-access ./path-to-release.accdb
+```
+
+#### Syntax Validation
+```bash
+poetry run pydpm syntax "{tT_01.00, r0010, c0010}"
+```
+
+#### Semantic Validation
+```bash
+poetry run pydpm semantic "{tT_01.00, r0010, c0010}"
+```
+
+### Python API
+
+#### DPM-XL Expression Processing
+
+```python
+from py_dpm.api import SyntaxAPI, SemanticAPI, ASTGenerator
+
+# Syntax validation
+syntax_api = SyntaxAPI()
+is_valid = syntax_api.is_valid_syntax("{tT_01.00, r0010, c0010}")
+print(f"Valid syntax: {is_valid}")
+
+# Get detailed syntax errors
+errors = syntax_api.validate_syntax("invalid expression")
+for error in errors:
+    print(f"Line {error.line}, Col {error.column}: {error.message}")
+
+# Generate AST
+ast_gen = ASTGenerator()
+ast = ast_gen.generate("{tT_01.00, r0010, c0010}")
+print(f"AST: {ast}")
+
+# Semantic validation (requires database)
+semantic_api = SemanticAPI()
+result = semantic_api.validate("{tT_01.00, r0010, c0010}", release_id=123)
+if result.is_valid:
+    print("Semantically valid!")
+else:
+    for error in result.errors:
+        print(f"Error: {error.message}")
+```
+
+#### Working with DPM Database
+
+```python
+from py_dpm.api import DataDictionaryAPI, DPMExplorer
+
+# Query data dictionary
+dd_api = DataDictionaryAPI()
+
+# Get all tables
+tables = dd_api.get_all_tables(release_id=123)
+for table in tables:
+    print(f"Table: {table.code} - {table.name}")
+
+# Get table details
+table = dd_api.get_table_by_code("T_01.00", release_id=123)
+print(f"Table headers: {len(table.headers)}")
+
+# Explore database structure
+explorer = DPMExplorer()
+modules = explorer.get_modules(release_id=123)
+for module in modules:
+    print(f"Module: {module.code}")
+```
+
+#### Operation Scopes
+
+```python
+from py_dpm.api import OperationScopesAPI, calculate_scopes_from_expression
+
+# Calculate scopes for an expression
+scopes_api = OperationScopesAPI()
+result = calculate_scopes_from_expression(
+    expression="{tT_01.00, r0010, c0010}",
+    release_id=123
+)
+
+if result.success:
+    for scope in result.scopes:
+        print(f"Table: {scope.table_code}")
+        print(f"Headers: {[h.header_code for h in scope.headers]}")
+else:
+    print(f"Error: {result.error}")
+```
+
+#### Complete AST Generation
+
+```python
+from py_dpm.api import generate_complete_ast, generate_enriched_ast
+
+# Generate complete AST with data fields
+result = generate_complete_ast("{tT_01.00, r0010, c0010}", release_id=123)
+if result.success:
+    print(f"AST: {result.ast}")
+    print(f"Data fields: {result.data_fields}")
+else:
+    print(f"Errors: {result.errors}")
+
+# Generate enriched AST (ready for execution engine)
+enriched = generate_enriched_ast("{tT_01.00, r0010, c0010}", release_id=123)
+print(f"Enriched AST: {enriched}")
+```
+
+#### Migration
+
+```python
+from py_dpm.api import MigrationAPI
+
+# Migrate Access database to SQLAlchemy
+migration_api = MigrationAPI()
+migration_api.migrate_from_access(
+    access_db_path="./release.accdb",
+    release_id=123
+)
+```
+
+## Development
+
+### Running Tests
+
+```bash
+poetry run pytest
+```
+
+### Code Structure
+
+- See [API_DOCUMENTATION.md](API_DOCUMENTATION.md) for detailed API reference
+- See [REORGANIZATION_SUMMARY.md](REORGANIZATION_SUMMARY.md) for reorganization details
+
+## Important Notes
+
+### ANTLR Version
+This project uses **ANTLR 4.9.2**. Always run Python scripts using Poetry to ensure the correct runtime version:
+
+```bash
+poetry run python your_script.py  # ✅ Correct
+python your_script.py             # ❌ May use wrong ANTLR version
+```
+
+### Database Sessions
+When using database APIs without explicit connection configuration, pyDPM uses global session management. For concurrent usage or testing, pass explicit database paths or connection URLs:
+
+```python
+api = DataDictionaryAPI(database_path="./test.db")
+# or
+api = DataDictionaryAPI(connection_url="postgresql://user:pass@localhost/db")
+```
+
+## License
+
+[Add your license information here]