pydpm_xl 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

py_dpm/__init__.py

@@ -41,7 +41,7 @@ Available packages:
 - pydpm.api: Main APIs for migration, syntax, and semantic analysis
 """
 
-__version__ = "0.2.4"
+__version__ = "0.2.5"
 __author__ = "MeaningfulData S.L."
 __email__ = "info@meaningfuldata.eu"
 __license__ = "GPL-3.0-or-later"

py_dpm/api/dpm_xl/ast_generator.py

@@ -369,6 +369,7 @@ class ASTGeneratorAPI:
         precondition: Optional[str] = None,
         release_id: Optional[int] = None,
         output_path: Optional[Union[str, Path]] = None,
+        primary_module_vid: Optional[int] = None,
     ) -> Dict[str, Any]:
         """
         Generate enriched, engine-ready AST with framework structure (Level 3).
@@ -381,13 +382,14 @@ class ASTGeneratorAPI:
         - Everything from generate_complete_ast() PLUS:
         - Framework structure: operations, variables, tables, preconditions
         - Module metadata: version, release info, dates
-        - Dependency information
+        - Dependency information (including cross-module dependencies)
         - Coordinates (x/y/z) added to data entries
 
         **Typical use case:**
         - Feeding AST to business rule execution engines
         - Validation framework integration
         - Production rule processing
+        - Module exports with cross-module dependency tracking
 
         Args:
             expression: DPM-XL expression string
@@ -399,6 +401,11 @@ class ASTGeneratorAPI:
                 If None, uses all available data (release-agnostic).
             output_path: Optional path (string or Path) to save the enriched_ast as JSON file.
                 If provided, the enriched_ast will be automatically saved to this location.
+            primary_module_vid: Optional module version ID of the module being exported.
+                When provided, enables detection of cross-module dependencies - tables from
+                other modules will be identified and added to dependency_modules and
+                cross_instance_dependencies fields. If None, cross-module detection uses
+                the first table's module as the primary module.
 
         Returns:
             dict: {
@@ -416,14 +423,17 @@ class ASTGeneratorAPI:
             ... )
             >>> # result['enriched_ast'] contains framework structure ready for engines
             >>>
-            >>> # Or save directly to a file:
+            >>> # For module exports with cross-module dependency tracking:
             >>> result = generator.generate_enriched_ast(
-            ...     "{tF_01.00, r0010, c0010}",
+            ...     "{tC_26.00, r030, c010} * {tC_01.00, r0015, c0010}",
             ...     dpm_version="4.2",
-            ...     operation_code="my_validation",
-            ...     output_path="./output/enriched_ast.json"
+            ...     operation_code="v2814_m",
+            ...     primary_module_vid=123,  # Module being exported
+            ...     release_id=42
             ... )
-            >>> # The enriched_ast is automatically saved to the specified path
+            >>> # result['enriched_ast']['dependency_modules'] contains external module info
+            >>> # result['enriched_ast']['dependency_information']['cross_instance_dependencies']
+            >>> # contains list of external module dependencies
         """
         try:
             # Generate complete AST first
@@ -447,6 +457,8 @@ class ASTGeneratorAPI:
                 dpm_version=dpm_version,
                 operation_code=operation_code,
                 precondition=precondition,
+                release_id=release_id,
+                primary_module_vid=primary_module_vid,
             )
 
             # Save to file if output_path is provided
@@ -723,11 +735,23 @@ class ASTGeneratorAPI:
         dpm_version: Optional[str] = None,
         operation_code: Optional[str] = None,
         precondition: Optional[str] = None,
+        release_id: Optional[int] = None,
+        primary_module_vid: Optional[int] = None,
     ) -> Dict[str, Any]:
         """
         Add framework structure (operations, variables, tables, preconditions) to complete AST.
 
         This creates the engine-ready format with all metadata sections.
+
+        Args:
+            ast_dict: Complete AST dictionary
+            expression: Original DPM-XL expression
+            context: Context dict with table, rows, columns, sheets, default, interval
+            dpm_version: DPM version code (e.g., "4.2")
+            operation_code: Operation code (defaults to "default_code")
+            precondition: Precondition variable reference (e.g., {v_F_44_04})
+            release_id: Optional release ID to filter database lookups
+            primary_module_vid: Module VID being exported (to identify external dependencies)
         """
         from py_dpm.dpm.utils import get_engine
         import copy
@@ -787,7 +811,7 @@ class ASTGeneratorAPI:
         preconditions = {}
         precondition_variables = {}
 
-        if precondition or (context and "table" in context):
+        if precondition:
             preconditions, precondition_variables = self._build_preconditions(
                 precondition=precondition,
                 context=context,
@@ -795,15 +819,21 @@ class ASTGeneratorAPI:
                 engine=engine,
             )
 
+        # Detect cross-module dependencies
+        dependency_modules, cross_instance_dependencies = self._detect_cross_module_dependencies(
+            expression=expression,
+            variables_by_table=variables_by_table,
+            primary_module_vid=primary_module_vid,
+            operation_code=operation_code,
+            release_id=release_id,
+        )
+
         # Build dependency information
         dependency_info = {
             "intra_instance_validations": [operation_code],
-            "cross_instance_dependencies": [],
+            "cross_instance_dependencies": cross_instance_dependencies,
         }
 
-        # Build dependency modules
-        dependency_modules = {}
-
         # Build complete structure
         namespace = "default_module"
 
@@ -944,8 +974,6 @@ class ASTGeneratorAPI:
             match = re.match(r"\{v_([^}]+)\}", precondition)
             if match:
                 table_code = match.group(1)
-        elif context and "table" in context:
-            table_code = context["table"]
 
         if table_code:
             # Query database for actual variable ID and version
@@ -1009,6 +1037,261 @@ class ASTGeneratorAPI:
         extract_from_node(ast_dict)
         return all_variables, variables_by_table
 
+    def _extract_time_shifts_by_table(self, expression: str) -> Dict[str, str]:
+        """
+        Extract time shift information for each table in the expression.
+
+        Uses the AST to properly parse the expression and find TimeShiftOp nodes
+        to determine the ref_period for each table reference.
+
+        Args:
+            expression: DPM-XL expression
+
+        Returns:
+            Dict mapping table codes to ref_period values (e.g., {"C_01.00": "T-1Q"})
+            Tables without time shifts default to "T".
+        """
+        from py_dpm.dpm_xl.ast.template import ASTTemplate
+
+        time_shifts = {}
+        current_period = ["t"]  # Use list to allow mutation in nested function
+
+        class TimeShiftExtractor(ASTTemplate):
+            """Lightweight AST visitor that extracts time shifts for each table."""
+
+            def visit_TimeShiftOp(self, node):
+                # Save current time period and compute new one
+                previous_period = current_period[0]
+
+                period_indicator = node.period_indicator
+                shift_number = node.shift_number
+
+                # Compute time period (same logic as ModuleDependencies)
+                if "-" in str(shift_number):
+                    current_period[0] = f"t+{period_indicator}{shift_number}"
+                else:
+                    current_period[0] = f"t-{period_indicator}{shift_number}"
+
+                # Visit operand (which contains the VarID)
+                self.visit(node.operand)
+
+                # Restore previous time period
+                current_period[0] = previous_period
+
+            def visit_VarID(self, node):
+                if node.table and current_period[0] != "t":
+                    time_shifts[node.table] = current_period[0]
+
+        def convert_to_ref_period(internal_period: str) -> str:
+            """Convert internal time period format to ref_period format.
+
+            Internal format: "t+Q-1" or "t-Q1"
+            Output format: "T-1Q" for one quarter back
+            """
+            if internal_period.startswith("t+"):
+                # e.g., "t+Q-1" -> "T-1Q"
+                indicator = internal_period[2]
+                number = internal_period[3:]
+                if number.startswith("-"):
+                    return f"T{number}{indicator}"
+                return f"T+{number}{indicator}"
+            elif internal_period.startswith("t-"):
+                # e.g., "t-Q1" -> "T-1Q"
+                indicator = internal_period[2]
+                number = internal_period[3:]
+                return f"T-{number}{indicator}"
+            return "T"
+
+        try:
+            ast = self.syntax_api.parse_expression(expression)
+            extractor = TimeShiftExtractor()
+            extractor.visit(ast)
+
+            return {table: convert_to_ref_period(period) for table, period in time_shifts.items()}
+
+        except Exception:
+            return {}
+
+    def _detect_cross_module_dependencies(
+        self,
+        expression: str,
+        variables_by_table: Dict[str, Dict[str, str]],
+        primary_module_vid: Optional[int],
+        operation_code: str,
+        release_id: Optional[int] = None,
+    ) -> tuple:
+        """
+        Detect cross-module dependencies for a single expression.
+
+        Uses existing OperationScopesAPI and ExplorerQuery to detect external module
+        references in cross-module expressions.
+
+        Args:
+            expression: DPM-XL expression
+            variables_by_table: Variables by table code (from _extract_variables_from_ast)
+            primary_module_vid: The module being exported (if known)
+            operation_code: Current operation code
+            release_id: Optional release ID for filtering
+
+        Returns:
+            Tuple of (dependency_modules, cross_instance_dependencies)
+            - dependency_modules: {uri: {tables: {...}, variables: {...}}}
+            - cross_instance_dependencies: [{modules: [...], affected_operations: [...], ...}]
+        """
+        from py_dpm.api.dpm_xl.operation_scopes import OperationScopesAPI
+        from py_dpm.dpm.queries.explorer_queries import ExplorerQuery
+        import logging
+
+        scopes_api = OperationScopesAPI(
+            database_path=self.database_path,
+            connection_url=self.connection_url
+        )
+
+        try:
+            # Get tables with module info (includes module_version)
+            tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
+                expression=expression,
+                release_id=release_id
+            )
+
+            # Check if cross-module
+            scope_result = scopes_api.calculate_scopes_from_expression(
+                expression=expression,
+                release_id=release_id,
+                read_only=True
+            )
+
+            if scope_result.has_error or not scope_result.is_cross_module:
+                return {}, []
+
+            # Extract time shifts for each table from expression
+            time_shifts_by_table = self._extract_time_shifts_by_table(expression)
+
+            # Determine primary module from first table if not provided
+            if primary_module_vid is None and tables_with_modules:
+                primary_module_vid = tables_with_modules[0].get("module_vid")
+
+            # Helper to normalize table code (remove 't' prefix if present)
+            def normalize_table_code(code: str) -> str:
+                return code[1:] if code and code.startswith('t') else code
+
+            # Helper to lookup ref_period for a table
+            def get_ref_period(table_code: str) -> str:
+                if not table_code:
+                    return "T"
+                ref = time_shifts_by_table.get(table_code)
+                if not ref:
+                    ref = time_shifts_by_table.get(normalize_table_code(table_code))
+                return ref or "T"
+
+            # Helper to lookup variables for a table
+            def get_table_variables(table_code: str) -> dict:
+                if not table_code:
+                    return {}
+                variables = variables_by_table.get(table_code)
+                if not variables:
+                    variables = variables_by_table.get(f"t{table_code}", {})
+                return variables or {}
+
+            # Group external tables by module
+            external_modules = {}
+            for table_info in tables_with_modules:
+                module_vid = table_info.get("module_vid")
+                if module_vid == primary_module_vid:
+                    continue  # Skip primary module
+
+                module_code = table_info.get("module_code")
+                if not module_code:
+                    continue
+
+                # Get module URI
+                try:
+                    module_uri = ExplorerQuery.get_module_url(
+                        scopes_api.session,
+                        module_code=module_code,
+                        release_id=release_id,
+                    )
+                    if module_uri.endswith(".json"):
+                        module_uri = module_uri[:-5]
+                except Exception:
+                    continue
+
+                table_code = table_info.get("code")
+                ref_period = get_ref_period(table_code)
+
+                if module_uri not in external_modules:
+                    external_modules[module_uri] = {
+                        "module_vid": module_vid,
+                        "module_version": table_info.get("module_version"),  # Already in table_info
+                        "ref_period": ref_period,
+                        "tables": {},
+                        "variables": {},
+                        "from_date": None,
+                        "to_date": None
+                    }
+                elif ref_period != "T":
+                    # Keep most specific ref_period (non-T takes precedence)
+                    external_modules[module_uri]["ref_period"] = ref_period
+
+                # Add table and variables
+                if table_code:
+                    table_variables = get_table_variables(table_code)
+                    external_modules[module_uri]["tables"][table_code] = {
+                        "variables": table_variables,
+                        "open_keys": {}
+                    }
+                    external_modules[module_uri]["variables"].update(table_variables)
+
+            # Get date info from scopes metadata
+            scopes_metadata = scopes_api.get_scopes_with_metadata_from_expression(
+                expression=expression,
+                release_id=release_id
+            )
+            for scope_info in scopes_metadata:
+                for module in scope_info.module_versions:
+                    mvid = module.get("module_vid")
+                    for uri, data in external_modules.items():
+                        if data["module_vid"] == mvid:
+                            data["from_date"] = module.get("from_reference_date")
+                            data["to_date"] = module.get("to_reference_date")
+
+            # Build output structures
+            dependency_modules = {}
+            cross_instance_dependencies = []
+
+            for uri, data in external_modules.items():
+                # dependency_modules entry
+                dependency_modules[uri] = {
+                    "tables": data["tables"],
+                    "variables": data["variables"]
+                }
+
+                # cross_instance_dependencies entry (one per external module)
+                from_date = data["from_date"]
+                to_date = data["to_date"]
+                module_entry = {
+                    "URI": uri,
+                    "ref_period": data["ref_period"]
+                }
+                # Add module_version if available
+                if data["module_version"]:
+                    module_entry["module_version"] = data["module_version"]
+
+                cross_instance_dependencies.append({
+                    "modules": [module_entry],
+                    "affected_operations": [operation_code],
+                    "from_reference_date": str(from_date) if from_date else "",
+                    "to_reference_date": str(to_date) if to_date else ""
+                })
+
+            return dependency_modules, cross_instance_dependencies
+
+        except Exception as e:
+            logging.warning(f"Failed to detect cross-module dependencies: {e}")
+            return {}, []
+        finally:
+            scopes_api.close()
+
     def _add_coordinates_to_ast(
         self, ast_dict: Dict[str, Any], context: Optional[Dict[str, Any]]
     ) -> Dict[str, Any]:

py_dpm/api/dpm_xl/complete_ast.py

@@ -117,6 +117,7 @@ def generate_enriched_ast(
     table_context: Optional[Dict[str, Any]] = None,
     precondition: Optional[str] = None,
     release_id: Optional[int] = None,
+    primary_module_vid: Optional[int] = None,
 ) -> Dict[str, Any]:
     """
     Generate enriched, engine-ready AST from DPM-XL expression.
@@ -133,6 +134,8 @@ def generate_enriched_ast(
         precondition: Optional precondition variable reference (e.g., {v_F_44_04})
         release_id: Optional release ID to filter database lookups by specific release.
             If None, uses all available data (release-agnostic).
+        primary_module_vid: Optional module version ID of the module being exported.
+            When provided, enables detection of cross-module dependencies.
 
     Returns:
         dict: {
@@ -153,6 +156,7 @@ def generate_enriched_ast(
         table_context=table_context,
         precondition=precondition,
         release_id=release_id,
+        primary_module_vid=primary_module_vid,
     )
 
 
@@ -165,6 +169,8 @@ def enrich_ast_with_metadata(
     dpm_version: Optional[str] = None,
     operation_code: Optional[str] = None,
     precondition: Optional[str] = None,
+    release_id: Optional[int] = None,
+    primary_module_vid: Optional[int] = None,
 ) -> Dict[str, Any]:
     """
     Add framework structure (operations, variables, tables, preconditions) to complete AST.
@@ -180,6 +186,8 @@ def enrich_ast_with_metadata(
         dpm_version: DPM version code (e.g., "4.2")
         operation_code: Operation code (defaults to "default_code")
         precondition: Precondition variable reference (e.g., {v_F_44_04})
+        release_id: Optional release ID to filter database lookups
+        primary_module_vid: Optional module VID of the module being exported
 
     Returns:
         dict: Engine-ready AST with framework structure
@@ -196,4 +204,6 @@ def enrich_ast_with_metadata(
         dpm_version=dpm_version,
         operation_code=operation_code,
         precondition=precondition,
+        release_id=release_id,
+        primary_module_vid=primary_module_vid,
     )

py_dpm/dpm/models.py

@@ -720,16 +720,20 @@ class ModuleVersion(Base):
             release_id: Optional release ID to filter modules by release range
 
         Returns:
-            pandas DataFrame with columns: ModuleVID, TableVID (as VARIABLE_VID),
-            FromReferenceDate, ToReferenceDate, EndReleaseID
+            pandas DataFrame with columns: ModuleVID, TableVID (as variable_vid),
+            ModuleCode, VersionNumber, FromReferenceDate, ToReferenceDate,
+            StartReleaseID, EndReleaseID
         """
         if not tables_vids:
             return pd.DataFrame(
                 columns=[
                     "ModuleVID",
                     "variable_vid",
+                    "ModuleCode",
+                    "VersionNumber",
                     "FromReferenceDate",
                     "ToReferenceDate",
+                    "StartReleaseID",
                     "EndReleaseID",
                 ]
             )
@@ -738,8 +742,11 @@ class ModuleVersion(Base):
             session.query(
                 cls.modulevid.label("ModuleVID"),
                 ModuleVersionComposition.tablevid.label("variable_vid"),
+                cls.code.label("ModuleCode"),
+                cls.versionnumber.label("VersionNumber"),
                 cls.fromreferencedate.label("FromReferenceDate"),
                 cls.toreferencedate.label("ToReferenceDate"),
+                cls.startreleaseid.label("StartReleaseID"),
                 cls.endreleaseid.label("EndReleaseID"),
             )
             .join(
@@ -759,16 +766,20 @@ class ModuleVersion(Base):
             )
 
         results = query.all()
-        return pd.DataFrame(
+        df = pd.DataFrame(
             results,
             columns=[
                 "ModuleVID",
                 "variable_vid",
+                "ModuleCode",
+                "VersionNumber",
                 "FromReferenceDate",
                 "ToReferenceDate",
+                "StartReleaseID",
                 "EndReleaseID",
             ],
         )
+        return cls._apply_fallback_for_equal_dates(session, df)
 
     @classmethod
     def get_from_table_codes(cls, session, table_codes, release_id=None):
@@ -790,6 +801,8 @@ class ModuleVersion(Base):
                 columns=[
                     "ModuleVID",
                     "variable_vid",
+                    "ModuleCode",
+                    "VersionNumber",
                     "FromReferenceDate",
                     "ToReferenceDate",
                     "StartReleaseID",
@@ -804,6 +817,8 @@ class ModuleVersion(Base):
             session.query(
                 cls.modulevid.label("ModuleVID"),
                 ModuleVersionComposition.tablevid.label("variable_vid"),
+                cls.code.label("ModuleCode"),
+                cls.versionnumber.label("VersionNumber"),
                 cls.fromreferencedate.label("FromReferenceDate"),
                 cls.toreferencedate.label("ToReferenceDate"),
                 cls.startreleaseid.label("StartReleaseID"),
@@ -831,11 +846,13 @@ class ModuleVersion(Base):
             )
 
         results = query.all()
-        return pd.DataFrame(
+        df = pd.DataFrame(
             results,
             columns=[
                 "ModuleVID",
                 "variable_vid",
+                "ModuleCode",
+                "VersionNumber",
                 "FromReferenceDate",
                 "ToReferenceDate",
                 "StartReleaseID",
@@ -843,6 +860,7 @@ class ModuleVersion(Base):
                 "TableCode",
             ],
         )
+        return cls._apply_fallback_for_equal_dates(session, df)
 
     @classmethod
     def get_precondition_module_versions(
@@ -857,16 +875,21 @@ class ModuleVersion(Base):
             release_id: Optional release ID to filter modules by release range
 
         Returns:
-            pandas DataFrame with columns: ModuleVID, VariableVID (as VARIABLE_VID),
-            FromReferenceDate, ToReferenceDate, Code
+            pandas DataFrame with columns: ModuleVID, variable_vid (VariableVID),
+            ModuleCode, VersionNumber, FromReferenceDate, ToReferenceDate,
+            StartReleaseID, EndReleaseID, Code
         """
         if not precondition_items:
             return pd.DataFrame(
                 columns=[
                     "ModuleVID",
                     "variable_vid",
+                    "ModuleCode",
+                    "VersionNumber",
                     "FromReferenceDate",
                     "ToReferenceDate",
+                    "StartReleaseID",
+                    "EndReleaseID",
                     "Code",
                 ]
             )
@@ -875,8 +898,12 @@ class ModuleVersion(Base):
             session.query(
                 cls.modulevid.label("ModuleVID"),
                 VariableVersion.variablevid.label("variable_vid"),
+                cls.code.label("ModuleCode"),
+                cls.versionnumber.label("VersionNumber"),
                 cls.fromreferencedate.label("FromReferenceDate"),
                 cls.toreferencedate.label("ToReferenceDate"),
+                cls.startreleaseid.label("StartReleaseID"),
+                cls.endreleaseid.label("EndReleaseID"),
                 VariableVersion.code.label("Code"),
             )
             .join(ModuleParameters, cls.modulevid == ModuleParameters.modulevid)
@@ -899,16 +926,21 @@ class ModuleVersion(Base):
             )
 
         results = query.all()
-        return pd.DataFrame(
+        df = pd.DataFrame(
             results,
             columns=[
                 "ModuleVID",
                 "variable_vid",
+                "ModuleCode",
+                "VersionNumber",
                 "FromReferenceDate",
                 "ToReferenceDate",
+                "StartReleaseID",
+                "EndReleaseID",
                 "Code",
             ],
         )
+        return cls._apply_fallback_for_equal_dates(session, df)
 
     @classmethod
     def get_module_version_by_vid(cls, session, vid):
@@ -946,6 +978,170 @@ class ModuleVersion(Base):
             ],
         )
 
+    @classmethod
+    def _apply_fallback_for_equal_dates(cls, session, df, module_vid_col="ModuleVID"):
+        """
+        Apply fallback logic for rows where FromReferenceDate == ToReferenceDate.
+
+        For each such row, find the previous module version (same moduleid,
+        highest startreleaseid less than current) and replace module-specific
+        columns while preserving association columns (variable_vid, TableCode, Code).
+
+        Args:
+            session: SQLAlchemy session
+            df: pandas DataFrame with module version data
+            module_vid_col: Column name for module version ID (default: "ModuleVID")
+
+        Returns:
+            pandas DataFrame with fallback logic applied
+        """
+        if df.empty:
+            return df
+
+        # Identify rows needing fallback
+        mask = df["FromReferenceDate"] == df["ToReferenceDate"]
+        rows_needing_fallback = df[mask]
+
+        if rows_needing_fallback.empty:
+            return df
+
+        # Get unique ModuleVIDs that need fallback
+        module_vids_needing_fallback = (
+            rows_needing_fallback[module_vid_col].unique().tolist()
+        )
+
+        # Batch query: get module info (moduleid, startreleaseid) for affected rows
+        current_modules = (
+            session.query(
+                cls.modulevid,
+                cls.moduleid,
+                cls.startreleaseid,
+            )
+            .filter(cls.modulevid.in_(module_vids_needing_fallback))
+            .all()
+        )
+
+        # Build mapping: current_modulevid -> (moduleid, startreleaseid)
+        current_module_info = {
+            row.modulevid: (row.moduleid, row.startreleaseid) for row in current_modules
+        }
+
+        # Get all potential previous versions for the affected modules
+        unique_module_ids = list(set(info[0] for info in current_module_info.values()))
+
+        previous_versions_query = (
+            session.query(cls)
+            .filter(cls.moduleid.in_(unique_module_ids))
+            .order_by(cls.moduleid, cls.startreleaseid.desc())
+            .all()
+        )
+
+        # Build lookup: moduleid -> list of versions sorted by startreleaseid desc
+        versions_by_moduleid = {}
+        for mv in previous_versions_query:
+            if mv.moduleid not in versions_by_moduleid:
+                versions_by_moduleid[mv.moduleid] = []
+            versions_by_moduleid[mv.moduleid].append(mv)
+
+        # For each current modulevid, find the previous version
+        replacement_map = {}  # current_modulevid -> previous_moduleversion
+        for current_vid, (moduleid, current_startreleaseid) in current_module_info.items():
+            versions = versions_by_moduleid.get(moduleid, [])
+            for mv in versions:
+                if mv.startreleaseid < current_startreleaseid:
+                    replacement_map[current_vid] = mv
+                    break  # Already sorted desc, so first match is highest
+
+        # Apply replacements to DataFrame
+        if not replacement_map:
+            return df
+
+        # Create a copy to avoid modifying original
+        result_df = df.copy()
+
+        for idx, row in result_df.iterrows():
+            if row["FromReferenceDate"] == row["ToReferenceDate"]:
+                current_vid = row[module_vid_col]
+                if current_vid in replacement_map:
+                    prev_mv = replacement_map[current_vid]
+                    result_df.at[idx, "ModuleVID"] = prev_mv.modulevid
+                    result_df.at[idx, "ModuleCode"] = prev_mv.code
+                    result_df.at[idx, "VersionNumber"] = prev_mv.versionnumber
+                    result_df.at[idx, "FromReferenceDate"] = prev_mv.fromreferencedate
+                    result_df.at[idx, "ToReferenceDate"] = prev_mv.toreferencedate
+                    if "StartReleaseID" in result_df.columns:
+                        result_df.at[idx, "StartReleaseID"] = prev_mv.startreleaseid
+                    if "EndReleaseID" in result_df.columns:
+                        result_df.at[idx, "EndReleaseID"] = prev_mv.endreleaseid
+
+        return result_df
+
+    @classmethod
+    def get_from_release_id(
+        cls, session, release_id, module_id=None, module_code=None
+    ):
+        """
+        Get the module version applicable to a given release for a specific module.
+
+        If the resulting module version has fromreferencedate == toreferencedate,
+        the previous module version for the same module is returned instead.
+
+        Args:
+            session: SQLAlchemy session
+            release_id: The release ID to filter for
+            module_id: Optional module ID (mutually exclusive with module_code)
+            module_code: Optional module code (mutually exclusive with module_id)
+
+        Returns:
+            ModuleVersion instance or None if not found
+
+        Raises:
+            ValueError: If neither module_id nor module_code is provided,
+                        or if both are provided
+        """
+        if module_id is None and module_code is None:
+            raise ValueError("Either module_id or module_code must be provided.")
+        if module_id is not None and module_code is not None:
+            raise ValueError(
+                "Specify only one of module_id or module_code, not both."
+            )
+
+        # Build the base query with release filtering
+        query = session.query(cls).filter(
+            and_(
+                cls.startreleaseid <= release_id,
+                or_(cls.endreleaseid > release_id, cls.endreleaseid.is_(None)),
+            )
+        )
+
+        # Apply module filter
+        if module_id is not None:
+            query = query.filter(cls.moduleid == module_id)
+        else:  # module_code
+            query = query.filter(cls.code == module_code)
+
+        module_version = query.first()
+
+        if module_version is None:
+            return None
+
+        # Check if fromreferencedate == toreferencedate
+        if module_version.fromreferencedate == module_version.toreferencedate:
+            # Get the previous module version for the same module
+            prev_query = (
+                session.query(cls)
+                .filter(
+                    cls.moduleid == module_version.moduleid,
+                    cls.startreleaseid < module_version.startreleaseid,
+                )
+                .order_by(cls.startreleaseid.desc())
+            )
+            prev_module_version = prev_query.first()
+            if prev_module_version:
+                return prev_module_version
+
+        return module_version
+
     @classmethod
     def get_last_release(cls, session):
         """
@@ -1119,6 +1315,60 @@ class OperationScope(Base):
         "OperationScopeComposition", back_populates="operation_scope"
     )
 
+    def to_dict(self):
+        """
+        Convert the operation scope to a dictionary representation.
+
+        Returns:
+            dict: A dictionary with module codes as keys and module details as values.
+                  Format: {
+                      "<module_code>": {
+                          "module_version_number": <versionnumber>,
+                          "from_reference_date": <fromreferencedate>,
+                          "to_reference_date": <toreferencedate>
+                      },
+                      ...
+                  }
+        """
+        from sqlalchemy.orm import object_session
+
+        def format_date(date_value):
+            """Format date to string (YYYY-MM-DD) or None if NaT/None."""
+            if date_value is None:
+                return None
+            if pd.isna(date_value):
+                return None
+            if hasattr(date_value, "strftime"):
+                return date_value.strftime("%Y-%m-%d")
+            return str(date_value)
+
+        result = {}
+        for composition in self.operation_scope_compositions:
+            # For new/proposed scopes, use transient _module_info attribute
+            if hasattr(composition, "_module_info") and composition._module_info:
+                info = composition._module_info
+                result[info["code"]] = {
+                    "module_version_number": info["version_number"],
+                    "from_reference_date": format_date(info["from_reference_date"]),
+                    "to_reference_date": format_date(info["to_reference_date"]),
+                }
+            else:
+                # For existing scopes from DB, use relationship or query
+                module_version = composition.module_version
+                if module_version is None:
+                    session = object_session(self)
+                    if session is not None:
+                        module_version = session.query(ModuleVersion).filter(
+                            ModuleVersion.modulevid == composition.modulevid
+                        ).first()
+                if module_version is not None:
+                    result[module_version.code] = {
+                        "module_version_number": module_version.versionnumber,
+                        "from_reference_date": format_date(module_version.fromreferencedate),
+                        "to_reference_date": format_date(module_version.toreferencedate),
+                    }
+        return result
+
 
 class OperationScopeComposition(Base):
     __tablename__ = "OperationScopeComposition"

py_dpm/dpm_xl/utils/scopes_calculator.py

@@ -71,9 +71,19 @@ class OperationScopeService:
         if len(modules_info_dataframe) == 1:
             module_vid = modules_vids[0]
             from_date = modules_info_dataframe["FromReferenceDate"].values[0]
+            to_date = modules_info_dataframe["ToReferenceDate"].values[0]
+            module_code = modules_info_dataframe["ModuleCode"].values[0]
+            version_number = modules_info_dataframe["VersionNumber"].values[0]
             operation_scope = self.create_operation_scope(from_date)
             self.create_operation_scope_composition(
-                operation_scope=operation_scope, module_vid=module_vid
+                operation_scope=operation_scope,
+                module_vid=module_vid,
+                module_info={
+                    "code": module_code,
+                    "version_number": version_number,
+                    "from_reference_date": from_date,
+                    "to_reference_date": to_date,
+                },
             )
         else:
             intra_modules = []
@@ -83,11 +93,10 @@ class OperationScopeService:
             if table_codes:
                 unique_operands_number = len(table_codes) + len(precondition_items)
 
-                # Categorize modules by lifecycle: starting vs ending in this release
-                starting_modules = (
-                    {}
-                )  # Modules that START in this release (replacements)
-                ending_modules = {}  # Modules that END in this release (being replaced)
+                # First pass: categorize modules by table code and lifecycle
+                # We track lifecycle to handle version transitions within the SAME module
+                starting_by_code = {}  # table_code -> [module_vids that START in this release]
+                ending_by_code = {}  # table_code -> [module_vids that END or are active]
 
                 for module_vid, group_df in modules_info_dataframe.groupby(MODULE_VID):
                     table_codes_in_module = (
@@ -105,31 +114,55 @@ class OperationScopeService:
                     end_release = group_df["EndReleaseID"].values[0]
 
                     # Determine if this is a "new" module starting in this release
-                    # or an "old" module ending in this release
                     is_starting = start_release == release_id
-                    is_ending = end_release == release_id or end_release == float(
-                        release_id
-                    )
 
                     if len(table_codes_in_module) == unique_operands_number:
                         # Intra-module: include ALL modules active in the release
-                        # (don't filter by lifecycle - that's only for cross-module)
                         intra_modules.append(module_vid)
                     else:
-                        # For cross-module, group by table code AND lifecycle stage
-                        target_dict = (
-                            starting_modules if is_starting else ending_modules
-                        )
+                        # Track modules by table code and lifecycle
                         for table_code in table_codes_in_module:
-                            if table_code not in target_dict:
-                                target_dict[table_code] = []
-                            target_dict[table_code].append(module_vid)
-
-                # Process cross-module scopes separately for each generation
-                if starting_modules:
-                    cross_modules["_starting"] = starting_modules
-                if ending_modules:
-                    cross_modules["_ending"] = ending_modules
+                            if is_starting:
+                                if table_code not in starting_by_code:
+                                    starting_by_code[table_code] = []
+                                starting_by_code[table_code].append(module_vid)
+                            else:
+                                if table_code not in ending_by_code:
+                                    ending_by_code[table_code] = []
+                                ending_by_code[table_code].append(module_vid)
+
+                # Second pass: determine if lifecycle separation is needed
+                # Only separate if a table code has modules in BOTH starting and ending
+                # (indicating a version transition for that table)
+                needs_lifecycle_separation = any(
+                    code in starting_by_code and code in ending_by_code
+                    for code in set(starting_by_code.keys()) | set(ending_by_code.keys())
+                )
+
+                if needs_lifecycle_separation:
+                    # Separate into starting and ending scopes
+                    starting_modules = {}
+                    ending_modules = {}
+                    for code, vids in starting_by_code.items():
+                        starting_modules[code] = vids
+                    for code, vids in ending_by_code.items():
+                        ending_modules[code] = vids
+                    if starting_modules:
+                        cross_modules["_starting"] = starting_modules
+                    if ending_modules:
+                        cross_modules["_ending"] = ending_modules
+                else:
+                    # No version transitions - combine all modules by table code
+                    all_by_code = {}
+                    for code, vids in starting_by_code.items():
+                        if code not in all_by_code:
+                            all_by_code[code] = []
+                        all_by_code[code].extend(vids)
+                    for code, vids in ending_by_code.items():
+                        if code not in all_by_code:
+                            all_by_code[code] = []
+                        all_by_code[code].extend(vids)
+                    cross_modules = all_by_code
             else:
                 # Original logic for table VIDs
                 unique_operands_number = len(tables_vids) + len(precondition_items)
@@ -268,12 +301,21 @@ class OperationScopeService:
         :param modules_vids: list with module version ids
         """
         for module_vid in modules_vids:
-            from_date = modules_info[modules_info["ModuleVID"] == module_vid][
-                "FromReferenceDate"
-            ].values[0]
+            module_row = modules_info[modules_info["ModuleVID"] == module_vid].iloc[0]
+            from_date = module_row["FromReferenceDate"]
+            to_date = module_row["ToReferenceDate"]
+            module_code = module_row["ModuleCode"]
+            version_number = module_row["VersionNumber"]
             operation_scope = self.create_operation_scope(from_date)
             self.create_operation_scope_composition(
-                operation_scope=operation_scope, module_vid=module_vid
+                operation_scope=operation_scope,
+                module_vid=module_vid,
+                module_info={
+                    "code": module_code,
+                    "version_number": version_number,
+                    "from_reference_date": from_date,
+                    "to_reference_date": to_date,
+                },
             )
 
     def process_cross_module(self, cross_modules, modules_dataframe):
@@ -313,8 +355,18 @@ class OperationScopeService:
             operation_scope = self.create_operation_scope(from_submission_date)
             combination = set(combination)
             for module in combination:
+                module_row = modules_dataframe[
+                    modules_dataframe[MODULE_VID] == module
+                ].iloc[0]
                 self.create_operation_scope_composition(
-                    operation_scope=operation_scope, module_vid=module
+                    operation_scope=operation_scope,
+                    module_vid=module,
+                    module_info={
+                        "code": module_row["ModuleCode"],
+                        "version_number": module_row["VersionNumber"],
+                        "from_reference_date": module_row[FROM_REFERENCE_DATE],
+                        "to_reference_date": module_row[TO_REFERENCE_DATE],
+                    },
                 )
 
     def create_operation_scope(self, submission_date):
@@ -340,17 +392,21 @@ class OperationScopeService:
         self.session.add(operation_scope)
         return operation_scope
 
-    def create_operation_scope_composition(self, operation_scope, module_vid):
+    def create_operation_scope_composition(self, operation_scope, module_vid, module_info=None):
         """
         Method to populate OperationScopeComposition table
         :param operation_scope: Operation scope data
         :param module_vid: Module version id
+        :param module_info: Optional dict with module info (code, from_reference_date, to_reference_date)
         """
         operation_scope_composition = OperationScopeComposition(
             operation_scope=operation_scope,
             modulevid=module_vid,
             rowguid=str(uuid.uuid4()),
         )
+        # Store module info as transient attribute for to_dict() access
+        if module_info:
+            operation_scope_composition._module_info = module_info
         self.session.add(operation_scope_composition)
 
     def get_scopes_with_status(self):

{pydpm_xl-0.2.4.dist-info → pydpm_xl-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydpm_xl
-Version: 0.2.4
+Version: 0.2.5
 Summary: Python library for DPM-XL data processing and analysis
 Author-email: "MeaningfulData S.L." <info@meaningfuldata.eu>
 License: GPL-3.0-or-later
@@ -241,6 +241,59 @@ migration_api.migrate_from_access(
 )
 ```
 
+#### XBRL-CSV Instance Generation
+
+```python
+from py_dpm.api import InstanceAPI
+
+api = InstanceAPI()
+
+# Build package from dictionary
+data = {
+    "module_code": "F_01.01",
+    "parameters": {"refPeriod": "2024-12-31"},
+    "facts": [
+        {"table_code": "t001", "row_code": "r010", "column_code": "c010", "value": 1000000}
+    ]
+}
+output_path = api.build_package_from_dict(data, "/tmp/output")
+
+# Build package from JSON file
+output_path = api.build_package_from_json("instance_data.json", "/tmp/output")
+```
+
+#### DPM Explorer - Introspection Queries
+
+```python
+from py_dpm.api import ExplorerQueryAPI
+
+with ExplorerQueryAPI() as api:
+    # Find all properties using a specific item
+    properties = api.get_properties_using_item("EUR")
+
+    # Get module URL for documentation
+    url = api.get_module_url(module_code="F_01.01")
+
+    # Explore variable usage
+    tables = api.get_tables_using_variable(variable_code="mi123")
+```
+
+#### Hierarchical Queries
+
+```python
+from py_dpm.api import HierarchicalQueryAPI
+
+with HierarchicalQueryAPI() as api:
+    # Get hierarchy for a domain
+    hierarchy = api.get_hierarchy(domain_code="DOM_001")
+
+    # Navigate parent-child relationships
+    children = api.get_children(item_code="PARENT_001")
+
+    # Get all ancestors
+    ancestors = api.get_ancestors(item_code="LEAF_001")
+```
+
 ## Development
 
 ### Running Tests

{pydpm_xl-0.2.4.dist-info → pydpm_xl-0.2.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-py_dpm/__init__.py,sha256=LkRi-JRS2aWrSr5G5J8eTVRvZt1J1WfIrS2dXyTemmw,1858
+py_dpm/__init__.py,sha256=SPXDScW5Pf4DjIIpaWm9UcOpbqRo_2u7nEZIIJaMs-k,1858
 py_dpm/api/__init__.py,sha256=n79vAD7qatlYaXaI2N5IAD9m_8Fgb00EOdapVXZYTpI,1081
 py_dpm/api/dpm/__init__.py,sha256=HQflgiRbs1eDi3KTadNhxS1NoaG6PGQDVMvFnuIEfXo,506
 py_dpm/api/dpm/data_dictionary.py,sha256=g0h6Yfschz7rboYly9LTbP-2SS5UxltU3AXu0v0tqrU,29457
@@ -7,8 +7,8 @@ py_dpm/api/dpm/hierarchical_queries.py,sha256=X4AbpsWy3iItOTVIdVbtaTmRgOHPf0Y64I
 py_dpm/api/dpm/instance.py,sha256=v3DWzdaM5gPCecLjwjZ49FGfqZzUR3dPC0U8zGwdttk,3795
 py_dpm/api/dpm/migration.py,sha256=9FT7zzz4QdUIRR6MD01gMODBtfq9HH_RF4hRgZqMcZc,2404
 py_dpm/api/dpm_xl/__init__.py,sha256=aRjaMAf_i2a33UAGTg-TF1BfO6miOOrbCydTUqAVvRU,910
-py_dpm/api/dpm_xl/ast_generator.py,sha256=maVFlxBnBBZ6UYgVxbvewPFTbrTUqXfdKpmjB0ue0qc,42979
-py_dpm/api/dpm_xl/complete_ast.py,sha256=sOaN9uSQo-Nns8twElVQHBg7MnaDSo_8EuZPSIqGdU0,7125
+py_dpm/api/dpm_xl/ast_generator.py,sha256=GHu_F3YKaMpqvu16ABIvj8BwlmWXehvk0eUJVQkCphE,55204
+py_dpm/api/dpm_xl/complete_ast.py,sha256=VkmcBatrydu97Inwp3pHjz93F38q2JRo-4Lohdu30RY,7684
 py_dpm/api/dpm_xl/operation_scopes.py,sha256=7AyOFAn9h012JPF9H5EtZ3sPzv6DOxkoinpj5ArzVOc,48492
 py_dpm/api/dpm_xl/semantic.py,sha256=Buo_t-sEv65r6RmYDy1xkCWGlU2pB2WQsDM-X-FX4cc,13629
 py_dpm/api/dpm_xl/syntax.py,sha256=Ke_kKd9ModoJ6siL3GPT9j9QClmopryCRcdDAT3M5-E,5954
@@ -17,7 +17,7 @@ py_dpm/cli/main.py,sha256=v8ZgIjg4Zqf6UWNv2bydYlZV6KfDLnPACyqplNIJyNE,22447
 py_dpm/cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py_dpm/dpm/__init__.py,sha256=moagUo5Gxf24-Tl9FL_3n2wmVoD_oXtpC-YIGktH_rc,212
 py_dpm/dpm/migration.py,sha256=ivO_ObvKzVomTns6qfo-o5FuciWxkXbMd_gJ4_tu7Xc,14110
-py_dpm/dpm/models.py,sha256=d70oZ_3wXsKBRGM2TJRlsuDrHWSVkzetLAZeMIoImwc,124821
+py_dpm/dpm/models.py,sha256=nv6KMk0nYlXGWFrJEcqyzKHCaIDnMF5IQh0l5Ww8PTI,135212
 py_dpm/dpm/utils.py,sha256=JNdAeOXjzQtye94jLPRHHGUMcvkGtTsjA5HFl92rWig,12783
 py_dpm/dpm/queries/base.py,sha256=EddMeJMwtp63DyyIFO7_XxGvdlCtJQWWpeOVImlKp4I,3648
 py_dpm/dpm/queries/basic_objects.py,sha256=JOXC235lMDfVENrFAhZAl7_nqePJ4RrwJhFF0WDyk0M,955
@@ -68,7 +68,7 @@ py_dpm/dpm_xl/utils/__init__.py,sha256=4-jXa7AdHjx2DpikAzjZVKqBktdrHgSAx6pibb4sM
 py_dpm/dpm_xl/utils/data_handlers.py,sha256=a0E-IaP_-CDKLcj-Gt2ggAziKIOUiwnT2D9IkWCS68o,4402
 py_dpm/dpm_xl/utils/operands_mapping.py,sha256=LG0hPlUuTM2X2uWOtiD6HkmNeDEJkWJ8gV-Fxej_8QM,2241
 py_dpm/dpm_xl/utils/operator_mapping.py,sha256=BFgbVbSCSuutFNHJ4gtgm5VuG38pcl8Kmfi-sefg6JU,1913
-py_dpm/dpm_xl/utils/scopes_calculator.py,sha256=nCx2mz_qtw61BESp38ORQYlF2uRT8SyUKawSX9OQljM,17832
+py_dpm/dpm_xl/utils/scopes_calculator.py,sha256=do_emsUqD1TbrjguKlOOqFleaVhxzqm-NnlgdrdIb6I,20906
 py_dpm/dpm_xl/utils/serialization.py,sha256=LPcmudFfzHeEjIIr57kr5BvGPZbxshOAAeUYOrLl7XM,32482
 py_dpm/dpm_xl/utils/tokens.py,sha256=VRIrPDi5ttwgH-on5Qt4-l4ho4bLA755-nfTalponcA,3496
 py_dpm/exceptions/__init__.py,sha256=yDERfUxYW7NUUEiTQChGpuJx6abr7IDe2XUpwVFPtvM,416
@@ -76,9 +76,9 @@ py_dpm/exceptions/exceptions.py,sha256=6S3p-_i5O1oStvSMixt_JQG0xwTeSfBcdzrwL8yBy
 py_dpm/exceptions/messages.py,sha256=UwY6QIK8c-POcDCc9HYbZFGArCIYAanUGNh2LNKPx3U,7534
 py_dpm/instance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 py_dpm/instance/instance.py,sha256=OPSEPgSYAxhgqhKuxbMpMPTfBnaFNzURTrUUT4kvGKc,10820
-pydpm_xl-0.2.4.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-pydpm_xl-0.2.4.dist-info/METADATA,sha256=4tYu82PigByb4NHsUhtYMN3YsXhy_2FKjlmn9r_XrH8,7974
-pydpm_xl-0.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pydpm_xl-0.2.4.dist-info/entry_points.txt,sha256=6DDmBfw-AjtgvMHgq_I730i_LAAs_7-N3C95HD_bRr4,47
-pydpm_xl-0.2.4.dist-info/top_level.txt,sha256=495PvWZRoKl2NvbQU25W7dqWIBHqY-mFMPt83uxPpcM,7
-pydpm_xl-0.2.4.dist-info/RECORD,,
+pydpm_xl-0.2.5.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+pydpm_xl-0.2.5.dist-info/METADATA,sha256=4zUBTstmgwrkyneDaFLnbB89XiG7fY67Prjj-_UOw3E,9302
+pydpm_xl-0.2.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pydpm_xl-0.2.5.dist-info/entry_points.txt,sha256=6DDmBfw-AjtgvMHgq_I730i_LAAs_7-N3C95HD_bRr4,47
+pydpm_xl-0.2.5.dist-info/top_level.txt,sha256=495PvWZRoKl2NvbQU25W7dqWIBHqY-mFMPt83uxPpcM,7
+pydpm_xl-0.2.5.dist-info/RECORD,,