vtlengine 1.1rc1__py3-none-any.whl → 1.1.1__py3-none-any.whl

vtlengine/AST/ASTTemplate.py

@@ -186,8 +186,6 @@ class ASTTemplate(NodeVisitor):
 
             return node.value
         """
-        if node.value in self.datasets:
-            return self.datasets[node.value]
         return node.value
 
     def visit_Optional(self, node: AST.Optional) -> AST.AST:
@@ -329,6 +327,18 @@ class ASTTemplate(NodeVisitor):
             self.visit(case.thenOp)
         self.visit(node.elseOp)
 
+    def visit_CaseObj(self, node: AST.CaseObj) -> Any:
+        """
+        CaseObj: (condition, thenOp)
+
+        Basic usage:
+
+            self.visit(node.condition)
+            self.visit(node.thenOp)
+        """
+        self.visit(node.condition)
+        self.visit(node.thenOp)
+
     def visit_Validation(self, node: AST.Validation) -> Any:
         """
         Validation: (op, validation, params, inbalance, invalid)
@@ -505,6 +515,12 @@ class ASTTemplate(NodeVisitor):
         for child in node.operands:
             self.visit(child)
 
+    def visit_ParFunction(self, node: AST.ParFunction) -> None:
+        """
+        ParFunction: (operand)
+        """
+        self.visit(node.operand)
+
     def visit_NoOp(self, node: AST.NoOp) -> None:  # pylint: disable=unused-argument
         """
         NoOp: ()
@@ -526,3 +542,13 @@ class ASTTemplate(NodeVisitor):
         """
         for param in node.params:
             self.visit(param)
+
+    def visit_Windowing(self, node: AST.Windowing) -> None:
+        """
+        Windowing: (type_, start, start_mode, stop, stop_mode)
+        """
+
+    def visit_Comment(self, node: AST.Comment) -> None:
+        """
+        Comment: (value)
+        """

vtlengine/AST/DAG/__init__.py

@@ -85,10 +85,13 @@ class DAGAnalyzer(ASTTemplate):
         all_output = []
         global_inputs = []
         inserted = []
+        persistent_datasets = []
         for key, statement in self.dependencies.items():
             outputs = statement[OUTPUTS]
             persistent = statement[PERSISTENT]
             reference = outputs + persistent
+            if len(persistent) == 1 and persistent[0] not in persistent_datasets:
+                persistent_datasets.append(persistent[0])
             deletion_key = key
             all_output.append(reference[0])
             for subKey, subStatement in self.dependencies.items():
@@ -127,10 +130,11 @@ class DAGAnalyzer(ASTTemplate):
                         statements[INSERT][key] = [element]
 
         statements[GLOBAL] = global_inputs
+        statements[PERSISTENT] = persistent_datasets
         return statements
 
     @classmethod
-    def createDAG(cls, ast: AST):
+    def createDAG(cls, ast: Start):
         """ """
         # Visit AST.
         dag = cls()
@@ -143,6 +147,11 @@ class DAGAnalyzer(ASTTemplate):
             # Create output dict.
             if len(dag.edges) != 0:
                 dag.sortAST(ast)
+            else:
+                MLStatements: list = [
+                    ML for ML in ast.children if not isinstance(ML, (HRuleset, DPRuleset, Operator))
+                ]
+                dag.check_overwriting(MLStatements)
             return dag
 
         except nx.NetworkXUnfeasible as error:

vtlengine/AST/__init__.py

@@ -11,7 +11,7 @@ from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Type, Union
 
 from vtlengine.DataTypes import ScalarType
-from vtlengine.Model import Role
+from vtlengine.Model import Dataset, Role
 
 
 @dataclass
@@ -20,6 +20,11 @@ class AST:
     AST: (children)
     """
 
+    line_start: int
+    column_start: int
+    line_stop: int
+    column_stop: int
+
     @classmethod
     def __all_annotations(cls) -> Dict[str, Any]:
         class_attributes = {}
@@ -47,6 +52,30 @@ class AST:
 
     __repr__ = __str__
 
+    def ast_equality(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        for k in self.__all_annotations():
+            if (
+                getattr(self, k) != getattr(other, k)
+                and k not in AST.__annotations__
+                and k != "children"  # We do not want to compare the children order here
+            ):
+                return False
+        return True
+
+    __eq__ = ast_equality
+
+
+@dataclass
+class Comment(AST):
+    """
+    Comment: (value)
+    """
+
+    value: str
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Start(AST):
@@ -56,6 +85,8 @@ class Start(AST):
 
     children: List[AST]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Assignment(AST):
@@ -67,6 +98,8 @@ class Assignment(AST):
     op: str
     right: AST
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class PersistentAssignment(Assignment):
@@ -85,7 +118,9 @@ class VarID(AST):
     Could be: DATASET or a COMPONENT.
     """
 
-    value: Any
+    value: str
+
+    __eq__ = AST.ast_equality
 
 
 @dataclass
@@ -100,6 +135,8 @@ class UnaryOp(AST):
     op: str
     operand: AST
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class BinOp(AST):
@@ -113,6 +150,8 @@ class BinOp(AST):
     op: str
     right: AST
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class MulOp(AST):
@@ -124,6 +163,8 @@ class MulOp(AST):
     op: str
     children: List[AST]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class ParamOp(AST):
@@ -137,12 +178,16 @@ class ParamOp(AST):
     children: List[AST]
     params: List[AST]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class UDOCall(AST):
     op: str
     params: List[AST]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class JoinOp(AST):
@@ -156,9 +201,11 @@ class JoinOp(AST):
 
     op: str
     clauses: List[AST]
-    using: Optional[List[AST]]
+    using: Optional[List[str]]
     isLast: bool = False
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Constant(AST):
@@ -172,6 +219,8 @@ class Constant(AST):
     type_: str
     value: Optional[Union[str, int, float, bool]]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class ParamConstant(Constant):
@@ -184,6 +233,8 @@ class ParamConstant(Constant):
     type_: str
     value: str
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Identifier(AST):
@@ -194,6 +245,8 @@ class Identifier(AST):
     value: str
     kind: str
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class ID(AST):
@@ -206,6 +259,8 @@ class ID(AST):
     type_: str
     value: str
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Collection(AST):
@@ -220,6 +275,8 @@ class Collection(AST):
     children: List[AST]
     kind: str = "Set"
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Windowing(AST):
@@ -234,11 +291,13 @@ class Windowing(AST):
     """
 
     type_: str
-    start: str
+    start: Union[int, str]
     start_mode: str
     stop: Union[int, str]
     stop_mode: str
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class OrderBy(AST):
@@ -249,6 +308,8 @@ class OrderBy(AST):
         if self.order not in ["asc", "desc"]:
             raise ValueError(f"Invalid order: {self.order}")
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Analytic(AST):
@@ -274,6 +335,8 @@ class Analytic(AST):
         if self.partition_by is None and self.order_by is None:
             raise ValueError("Partition by or order by must be provided on Analytic.")
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class RegularAggregation(AST):
@@ -288,6 +351,8 @@ class RegularAggregation(AST):
     dataset: Optional[AST] = None
     isLast: bool = False
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class RenameNode(AST):
@@ -298,6 +363,8 @@ class RenameNode(AST):
     old_name: str
     new_name: str
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Aggregation(AST):
@@ -316,6 +383,8 @@ class Aggregation(AST):
     grouping: Optional[List[AST]] = None
     having_clause: Optional[AST] = None
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class TimeAggregation(AST):
@@ -331,6 +400,8 @@ class TimeAggregation(AST):
     operand: Optional[AST] = None
     conf: Optional[str] = None
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class If(AST):
@@ -342,15 +413,14 @@ class If(AST):
     thenOp: AST
     elseOp: AST
 
+    __eq__ = AST.ast_equality
+
 
-class CaseObj:
+@dataclass
+class CaseObj(AST):
     condition: AST
     thenOp: AST
 
-    def __init__(self, condition: AST, thenOp: AST):
-        self.condition = condition
-        self.thenOp = thenOp
-
 
 @dataclass
 class Case(AST):
@@ -361,6 +431,8 @@ class Case(AST):
     cases: List[CaseObj]
     elseOp: AST
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Validation(AST):
@@ -375,6 +447,8 @@ class Validation(AST):
     imbalance: Optional[AST]
     invalid: bool
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class ComponentType(AST):
@@ -386,6 +460,8 @@ class ComponentType(AST):
     data_type: Optional[Type[ScalarType]] = None
     role: Optional[Role] = None
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class ASTScalarType(AST):
@@ -400,6 +476,8 @@ class DatasetType(AST):
 
     components: List[ComponentType]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Types(AST):
@@ -418,6 +496,8 @@ class Types(AST):
     nullable: Optional[bool]
     name: Optional[str] = None
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Argument(AST):
@@ -429,6 +509,8 @@ class Argument(AST):
     type_: Type[ScalarType]
     default: Optional[AST]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class Operator(AST):
@@ -441,6 +523,8 @@ class Operator(AST):
     output_type: str
     expression: AST
 
+    __eq__ = AST.ast_equality
+
 
 # TODO: Is this class necessary?
 @dataclass
@@ -452,6 +536,8 @@ class DefIdentifier(AST):
     value: str
     kind: str
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class DPRIdentifier(AST):
@@ -463,6 +549,8 @@ class DPRIdentifier(AST):
     kind: str
     alias: Optional[str] = None
 
+    __eq__ = AST.ast_equality
+
 
 # TODO: Are HRBinOp and HRUnOp necessary?
 @dataclass
@@ -476,6 +564,8 @@ class HRBinOp(AST):
     op: str
     right: DefIdentifier
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class HRUnOp(AST):
@@ -487,6 +577,8 @@ class HRUnOp(AST):
     op: str
     operand: DefIdentifier
 
+    __eq__ = AST.ast_equality
+
 
 # TODO: Unify HRule and DPRule?
 @dataclass
@@ -497,8 +589,10 @@ class HRule(AST):
 
     name: Optional[str]
     rule: HRBinOp
-    erCode: Optional[Constant]
-    erLevel: Optional[Constant]
+    erCode: Optional[str]
+    erLevel: Optional[int]
+
+    __eq__ = AST.ast_equality
 
 
 @dataclass
@@ -509,8 +603,10 @@ class DPRule(AST):
 
     name: Optional[str]
     rule: HRBinOp
-    erCode: Optional[Constant]
-    erLevel: Optional[Constant]
+    erCode: Optional[str]
+    erLevel: Optional[int]
+
+    __eq__ = AST.ast_equality
 
 
 # TODO: Unify HRuleset and DPRuleset?
@@ -525,6 +621,8 @@ class HRuleset(AST):
     element: DefIdentifier
     rules: List[HRule]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class DPRuleset(AST):
@@ -537,6 +635,8 @@ class DPRuleset(AST):
     params: Union[DefIdentifier, list]
     rules: List[DPRule]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class EvalOp(AST):
@@ -548,9 +648,11 @@ class EvalOp(AST):
 
     name: str
     operands: List[AST]
-    output: Optional[str]
+    output: Optional[Dataset]
     language: Optional[str]
 
+    __eq__ = AST.ast_equality
+
 
 @dataclass
 class NoOp(AST):
@@ -559,3 +661,14 @@ class NoOp(AST):
     """
 
     pass
+
+
+@dataclass
+class ParFunction(AST):
+    """
+    ParFunction: (operand)
+    """
+
+    operand: AST
+
+    __eq__ = AST.ast_equality

vtlengine/Exceptions/messages.py

@@ -15,6 +15,13 @@ centralised_messages = {
     "0-1-2-3": "Component {component} is duplicated.",
     "0-1-2-4": "Invalid json structure because {err} on file {filename}.",
     "0-1-2-5": "File {file} must be encoded in utf-8 (without BOM).",
+    # Run SDMX errors
+    "0-1-3-1": "Expected exactly one input dataset in the whole script, found: {number_datasets}",
+    "0-1-3-2": "SDMX Dataset {schema} requires to have a Schema object defined as structure",
+    "0-1-3-3": "If no mappings are provided, only one dataset is allowed.",
+    "0-1-3-4": "Dataset {short_urn} not found in mapping dictionary.",
+    "0-1-3-5": "Dataset {dataset_name} not found in the input datasets.",
+    "0-1-3-6": "Input name {missing} not found in the input datasets.",
     # JSON Schema validations
     "0-3-1-1": "Dataset {dataset} is not valid according to JSON schema",
     # Infer Data Structure errors
@@ -206,6 +213,8 @@ centralised_messages = {
     "1-1-19-9": "At op {op}: {op} can only be applied to a {comp_type} with a {param}",
     # New Unary time operators
     "1-1-19-10": "{op} can only be applied to operands with data type as Date or Time Period",
+    "1-1-19-11": "The time aggregation operand has to be "
+    "defined if not used inside an aggregation.",
     # Other time operators
     "2-1-19-1": "At op {op}: Invalid values {value_1} and {value_2} for duration, "
     "periodIndTo parameter must be a larger duration value than the "

vtlengine/Interpreter/__init__.py

@@ -8,9 +8,10 @@ import pandas as pd
 import vtlengine.AST as AST
 import vtlengine.Exceptions
 import vtlengine.Operators as Operators
+from vtlengine.AST import VarID
 from vtlengine.AST.ASTTemplate import ASTTemplate
 from vtlengine.AST.DAG import HRDAGAnalyzer
-from vtlengine.AST.DAG._words import DELETE, GLOBAL, INSERT
+from vtlengine.AST.DAG._words import DELETE, GLOBAL, INSERT, PERSISTENT
 from vtlengine.AST.Grammar.tokens import (
     AGGREGATE,
     ALL,
@@ -113,6 +114,8 @@ class InterpreterAnalyzer(ASTTemplate):
     output_path: Optional[Union[str, Path]] = None
     # Time Period Representation
     time_period_representation: Optional[TimePeriodRepresentation] = None
+    # Return only persistent
+    return_only_persistent: bool = True
     # Flags to change behavior
     nested_condition: Union[str, bool] = False
     is_from_assignment: bool = False
@@ -188,7 +191,9 @@ class InterpreterAnalyzer(ASTTemplate):
                 # We do not save global input datasets, only results of transformations
                 self.datasets[ds_name].data = None
                 continue
-
+            if self.return_only_persistent and ds_name not in self.ds_analysis[PERSISTENT]:
+                self.datasets[ds_name].data = None
+                continue
             # Saving only datasets, no scalars
             save_datapoints(
                 self.time_period_representation,
@@ -335,7 +340,7 @@ class InterpreterAnalyzer(ASTTemplate):
             for i, rule in enumerate(node.rules):
                 rule.name = (i + 1).__str__()
 
-        cond_comp = []
+        cond_comp: List[Any] = []
         if isinstance(node.element, list):
             cond_comp = [x.value for x in node.element[:-1]]
             node.element = node.element[-1]
@@ -369,6 +374,9 @@ class InterpreterAnalyzer(ASTTemplate):
     def visit_PersistentAssignment(self, node: AST.PersistentAssignment) -> Any:
         return self.visit_Assignment(node)
 
+    def visit_ParFunction(self, node: AST.ParFunction) -> Any:
+        return self.visit(node.operand)
+
     def visit_BinOp(self, node: AST.BinOp) -> Any:
         is_from_if = False
         if (
@@ -390,7 +398,13 @@ class InterpreterAnalyzer(ASTTemplate):
                 comp_name = f"{node.left.value}#{self.udo_params[-1][node.right.value]}"
             else:
                 comp_name = f"{node.left.value}#{node.right.value}"
-            ast_var_id = AST.VarID(value=comp_name)
+            ast_var_id = AST.VarID(
+                value=comp_name,
+                line_start=node.right.line_start,
+                line_stop=node.right.line_stop,
+                column_start=node.right.column_start,
+                column_stop=node.right.column_stop,
+            )
             return self.visit(ast_var_id)
         left_operand = self.visit(node.left)
         right_operand = self.visit(node.right)
@@ -572,6 +586,8 @@ class InterpreterAnalyzer(ASTTemplate):
                 for comp_name in node.partition_by:
                     if comp_name in self.udo_params[-1]:
                         partitioning.append(self.udo_params[-1][comp_name])
+                    elif comp_name in operand.get_identifiers_names():
+                        partitioning.append(comp_name)
                     else:
                         raise SemanticError(
                             "2-3-9",
@@ -583,7 +599,7 @@ class InterpreterAnalyzer(ASTTemplate):
                 for o in node.order_by:
                     if o.component in self.udo_params[-1]:
                         o.component = self.udo_params[-1][o.component]
-                    else:
+                    elif o.component not in operand.get_identifiers_names():
                         raise SemanticError(
                             "2-3-9",
                             comp_type="Component",
@@ -778,7 +794,7 @@ class InterpreterAnalyzer(ASTTemplate):
                             )
                         elif is_partial_present == 2:
                             raise SemanticError("1-1-13-9", comp_name=node.value)
-                        node.value = found_comp
+                        node.value = found_comp  # type:ignore[assignment]
                     if node.value not in self.regular_aggregation_dataset.components:
                         raise SemanticError(
                             "1-1-1-10",
@@ -1273,6 +1289,10 @@ class InterpreterAnalyzer(ASTTemplate):
                         signature_type=hr_info["node"].signature_type,
                         element=hr_info["node"].element,
                         rules=aux,
+                        line_start=node.line_start,
+                        line_stop=node.line_stop,
+                        column_start=node.column_start,
+                        column_stop=node.column_stop,
                     )
                     HRDAGAnalyzer().visit(hierarchy_ast)
 
@@ -1580,8 +1600,19 @@ class InterpreterAnalyzer(ASTTemplate):
                 )
             }
         )
+
+        if self.condition_stack and len(self.condition_stack) > 0:
+            last_condition_dataset = (
+                self.then_condition_dataset[-1]
+                if self.condition_stack[-1] == THEN_ELSE["then"]
+                else (self.else_condition_dataset[-1])
+            )
+            measure_name = last_condition_dataset.get_measures_names()[0]
+            then_data = then_data[then_data[name].isin(last_condition_dataset.data[measure_name])]
+            else_data = else_data[else_data[name].isin(last_condition_dataset.data[measure_name])]
         then_dataset = Dataset(name=name, components=components, data=then_data)
         else_dataset = Dataset(name=name, components=components, data=else_data)
+
         self.then_condition_dataset.append(then_dataset)
         self.else_condition_dataset.append(else_dataset)
 
@@ -1592,11 +1623,13 @@ class InterpreterAnalyzer(ASTTemplate):
             or self.condition_stack is None
         ):
             return left_operand, right_operand
+
         merge_dataset = (
             self.then_condition_dataset.pop()
             if self.condition_stack.pop() == THEN_ELSE["then"]
             else (self.else_condition_dataset.pop())
         )
+
         merge_index = merge_dataset.data[merge_dataset.get_measures_names()[0]].to_list()
         ids = merge_dataset.get_identifiers_names()
         if isinstance(left_operand, (Dataset, DataComponent)):
@@ -1873,8 +1906,20 @@ class InterpreterAnalyzer(ASTTemplate):
         return result
 
     def visit_TimeAggregation(self, node: AST.TimeAggregation) -> None:
-        operand = self.visit(node.operand)
-
+        if node.operand is not None:
+            operand = self.visit(node.operand)
+        else:
+            if self.aggregation_dataset is None:
+                raise SemanticError("1-1-19-11")
+            component_name = Time_Aggregation._get_time_id(self.aggregation_dataset)
+            ast_operand = VarID(
+                value=component_name,
+                line_start=node.line_start,
+                line_stop=node.line_stop,
+                column_start=node.column_start,
+                column_stop=node.column_stop,
+            )
+            operand = self.visit(ast_operand)
         return Time_Aggregation.analyze(
             operand=operand,
             period_from=node.period_from,