vtlengine 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

vtlengine/AST/Grammar/tokens.py

@@ -32,6 +32,7 @@ LN = "ln"
 POWER = "power"
 LOG = "log"
 SQRT = "sqrt"
+RANDOM = "random"
 # Boolean operators.
 AND = "and"
 OR = "or"
@@ -70,6 +71,7 @@ SYMDIFF = "symdiff"
 IF = "if"
 THEN = "then"
 ELSE = "else"
+CASE = "case"
 NVL = "nvl"
 # Clause Operators.
 FILTER = "filter"
@@ -89,6 +91,16 @@ STOCK_TO_FLOW = "stock_to_flow"
 TIMESHIFT = "timeshift"
 TIME_AGG = "time_agg"
 CURRENT_DATE = "current_date"
+DATE_DIFF = "date_diff"
+DATE_ADD = "date_add"
+YEAR = "year"
+MONTH = "month"
+DAYOFMONTH = "dayofmonth"
+DAYOFYEAR = "dayofyear"
+DAYTOYEAR = "daytoyear"
+DAYTOMONTH = "daytomonth"
+YEARTODAY = "yeartoday"
+MONTHTODAY = "monthtoday"
 # Join Operators.
 INNER_JOIN = "inner_join"
 LEFT_JOIN = "left_join"

vtlengine/AST/VtlVisitor.py

@@ -62,6 +62,10 @@ class VtlVisitor(ParseTreeVisitor):
     def visitIfExpr(self, ctx: Parser.IfExprContext):
         return self.visitChildren(ctx)
 
+    # Visit a parse tree produced by Parser#caseExpr.
+    def visitCaseExpr(self, ctx: Parser.CaseExprContext):
+        return self.visitChildren(ctx)
+
     # Visit a parse tree produced by Parser#clauseExpr.
     def visitClauseExpr(self, ctx: Parser.ClauseExprContext):
         return self.visitChildren(ctx)
@@ -90,6 +94,10 @@ class VtlVisitor(ParseTreeVisitor):
     def visitIfExprComp(self, ctx: Parser.IfExprCompContext):
         return self.visitChildren(ctx)
 
+    # Visit a parse tree produced by Parser#caseExprComp.
+    def visitCaseExprComp(self, ctx: Parser.CaseExprCompContext):
+        return self.visitChildren(ctx)
+
     # Visit a parse tree produced by Parser#comparisonExprComp.
     def visitComparisonExprComp(self, ctx: Parser.ComparisonExprCompContext):
         return self.visitChildren(ctx)
@@ -399,7 +407,7 @@ class VtlVisitor(ParseTreeVisitor):
         return self.visitChildren(ctx)
 
     # Visit a parse tree produced by Parser#periodAtomComponent.
-    def visitPeriodAtomComponent(self, ctx: Parser.PeriodAtomComponentContext):
+    def visitTimeUnaryAtomComponent(self, ctx: Parser.PeriodAtomComponentContext):
         return self.visitChildren(ctx)
 
     # Visit a parse tree produced by Parser#fillTimeAtomComponent.

vtlengine/AST/__init__.py

@@ -344,6 +344,25 @@ class If(AST):
     elseOp: AST
 
 
+class CaseObj:
+    condition: AST
+    thenOp: AST
+
+    def __init__(self, condition: AST, thenOp: AST):
+        self.condition = condition
+        self.thenOp = thenOp
+
+
+@dataclass
+class Case(AST):
+    """
+    Case: (condition, thenOp, elseOp)
+    """
+
+    cases: List[CaseObj]
+    elseOp: AST
+
+
 @dataclass
 class Validation(AST):
     """

vtlengine/DataTypes/TimeHandling.py

@@ -145,6 +145,8 @@ class TimePeriodHandler:
     _period_number: int
 
     def __init__(self, period: str) -> None:
+        if isinstance(period, int):
+            period = str(period)
         if "-" in period:
             self.year, self.period_indicator, self.period_number = (
                 from_input_customer_support_to_internal(period)

vtlengine/Exceptions/__init__.py

@@ -130,3 +130,47 @@ class InputValidationException(VTLEngineException):
             super().__init__(message, lino, colno, code)
         else:
             super().__init__(message, lino, colno)
+
+
+def check_key(field: str, dict_keys: Any, key: str) -> None:
+    if key not in dict_keys:
+        closest_key = find_closest_key(dict_keys, key)
+        message_append = f". Did you mean {closest_key}?" if closest_key else ""
+        raise SemanticError("0-1-1-13", field=field, key=key, closest_key=message_append)
+
+
+def find_closest_key(dict_keys: Any, key: str) -> Optional[str]:
+    closest_key = None
+    max_distance = 3
+    min_distance = float('inf')
+
+    for dict_key in dict_keys:
+        distance = key_distance(key, dict_key)
+        if distance < min_distance:
+            min_distance = distance
+            closest_key = dict_key
+
+    if min_distance <= max_distance:
+        return closest_key
+    return None
+
+
+def key_distance(key: str, objetive: str) -> int:
+    dp = [[0] * (len(objetive) + 1) for _ in range(len(key) + 1)]
+
+    for i in range(len(key) + 1):
+        dp[i][0] = i
+    for j in range(len(objetive) + 1):
+        dp[0][j] = j
+
+    for i in range(1, len(key) + 1):
+        for j in range(1, len(objetive) + 1):
+            if key[i - 1] == objetive[j - 1]:
+                cost = 0
+            else:
+                cost = 1
+            dp[i][j] = min(dp[i - 1][j] + 1,
+                           dp[i][j - 1] + 1,
+                           dp[i - 1][j - 1] + cost)
+
+    return dp[-1][-1]

vtlengine/Exceptions/messages.py

@@ -20,21 +20,21 @@ centralised_messages = {
     # Infer Data Structure errors
     # "0-1-1-1": "A csv file or a dataframe is required.",
     "0-1-1-2": "The provided {source} must have data to can infer the data structure.",
-    "0-1-1-3": "Can not infer data structure: {errors}",
+    "0-1-1-3": "Can not infer data structure: {errors}.",
     "0-1-1-4": "On Dataset {name} loading:  An identifier cannot have null values, found null "
     "values on {null_identifier}.",
     "0-1-1-5": "On Dataset {name} loading: Datasets without identifiers must have 0 or "
     "1 datapoints.",
     "0-1-1-6": "Duplicated records. Combination of identifiers are repeated.",
-    "0-1-1-7": "G1 - The provided CSV file is empty",
-    "0-1-1-8": "The following identifiers {ids} were not found , review file {file}",
+    "0-1-1-7": "G1 - The provided CSV file is empty.",
+    "0-1-1-8": "The following identifiers {ids} were not found , review file {file}.",
     "0-1-1-9": "You have a problem related with commas, review rfc4180 standard, review file "
-    "{file}",
+    "{file}.",
     "0-1-1-10": "On Dataset {name} loading: Component {comp_name} is missing in Datapoints.",
-    "0-1-1-11": "Wrong data in the file for this scalardataset {name}",
-    "0-1-1-12": "On Dataset {name} loading: not possible to cast column {column} to {type}",
-    #
-    "0-1-0-1": " Trying to redefine input datasets {dataset}",  # Semantic Error
+    "0-1-1-11": "Wrong data in the file for this scalardataset {name}.",
+    "0-1-1-12": "On Dataset {name} loading: not possible to cast column {column} to {type}.",
+    "0-1-1-13": "Invalid key on {field} field: {key}{closest_key}.",
+    "0-1-0-1": " Trying to redefine input datasets {dataset}.",  # Semantic Error
     # ------------Operators-------------
     # General Semantic errors
     # "1-1-1-1": "At op {op}. Unable to validate types.",
@@ -134,6 +134,14 @@ centralised_messages = {
     "be Datasets or at least one of them a Scalar.",
     "1-1-9-13": "At op {op}: then {then} and else {else_clause} datasets must contain the same "
     "number of components.",
+    "2-1-9-1": "At op {op}: Condition operators must have the same operator type.",
+    "2-1-9-2": "At op {op}: Condition {name} it's not a boolean.",
+    "2-1-9-3": "At op {op}: All then and else operands must be scalars.",
+    "2-1-9-4": "At op {op}: Condition {name} must be boolean type.",
+    "2-1-9-5": "At op {op}: Condition Dataset {name} measure must be Boolean.",
+    "2-1-9-6": "At op {op}: At least a then or else operand must be Dataset.",
+    "2-1-9-7": "At op {op}: All Dataset operands must have the same components.",
+
     # Data Validation errors
     "1-1-10-1": "At op {op}: The {op_type} operand must have exactly one measure of type {me_type}",
     "1-1-10-2": "At op {op}: Number of variable has to be equal between the call and signature.",

vtlengine/Interpreter/__init__.py

@@ -18,7 +18,7 @@ from vtlengine.Operators.Aggregation import extract_grouping_identifiers
 from vtlengine.Operators.Assignment import Assignment
 from vtlengine.Operators.CastOperator import Cast
 from vtlengine.Operators.Comparison import Between, ExistIn
-from vtlengine.Operators.Conditional import If
+from vtlengine.Operators.Conditional import If, Case
 from vtlengine.Operators.General import Eval
 from vtlengine.Operators.HROperators import get_measure_from_dataset, HAAssignment, Hierarchy
 from vtlengine.Operators.Numeric import Round, Trunc
@@ -73,6 +73,7 @@ from vtlengine.AST.Grammar.tokens import (
     EQ,
     CURRENT_DATE,
     CALC,
+    COUNT,
 )
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import (
@@ -103,7 +104,7 @@ class InterpreterAnalyzer(ASTTemplate):
     # Time Period Representation
     time_period_representation: Optional[TimePeriodRepresentation] = None
     # Flags to change behavior
-    nested_if: Union[str, bool] = False
+    nested_condition: Union[str, bool] = False
     is_from_assignment: bool = False
     is_from_component_assignment: bool = False
     is_from_regular_aggregation: bool = False
@@ -115,7 +116,7 @@ class InterpreterAnalyzer(ASTTemplate):
     is_from_condition: bool = False
     is_from_hr_val: bool = False
     is_from_hr_agg: bool = False
-    if_stack: Optional[List[str]] = None
+    condition_stack: Optional[List[str]] = None
     # Handlers for simplicity
     regular_aggregation_dataset: Optional[Dataset] = None
     aggregation_grouping: Optional[List[str]] = None
@@ -206,10 +207,10 @@ class InterpreterAnalyzer(ASTTemplate):
 
             # Reset some handlers (joins and if)
             self.is_from_join = False
-            self.if_stack = None
+            self.condition_stack = None
             self.then_condition_dataset = None
             self.else_condition_dataset = None
-            self.nested_if = False
+            self.nested_condition = False
 
             if result is None:
                 continue
@@ -361,8 +362,8 @@ class InterpreterAnalyzer(ASTTemplate):
         if (
             not self.is_from_condition
             and node.op != MEMBERSHIP
-            and self.if_stack is not None
-            and len(self.if_stack) > 0
+            and self.condition_stack is not None
+            and len(self.condition_stack) > 0
         ):
             is_from_if = self.is_from_if
             self.is_from_if = False
@@ -520,6 +521,7 @@ class InterpreterAnalyzer(ASTTemplate):
         return having
 
     def visit_Analytic(self, node: AST.Analytic) -> Any:  # noqa: C901
+        component_name = None
         if self.is_from_regular_aggregation:
             if self.regular_aggregation_dataset is None:
                 raise SemanticError("1-1-6-10")
@@ -527,6 +529,7 @@ class InterpreterAnalyzer(ASTTemplate):
                 operand = self.regular_aggregation_dataset
             else:
                 operand_comp = self.visit(node.operand)
+                component_name = operand_comp.name
                 measure_names = self.regular_aggregation_dataset.get_measures_names()
                 dataset_components = self.regular_aggregation_dataset.components.copy()
                 for name in measure_names:
@@ -598,6 +601,7 @@ class InterpreterAnalyzer(ASTTemplate):
             ordering=ordering,
             window=node.window,
             params=params,
+            component_name=component_name,
         )
         if not self.is_from_regular_aggregation:
             return result
@@ -610,7 +614,10 @@ class InterpreterAnalyzer(ASTTemplate):
         )
 
         # # Extracting the component we need (only measure)
-        measure_name = result.get_measures_names()[0]
+        if component_name is None or node.op == COUNT:
+            measure_name = result.get_measures_names()[0]
+        else:
+            measure_name = component_name
         # Joining the result with the original dataset
         if self.only_semantic:
             data = None
@@ -961,36 +968,52 @@ class InterpreterAnalyzer(ASTTemplate):
 
         # Analysis for data component and dataset
         else:
-            if self.if_stack is None:
-                self.if_stack = []
+            if self.condition_stack is None:
+                self.condition_stack = []
             if self.then_condition_dataset is None:
                 self.then_condition_dataset = []
             if self.else_condition_dataset is None:
                 self.else_condition_dataset = []
             self.generate_then_else_datasets(copy(condition))
 
-        self.if_stack.append(THEN_ELSE["then"])
+        self.condition_stack.append(THEN_ELSE["then"])
         self.is_from_if = True
-        self.nested_if = "T" if isinstance(node.thenOp, AST.If) else False
+        self.nested_condition = "T" if isinstance(node.thenOp, AST.If) else False
         thenOp = self.visit(node.thenOp)
         if isinstance(thenOp, Scalar) or not isinstance(node.thenOp, AST.BinOp):
             self.then_condition_dataset.pop()
-            self.if_stack.pop()
+            self.condition_stack.pop()
 
-        self.if_stack.append(THEN_ELSE["else"])
+        self.condition_stack.append(THEN_ELSE["else"])
         self.is_from_if = True
-        self.nested_if = "E" if isinstance(node.elseOp, AST.If) else False
+        self.nested_condition = "E" if isinstance(node.elseOp, AST.If) else False
         elseOp = self.visit(node.elseOp)
         if isinstance(elseOp, Scalar) or (
             not isinstance(node.elseOp, AST.BinOp) and not isinstance(node.elseOp, AST.If)
         ):
             if len(self.else_condition_dataset) > 0:
                 self.else_condition_dataset.pop()
-            if len(self.if_stack) > 0:
-                self.if_stack.pop()
+            if len(self.condition_stack) > 0:
+                self.condition_stack.pop()
 
         return If.analyze(condition, thenOp, elseOp)
 
+    def visit_Case(self, node: AST.Case) -> Any:
+        conditions: List[Any] = []
+        thenOps: List[Any] = []
+
+        if self.condition_stack is None:
+            self.condition_stack = []
+
+        while node.cases:
+            case = node.cases.pop(0)
+            self.is_from_condition = True
+            conditions.append(self.visit(case.condition))
+            self.is_from_condition = False
+            thenOps.append(self.visit(case.thenOp))
+
+        return Case.analyze(conditions, thenOps, self.visit(node.elseOp))
+
     def visit_RenameNode(self, node: AST.RenameNode) -> Any:
         if self.udo_params is not None:
             if "#" in node.old_name:
@@ -1323,9 +1346,11 @@ class InterpreterAnalyzer(ASTTemplate):
             if self.rule_data is None:
                 return None
             filtering_indexes = list(filter_comp.data[filter_comp.data == True].index)
+            nan_indexes = list(filter_comp.data[filter_comp.data.isnull()].index)
             # If no filtering indexes, then all datapoints are valid on DPR and HR
             if len(filtering_indexes) == 0 and not (self.is_from_hr_agg or self.is_from_hr_val):
                 self.rule_data["bool_var"] = True
+                self.rule_data.loc[nan_indexes, "bool_var"] = None
                 return self.rule_data
             non_filtering_indexes = list(set(filter_comp.data.index) - set(filtering_indexes))
 
@@ -1340,6 +1365,7 @@ class InterpreterAnalyzer(ASTTemplate):
                 self.rule_data, how="left", on=original_data.columns.tolist()
             )
             original_data.loc[non_filtering_indexes, "bool_var"] = True
+            original_data.loc[nan_indexes, "bool_var"] = None
             return original_data
         elif node.op in HR_COMP_MAPPING:
             self.is_from_assignment = True
@@ -1484,10 +1510,10 @@ class InterpreterAnalyzer(ASTTemplate):
                 data = condition.data
 
         if data is not None:
-            if self.nested_if and self.if_stack is not None:
+            if self.nested_condition and self.condition_stack is not None:
                 merge_df = (
                     self.then_condition_dataset[-1]
-                    if self.if_stack[-1] == THEN_ELSE["then"]
+                    if self.condition_stack[-1] == THEN_ELSE["then"]
                     else self.else_condition_dataset[-1]
                 )
                 indexes = merge_df.data[merge_df.data.columns[-1]]
@@ -1540,12 +1566,12 @@ class InterpreterAnalyzer(ASTTemplate):
         if (
             self.then_condition_dataset is None
             or self.else_condition_dataset is None
-            or self.if_stack is None
+            or self.condition_stack is None
         ):
             return left_operand, right_operand
         merge_dataset = (
             self.then_condition_dataset.pop()
-            if self.if_stack.pop() == THEN_ELSE["then"]
+            if self.condition_stack.pop() == THEN_ELSE["then"]
             else (self.else_condition_dataset.pop())
         )
         merge_index = merge_dataset.data[merge_dataset.get_measures_names()[0]].to_list()
@@ -1617,8 +1643,9 @@ class InterpreterAnalyzer(ASTTemplate):
 
         # Getting Dataset elements
         result_components = {
-            c_name: copy(comp)
-            for c_name, comp in self.ruleset_dataset.components.items()  # type: ignore[union-attr]
+            comp_name: copy(comp)
+            for comp_name, comp in
+            self.ruleset_dataset.components.items()  # type: ignore[union-attr]
         }
         if self.ruleset_signature is not None:
             hr_component = self.ruleset_signature["RULE_COMPONENT"]
@@ -1732,9 +1759,8 @@ class InterpreterAnalyzer(ASTTemplate):
                         signature_values[param["name"]] = self.visit(node.params[i])
                     elif param["type"] in ["Dataset", "Component"]:
                         if isinstance(node.params[i], AST.VarID):
-                            signature_values[param["name"]] = node.params[
-                                i
-                            ].value  # type: ignore[attr-defined]
+                            signature_values[param["name"]] = (
+                                node.params[i].value)  # type: ignore[attr-defined]
                         else:
                             param_element = self.visit(node.params[i])
                             if isinstance(param_element, Dataset):

vtlengine/Model/__init__.py

@@ -41,6 +41,13 @@ class Scalar:
         return same_name and same_type and same_value
 
 
+Role_keys = [
+    "Identifier",
+    "Attribute",
+    "Measure",
+]
+
+
 class Role(Enum):
     """
     Enum class for the role of a component  (Identifier, Attribute, Measure)

vtlengine/Operators/Analytic.py

@@ -58,6 +58,7 @@ class Analytic(Operator.Unary):
         ordering: Optional[List[OrderBy]],
         window: Optional[Windowing],
         params: Optional[List[int]],
+        component_name: Optional[str] = None,
     ) -> Dataset:
         if ordering is None:
             order_components = []
@@ -83,25 +84,51 @@ class Analytic(Operator.Unary):
                 raise SemanticError(
                     "1-1-1-10", op=cls.op, comp_name=comp_name, dataset_name=operand.name
                 )
-        measures = operand.get_measures()
-        if measures is None:
-            raise SemanticError("1-1-1-8", op=cls.op, name=operand.name)
-        if cls.type_to_check is not None:
-            for measure in measures:
-                unary_implicit_promotion(measure.data_type, cls.type_to_check)
-        if cls.return_type is not None:
-            for measure in measures:
-                new_measure = copy(measure)
-                new_measure.data_type = cls.return_type
-                result_components[measure.name] = new_measure
-        if cls.op == COUNT and len(measures) <= 1:
-            measure_name = COMP_NAME_MAPPING[cls.return_type]
-            nullable = False if len(measures) == 0 else measures[0].nullable
-            if len(measures) == 1:
-                del result_components[measures[0].name]
-            result_components[measure_name] = Component(
-                name=measure_name, data_type=cls.return_type, role=Role.MEASURE, nullable=nullable
-            )
+        if component_name is not None:
+            if cls.type_to_check is not None:
+                unary_implicit_promotion(
+                    operand.components[component_name].data_type, cls.type_to_check
+                )
+            if cls.return_type is not None:
+                result_components[component_name] = Component(
+                    name=component_name,
+                    data_type=cls.return_type,
+                    role=operand.components[component_name].role,
+                    nullable=operand.components[component_name].nullable,
+                )
+            if cls.op == COUNT:
+                measure_name = COMP_NAME_MAPPING[cls.return_type]
+                result_components[measure_name] = Component(
+                    name=measure_name,
+                    data_type=cls.return_type,
+                    role=Role.MEASURE,
+                    nullable=operand.components[component_name].nullable,
+                )
+                if component_name in result_components:
+                    del result_components[component_name]
+        else:
+            measures = operand.get_measures()
+            if len(measures) == 0:
+                raise SemanticError("1-1-1-8", op=cls.op, name=operand.name)
+            if cls.type_to_check is not None:
+                for measure in measures:
+                    unary_implicit_promotion(measure.data_type, cls.type_to_check)
+            if cls.return_type is not None:
+                for measure in measures:
+                    new_measure = copy(measure)
+                    new_measure.data_type = cls.return_type
+                    result_components[measure.name] = new_measure
+            if cls.op == COUNT and len(measures) <= 1:
+                measure_name = COMP_NAME_MAPPING[cls.return_type]
+                nullable = False if len(measures) == 0 else measures[0].nullable
+                if len(measures) == 1:
+                    del result_components[measures[0].name]
+                result_components[measure_name] = Component(
+                    name=measure_name,
+                    data_type=cls.return_type,
+                    role=Role.MEASURE,
+                    nullable=nullable,
+                )
 
         return Dataset(name="result", components=result_components, data=None)
 
@@ -205,12 +232,17 @@ class Analytic(Operator.Unary):
         ordering: Optional[List[OrderBy]],
         window: Optional[Windowing],
         params: Optional[List[int]],
+        component_name: Optional[str] = None,
     ) -> Dataset:
-        result = cls.validate(operand, partitioning, ordering, window, params)
+        result = cls.validate(operand, partitioning, ordering, window, params, component_name)
         df = operand.data.copy() if operand.data is not None else pd.DataFrame()
-        measure_names = operand.get_measures_names()
         identifier_names = operand.get_identifiers_names()
 
+        if component_name is not None:
+            measure_names = [component_name]
+        else:
+            measure_names = operand.get_measures_names()
+
         result.data = cls.analyticfunc(
             df=df,
             partitioning=partitioning,

vtlengine/Operators/Boolean.py

@@ -31,7 +31,7 @@ class Binary(Operator.Binary):
     @classmethod
     def apply_operation_two_series(cls, left_series: Any, right_series: Any) -> Any:
         result = cls.comp_op(
-            left_series.astype("bool[pyarrow]"), right_series.astype("bool[pyarrow]")
+            left_series.astype("boolean"), right_series.astype("boolean")
         )
         return result.replace({pd.NA: None}).astype(object)