vtlengine 1.0.3rc2__py3-none-any.whl → 1.0.4__py3-none-any.whl

vtlengine/Interpreter/__init__.py

@@ -1,7 +1,7 @@
 from copy import copy, deepcopy
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
 import pandas as pd
 
@@ -68,10 +68,19 @@ from vtlengine.Operators.CastOperator import Cast
 from vtlengine.Operators.Comparison import Between, ExistIn
 from vtlengine.Operators.Conditional import Case, If
 from vtlengine.Operators.General import Eval
-from vtlengine.Operators.HROperators import HAAssignment, Hierarchy, get_measure_from_dataset
+from vtlengine.Operators.HROperators import (
+    HAAssignment,
+    Hierarchy,
+    get_measure_from_dataset,
+)
 from vtlengine.Operators.Numeric import Round, Trunc
 from vtlengine.Operators.String import Instr, Replace, Substr
-from vtlengine.Operators.Time import Current_Date, Date_Add, Fill_time_series, Time_Aggregation
+from vtlengine.Operators.Time import (
+    Current_Date,
+    Date_Add,
+    Fill_time_series,
+    Time_Aggregation,
+)
 from vtlengine.Operators.Validation import Check, Check_Datapoint, Check_Hierarchy
 from vtlengine.Utils import (
     AGGREGATION_MAPPING,
@@ -153,7 +162,9 @@ class InterpreterAnalyzer(ASTTemplate):
         for ds_name in self.ds_analysis[INSERT][statement_num]:
             if ds_name in self.datapoints_paths:
                 self.datasets[ds_name].data = load_datapoints(
-                    self.datasets[ds_name].components, ds_name, self.datapoints_paths[ds_name]
+                    self.datasets[ds_name].components,
+                    ds_name,
+                    self.datapoints_paths[ds_name],
                 )
             elif ds_name in self.datasets and self.datasets[ds_name].data is None:
                 _fill_dataset_empty_data(self.datasets[ds_name])
@@ -180,7 +191,9 @@ class InterpreterAnalyzer(ASTTemplate):
 
             # Saving only datasets, no scalars
             save_datapoints(
-                self.time_period_representation, self.datasets[ds_name], self.output_path
+                self.time_period_representation,
+                self.datasets[ds_name],
+                self.output_path,
             )
             self.datasets[ds_name].data = None
 
@@ -201,8 +214,9 @@ class InterpreterAnalyzer(ASTTemplate):
             if isinstance(child, (AST.Assignment, AST.PersistentAssignment)):
                 vtlengine.Exceptions.dataset_output = child.left.value  # type: ignore[attr-defined]
                 self._load_datapoints_efficient(statement_num)
-            if (not isinstance(child, (AST.HRuleset, AST.DPRuleset, AST.Operator)) and
-                    not isinstance(child, (AST.Assignment, AST.PersistentAssignment))):
+            if not isinstance(
+                child, (AST.HRuleset, AST.DPRuleset, AST.Operator)
+            ) and not isinstance(child, (AST.Assignment, AST.PersistentAssignment)):
                 raise SemanticError("1-3-17")
             result = self.visit(child)
 
@@ -229,15 +243,14 @@ class InterpreterAnalyzer(ASTTemplate):
     # Definition Language
 
     def visit_Operator(self, node: AST.Operator) -> None:
-
         if self.udos is None:
             self.udos = {}
         elif node.op in self.udos:
             raise ValueError(f"User Defined Operator {node.op} already exists")
 
-        param_info = []
+        param_info: List[Dict[str, Union[str, Type[ScalarType], AST.AST]]] = []
         for param in node.parameters:
-            if param.name in param_info:
+            if param.name in [x["name"] for x in param_info]:
                 raise ValueError(f"Duplicated Parameter {param.name} in UDO {node.op}")
             # We use a string for model types, but the data type class for basic types
             # (Integer, Number, String, Boolean, ...)
@@ -262,7 +275,6 @@ class InterpreterAnalyzer(ASTTemplate):
         }
 
     def visit_DPRuleset(self, node: AST.DPRuleset) -> None:
-
         # Rule names are optional, if not provided, they are generated.
         # If provided, all must be provided
         rule_names = [rule.name for rule in node.rules if rule.name is not None]
@@ -358,7 +370,6 @@ class InterpreterAnalyzer(ASTTemplate):
         return self.visit_Assignment(node)
 
     def visit_BinOp(self, node: AST.BinOp) -> Any:
-
         is_from_if = False
         if (
             not self.is_from_condition
@@ -369,8 +380,12 @@ class InterpreterAnalyzer(ASTTemplate):
             is_from_if = self.is_from_if
             self.is_from_if = False
 
-        if (self.is_from_join and node.op in [MEMBERSHIP, AGGREGATE] and
-                hasattr(node.left, "value") and hasattr(node.right, "value")):
+        if (
+            self.is_from_join
+            and node.op in [MEMBERSHIP, AGGREGATE]
+            and hasattr(node.left, "value")
+            and hasattr(node.right, "value")
+        ):
             if self.udo_params is not None and node.right.value in self.udo_params[-1]:
                 comp_name = f"{node.left.value}#{self.udo_params[-1][node.right.value]}"
             else:
@@ -421,7 +436,10 @@ class InterpreterAnalyzer(ASTTemplate):
             if node.operand is not None and operand is not None:
                 op_comp: DataComponent = self.visit(node.operand)
                 comps_to_keep = {}
-                for comp_name, comp in self.regular_aggregation_dataset.components.items():
+                for (
+                    comp_name,
+                    comp,
+                ) in self.regular_aggregation_dataset.components.items():
                     if comp.role == Role.IDENTIFIER:
                         comps_to_keep[comp_name] = copy(comp)
                 comps_to_keep[op_comp.name] = Component(
@@ -744,8 +762,12 @@ class InterpreterAnalyzer(ASTTemplate):
                         is_partial_present = 0
                         found_comp = None
                         for comp_name in self.regular_aggregation_dataset.get_components_names():
-                            if ("#" in comp_name and comp_name.split("#")[1] == node.value or "#"
-                                    in node.value and node.value.split("#")[1] == comp_name):
+                            if (
+                                "#" in comp_name
+                                and comp_name.split("#")[1] == node.value
+                                or "#" in node.value
+                                and node.value.split("#")[1] == comp_name
+                            ):
                                 is_partial_present += 1
                                 found_comp = comp_name
                         if is_partial_present == 0:
@@ -783,7 +805,9 @@ class InterpreterAnalyzer(ASTTemplate):
             comp_name = self.ruleset_signature[node.value]
             if comp_name not in self.ruleset_dataset.components:
                 raise SemanticError(
-                    "1-1-1-10", comp_name=node.value, dataset_name=self.ruleset_dataset.name
+                    "1-1-1-10",
+                    comp_name=node.value,
+                    dataset_name=self.ruleset_dataset.name,
                 )
             data = None if self.rule_data is None else self.rule_data[comp_name]
             return DataComponent(
@@ -938,7 +962,6 @@ class InterpreterAnalyzer(ASTTemplate):
         return REGULAR_AGGREGATION_MAPPING[node.op].analyze(operands, dataset)
 
     def visit_If(self, node: AST.If) -> Dataset:
-
         self.is_from_condition = True
         condition = self.visit(node.condition)
         self.is_from_condition = False
@@ -948,7 +971,10 @@ class InterpreterAnalyzer(ASTTemplate):
             elseValue = self.visit(node.elseOp)
             if not isinstance(thenValue, Scalar) or not isinstance(elseValue, Scalar):
                 raise SemanticError(
-                    "1-1-9-3", op="If_op", then_name=thenValue.name, else_name=elseValue.name
+                    "1-1-9-3",
+                    op="If_op",
+                    then_name=thenValue.name,
+                    else_name=elseValue.name,
                 )
             if condition.value:
                 return self.visit(node.thenOp)
@@ -1024,7 +1050,9 @@ class InterpreterAnalyzer(ASTTemplate):
 
     def visit_Constant(self, node: AST.Constant) -> Any:
         return Scalar(
-            name=str(node.value), value=node.value, data_type=BASIC_TYPES[type(node.value)]
+            name=str(node.value),
+            value=node.value,
+            data_type=BASIC_TYPES[type(node.value)],
         )
 
     def visit_JoinOp(self, node: AST.JoinOp) -> None:
@@ -1130,7 +1158,9 @@ class InterpreterAnalyzer(ASTTemplate):
                 for comp_name in node.children[2:]:
                     if comp_name.__str__() not in dataset_element.components:
                         raise SemanticError(
-                            "1-1-1-10", comp_name=comp_name, dataset_name=dataset_element.name
+                            "1-1-1-10",
+                            comp_name=comp_name,
+                            dataset_name=dataset_element.name,
                         )
                 if dpr_info is not None and dpr_info["signature_type"] == "variable":
                     for i, comp_name in enumerate(node.children[2:]):
@@ -1164,7 +1194,9 @@ class InterpreterAnalyzer(ASTTemplate):
 
             # Datapoint Ruleset final evaluation
             return Check_Datapoint.analyze(
-                dataset_element=dataset_element, rule_info=rule_output_values, output=output
+                dataset_element=dataset_element,
+                rule_info=rule_output_values,
+                output=output,
             )
         elif node.op in (CHECK_HIERARCHY, HIERARCHY):
             if len(node.children) == 3:
@@ -1203,7 +1235,10 @@ class InterpreterAnalyzer(ASTTemplate):
                     and hr_info["signature"] != component
                 ):
                     raise SemanticError(
-                        "1-1-10-3", op=node.op, found=component, expected=hr_info["signature"]
+                        "1-1-10-3",
+                        op=node.op,
+                        found=component,
+                        expected=hr_info["signature"],
                     )
                 elif hr_info["node"].signature_type == "valuedomain" and component is None:
                     raise SemanticError("1-1-10-4", op=node.op)
@@ -1215,7 +1250,10 @@ class InterpreterAnalyzer(ASTTemplate):
                         and cond_components[i] != cond_comp
                     ):
                         raise SemanticError(
-                            "1-1-10-6", op=node.op, expected=cond_comp, found=cond_components[i]
+                            "1-1-10-6",
+                            op=node.op,
+                            expected=cond_comp,
+                            found=cond_components[i],
                         )
                     cond_info[cond_comp] = cond_components[i]
 
@@ -1270,7 +1308,9 @@ class InterpreterAnalyzer(ASTTemplate):
                 # Final evaluation
                 if node.op == CHECK_HIERARCHY:
                     result = Check_Hierarchy.analyze(
-                        dataset_element=dataset, rule_info=rule_output_values, output=output
+                        dataset_element=dataset,
+                        rule_info=rule_output_values,
+                        output=output,
                     )
                     del rule_output_values
                 else:
@@ -1396,10 +1436,12 @@ class InterpreterAnalyzer(ASTTemplate):
                     left_operand.data = pd.DataFrame({measure_name: []})
                 if right_operand.data is None:
                     right_operand.data = pd.DataFrame({measure_name: []})
-                left_null_indexes = set(left_operand.data[left_operand.data[
-                    measure_name].isnull()].index)
-                right_null_indexes = set(right_operand.data[right_operand.data[
-                    measure_name].isnull()].index)
+                left_null_indexes = set(
+                    left_operand.data[left_operand.data[measure_name].isnull()].index
+                )
+                right_null_indexes = set(
+                    right_operand.data[right_operand.data[measure_name].isnull()].index
+                )
                 # If no indexes are in common, then one datapoint is not null
                 invalid_indexes = list(left_null_indexes.intersection(right_null_indexes))
                 if len(invalid_indexes) > 0:
@@ -1415,7 +1457,6 @@ class InterpreterAnalyzer(ASTTemplate):
         return HR_UNARY_MAPPING[node.op].analyze(operand)
 
     def visit_Validation(self, node: AST.Validation) -> Dataset:
-
         validation_element = self.visit(node.validation)
         if not isinstance(validation_element, Dataset):
             raise ValueError(f"Expected dataset, got {type(validation_element).__name__}")
@@ -1532,7 +1573,10 @@ class InterpreterAnalyzer(ASTTemplate):
         components.update(
             {
                 name: Component(
-                    name=name, data_type=BASIC_TYPES[int], role=Role.MEASURE, nullable=True
+                    name=name,
+                    data_type=BASIC_TYPES[int],
+                    role=Role.MEASURE,
+                    nullable=True,
                 )
             }
         )
@@ -1555,7 +1599,7 @@ class InterpreterAnalyzer(ASTTemplate):
         )
         merge_index = merge_dataset.data[merge_dataset.get_measures_names()[0]].to_list()
         ids = merge_dataset.get_identifiers_names()
-        if isinstance(left_operand, Dataset | DataComponent):
+        if isinstance(left_operand, (Dataset, DataComponent)):
             if left_operand.data is None:
                 return left_operand, right_operand
             if isinstance(left_operand, Dataset):
@@ -1571,7 +1615,7 @@ class InterpreterAnalyzer(ASTTemplate):
             else:
                 left = left_operand.data
                 left_operand.data = left.reindex(merge_index, fill_value=None)
-        if isinstance(right_operand, Dataset | DataComponent):
+        if isinstance(right_operand, (Dataset, DataComponent)):
             if right_operand.data is None:
                 return left_operand, right_operand
             if isinstance(right_operand, Dataset):
@@ -1737,9 +1781,7 @@ class InterpreterAnalyzer(ASTTemplate):
                         signature_values[param["name"]] = self.visit(node.params[i])
                     elif param["type"] in ["Dataset", "Component"]:
                         if isinstance(node.params[i], AST.VarID):
-                            signature_values[param["name"]] = node.params[
-                                i
-                            ].value  # type: ignore[attr-defined]
+                            signature_values[param["name"]] = node.params[i].value  # type: ignore[attr-defined]
                         else:
                             param_element = self.visit(node.params[i])
                             if isinstance(param_element, Dataset):
@@ -1834,5 +1876,8 @@ class InterpreterAnalyzer(ASTTemplate):
         operand = self.visit(node.operand)
 
         return Time_Aggregation.analyze(
-            operand=operand, period_from=node.period_from, period_to=node.period_to, conf=node.conf
+            operand=operand,
+            period_from=node.period_from,
+            period_to=node.period_to,
+            conf=node.conf,
         )

vtlengine/Model/__init__.py

@@ -229,23 +229,21 @@ class Dataset:
                 self.data[comp.name] = self.data[comp.name].astype(str)
                 other.data[comp.name] = other.data[comp.name].astype(str)
                 self.data[comp.name] = self.data[comp.name].map(
-                    lambda x: str(TimePeriodHandler(x)) if x != "" else "", na_action="ignore"
+                    lambda x: str(TimePeriodHandler(x)) if x != "" else "",
+                    na_action="ignore",
                 )
                 other.data[comp.name] = other.data[comp.name].map(
-                    lambda x: str(TimePeriodHandler(x)) if x != "" else "", na_action="ignore"
+                    lambda x: str(TimePeriodHandler(x)) if x != "" else "",
+                    na_action="ignore",
                 )
             elif type_name in ["Integer", "Number"]:
                 type_ = "int64" if type_name == "Integer" else "float32"
                 # We use here a number to avoid errors on equality on empty strings
                 self.data[comp.name] = (
-                    self.data[comp.name]
-                    .replace("", -1234997)
-                    .astype(type_)  # type: ignore[call-overload]
+                    self.data[comp.name].replace("", -1234997).astype(type_)  # type: ignore[call-overload]
                 )
                 other.data[comp.name] = (
-                    other.data[comp.name]
-                    .replace("", -1234997)
-                    .astype(type_)  # type: ignore[call-overload]
+                    other.data[comp.name].replace("", -1234997).astype(type_)  # type: ignore[call-overload]
                 )
         try:
             assert_frame_equal(
@@ -334,7 +332,7 @@ class Dataset:
         return {
             "name": self.name,
             "components": {k: v.to_dict() for k, v in self.components.items()},
-            "data": self.data.to_dict(orient="records") if self.data is not None else None,
+            "data": (self.data.to_dict(orient="records") if self.data is not None else None),
         }
 
     def to_json(self) -> str:

vtlengine/Operators/Aggregation.py

@@ -29,8 +29,8 @@ from vtlengine.DataTypes import (
     unary_implicit_promotion,
 )
 from vtlengine.DataTypes.TimeHandling import (
-    DURATION_MAPPING,
-    DURATION_MAPPING_REVERSED,
+    PERIOD_IND_MAPPING,
+    PERIOD_IND_MAPPING_REVERSE,
     TimeIntervalHandler,
     TimePeriodHandler,
 )
@@ -89,7 +89,10 @@ class Aggregation(Operator.Unary):
                     data[measure.name] = (
                         data[measure.name]
                         .astype(object)
-                        .map(lambda x: TimeIntervalHandler.from_iso_format(x), na_action="ignore")
+                        .map(
+                            lambda x: TimeIntervalHandler.from_iso_format(x),
+                            na_action="ignore",
+                        )
                     )
                 else:
                     data[measure.name] = data[measure.name].map(
@@ -100,11 +103,11 @@ class Aggregation(Operator.Unary):
             elif measure.data_type == Duration:
                 if mode == "input":
                     data[measure.name] = data[measure.name].map(
-                        lambda x: DURATION_MAPPING[x], na_action="ignore"
+                        lambda x: PERIOD_IND_MAPPING[x], na_action="ignore"
                     )
                 else:
                     data[measure.name] = data[measure.name].map(
-                        lambda x: DURATION_MAPPING_REVERSED[x], na_action="ignore"
+                        lambda x: PERIOD_IND_MAPPING_REVERSE[x], na_action="ignore"
                     )
             elif measure.data_type == Boolean:
                 if mode == "result":
@@ -128,7 +131,10 @@ class Aggregation(Operator.Unary):
             for comp_name in grouping_columns:
                 if comp_name not in operand.components:
                     raise SemanticError(
-                        "1-1-1-10", op=cls.op, comp_name=comp_name, dataset_name=operand.name
+                        "1-1-1-10",
+                        op=cls.op,
+                        comp_name=comp_name,
+                        dataset_name=operand.name,
                     )
                 if operand.components[comp_name].role != Role.IDENTIFIER:
                     raise SemanticError(
@@ -223,7 +229,7 @@ class Aggregation(Operator.Unary):
             )
 
         try:
-            return duckdb.query(query).to_df()
+            return duckdb.query(query).to_df().astype(object)
         except RuntimeError as e:
             if "Conversion" in e.args[0]:
                 raise SemanticError("2-3-8", op=cls.op, msg=e.args[0].split(":")[-1])

vtlengine/Operators/Analytic.py

@@ -29,10 +29,17 @@ from vtlengine.AST.Grammar.tokens import (
     VAR_POP,
     VAR_SAMP,
 )
-from vtlengine.DataTypes import COMP_NAME_MAPPING, Integer, Number, unary_implicit_promotion
+from vtlengine.DataTypes import (
+    COMP_NAME_MAPPING,
+    Integer,
+    Number,
+    unary_implicit_promotion,
+)
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, Dataset, Role
 
+return_integer_operators = [MAX, MIN, SUM]
+
 
 # noinspection PyMethodOverriding
 class Analytic(Operator.Unary):
@@ -47,10 +54,11 @@ class Analytic(Operator.Unary):
         Evaluate: Ensures the type of data is the correct one to perform the Analytic operators.
     """
 
+    return_integer = None
     sql_op: Optional[str] = None
 
     @classmethod
-    def validate(  # type: ignore[override]
+    def validate(  # type: ignore[override]  # noqa: C901
         cls,
         operand: Dataset,
         partitioning: List[str],
@@ -66,7 +74,10 @@ class Analytic(Operator.Unary):
         for comp_name in partitioning:
             if comp_name not in operand.components:
                 raise SemanticError(
-                    "1-1-1-10", op=cls.op, comp_name=comp_name, dataset_name=operand.name
+                    "1-1-1-10",
+                    op=cls.op,
+                    comp_name=comp_name,
+                    dataset_name=operand.name,
                 )
             if comp_name not in identifier_names:
                 raise SemanticError(
@@ -78,14 +89,21 @@ class Analytic(Operator.Unary):
         for comp_name in order_components:
             if comp_name not in operand.components:
                 raise SemanticError(
-                    "1-1-1-10", op=cls.op, comp_name=comp_name, dataset_name=operand.name
+                    "1-1-1-10",
+                    op=cls.op,
+                    comp_name=comp_name,
+                    dataset_name=operand.name,
                 )
         if component_name is not None:
             if cls.type_to_check is not None:
                 unary_implicit_promotion(
                     operand.components[component_name].data_type, cls.type_to_check
                 )
-            if cls.return_type is not None:
+
+            if cls.op in return_integer_operators:
+                cls.return_integer = isinstance(cls.return_type, Integer)
+
+            elif cls.return_type is not None:
                 result_components[component_name] = Component(
                     name=component_name,
                     data_type=cls.return_type,
@@ -106,14 +124,28 @@ class Analytic(Operator.Unary):
             measures = operand.get_measures()
             if len(measures) == 0:
                 raise SemanticError("1-1-1-8", op=cls.op, name=operand.name)
+
+            if cls.op in return_integer_operators:
+                isNumber = False
+                for measure in measures:
+                    isNumber |= isinstance(measure.data_type, Number)
+                cls.return_integer = not isNumber
+
             if cls.type_to_check is not None:
                 for measure in measures:
                     unary_implicit_promotion(measure.data_type, cls.type_to_check)
-            if cls.return_type is not None:
+
+            if cls.op in return_integer_operators:
+                for measure in measures:
+                    new_measure = copy(measure)
+                    new_measure.data_type = Integer if cls.return_integer else Number
+                    result_components[measure.name] = new_measure
+            elif cls.return_type is not None:
                 for measure in measures:
                     new_measure = copy(measure)
                     new_measure.data_type = cls.return_type
                     result_components[measure.name] = new_measure
+
             if cls.op == COUNT and len(measures) <= 1:
                 measure_name = COMP_NAME_MAPPING[cls.return_type]
                 nullable = False if len(measures) == 0 else measures[0].nullable
@@ -199,6 +231,8 @@ class Analytic(Operator.Unary):
                 measure_query = f"{cls.sql_op}({measure})"
             if cls.op == COUNT and len(measure_names) == 1:
                 measure_query += f" {analytic_str} as {COMP_NAME_MAPPING[cls.return_type]}"
+            elif cls.op in return_integer_operators and cls.return_integer:
+                measure_query = f"CAST({measure_query} {analytic_str} AS INTEGER) as {measure}"
             else:
                 measure_query += f" {analytic_str} as {measure}"
             measure_queries.append(measure_query)
@@ -215,7 +249,7 @@ class Analytic(Operator.Unary):
             df[measure_names] = df[measure_names].fillna(-1)
         # if os.getenv("SPARK", False):
         #     df = df.to_pandas()
-        return duckdb.query(query).to_df()
+        return duckdb.query(query).to_df().astype(object)
 
     @classmethod
     def evaluate(  # type: ignore[override]
@@ -245,6 +279,10 @@ class Analytic(Operator.Unary):
             window=window,
             params=params,
         )
+
+        # if cls.return_type == Integer:
+        #     result.data[measure_names] = result.data[measure_names].astype('Int64')
+
         return result
 
 
@@ -255,6 +293,7 @@ class Max(Analytic):
 
     op = MAX
     sql_op = "MAX"
+    return_integer = False
 
 
 class Min(Analytic):
@@ -264,6 +303,7 @@ class Min(Analytic):
 
     op = MIN
     sql_op = "MIN"
+    return_integer = False
 
 
 class Sum(Analytic):
@@ -272,9 +312,8 @@ class Sum(Analytic):
     """
 
     op = SUM
-    type_to_check = Number
-    return_type = Number
     sql_op = "SUM"
+    return_integer = False
 
 
 class Count(Analytic):

vtlengine/Operators/Assignment.py

@@ -8,7 +8,6 @@ ALL_MODEL_TYPES = Union[DataComponent, Dataset]
 
 
 class Assignment(Binary):
-
     @classmethod
     def validate(cls, left_operand: Any, right_operand: Any) -> ALL_MODEL_TYPES:
         if (