vtlengine 1.0.3rc3__py3-none-any.whl → 1.1__py3-none-any.whl

vtlengine/Operators/Analytic.py

@@ -29,10 +29,17 @@ from vtlengine.AST.Grammar.tokens import (
     VAR_POP,
     VAR_SAMP,
 )
-from vtlengine.DataTypes import COMP_NAME_MAPPING, Integer, Number, unary_implicit_promotion
+from vtlengine.DataTypes import (
+    COMP_NAME_MAPPING,
+    Integer,
+    Number,
+    unary_implicit_promotion,
+)
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, Dataset, Role
 
+return_integer_operators = [MAX, MIN, SUM]
+
 
 # noinspection PyMethodOverriding
 class Analytic(Operator.Unary):
@@ -47,10 +54,11 @@ class Analytic(Operator.Unary):
         Evaluate: Ensures the type of data is the correct one to perform the Analytic operators.
     """
 
+    return_integer = None
     sql_op: Optional[str] = None
 
     @classmethod
-    def validate(  # type: ignore[override]
+    def validate(  # type: ignore[override]  # noqa: C901
         cls,
         operand: Dataset,
         partitioning: List[str],
@@ -66,7 +74,10 @@ class Analytic(Operator.Unary):
         for comp_name in partitioning:
             if comp_name not in operand.components:
                 raise SemanticError(
-                    "1-1-1-10", op=cls.op, comp_name=comp_name, dataset_name=operand.name
+                    "1-1-1-10",
+                    op=cls.op,
+                    comp_name=comp_name,
+                    dataset_name=operand.name,
                 )
             if comp_name not in identifier_names:
                 raise SemanticError(
@@ -78,14 +89,21 @@ class Analytic(Operator.Unary):
         for comp_name in order_components:
             if comp_name not in operand.components:
                 raise SemanticError(
-                    "1-1-1-10", op=cls.op, comp_name=comp_name, dataset_name=operand.name
+                    "1-1-1-10",
+                    op=cls.op,
+                    comp_name=comp_name,
+                    dataset_name=operand.name,
                 )
         if component_name is not None:
             if cls.type_to_check is not None:
                 unary_implicit_promotion(
                     operand.components[component_name].data_type, cls.type_to_check
                 )
-            if cls.return_type is not None:
+
+            if cls.op in return_integer_operators:
+                cls.return_integer = isinstance(cls.return_type, Integer)
+
+            elif cls.return_type is not None:
                 result_components[component_name] = Component(
                     name=component_name,
                     data_type=cls.return_type,
@@ -106,14 +124,28 @@ class Analytic(Operator.Unary):
             measures = operand.get_measures()
             if len(measures) == 0:
                 raise SemanticError("1-1-1-8", op=cls.op, name=operand.name)
+
+            if cls.op in return_integer_operators:
+                isNumber = False
+                for measure in measures:
+                    isNumber |= isinstance(measure.data_type, Number)
+                cls.return_integer = not isNumber
+
             if cls.type_to_check is not None:
                 for measure in measures:
                     unary_implicit_promotion(measure.data_type, cls.type_to_check)
-            if cls.return_type is not None:
+
+            if cls.op in return_integer_operators:
+                for measure in measures:
+                    new_measure = copy(measure)
+                    new_measure.data_type = Integer if cls.return_integer else Number
+                    result_components[measure.name] = new_measure
+            elif cls.return_type is not None:
                 for measure in measures:
                     new_measure = copy(measure)
                     new_measure.data_type = cls.return_type
                     result_components[measure.name] = new_measure
+
             if cls.op == COUNT and len(measures) <= 1:
                 measure_name = COMP_NAME_MAPPING[cls.return_type]
                 nullable = False if len(measures) == 0 else measures[0].nullable
@@ -157,21 +189,26 @@ class Analytic(Operator.Unary):
         if window is not None:
             mode = "ROWS" if window.type_ == "data" else "RANGE"
             start_mode = (
-                window.start_mode
-                if window.start_mode != "current" and window.start != "CURRENT ROW"
+                window.start_mode.upper()
+                if (isinstance(window.start, int) and window.start != 0)
+                or (isinstance(window.start, str) and window.start == "unbounded")
                 else ""
             )
             stop_mode = (
-                window.stop_mode
-                if window.stop_mode != "current" and window.stop != "CURRENT ROW"
+                window.stop_mode.upper()
+                if (isinstance(window.stop, int) and window.stop != 0)
+                or (isinstance(window.stop, str) and window.stop == "unbounded")
                 else ""
             )
-            if isinstance(window.start, int) and window.start == -1:
-                window.start = "UNBOUNDED"
-
-            if stop_mode == "" and window.stop == 0:
-                window.stop = "CURRENT ROW"
-            window_str = f"{mode} BETWEEN {window.start} {start_mode} AND {window.stop} {stop_mode}"
+            start = (
+                "UNBOUNDED"
+                if window.start == "unbounded" or window.start == -1
+                else str(window.start)
+            )
+            stop = (
+                "CURRENT ROW" if window.stop == "current" or window.stop == 0 else str(window.stop)
+            )
+            window_str = f"{mode} BETWEEN {start} {start_mode} AND {stop} {stop_mode}"
 
         # Partitioning
         partition = "PARTITION BY " + ", ".join(partitioning) if len(partitioning) > 0 else ""
@@ -192,13 +229,15 @@ class Analytic(Operator.Unary):
             if cls.op == RANK:
                 measure_query = f"{cls.sql_op}()"
             elif cls.op == RATIO_TO_REPORT:
-                measure_query = f"CAST({measure} AS REAL) / SUM(CAST({measure} AS REAL))"
+                measure_query = f"CAST({measure} AS DOUBLE) / SUM(CAST({measure} AS DOUBLE))"
             elif cls.op in [LAG, LEAD]:
                 measure_query = f"{cls.sql_op}({measure}, {','.join(map(str, params or []))})"
             else:
                 measure_query = f"{cls.sql_op}({measure})"
             if cls.op == COUNT and len(measure_names) == 1:
                 measure_query += f" {analytic_str} as {COMP_NAME_MAPPING[cls.return_type]}"
+            elif cls.op in return_integer_operators and cls.return_integer:
+                measure_query = f"CAST({measure_query} {analytic_str} AS INTEGER) as {measure}"
             else:
                 measure_query += f" {analytic_str} as {measure}"
             measure_queries.append(measure_query)
@@ -215,7 +254,7 @@ class Analytic(Operator.Unary):
             df[measure_names] = df[measure_names].fillna(-1)
         # if os.getenv("SPARK", False):
         #     df = df.to_pandas()
-        return duckdb.query(query).to_df()
+        return duckdb.query(query).to_df().astype(object)
 
     @classmethod
     def evaluate(  # type: ignore[override]
@@ -245,6 +284,10 @@ class Analytic(Operator.Unary):
             window=window,
             params=params,
         )
+
+        # if cls.return_type == Integer:
+        #     result.data[measure_names] = result.data[measure_names].astype('Int64')
+
         return result
 
 
@@ -255,6 +298,7 @@ class Max(Analytic):
 
     op = MAX
     sql_op = "MAX"
+    return_integer = False
 
 
 class Min(Analytic):
@@ -264,6 +308,7 @@ class Min(Analytic):
 
     op = MIN
     sql_op = "MIN"
+    return_integer = False
 
 
 class Sum(Analytic):
@@ -272,9 +317,8 @@ class Sum(Analytic):
     """
 
     op = SUM
-    type_to_check = Number
-    return_type = Number
     sql_op = "SUM"
+    return_integer = False
 
 
 class Count(Analytic):

vtlengine/Operators/Assignment.py

@@ -8,7 +8,6 @@ ALL_MODEL_TYPES = Union[DataComponent, Dataset]
 
 
 class Assignment(Binary):
-
     @classmethod
     def validate(cls, left_operand: Any, right_operand: Any) -> ALL_MODEL_TYPES:
         if (

vtlengine/Operators/CastOperator.py

@@ -34,43 +34,43 @@ class Cast(Operator.Unary):
     # CASTS VALUES
     # Converts the value from one type to another in a way that is according to the mask
     @classmethod
-    def cast_string_to_number(cls, *args: Any) -> Any:
+    def cast_string_to_number(cls, value: Any, mask: str) -> Any:
         """
         This method casts a string to a number, according to the mask.
 
         """
 
-        raise NotImplementedError("How this cast should be implemented is not yet defined.")
+        raise NotImplementedError("How this mask should be implemented is not yet defined.")
 
     @classmethod
-    def cast_string_to_date(cls, *args: Any) -> Any:
+    def cast_string_to_date(cls, value: Any, mask: str) -> Any:
         """
         This method casts a string to a number, according to the mask.
 
         """
 
-        raise NotImplementedError("How this cast should be implemented is not yet defined.")
+        raise NotImplementedError("How this mask should be implemented is not yet defined.")
 
     @classmethod
-    def cast_string_to_duration(cls, *args: Any) -> Any:
+    def cast_string_to_duration(cls, value: Any, mask: str) -> Any:
         """
         This method casts a string to a duration, according to the mask.
 
         """
 
-        raise NotImplementedError("How this cast should be implemented is not yet defined.")
+        raise NotImplementedError("How this mask should be implemented is not yet defined.")
 
     @classmethod
-    def cast_string_to_time_period(cls, *args: Any) -> Any:
+    def cast_string_to_time_period(cls, value: Any, mask: str) -> Any:
         """
         This method casts a string to a time period, according to the mask.
 
         """
 
-        raise NotImplementedError("How this cast should be implemented is not yet defined.")
+        raise NotImplementedError("How this mask should be implemented is not yet defined.")
 
     @classmethod
-    def cast_string_to_time(cls, *args: Any) -> Any:
+    def cast_string_to_time(cls, value: Any, mask: str) -> Any:
         """
         This method casts a string to a time, according to the mask.
 
@@ -78,20 +78,21 @@ class Cast(Operator.Unary):
 
         raise NotImplementedError("How this cast should be implemented is not yet defined.")
 
-    @classmethod
-    def cast_date_to_string(cls, *args: Any) -> Any:
-        """ """
-        return NotImplementedError("How this cast should be implemented is not yet defined.")
-
-    @classmethod
-    def cast_duration_to_string(cls, *args: Any) -> Any:
-        """ """
-        return NotImplementedError("How this cast should be implemented is not yet defined.")
-
-    @classmethod
-    def cast_time_to_string(cls, *args: Any) -> Any:
-        """ """
-        return NotImplementedError("How this cast should be implemented is not yet defined.")
+    #
+    # @classmethod
+    # def cast_date_to_string(cls, value: Any, mask: str) -> Any:
+    #     """ """
+    #     return NotImplementedError("How this cast should be implemented is not yet defined.")
+    #
+    # @classmethod
+    # def cast_duration_to_string(cls, value: Any, mask: str) -> Any:
+    #     """ """
+    #     return NotImplementedError("How this cast should be implemented is not yet defined.")
+    #
+    # @classmethod
+    # def cast_time_to_string(cls, value: Any, mask: str) -> Any:
+    #     """ """
+    #     return NotImplementedError("How this cast should be implemented is not yet defined.")
 
     @classmethod
     def cast_time_period_to_date(cls, value: Any, mask_value: str) -> Any:
@@ -142,7 +143,6 @@ class Cast(Operator.Unary):
 
     @classmethod
     def check_mask_value_from_time_period_to_date(cls, mask_value: str) -> None:
-
         if mask_value not in ["START", "END"]:
             raise SemanticError("1-1-5-4", op=cls.op, type_1="Time_Period", type_2="Date")
 
@@ -180,9 +180,11 @@ class Cast(Operator.Unary):
 
     @classmethod
     def check_cast(
-        cls, from_type: Type[ScalarType], to_type: Type[ScalarType], mask_value: Optional[str]
+        cls,
+        from_type: Type[ScalarType],
+        to_type: Type[ScalarType],
+        mask_value: Optional[str],
     ) -> None:
-
         if mask_value is not None:
             cls.check_with_mask(from_type, to_type, mask_value)
         else:
@@ -192,7 +194,6 @@ class Cast(Operator.Unary):
     def check_with_mask(
         cls, from_type: Type[ScalarType], to_type: Type[ScalarType], mask_value: str
     ) -> None:
-
         explicit_promotion = EXPLICIT_WITH_MASK_TYPE_PROMOTION_MAPPING[from_type]
         if to_type.is_included(explicit_promotion):
             return cls.check_mask_value(from_type, to_type, mask_value)
@@ -207,7 +208,6 @@ class Cast(Operator.Unary):
 
     @classmethod
     def check_without_mask(cls, from_type: Type[ScalarType], to_type: Type[ScalarType]) -> None:
-
         explicit_promotion = EXPLICIT_WITHOUT_MASK_TYPE_PROMOTION_MAPPING[from_type]
         implicit_promotion = IMPLICIT_TYPE_PROMOTION_MAPPING[from_type]
         if not (to_type.is_included(explicit_promotion) or to_type.is_included(implicit_promotion)):
@@ -231,7 +231,7 @@ class Cast(Operator.Unary):
         cls, data: Any, from_type: Type[ScalarType], to_type: Type[ScalarType]
     ) -> Any:
         """
-        cast the component to the type to_type without mask
+        Cast the component to the type to_type without mask
         """
 
         if to_type.is_included(IMPLICIT_TYPE_PROMOTION_MAPPING[from_type]):
@@ -242,15 +242,17 @@ class Cast(Operator.Unary):
 
     @classmethod
     def cast_mask_component(cls, data: Any, from_type: Any, to_type: Any, mask: str) -> Any:
-
         result = data.map(lambda x: cls.cast_value(x, from_type, to_type, mask), na_action="ignore")
         return result
 
     @classmethod
     def cast_value(
-        cls, value: Any, provided_type: Type[ScalarType], to_type: Type[ScalarType], mask_value: str
+        cls,
+        value: Any,
+        provided_type: Type[ScalarType],
+        to_type: Type[ScalarType],
+        mask_value: str,
     ) -> Any:
-
         if provided_type == String and to_type == Number:
             return cls.cast_string_to_number(value, mask_value)
         if provided_type == String and to_type == Date:
@@ -261,12 +263,12 @@ class Cast(Operator.Unary):
             return cls.cast_string_to_time_period(value, mask_value)
         if provided_type == String and to_type == TimeInterval:
             return cls.cast_string_to_time(value, mask_value)
-        if provided_type == Date and to_type == String:
-            return cls.cast_date_to_string(value, mask_value)
-        if provided_type == Duration and to_type == String:
-            return cls.cast_duration_to_string(value, mask_value)
-        if provided_type == TimeInterval and to_type == String:
-            return cls.cast_time_to_string(value, mask_value)
+        # if provided_type == Date and to_type == String:
+        #     return cls.cast_date_to_string(value, mask_value)
+        # if provided_type == Duration and to_type == String:
+        #     return cls.cast_duration_to_string(value, mask_value)
+        # if provided_type == TimeInterval and to_type == String:
+        #     return cls.cast_time_to_string(value, mask_value)
         if provided_type == TimePeriod and to_type == Date:
             return cls.cast_time_period_to_date(value, mask_value)
 
@@ -285,7 +287,6 @@ class Cast(Operator.Unary):
         scalarType: Type[ScalarType],
         mask: Optional[str] = None,
     ) -> Any:
-
         if mask is not None and not isinstance(mask, str):
             raise Exception(f"{cls.op} mask must be a string")
 
@@ -325,7 +326,10 @@ class Cast(Operator.Unary):
         else:
             measure_name = measure.name
         result_components[measure_name] = Component(
-            name=measure_name, data_type=to_type, role=Role.MEASURE, nullable=measure.nullable
+            name=measure_name,
+            data_type=to_type,
+            role=Role.MEASURE,
+            nullable=measure.nullable,
         )
         return Dataset(name="result", components=result_components, data=None)
 
@@ -366,7 +370,6 @@ class Cast(Operator.Unary):
         scalarType: Type[ScalarType],
         mask: Optional[str] = None,
     ) -> Any:
-
         if isinstance(operand, Dataset):
             return cls.dataset_evaluation(operand, scalarType, mask)
         if isinstance(operand, Scalar):
@@ -381,7 +384,6 @@ class Cast(Operator.Unary):
         to_type: Type[ScalarType],
         mask: Optional[str] = None,
     ) -> Dataset:
-
         from_type = operand.get_measures()[0].data_type
         original_measure = operand.get_measures()[0]
         result_dataset = cls.dataset_validation(operand, to_type, mask)
@@ -410,7 +412,6 @@ class Cast(Operator.Unary):
         to_type: Type[ScalarType],
         mask: Optional[str] = None,
     ) -> Scalar:
-
         from_type = operand.data_type
         result_scalar = cls.scalar_validation(operand, to_type, mask)
         if pd.isna(operand.value):
@@ -431,7 +432,6 @@ class Cast(Operator.Unary):
         to_type: Type[ScalarType],
         mask: Optional[str] = None,
     ) -> DataComponent:
-
         from_type = operand.data_type
         result_component = cls.component_validation(operand, to_type, mask)
         if mask:

vtlengine/Operators/Clause.py

@@ -22,12 +22,10 @@ class Calc(Operator):
 
     @classmethod
     def validate(cls, operands: List[Union[DataComponent, Scalar]], dataset: Dataset) -> Dataset:
-
         result_components = {name: copy(comp) for name, comp in dataset.components.items()}
         result_dataset = Dataset(name=dataset.name, components=result_components, data=None)
 
         for operand in operands:
-
             if operand.name in result_dataset.components:
                 if result_dataset.components[operand.name].role == Role.IDENTIFIER:
                     raise SemanticError("1-1-6-13", op=cls.op, comp_name=operand.name)
@@ -72,7 +70,6 @@ class Aggregate(Operator):
 
     @classmethod
     def validate(cls, operands: List[Union[DataComponent, Scalar]], dataset: Dataset) -> Dataset:
-
         result_dataset = Dataset(name=dataset.name, components=dataset.components, data=None)
 
         for operand in operands:
@@ -121,7 +118,6 @@ class Aggregate(Operator):
 
 
 class Filter(Operator):
-
     @classmethod
     def validate(cls, condition: DataComponent, dataset: Dataset) -> Dataset:
         if condition.data_type != Boolean:
@@ -212,11 +208,17 @@ class Rename(Operator):
         for operand in operands:
             if operand.old_name not in dataset.components:
                 raise SemanticError(
-                    "1-1-1-10", op=cls.op, comp_name=operand.old_name, dataset_name=dataset.name
+                    "1-1-1-10",
+                    op=cls.op,
+                    comp_name=operand.old_name,
+                    dataset_name=dataset.name,
                 )
             if operand.new_name in dataset.components:
                 raise SemanticError(
-                    "1-1-6-8", op=cls.op, comp_name=operand.new_name, dataset_name=dataset.name
+                    "1-1-6-8",
+                    op=cls.op,
+                    comp_name=operand.new_name,
+                    dataset_name=dataset.name,
                 )
 
         result_components = {comp.name: comp for comp in dataset.components.values()}
@@ -242,7 +244,6 @@ class Rename(Operator):
 
 
 class Pivot(Operator):
-
     @classmethod
     def validate(cls, operands: List[str], dataset: Dataset) -> Dataset:
         raise NotImplementedError
@@ -253,7 +254,6 @@ class Pivot(Operator):
 
 
 class Unpivot(Operator):
-
     @classmethod
     def validate(cls, operands: List[str], dataset: Dataset) -> Dataset:
         if len(operands) != 2:
@@ -311,11 +311,17 @@ class Sub(Operator):
         for operand in operands:
             if operand.name not in dataset.components:
                 raise SemanticError(
-                    "1-1-1-10", op=cls.op, comp_name=operand.name, dataset_name=dataset.name
+                    "1-1-1-10",
+                    op=cls.op,
+                    comp_name=operand.name,
+                    dataset_name=dataset.name,
                 )
             if operand.role != Role.IDENTIFIER:
                 raise SemanticError(
-                    "1-1-6-10", op=cls.op, operand=operand.name, dataset_name=dataset.name
+                    "1-1-6-10",
+                    op=cls.op,
+                    operand=operand.name,
+                    dataset_name=dataset.name,
                 )
             if isinstance(operand, Scalar):
                 raise SemanticError("1-1-6-5", op=cls.op, name=operand.name)

vtlengine/Operators/Comparison.py

@@ -74,10 +74,11 @@ class Binary(Operator.Binary):
     return_type = Boolean
 
     @classmethod
-    def _cast_values(cls,
-                     x: Optional[Union[int, float, str, bool]],
-                     y: Optional[Union[int, float, str, bool]]
-                     ) -> Any:
+    def _cast_values(
+        cls,
+        x: Optional[Union[int, float, str, bool]],
+        y: Optional[Union[int, float, str, bool]],
+    ) -> Any:
         # Cast values to compatible types for comparison
         try:
             if isinstance(x, str) and isinstance(y, bool):
@@ -247,9 +248,7 @@ class Between(Operator.Operator):
         z: Optional[Union[int, float, bool, str]],
     ) -> Optional[bool]:
         return (
-            None
-            if (pd.isnull(x) or pd.isnull(y) or pd.isnull(z))
-            else y <= x <= z  # type: ignore[operator]
+            None if (pd.isnull(x) or pd.isnull(y) or pd.isnull(z)) else y <= x <= z  # type: ignore[operator]
         )
 
     @classmethod
@@ -264,7 +263,8 @@ class Between(Operator.Operator):
                 to_data = pd.Series(to_data, index=series.index)
             df = pd.DataFrame({"operand": series, "from_data": from_data, "to_data": to_data})
             return df.apply(
-                lambda x: cls.op_func(x["operand"], x["from_data"], x["to_data"]), axis=1
+                lambda x: cls.op_func(x["operand"], x["from_data"], x["to_data"]),
+                axis=1,
             )
 
         return series.map(lambda x: cls.op_func(x, from_data, to_data))
@@ -310,13 +310,19 @@ class Between(Operator.Operator):
             result = Dataset(name=operand.name, components=result_components, data=None)
         elif isinstance(operand, DataComponent):
             result = DataComponent(
-                name=operand.name, data=None, data_type=cls.return_type, role=operand.role
+                name=operand.name,
+                data=None,
+                data_type=cls.return_type,
+                role=operand.role,
             )
         elif isinstance(from_, Scalar) and isinstance(to, Scalar):
             result = Scalar(name=operand.name, value=None, data_type=cls.return_type)
         else:  # From or To is a DataComponent, or both
             result = DataComponent(
-                name=operand.name, data=None, data_type=cls.return_type, role=Role.MEASURE
+                name=operand.name,
+                data=None,
+                data_type=cls.return_type,
+                role=Role.MEASURE,
             )
 
         if isinstance(operand, Dataset):
@@ -369,14 +375,16 @@ class Between(Operator.Operator):
         elif isinstance(operand, Scalar) and (
             isinstance(from_data, pd.Series) or isinstance(to_data, pd.Series)
         ):  # From or To is a DataComponent, or both
-
             if isinstance(from_data, pd.Series):
                 series = pd.Series(operand.value, index=from_data.index, dtype=object)
             elif isinstance(to_data, pd.Series):
                 series = pd.Series(operand.value, index=to_data.index, dtype=object)
             result_series = cls.apply_operation_component(series, from_data, to_data)
             result = DataComponent(
-                name=operand.name, data=result_series, data_type=cls.return_type, role=Role.MEASURE
+                name=operand.name,
+                data=result_series,
+                data_type=cls.return_type,
+                role=Role.MEASURE,
             )
         return result