vtlengine 1.4.0rc2__py3-none-any.whl

vtlengine/Operators/Numeric.py

@@ -0,0 +1,422 @@
+import _random
+import math
+import operator
+import warnings
+from decimal import Decimal, getcontext
+from typing import Any, Optional, Union
+
+import pandas as pd
+
+import vtlengine.Operators as Operator
+from vtlengine.AST.Grammar.tokens import (
+    ABS,
+    CEIL,
+    DIV,
+    EXP,
+    FLOOR,
+    LN,
+    LOG,
+    MINUS,
+    MOD,
+    MULT,
+    PLUS,
+    POWER,
+    RANDOM,
+    ROUND,
+    SQRT,
+    TRUNC,
+)
+from vtlengine.DataTypes import Integer, Number, binary_implicit_promotion
+from vtlengine.Exceptions import SemanticError
+from vtlengine.Model import DataComponent, Dataset, Scalar
+from vtlengine.Operators import ALL_MODEL_DATA_TYPES
+
+
+class Unary(Operator.Unary):
+    """
+    Checks that the unary operation is performed with a number.
+    """
+
+    type_to_check = Number
+
+
+class Binary(Operator.Binary):
+    """
+    Checks that the binary operation is performed with numbers.
+    """
+
+    type_to_check = Number
+
+    @classmethod
+    def op_func(cls, x: Any, y: Any) -> Any:
+        if pd.isnull(x) or pd.isnull(y):
+            return None
+        if isinstance(x, int) and isinstance(y, int):
+            if cls.op == DIV and y == 0:
+                raise SemanticError("2-1-15-6", op=cls.op, value=y)
+            if cls.op == RANDOM:
+                return cls.py_op(x, y)
+        x = float(x)
+        y = float(y)
+        # Handles precision to avoid floating point errors
+        if cls.op == DIV and y == 0:
+            raise SemanticError("2-1-15-6", op=cls.op, value=y)
+
+        decimal_value = cls.py_op(Decimal(x), Decimal(y))
+        getcontext().prec = 10
+        result = float(decimal_value)
+        if result.is_integer():
+            return int(result)
+        return result
+
+
+class UnPlus(Unary):
+    """
+    `Plus <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=94&zoom=100,72,142> `_ unary operator
+    """  # noqa E501
+
+    op = PLUS
+    py_op = operator.pos
+
+    @classmethod
+    def apply_operation_component(cls, series: Any) -> Any:
+        return series
+
+
+class UnMinus(Unary):
+    """
+    `Minus <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=95&zoom=100,72,414> `_unary operator
+    """  # noqa E501
+
+    op = MINUS
+    py_op = operator.neg
+
+
+class AbsoluteValue(Unary):
+    """
+    `Absolute <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=112&zoom=100,72,801> `_ unary operator
+    """  # noqa E501
+
+    op = ABS
+    py_op = operator.abs
+
+
+class Exponential(Unary):
+    """
+    `Exponential <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=114&zoom=100,72,94>`_ unary operator
+    """  # noqa E501
+
+    op = EXP
+    py_op = math.exp
+    return_type = Number
+
+
+class NaturalLogarithm(Unary):
+    """
+    `Natural logarithm <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=115&zoom=100,72,394> `_
+    unary operator
+    """  # noqa E501
+
+    op = LN
+    py_op = math.log
+    return_type = Number
+
+
+class SquareRoot(Unary):
+    """
+    `Square Root <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=119&zoom=100,72,556> '_
+    unary operator
+    """  # noqa E501
+
+    op = SQRT
+    py_op = math.sqrt
+    return_type = Number
+
+
+class Ceil(Unary):
+    """
+    `Ceilling <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=110&zoom=100,72,94> `_ unary operator
+    """  # noqa E501
+
+    op = CEIL
+    py_op = math.ceil
+    return_type = Integer
+
+
+class Floor(Unary):
+    """
+    `Floor <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=111&zoom=100,72,442> `_ unary operator
+    """  # noqa E501
+
+    op = FLOOR
+    py_op = math.floor
+    return_type = Integer
+
+
+class BinPlus(Binary):
+    """
+    `Addition <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=96&zoom=100,72,692> `_ binary operator
+    """  # noqa E501
+
+    op = PLUS
+    py_op = operator.add
+    type_to_check = Number
+
+
+class BinMinus(Binary):
+    """
+    `Subtraction <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=98&zoom=100,72,448> `_ binary operator
+    """  # noqa E501
+
+    op = MINUS
+    py_op = operator.sub
+    type_to_check = Number
+
+
+class Mult(Binary):
+    """
+    `Multiplication <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=100&zoom=100,72,254>`_
+    binary operator
+    """  # noqa E501
+
+    op = MULT
+    py_op = operator.mul
+
+
+class Div(Binary):
+    """
+    `Division <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=102&zoom=100,72,94>`_
+    binary operator
+    """  # noqa E501
+
+    op = DIV
+    py_op = operator.truediv
+    return_type = Number
+
+
+class Logarithm(Binary):
+    """
+    `Logarithm <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=118&zoom=100,72,228>`_ operator
+    """  # noqa E501
+
+    op = LOG
+    return_type = Number
+
+    @classmethod
+    def py_op(cls, x: Any, param: Any) -> Any:
+        if pd.isnull(param):
+            return None
+        if param <= 0:
+            raise SemanticError("2-1-15-3", op=cls.op, value=param)
+
+        return math.log(x, param)
+
+
+class Modulo(Binary):
+    """
+    `Module <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=104&zoom=100,72,94>`_ operator
+    """  # noqa E501
+
+    op = MOD
+    py_op = operator.mod
+
+
+class Power(Binary):
+    """
+    `Power <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=116&zoom=100,72,693>`_ operator
+    """  # noqa E501
+
+    op = POWER
+    return_type = Number
+
+    @classmethod
+    def py_op(cls, x: Any, param: Any) -> Any:
+        if pd.isnull(param):
+            return None
+        return x**param
+
+
+class Parameterized(Unary):
+    """Parametrized class
+    Inherits from Unary class, to validate the data type and evaluate if it is the correct one to
+    perform the operation. Similar to Unary, but in the end, the param validation is added.
+    """
+
+    @classmethod
+    def validate(
+        cls,
+        operand: Operator.ALL_MODEL_DATA_TYPES,
+        param: Optional[Union[DataComponent, Scalar]] = None,
+    ) -> Any:
+        if param is not None:
+            if isinstance(param, Dataset):
+                raise SemanticError("1-1-15-8", op=cls.op, comp_type="Dataset")
+            if isinstance(param, DataComponent):
+                if isinstance(operand, Scalar):
+                    raise SemanticError(
+                        "1-1-15-8",
+                        op=cls.op,
+                        comp_type="DataComponent and an Scalar operand",
+                    )
+                cls.validate_type_compatibility(param.data_type)
+            else:
+                cls.validate_scalar_type(param)
+        if param is None:
+            cls.return_type = Integer
+        else:
+            cls.return_type = Number
+
+        return super().validate(operand)
+
+    @classmethod
+    def op_func(cls, x: Any, param: Optional[Any]) -> Any:
+        return None if pd.isnull(x) else cls.py_op(x, param)
+
+    @classmethod
+    def apply_operation_two_series(cls, left_series: Any, right_series: Any) -> Any:
+        return left_series.combine(right_series, cls.op_func)
+
+    @classmethod
+    def apply_operation_series_scalar(cls, series: Any, param: Any) -> Any:
+        return series.map(lambda x: cls.op_func(x, param))
+
+    @classmethod
+    def dataset_evaluation(
+        cls, operand: Dataset, param: Optional[Union[DataComponent, Scalar]] = None
+    ) -> Dataset:
+        result = cls.validate(operand, param)
+        result.data = operand.data.copy() if operand.data is not None else pd.DataFrame()
+        for measure_name in result.get_measures_names():
+            try:
+                if isinstance(param, DataComponent):
+                    result.data[measure_name] = cls.apply_operation_two_series(
+                        result.data[measure_name], param.data
+                    )
+                else:
+                    param_value = param.value if param is not None else None
+                    result.data[measure_name] = cls.apply_operation_series_scalar(
+                        result.data[measure_name], param_value
+                    )
+            except ValueError:
+                raise SemanticError(
+                    "2-1-15-1",
+                    op=cls.op,
+                    comp_name=measure_name,
+                    dataset_name=operand.name,
+                ) from None
+        result.data = result.data[result.get_components_names()]
+        return result
+
+    @classmethod
+    def component_evaluation(
+        cls,
+        operand: DataComponent,
+        param: Optional[Union[DataComponent, Scalar]] = None,
+    ) -> DataComponent:
+        result = cls.validate(operand, param)
+        if operand.data is None:
+            operand.data = pd.Series()
+        result.data = operand.data.copy()
+        if isinstance(param, DataComponent):
+            result.data = cls.apply_operation_two_series(operand.data, param.data)
+        else:
+            param_value = param.value if param is not None else None
+            result.data = cls.apply_operation_series_scalar(operand.data, param_value)
+        return result
+
+    @classmethod
+    def scalar_evaluation(cls, operand: Scalar, param: Optional[Any] = None) -> Scalar:
+        result = cls.validate(operand, param)
+        param_value = param.value if param is not None else None
+        result.value = cls.op_func(operand.value, param_value)
+        return result
+
+    @classmethod
+    def evaluate(
+        cls,
+        operand: ALL_MODEL_DATA_TYPES,
+        param: Optional[Union[DataComponent, Scalar]] = None,
+    ) -> Union[DataComponent, Dataset, Scalar]:
+        if isinstance(operand, Dataset):
+            return cls.dataset_evaluation(operand, param)
+        elif isinstance(operand, DataComponent):
+            return cls.component_evaluation(operand, param)
+        else:
+            return cls.scalar_evaluation(operand, param)
+
+
+class Round(Parameterized):
+    """
+    `Round <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=106&zoom=100,72,94>`_ operator
+    """  # noqa E501
+
+    op = ROUND
+    return_type = Integer
+
+    @classmethod
+    def py_op(cls, x: Any, param: Any) -> Any:
+        multiplier = 1.0
+        if not pd.isnull(param):
+            multiplier = 10**param
+
+        if x >= 0.0:
+            rounded_value = math.floor(x * multiplier + 0.5) / multiplier
+        else:
+            rounded_value = math.ceil(x * multiplier - 0.5) / multiplier
+
+        if param is not None:
+            return rounded_value
+
+        return int(rounded_value)
+
+
+class Trunc(Parameterized):
+    """
+    `Trunc <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=108&zoom=100,72,94>`_ operator.
+    """  # noqa E501
+
+    op = TRUNC
+
+    @classmethod
+    def py_op(cls, x: float, param: Optional[float]) -> Any:
+        multiplier = 1.0
+        if not pd.isnull(param) and param is not None:
+            multiplier = 10**param
+
+        truncated_value = int(x * multiplier) / multiplier
+
+        if not pd.isnull(param):
+            return truncated_value
+
+        return int(truncated_value)
+
+
+class PseudoRandom(_random.Random):
+    def __init__(self, seed: Union[int, float]) -> None:
+        super().__init__()
+        self.seed(seed)
+
+
+class Random(Parameterized):
+    op = RANDOM
+    return_type = Number
+
+    @classmethod
+    def validate(cls, seed: Any, index: Any = None) -> Any:
+        if index.data_type != Integer:
+            index.data_type = binary_implicit_promotion(index.data_type, Integer)
+        if index.value < 0:
+            raise SemanticError("2-1-15-2", op=cls.op, value=index)
+        if index.value > 10000:
+            warnings.warn(
+                "Random: The value of 'index' is very big. This can affect performance.",
+                UserWarning,
+            )
+        return super().validate(seed, index)
+
+    @classmethod
+    def py_op(cls, seed: Union[int, float], index: int) -> float:
+        instance: PseudoRandom = PseudoRandom(seed)
+        for _ in range(index):
+            instance.random()
+        return instance.random().__round__(6)

vtlengine/Operators/RoleSetter.py

@@ -0,0 +1,77 @@
+from copy import copy
+from typing import Any, Union
+
+# if os.environ.get("SPARK", False):
+#     import pyspark.pandas as pd
+# else:
+#     import pandas as pd
+import pandas as pd
+
+from vtlengine.Exceptions import SemanticError
+from vtlengine.Model import DataComponent, Role, Scalar
+from vtlengine.Operators import Unary
+
+ALLOWED_MODEL_TYPES = Union[DataComponent, Scalar]
+
+
+class RoleSetter(Unary):
+    role: Role
+
+    @classmethod
+    def validate(cls, operand: ALLOWED_MODEL_TYPES, data_size: int = 0) -> DataComponent:
+        if isinstance(operand, Scalar):
+            nullable = True
+            if cls.role == Role.IDENTIFIER or operand.value is not None:
+                nullable = False
+            return DataComponent(
+                name=operand.name,
+                data_type=operand.data_type,
+                role=cls.role,
+                nullable=nullable,
+                data=None,
+            )
+        operand.role = cls.role
+        return copy(operand)
+
+    @classmethod
+    def evaluate(cls, operand: Any, data_size: int = 0) -> DataComponent:
+        if (
+            isinstance(operand, DataComponent)
+            and operand.data is not None
+            and not operand.nullable
+            and any(operand.data.isnull())
+        ):
+            raise SemanticError("1-1-1-16")
+        result = cls.validate(operand, data_size)
+        if isinstance(operand, Scalar):
+            result.data = pd.Series([operand.value] * data_size, dtype=object)
+        else:
+            result.data = operand.data
+        return result
+
+
+class Identifier(RoleSetter):
+    role = Role.IDENTIFIER
+
+    @classmethod
+    def validate(cls, operand: ALLOWED_MODEL_TYPES, data_size: int = 0) -> DataComponent:
+        result = super().validate(operand)
+        if result.nullable:
+            raise SemanticError("1-1-1-16")
+        return result
+
+    @classmethod
+    def evaluate(  # type: ignore[override]
+        cls, operand: ALLOWED_MODEL_TYPES, data_size: int = 0
+    ) -> DataComponent:
+        if isinstance(operand, Scalar) and operand.value is None:
+            raise SemanticError("1-1-1-16")
+        return super().evaluate(operand, data_size)
+
+
+class Attribute(RoleSetter):
+    role = Role.ATTRIBUTE
+
+
+class Measure(RoleSetter):
+    role = Role.MEASURE

vtlengine/Operators/Set.py

@@ -0,0 +1,176 @@
+from typing import Any, Dict, List
+
+# if os.environ.get("SPARK"):
+#     import pyspark.pandas as pd
+# else:
+#     import pandas as pd
+import pandas as pd
+
+from vtlengine.DataTypes import binary_implicit_promotion
+from vtlengine.Exceptions import SemanticError
+from vtlengine.Model import Dataset
+from vtlengine.Operators import Operator
+
+
+class Set(Operator):
+    @classmethod
+    def check_same_structure(cls, dataset_1: Dataset, dataset_2: Dataset) -> None:
+        if len(dataset_1.components) != len(dataset_2.components):
+            raise SemanticError(
+                "1-1-17-1",
+                op=cls.op,
+                dataset_1=dataset_1.name,
+                dataset_2=dataset_2.name,
+            )
+
+        for comp in dataset_1.components.values():
+            if comp.name not in dataset_2.components:
+                raise Exception(f"Component {comp.name} not found in dataset {dataset_2.name}")
+            second_comp = dataset_2.components[comp.name]
+            binary_implicit_promotion(
+                comp.data_type,
+                second_comp.data_type,
+                cls.type_to_check,
+                cls.return_type,
+            )
+            if comp.role != second_comp.role:
+                raise Exception(
+                    f"Component {comp.name} has different roles "
+                    f"in datasets {dataset_1.name} and {dataset_2.name}"
+                )
+
+    @classmethod
+    def validate(cls, operands: List[Dataset]) -> Dataset:
+        base_operand = operands[0]
+        for operand in operands[1:]:
+            cls.check_same_structure(base_operand, operand)
+
+        result_components: Dict[str, Any] = {}
+        for operand in operands:
+            if len(result_components) == 0:
+                result_components = operand.components
+            else:
+                for comp_name, comp in operand.components.items():
+                    current_comp = result_components[comp_name]
+                    result_components[comp_name].data_type = binary_implicit_promotion(
+                        current_comp.data_type, comp.data_type
+                    )
+                    result_components[comp_name].nullable = current_comp.nullable or comp.nullable
+
+        result = Dataset(name="result", components=result_components, data=None)
+        return result
+
+
+class Union(Set):
+    @classmethod
+    def evaluate(cls, operands: List[Dataset]) -> Dataset:
+        result = cls.validate(operands)
+        all_datapoints = [ds.data for ds in operands]
+        result.data = pd.concat(all_datapoints, sort=True, ignore_index=True)
+        identifiers_names = result.get_identifiers_names()
+        result.data = result.data.drop_duplicates(subset=identifiers_names, keep="first")
+        result.data.reset_index(drop=True, inplace=True)
+        return result
+
+
+class Intersection(Set):
+    @classmethod
+    def evaluate(cls, operands: List[Dataset]) -> Dataset:
+        result = cls.validate(operands)
+        all_datapoints = [ds.data for ds in operands]
+        for data in all_datapoints:
+            if result.data is None:
+                result.data = data
+            else:
+                if data is None:
+                    result.data = pd.DataFrame(columns=result.get_identifiers_names())
+                    break
+                result.data = result.data.merge(
+                    data, how="inner", on=result.get_identifiers_names()
+                )
+
+                not_identifiers = result.get_measures_names() + result.get_attributes_names()
+
+                for col in not_identifiers:
+                    result.data[col] = result.data[col + "_x"]
+                result.data = result.data[result.get_identifiers_names() + not_identifiers]
+        if result.data is not None:
+            result.data.reset_index(drop=True, inplace=True)
+        return result
+
+
+class Symdiff(Set):
+    @classmethod
+    def evaluate(cls, operands: List[Dataset]) -> Dataset:
+        result = cls.validate(operands)
+        all_datapoints = [ds.data for ds in operands]
+        for data in all_datapoints:
+            if data is None:
+                data = pd.DataFrame(columns=result.get_identifiers_names())
+            if result.data is None:
+                result.data = data
+            else:
+                # Realiza la operación equivalente en pyspark.pandas
+                result.data = result.data.merge(
+                    data,
+                    how="outer",
+                    on=result.get_identifiers_names(),
+                    suffixes=("_x", "_y"),
+                )
+
+                for measure in result.get_measures_names():
+                    result.data["_merge"] = result.data.apply(
+                        lambda row: (
+                            "left_only"
+                            if pd.isnull(row[f"{measure}_y"])
+                            else ("right_only" if pd.isnull(row[f"{measure}_x"]) else "both")
+                        ),
+                        axis=1,
+                    )
+
+                not_identifiers = result.get_measures_names() + result.get_attributes_names()
+                for col in not_identifiers:
+                    result.data[col] = result.data.apply(
+                        lambda x, c=col: (
+                            x[c + "_x"]
+                            if x["_merge"] == "left_only"
+                            else (x[c + "_y"] if x["_merge"] == "right_only" else None)
+                        ),
+                        axis=1,
+                    )
+                result.data = result.data[result.get_identifiers_names() + not_identifiers].dropna()
+        if result.data is not None:
+            result.data = result.data.reset_index(drop=True)
+        return result
+
+
+class Setdiff(Set):
+    @staticmethod
+    def has_null(row: Any) -> bool:
+        return row.isnull().any()
+
+    @classmethod
+    def evaluate(cls, operands: List[Dataset]) -> Dataset:
+        result = cls.validate(operands)
+        all_datapoints = [ds.data for ds in operands]
+        for data in all_datapoints:
+            if result.data is None:
+                result.data = data
+            else:
+                if data is None:
+                    data = pd.DataFrame(columns=result.get_identifiers_names())
+                result.data = result.data.merge(data, how="left", on=result.get_identifiers_names())
+                if len(result.data) > 0:
+                    result.data = result.data[result.data.apply(cls.has_null, axis=1)]
+
+                not_identifiers = result.get_measures_names() + result.get_attributes_names()
+                for col in not_identifiers:
+                    if col + "_x" in result.data:
+                        result.data[col] = result.data[col + "_x"]
+                        del result.data[col + "_x"]
+                    if col + "_y" in result.data:
+                        del result.data[col + "_y"]
+                result.data = result.data[result.get_identifiers_names() + not_identifiers]
+        if result.data is not None:
+            result.data.reset_index(drop=True, inplace=True)
+        return result