vtlengine 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

vtlengine/Operators/Analytic.py

@@ -3,8 +3,6 @@ from typing import List, Optional
 
 import duckdb
 
-from vtlengine.Exceptions import SemanticError
-
 # if os.environ.get("SPARK"):
 #     import pyspark.pandas as pd
 # else:
@@ -32,6 +30,7 @@ from vtlengine.AST.Grammar.tokens import (
     VAR_SAMP,
 )
 from vtlengine.DataTypes import COMP_NAME_MAPPING, Integer, Number, unary_implicit_promotion
+from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, Dataset, Role
 
 
@@ -58,11 +57,9 @@ class Analytic(Operator.Unary):
         ordering: Optional[List[OrderBy]],
         window: Optional[Windowing],
         params: Optional[List[int]],
+        component_name: Optional[str] = None,
     ) -> Dataset:
-        if ordering is None:
-            order_components = []
-        else:
-            order_components = [o.component for o in ordering]
+        order_components = [] if ordering is None else [o.component for o in ordering]
         identifier_names = operand.get_identifiers_names()
         result_components = operand.components.copy()
 
@@ -83,25 +80,51 @@ class Analytic(Operator.Unary):
                 raise SemanticError(
                     "1-1-1-10", op=cls.op, comp_name=comp_name, dataset_name=operand.name
                 )
-        measures = operand.get_measures()
-        if measures is None:
-            raise SemanticError("1-1-1-8", op=cls.op, name=operand.name)
-        if cls.type_to_check is not None:
-            for measure in measures:
-                unary_implicit_promotion(measure.data_type, cls.type_to_check)
-        if cls.return_type is not None:
-            for measure in measures:
-                new_measure = copy(measure)
-                new_measure.data_type = cls.return_type
-                result_components[measure.name] = new_measure
-        if cls.op == COUNT and len(measures) <= 1:
-            measure_name = COMP_NAME_MAPPING[cls.return_type]
-            nullable = False if len(measures) == 0 else measures[0].nullable
-            if len(measures) == 1:
-                del result_components[measures[0].name]
-            result_components[measure_name] = Component(
-                name=measure_name, data_type=cls.return_type, role=Role.MEASURE, nullable=nullable
-            )
+        if component_name is not None:
+            if cls.type_to_check is not None:
+                unary_implicit_promotion(
+                    operand.components[component_name].data_type, cls.type_to_check
+                )
+            if cls.return_type is not None:
+                result_components[component_name] = Component(
+                    name=component_name,
+                    data_type=cls.return_type,
+                    role=operand.components[component_name].role,
+                    nullable=operand.components[component_name].nullable,
+                )
+            if cls.op == COUNT:
+                measure_name = COMP_NAME_MAPPING[cls.return_type]
+                result_components[measure_name] = Component(
+                    name=measure_name,
+                    data_type=cls.return_type,
+                    role=Role.MEASURE,
+                    nullable=operand.components[component_name].nullable,
+                )
+                if component_name in result_components:
+                    del result_components[component_name]
+        else:
+            measures = operand.get_measures()
+            if len(measures) == 0:
+                raise SemanticError("1-1-1-8", op=cls.op, name=operand.name)
+            if cls.type_to_check is not None:
+                for measure in measures:
+                    unary_implicit_promotion(measure.data_type, cls.type_to_check)
+            if cls.return_type is not None:
+                for measure in measures:
+                    new_measure = copy(measure)
+                    new_measure.data_type = cls.return_type
+                    result_components[measure.name] = new_measure
+            if cls.op == COUNT and len(measures) <= 1:
+                measure_name = COMP_NAME_MAPPING[cls.return_type]
+                nullable = False if len(measures) == 0 else measures[0].nullable
+                if len(measures) == 1:
+                    del result_components[measures[0].name]
+                result_components[measure_name] = Component(
+                    name=measure_name,
+                    data_type=cls.return_type,
+                    role=Role.MEASURE,
+                    nullable=nullable,
+                )
 
         return Dataset(name="result", components=result_components, data=None)
 
@@ -151,10 +174,7 @@ class Analytic(Operator.Unary):
             window_str = f"{mode} BETWEEN {window.start} {start_mode} AND {window.stop} {stop_mode}"
 
         # Partitioning
-        if len(partitioning) > 0:
-            partition = "PARTITION BY " + ", ".join(partitioning)
-        else:
-            partition = ""
+        partition = "PARTITION BY " + ", ".join(partitioning) if len(partitioning) > 0 else ""
 
         # Ordering
         order_str = ""
@@ -205,12 +225,17 @@ class Analytic(Operator.Unary):
         ordering: Optional[List[OrderBy]],
         window: Optional[Windowing],
         params: Optional[List[int]],
+        component_name: Optional[str] = None,
     ) -> Dataset:
-        result = cls.validate(operand, partitioning, ordering, window, params)
+        result = cls.validate(operand, partitioning, ordering, window, params, component_name)
         df = operand.data.copy() if operand.data is not None else pd.DataFrame()
-        measure_names = operand.get_measures_names()
         identifier_names = operand.get_identifiers_names()
 
+        if component_name is not None:
+            measure_names = [component_name]
+        else:
+            measure_names = operand.get_measures_names()
+
         result.data = cls.analyticfunc(
             df=df,
             partitioning=partitioning,

vtlengine/Operators/Assignment.py

@@ -1,9 +1,8 @@
-from typing import Union, Any
-
-from vtlengine.Operators import Binary
+from typing import Any, Union
 
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import DataComponent, Dataset
+from vtlengine.Operators import Binary
 
 ALL_MODEL_TYPES = Union[DataComponent, Dataset]
 

vtlengine/Operators/Boolean.py

@@ -2,13 +2,13 @@
 #     import pyspark.pandas as pd
 # else:
 #     import pandas as pd
-import pandas as pd
+from typing import Any, Optional
 
-from typing import Optional, Any
+import pandas as pd
 
-from vtlengine.AST.Grammar.tokens import AND, OR, XOR, NOT
-from vtlengine.DataTypes import Boolean
 import vtlengine.Operators as Operator
+from vtlengine.AST.Grammar.tokens import AND, NOT, OR, XOR
+from vtlengine.DataTypes import Boolean
 
 
 class Unary(Operator.Unary):
@@ -30,9 +30,7 @@ class Binary(Operator.Binary):
 
     @classmethod
     def apply_operation_two_series(cls, left_series: Any, right_series: Any) -> Any:
-        result = cls.comp_op(
-            left_series.astype("bool[pyarrow]"), right_series.astype("bool[pyarrow]")
-        )
+        result = cls.comp_op(left_series.astype("boolean"), right_series.astype("boolean"))
         return result.replace({pd.NA: None}).astype(object)
 
     @classmethod

vtlengine/Operators/CastOperator.py

@@ -1,27 +1,27 @@
 from copy import copy
-from typing import Optional, Any, Union, Type
+from typing import Any, Optional, Type, Union
 
-import vtlengine.Operators as Operator
 import pandas as pd
+
+import vtlengine.Operators as Operator
+from vtlengine.AST.Grammar.tokens import CAST
 from vtlengine.DataTypes import (
     COMP_NAME_MAPPING,
     EXPLICIT_WITH_MASK_TYPE_PROMOTION_MAPPING,
     EXPLICIT_WITHOUT_MASK_TYPE_PROMOTION_MAPPING,
     IMPLICIT_TYPE_PROMOTION_MAPPING,
-    String,
-    Number,
-    TimeInterval,
+    SCALAR_TYPES_CLASS_REVERSE,
     Date,
-    TimePeriod,
     Duration,
-    SCALAR_TYPES_CLASS_REVERSE,
+    Number,
     ScalarType,
+    String,
+    TimeInterval,
+    TimePeriod,
 )
 from vtlengine.DataTypes.TimeHandling import str_period_to_date
-
-from vtlengine.AST.Grammar.tokens import CAST
 from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Component, Dataset, Role, Scalar, DataComponent
+from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar
 
 duration_mapping = {"A": 6, "S": 5, "Q": 4, "M": 3, "W": 2, "D": 1}
 
@@ -286,9 +286,8 @@ class Cast(Operator.Unary):
         mask: Optional[str] = None,
     ) -> Any:
 
-        if mask is not None:
-            if not isinstance(mask, str):
-                raise Exception(f"{cls.op} mask must be a string")
+        if mask is not None and not isinstance(mask, str):
+            raise Exception(f"{cls.op} mask must be a string")
 
         if isinstance(operand, Dataset):
             return cls.dataset_validation(operand, scalarType, mask)

vtlengine/Operators/Clause.py

@@ -1,21 +1,20 @@
-import pandas as pd
-
 from copy import copy
-from typing import List, Union, Type
+from typing import List, Type, Union
+
+import pandas as pd
 
+from vtlengine.AST import RenameNode
+from vtlengine.AST.Grammar.tokens import AGGREGATE, CALC, DROP, KEEP, RENAME, SUBSPACE
 from vtlengine.DataTypes import (
     Boolean,
+    ScalarType,
     String,
     check_unary_implicit_promotion,
     unary_implicit_promotion,
-    ScalarType,
 )
-from vtlengine.Operators import Operator
-
-from vtlengine.AST import RenameNode
-from vtlengine.AST.Grammar.tokens import KEEP, DROP, RENAME, SUBSPACE, CALC, AGGREGATE
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar
+from vtlengine.Operators import Operator
 
 
 class Calc(Operator):
@@ -162,9 +161,8 @@ class Keep(Operator):
     def evaluate(cls, operands: List[str], dataset: Dataset) -> Dataset:
         if len(operands) == 0:
             raise ValueError("Keep clause requires at least one operand")
-        if dataset is None:
-            if sum(isinstance(operand, Dataset) for operand in operands) != 1:
-                raise ValueError("Keep clause requires at most one dataset operand")
+        if dataset is None and sum(isinstance(operand, Dataset) for operand in operands) != 1:
+            raise ValueError("Keep clause requires at most one dataset operand")
         result_dataset = cls.validate(operands, dataset)
         if dataset.data is not None:
             result_dataset.data = dataset.data[dataset.get_identifiers_names() + operands]
@@ -212,11 +210,11 @@ class Rename(Operator):
             raise SemanticError("1-3-1", alias=duplicates)
 
         for operand in operands:
-            if operand.old_name not in dataset.components.keys():
+            if operand.old_name not in dataset.components:
                 raise SemanticError(
                     "1-1-1-10", op=cls.op, comp_name=operand.old_name, dataset_name=dataset.name
                 )
-            if operand.new_name in dataset.components.keys():
+            if operand.new_name in dataset.components:
                 raise SemanticError(
                     "1-1-6-8", op=cls.op, comp_name=operand.new_name, dataset_name=dataset.name
                 )

vtlengine/Operators/Comparison.py

@@ -3,15 +3,13 @@ import re
 from copy import copy
 from typing import Any, Optional, Union
 
-from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar, ScalarSet
-
 # if os.environ.get("SPARK"):
 #     import pyspark.pandas as pd
 # else:
 #     import pandas as pd
 import pandas as pd
 
+import vtlengine.Operators as Operator
 from vtlengine.AST.Grammar.tokens import (
     CHARSET_MATCH,
     EQ,
@@ -24,8 +22,9 @@ from vtlengine.AST.Grammar.tokens import (
     NEQ,
     NOT_IN,
 )
-from vtlengine.DataTypes import Boolean, COMP_NAME_MAPPING, String, Number, Null
-import vtlengine.Operators as Operator
+from vtlengine.DataTypes import COMP_NAME_MAPPING, Boolean, Null, Number, String
+from vtlengine.Exceptions import SemanticError
+from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar, ScalarSet
 
 
 class Unary(Operator.Unary):
@@ -75,11 +74,11 @@ class Binary(Operator.Binary):
     return_type = Boolean
 
     @classmethod
-    def _cast_values(
-        cls, x: Optional[Union[int, float, str, bool]], y: Optional[Union[int, float, str, bool]]
-    ) -> Any:
-        # Cast both values to the same data type
-        # An integer can be considered a bool, we must check first boolean, then numbers
+    def _cast_values(cls,
+                     x: Optional[Union[int, float, str, bool]],
+                     y: Optional[Union[int, float, str, bool]]
+                     ) -> Any:
+        # Cast values to compatible types for comparison
         try:
             if isinstance(x, str) and isinstance(y, bool):
                 y = String.cast(y)
@@ -97,6 +96,7 @@ class Binary(Operator.Binary):
 
     @classmethod
     def op_func(cls, x: Any, y: Any) -> Any:
+        # Return None if any of the values are NaN
         if pd.isnull(x) or pd.isnull(y):
             return None
         x, y = cls._cast_values(x, y)
@@ -104,12 +104,29 @@ class Binary(Operator.Binary):
 
     @classmethod
     def apply_operation_series_scalar(cls, series: Any, scalar: Any, series_left: bool) -> Any:
-        if scalar is None:
+        if pd.isnull(scalar):
             return pd.Series(None, index=series.index)
+
+        first_non_null = series.dropna().iloc[0] if not series.dropna().empty else None
+        if first_non_null is not None:
+            scalar, first_non_null = cls._cast_values(scalar, first_non_null)
+
+            series_type = pd.api.types.infer_dtype(series, skipna=True)
+            first_non_null_type = pd.api.types.infer_dtype([first_non_null])
+
+            if series_type != first_non_null_type:
+                if isinstance(first_non_null, str):
+                    series = series.astype(str)
+                elif isinstance(first_non_null, (int, float)):
+                    series = series.astype(float)
+
+        op = cls.py_op if cls.py_op is not None else cls.op_func
         if series_left:
-            return series.map(lambda x: cls.op_func(x, scalar), na_action="ignore")
+            result = series.map(lambda x: op(x, scalar), na_action="ignore")
         else:
-            return series.map(lambda x: cls.op_func(scalar, x), na_action="ignore")
+            result = series.map(lambda x: op(scalar, x), na_action="ignore")
+
+        return result
 
     @classmethod
     def apply_return_type_dataset(
@@ -408,10 +425,7 @@ class ExistIn(Operator.Operator):
         reference_identifiers_names = left_id_names
 
         # Checking if the left dataset is a subset of the right dataset
-        if is_subset_left:
-            common_columns = left_id_names
-        else:
-            common_columns = right_id_names
+        common_columns = left_id_names if is_subset_left else right_id_names
 
         # Check if the common identifiers are equal between the two datasets
         if dataset_1.data is not None and dataset_2.data is not None:

vtlengine/Operators/Conditional.py

@@ -1,31 +1,30 @@
 from copy import copy
-from typing import Union, Any
+from typing import Any, List, Union
 
 import numpy as np
 
+# if os.environ.get("SPARK", False):
+#     import pyspark.pandas as pd
+# else:
+#     import pandas as pd
+import pandas as pd
+
 from vtlengine.DataTypes import (
-    Boolean,
     COMP_NAME_MAPPING,
-    binary_implicit_promotion,
     SCALAR_TYPES_CLASS_REVERSE,
+    Boolean,
     Null,
+    binary_implicit_promotion,
 )
-from vtlengine.Operators import Operator, Binary
-
 from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Scalar, DataComponent, Dataset, Role
-
-# if os.environ.get("SPARK", False):
-#     import pyspark.pandas as pd
-# else:
-#     import pandas as pd
-import pandas as pd
+from vtlengine.Model import DataComponent, Dataset, Role, Scalar
+from vtlengine.Operators import Binary, Operator
 
 
 class If(Operator):
     """
     If class:
-        `If-then-else <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=225&zoom=100,72,142>`_ operator # noqa E501
+        `If-then-else <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=225&zoom=100,72,142>`_ operator
         inherits from Operator, a superclass that contains general validate and evaluate class methods.
         It has the following class methods:
     Class methods:
@@ -40,7 +39,7 @@ class If(Operator):
         validate: Class method that has two branches so datacomponent and datasets can be validated. With datacomponent,
         the code reviews if it is actually a Measure and if it is a binary operation. Dataset branch reviews if the
         identifiers are the same in 'if', 'then' and 'else'.
-    """
+    """ # noqa E501
 
     @classmethod
     def evaluate(cls, condition: Any, true_branch: Any, false_branch: Any) -> Any:
@@ -108,7 +107,7 @@ class If(Operator):
         )
         if isinstance(result, Dataset):
             drop_columns = [
-                column for column in result.data.columns if column not in result.components.keys()
+                column for column in result.data.columns if column not in result.components
             ]
             result.data = result.data.dropna(subset=drop_columns).drop(columns=drop_columns)
         if isinstance(true_branch, Scalar) and isinstance(false_branch, Scalar):
@@ -213,14 +212,14 @@ class If(Operator):
 class Nvl(Binary):
     """
     Null class:
-        `Nvl <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=229&zoom=100,72,370>`_operator class. # noqa E501
+        `Nvl <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=229&zoom=100,72,370>`_operator class.
         It has the following class methods:
 
     Class methods:
         Validate: Class method that validates if the operation at scalar,
         datacomponent or dataset level can be performed.
         Evaluate: Evaluates the actual operation, returning the result.
-    """
+    """ # noqa E501
 
     @classmethod
     def evaluate(cls, left: Any, right: Any) -> Union[Scalar, DataComponent, Dataset]:
@@ -288,3 +287,144 @@ class Nvl(Binary):
             for comp in result_components.values():
                 comp.nullable = False
         return Dataset(name="result", components=result_components, data=None)
+
+
+class Case(Operator):
+
+    @classmethod
+    def evaluate(
+        cls, conditions: List[Any], thenOps: List[Any], elseOp: Any
+    ) -> Union[Scalar, DataComponent, Dataset]:
+
+        result = cls.validate(conditions, thenOps, elseOp)
+
+        if isinstance(result, Scalar):
+            result.value = elseOp.value
+            for i in range(len(conditions)):
+                if conditions[i].value:
+                    result.value = thenOps[i].value
+
+        if isinstance(result, DataComponent):
+            result.data = pd.Series(None, index=conditions[0].data.index)
+
+            for i, condition in enumerate(conditions):
+                value = thenOps[i].value if isinstance(thenOps[i], Scalar) else thenOps[i].data
+                result.data = np.where(
+                    condition.data, value, result.data  # type: ignore[call-overload]
+                )
+
+            condition_mask_else = ~np.any([condition.data for condition in conditions], axis=0)
+            else_value = elseOp.value if isinstance(elseOp, Scalar) else elseOp.data
+            result.data = pd.Series(
+                np.where(condition_mask_else, else_value, result.data),
+                index=conditions[0].data.index,
+            )
+
+        if isinstance(result, Dataset):
+            identifiers = result.get_identifiers_names()
+            columns = [col for col in result.get_components_names() if col not in identifiers]
+            result.data = (
+                conditions[0].data[identifiers]
+                if conditions[0].data is not None
+                else pd.DataFrame(columns=identifiers)
+            )
+
+            for i in range(len(conditions)):
+                condition = conditions[i]
+                bool_col = next(x.name for x in condition.get_measures() if x.data_type == Boolean)
+                condition_mask = condition.data[bool_col]
+
+                result.data.loc[condition_mask, columns] = (
+                    thenOps[i].value
+                    if isinstance(thenOps[i], Scalar)
+                    else thenOps[i].data.loc[condition_mask, columns]
+                )
+
+            condition_mask_else = ~np.logical_or.reduce(
+                [
+                    condition.data[
+                        next(x.name for x in condition.get_measures() if x.data_type == Boolean)
+                    ].astype(bool)
+                    for condition in conditions
+                ]
+            )
+
+            result.data.loc[condition_mask_else, columns] = (
+                elseOp.value
+                if isinstance(elseOp, Scalar)
+                else elseOp.data.loc[condition_mask_else, columns]
+            )
+
+        return result
+
+    @classmethod
+    def validate(
+        cls, conditions: List[Any], thenOps: List[Any], elseOp: Any
+    ) -> Union[Scalar, DataComponent, Dataset]:
+
+        if len(set(map(type, conditions))) > 1:
+            raise SemanticError("2-1-9-1", op=cls.op)
+
+        ops = thenOps + [elseOp]
+        then_else_types = set(map(type, ops))
+        condition_type = type(conditions[0])
+
+        if condition_type is Scalar:
+            for condition in conditions:
+                if condition.data_type != Boolean:
+                    raise SemanticError("2-1-9-2", op=cls.op, name=condition.name)
+            if list(then_else_types) != [Scalar]:
+                raise SemanticError("2-1-9-3", op=cls.op)
+
+            # The output data type is the data type of the last then operation that has a true
+            # condition, defaulting to the data type of the else operation if no condition is true
+            output_data_type = elseOp.data_type
+            for i in range(len(conditions)):
+                if conditions[i].value:
+                    output_data_type = thenOps[i].data_type
+
+            return Scalar(
+                name="result",
+                value=None,
+                data_type=output_data_type,
+            )
+
+        elif condition_type is DataComponent:
+            for condition in conditions:
+                if not condition.data_type == Boolean:
+                    raise SemanticError("2-1-9-4", op=cls.op, name=condition.name)
+
+            nullable = any(
+                thenOp.nullable if isinstance(thenOp, DataComponent) else thenOp.data_type == Null
+                for thenOp in ops
+            )
+
+            data_type = ops[0].data_type
+            for op in ops[1:]:
+                data_type = binary_implicit_promotion(data_type, op.data_type)
+
+            return DataComponent(
+                name="result",
+                data=None,
+                data_type=data_type,
+                role=Role.MEASURE,
+                nullable=nullable,
+            )
+
+        # Dataset
+        for condition in conditions:
+            if len(condition.get_measures_names()) != 1:
+                raise SemanticError("1-1-1-4", op=cls.op)
+            if condition.get_measures()[0].data_type != Boolean:
+                raise SemanticError("2-1-9-5", op=cls.op, name=condition.name)
+
+        if Dataset not in then_else_types:
+            raise SemanticError("2-1-9-6", op=cls.op)
+
+        components = next(op for op in ops if isinstance(op, Dataset)).components
+        comp_names = [comp.name for comp in components.values()]
+        for op in ops:
+            if isinstance(op, Dataset) and op.get_components_names() != comp_names:
+                raise SemanticError("2-1-9-7", op=cls.op)
+
+        return Dataset(name="result", components=components, data=None)

vtlengine/Operators/General.py

@@ -1,11 +1,11 @@
-from typing import Dict, List, Any, Union
+import sqlite3
+from typing import Any, Dict, List, Union
 
 import pandas as pd
-import sqlite3
 
 from vtlengine.DataTypes import COMP_NAME_MAPPING
 from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Dataset, ExternalRoutine, Role, Component, DataComponent
+from vtlengine.Model import Component, DataComponent, Dataset, ExternalRoutine, Role
 from vtlengine.Operators import Binary, Unary
 
 
@@ -143,7 +143,7 @@ class Eval(Unary):
         df = cls._execute_query(
             external_routine.query, external_routine.dataset_names, empty_data_dict
         )
-        component_names = [name for name in df.columns]
+        component_names = df.columns.tolist()
         for comp_name in component_names:
             if comp_name not in output.components:
                 raise SemanticError(