vtlengine 1.0.2__py3-none-any.whl → 1.0.3rc1__py3-none-any.whl

vtlengine/Model/__init__.py

@@ -2,17 +2,18 @@ import json
 from collections import Counter
 from dataclasses import dataclass
 from enum import Enum
-from typing import Dict, List, Optional, Union, Any, Type
+from typing import Any, Dict, List, Optional, Type, Union
 
-import vtlengine.DataTypes as DataTypes
 import pandas as pd
 import sqlglot
 import sqlglot.expressions as exp
-from vtlengine.DataTypes import SCALAR_TYPES, ScalarType
-from vtlengine.DataTypes.TimeHandling import TimePeriodHandler
 from pandas import DataFrame as PandasDataFrame
 from pandas._testing import assert_frame_equal
 
+import vtlengine.DataTypes as DataTypes
+from vtlengine.DataTypes import SCALAR_TYPES, ScalarType
+from vtlengine.DataTypes.TimeHandling import TimePeriodHandler
+from vtlengine.Exceptions import SemanticError
 
 # from pyspark.pandas import DataFrame as SparkDataFrame, Series as SparkSeries
 
@@ -159,7 +160,7 @@ class Dataset:
                 raise ValueError(
                     "The number of components must match the number of columns in the data"
                 )
-            for name, component in self.components.items():
+            for name, _ in self.components.items():
                 if name not in self.data.columns:
                     raise ValueError(f"Component {name} not found in the data")
 
@@ -209,8 +210,8 @@ class Dataset:
             return True
         elif self.data is None or other.data is None:
             return False
-        if len(self.data) == len(other.data) == 0:
-            assert self.data.shape == other.data.shape
+        if len(self.data) == len(other.data) == 0 and self.data.shape != other.data.shape:
+            raise SemanticError("0-1-1-14", dataset1=self.name, dataset2=other.name)
 
         self.data.fillna("", inplace=True)
         other.data.fillna("", inplace=True)
@@ -234,11 +235,8 @@ class Dataset:
                     lambda x: str(TimePeriodHandler(x)) if x != "" else "", na_action="ignore"
                 )
             elif type_name in ["Integer", "Number"]:
-                if type_name == "Integer":
-                    type_ = "int64"
-                else:
-                    type_ = "float32"
-                    # We use here a number to avoid errors on equality on empty strings
+                type_ = "int64" if type_name == "Integer" else "float32"
+                # We use here a number to avoid errors on equality on empty strings
                 self.data[comp.name] = (
                     self.data[comp.name]
                     .replace("", -1234997)

vtlengine/Operators/Aggregation.py

@@ -1,19 +1,8 @@
 from copy import copy
-from typing import List, Optional, Any
+from typing import Any, List, Optional
 
 import duckdb
 import pandas as pd
-from vtlengine.DataTypes import (
-    Integer,
-    Number,
-    unary_implicit_promotion,
-    Boolean,
-    String,
-    Duration,
-    TimeInterval,
-    TimePeriod,
-    Date,
-)
 
 import vtlengine.Operators as Operator
 from vtlengine.AST.Grammar.tokens import (
@@ -28,11 +17,22 @@ from vtlengine.AST.Grammar.tokens import (
     VAR_POP,
     VAR_SAMP,
 )
+from vtlengine.DataTypes import (
+    Boolean,
+    Date,
+    Duration,
+    Integer,
+    Number,
+    String,
+    TimeInterval,
+    TimePeriod,
+    unary_implicit_promotion,
+)
 from vtlengine.DataTypes.TimeHandling import (
     DURATION_MAPPING,
     DURATION_MAPPING_REVERSED,
-    TimePeriodHandler,
     TimeIntervalHandler,
+    TimePeriodHandler,
 )
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, Dataset, Role
@@ -153,7 +153,7 @@ class Aggregation(Operator.Unary):
             if comp.role == Role.ATTRIBUTE:
                 del result_components[comp_name]
         # Change Measure data type
-        for comp_name, comp in result_components.items():
+        for _, comp in result_components.items():
             if comp.role == Role.MEASURE:
                 unary_implicit_promotion(comp.data_type, cls.type_to_check)
                 if cls.return_type is not None:

vtlengine/Operators/Analytic.py

@@ -3,8 +3,6 @@ from typing import List, Optional
 
 import duckdb
 
-from vtlengine.Exceptions import SemanticError
-
 # if os.environ.get("SPARK"):
 #     import pyspark.pandas as pd
 # else:
@@ -32,6 +30,7 @@ from vtlengine.AST.Grammar.tokens import (
     VAR_SAMP,
 )
 from vtlengine.DataTypes import COMP_NAME_MAPPING, Integer, Number, unary_implicit_promotion
+from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, Dataset, Role
 
 
@@ -60,10 +59,7 @@ class Analytic(Operator.Unary):
         params: Optional[List[int]],
         component_name: Optional[str] = None,
     ) -> Dataset:
-        if ordering is None:
-            order_components = []
-        else:
-            order_components = [o.component for o in ordering]
+        order_components = [] if ordering is None else [o.component for o in ordering]
         identifier_names = operand.get_identifiers_names()
         result_components = operand.components.copy()
 
@@ -178,10 +174,7 @@ class Analytic(Operator.Unary):
             window_str = f"{mode} BETWEEN {window.start} {start_mode} AND {window.stop} {stop_mode}"
 
         # Partitioning
-        if len(partitioning) > 0:
-            partition = "PARTITION BY " + ", ".join(partitioning)
-        else:
-            partition = ""
+        partition = "PARTITION BY " + ", ".join(partitioning) if len(partitioning) > 0 else ""
 
         # Ordering
         order_str = ""

vtlengine/Operators/Assignment.py

@@ -1,9 +1,8 @@
-from typing import Union, Any
-
-from vtlengine.Operators import Binary
+from typing import Any, Union
 
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import DataComponent, Dataset
+from vtlengine.Operators import Binary
 
 ALL_MODEL_TYPES = Union[DataComponent, Dataset]
 

vtlengine/Operators/Boolean.py

@@ -2,13 +2,13 @@
 #     import pyspark.pandas as pd
 # else:
 #     import pandas as pd
-import pandas as pd
+from typing import Any, Optional
 
-from typing import Optional, Any
+import pandas as pd
 
-from vtlengine.AST.Grammar.tokens import AND, OR, XOR, NOT
-from vtlengine.DataTypes import Boolean
 import vtlengine.Operators as Operator
+from vtlengine.AST.Grammar.tokens import AND, NOT, OR, XOR
+from vtlengine.DataTypes import Boolean
 
 
 class Unary(Operator.Unary):
@@ -30,9 +30,7 @@ class Binary(Operator.Binary):
 
     @classmethod
     def apply_operation_two_series(cls, left_series: Any, right_series: Any) -> Any:
-        result = cls.comp_op(
-            left_series.astype("boolean"), right_series.astype("boolean")
-        )
+        result = cls.comp_op(left_series.astype("boolean"), right_series.astype("boolean"))
         return result.replace({pd.NA: None}).astype(object)
 
     @classmethod

vtlengine/Operators/CastOperator.py

@@ -1,27 +1,27 @@
 from copy import copy
-from typing import Optional, Any, Union, Type
+from typing import Any, Optional, Type, Union
 
-import vtlengine.Operators as Operator
 import pandas as pd
+
+import vtlengine.Operators as Operator
+from vtlengine.AST.Grammar.tokens import CAST
 from vtlengine.DataTypes import (
     COMP_NAME_MAPPING,
     EXPLICIT_WITH_MASK_TYPE_PROMOTION_MAPPING,
     EXPLICIT_WITHOUT_MASK_TYPE_PROMOTION_MAPPING,
     IMPLICIT_TYPE_PROMOTION_MAPPING,
-    String,
-    Number,
-    TimeInterval,
+    SCALAR_TYPES_CLASS_REVERSE,
     Date,
-    TimePeriod,
     Duration,
-    SCALAR_TYPES_CLASS_REVERSE,
+    Number,
     ScalarType,
+    String,
+    TimeInterval,
+    TimePeriod,
 )
 from vtlengine.DataTypes.TimeHandling import str_period_to_date
-
-from vtlengine.AST.Grammar.tokens import CAST
 from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Component, Dataset, Role, Scalar, DataComponent
+from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar
 
 duration_mapping = {"A": 6, "S": 5, "Q": 4, "M": 3, "W": 2, "D": 1}
 
@@ -286,9 +286,8 @@ class Cast(Operator.Unary):
         mask: Optional[str] = None,
     ) -> Any:
 
-        if mask is not None:
-            if not isinstance(mask, str):
-                raise Exception(f"{cls.op} mask must be a string")
+        if mask is not None and not isinstance(mask, str):
+            raise Exception(f"{cls.op} mask must be a string")
 
         if isinstance(operand, Dataset):
             return cls.dataset_validation(operand, scalarType, mask)

vtlengine/Operators/Clause.py

@@ -1,21 +1,20 @@
-import pandas as pd
-
 from copy import copy
-from typing import List, Union, Type
+from typing import List, Type, Union
+
+import pandas as pd
 
+from vtlengine.AST import RenameNode
+from vtlengine.AST.Grammar.tokens import AGGREGATE, CALC, DROP, KEEP, RENAME, SUBSPACE
 from vtlengine.DataTypes import (
     Boolean,
+    ScalarType,
     String,
     check_unary_implicit_promotion,
     unary_implicit_promotion,
-    ScalarType,
 )
-from vtlengine.Operators import Operator
-
-from vtlengine.AST import RenameNode
-from vtlengine.AST.Grammar.tokens import KEEP, DROP, RENAME, SUBSPACE, CALC, AGGREGATE
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar
+from vtlengine.Operators import Operator
 
 
 class Calc(Operator):
@@ -162,9 +161,8 @@ class Keep(Operator):
     def evaluate(cls, operands: List[str], dataset: Dataset) -> Dataset:
         if len(operands) == 0:
             raise ValueError("Keep clause requires at least one operand")
-        if dataset is None:
-            if sum(isinstance(operand, Dataset) for operand in operands) != 1:
-                raise ValueError("Keep clause requires at most one dataset operand")
+        if dataset is None and sum(isinstance(operand, Dataset) for operand in operands) != 1:
+            raise ValueError("Keep clause requires at most one dataset operand")
         result_dataset = cls.validate(operands, dataset)
         if dataset.data is not None:
             result_dataset.data = dataset.data[dataset.get_identifiers_names() + operands]
@@ -212,11 +210,11 @@ class Rename(Operator):
             raise SemanticError("1-3-1", alias=duplicates)
 
         for operand in operands:
-            if operand.old_name not in dataset.components.keys():
+            if operand.old_name not in dataset.components:
                 raise SemanticError(
                     "1-1-1-10", op=cls.op, comp_name=operand.old_name, dataset_name=dataset.name
                 )
-            if operand.new_name in dataset.components.keys():
+            if operand.new_name in dataset.components:
                 raise SemanticError(
                     "1-1-6-8", op=cls.op, comp_name=operand.new_name, dataset_name=dataset.name
                 )

vtlengine/Operators/Comparison.py

@@ -3,15 +3,13 @@ import re
 from copy import copy
 from typing import Any, Optional, Union
 
-from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar, ScalarSet
-
 # if os.environ.get("SPARK"):
 #     import pyspark.pandas as pd
 # else:
 #     import pandas as pd
 import pandas as pd
 
+import vtlengine.Operators as Operator
 from vtlengine.AST.Grammar.tokens import (
     CHARSET_MATCH,
     EQ,
@@ -24,8 +22,9 @@ from vtlengine.AST.Grammar.tokens import (
     NEQ,
     NOT_IN,
 )
-from vtlengine.DataTypes import Boolean, COMP_NAME_MAPPING, String, Number, Null
-import vtlengine.Operators as Operator
+from vtlengine.DataTypes import COMP_NAME_MAPPING, Boolean, Null, Number, String
+from vtlengine.Exceptions import SemanticError
+from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar, ScalarSet
 
 
 class Unary(Operator.Unary):
@@ -75,11 +74,11 @@ class Binary(Operator.Binary):
     return_type = Boolean
 
     @classmethod
-    def _cast_values(
-        cls, x: Optional[Union[int, float, str, bool]], y: Optional[Union[int, float, str, bool]]
-    ) -> Any:
-        # Cast both values to the same data type
-        # An integer can be considered a bool, we must check first boolean, then numbers
+    def _cast_values(cls,
+                     x: Optional[Union[int, float, str, bool]],
+                     y: Optional[Union[int, float, str, bool]]
+                     ) -> Any:
+        # Cast values to compatible types for comparison
         try:
             if isinstance(x, str) and isinstance(y, bool):
                 y = String.cast(y)
@@ -97,6 +96,7 @@ class Binary(Operator.Binary):
 
     @classmethod
     def op_func(cls, x: Any, y: Any) -> Any:
+        # Return None if any of the values are NaN
         if pd.isnull(x) or pd.isnull(y):
             return None
         x, y = cls._cast_values(x, y)
@@ -104,12 +104,29 @@ class Binary(Operator.Binary):
 
     @classmethod
     def apply_operation_series_scalar(cls, series: Any, scalar: Any, series_left: bool) -> Any:
-        if scalar is None:
+        if pd.isnull(scalar):
             return pd.Series(None, index=series.index)
+
+        first_non_null = series.dropna().iloc[0] if not series.dropna().empty else None
+        if first_non_null is not None:
+            scalar, first_non_null = cls._cast_values(scalar, first_non_null)
+
+            series_type = pd.api.types.infer_dtype(series, skipna=True)
+            first_non_null_type = pd.api.types.infer_dtype([first_non_null])
+
+            if series_type != first_non_null_type:
+                if isinstance(first_non_null, str):
+                    series = series.astype(str)
+                elif isinstance(first_non_null, (int, float)):
+                    series = series.astype(float)
+
+        op = cls.py_op if cls.py_op is not None else cls.op_func
         if series_left:
-            return series.map(lambda x: cls.op_func(x, scalar), na_action="ignore")
+            result = series.map(lambda x: op(x, scalar), na_action="ignore")
         else:
-            return series.map(lambda x: cls.op_func(scalar, x), na_action="ignore")
+            result = series.map(lambda x: op(scalar, x), na_action="ignore")
+
+        return result
 
     @classmethod
     def apply_return_type_dataset(
@@ -408,10 +425,7 @@ class ExistIn(Operator.Operator):
         reference_identifiers_names = left_id_names
 
         # Checking if the left dataset is a subset of the right dataset
-        if is_subset_left:
-            common_columns = left_id_names
-        else:
-            common_columns = right_id_names
+        common_columns = left_id_names if is_subset_left else right_id_names
 
         # Check if the common identifiers are equal between the two datasets
         if dataset_1.data is not None and dataset_2.data is not None:

vtlengine/Operators/Conditional.py

@@ -1,31 +1,30 @@
 from copy import copy
-from typing import Union, Any, List
+from typing import Any, List, Union
 
 import numpy as np
 
+# if os.environ.get("SPARK", False):
+#     import pyspark.pandas as pd
+# else:
+#     import pandas as pd
+import pandas as pd
+
 from vtlengine.DataTypes import (
-    Boolean,
     COMP_NAME_MAPPING,
-    binary_implicit_promotion,
     SCALAR_TYPES_CLASS_REVERSE,
+    Boolean,
     Null,
+    binary_implicit_promotion,
 )
-from vtlengine.Operators import Operator, Binary
-
 from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Scalar, DataComponent, Dataset, Role
-
-# if os.environ.get("SPARK", False):
-#     import pyspark.pandas as pd
-# else:
-#     import pandas as pd
-import pandas as pd
+from vtlengine.Model import DataComponent, Dataset, Role, Scalar
+from vtlengine.Operators import Binary, Operator
 
 
 class If(Operator):
     """
     If class:
-        `If-then-else <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=225&zoom=100,72,142>`_ operator # noqa E501
+        `If-then-else <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=225&zoom=100,72,142>`_ operator
         inherits from Operator, a superclass that contains general validate and evaluate class methods.
         It has the following class methods:
     Class methods:
@@ -40,7 +39,7 @@ class If(Operator):
         validate: Class method that has two branches so datacomponent and datasets can be validated. With datacomponent,
         the code reviews if it is actually a Measure and if it is a binary operation. Dataset branch reviews if the
         identifiers are the same in 'if', 'then' and 'else'.
-    """
+    """ # noqa E501
 
     @classmethod
     def evaluate(cls, condition: Any, true_branch: Any, false_branch: Any) -> Any:
@@ -108,7 +107,7 @@ class If(Operator):
         )
         if isinstance(result, Dataset):
             drop_columns = [
-                column for column in result.data.columns if column not in result.components.keys()
+                column for column in result.data.columns if column not in result.components
             ]
             result.data = result.data.dropna(subset=drop_columns).drop(columns=drop_columns)
         if isinstance(true_branch, Scalar) and isinstance(false_branch, Scalar):
@@ -213,14 +212,14 @@ class If(Operator):
 class Nvl(Binary):
     """
     Null class:
-        `Nvl <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=229&zoom=100,72,370>`_operator class. # noqa E501
+        `Nvl <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=229&zoom=100,72,370>`_operator class.
         It has the following class methods:
 
     Class methods:
         Validate: Class method that validates if the operation at scalar,
         datacomponent or dataset level can be performed.
         Evaluate: Evaluates the actual operation, returning the result.
-    """
+    """ # noqa E501
 
     @classmethod
     def evaluate(cls, left: Any, right: Any) -> Union[Scalar, DataComponent, Dataset]:
@@ -287,21 +286,15 @@ class Nvl(Binary):
             }
             for comp in result_components.values():
                 comp.nullable = False
-        return Dataset(
-            name="result",
-            components=result_components,
-            data=None
-        )
+        return Dataset(name="result", components=result_components, data=None)
 
 
 class Case(Operator):
 
     @classmethod
-    def evaluate(cls,
-                 conditions: List[Any],
-                 thenOps: List[Any],
-                 elseOp: Any
-                 ) -> Union[Scalar, DataComponent, Dataset]:
+    def evaluate(
+        cls, conditions: List[Any], thenOps: List[Any], elseOp: Any
+    ) -> Union[Scalar, DataComponent, Dataset]:
 
         result = cls.validate(conditions, thenOps, elseOp)
 
@@ -316,19 +309,25 @@ class Case(Operator):
 
             for i, condition in enumerate(conditions):
                 value = thenOps[i].value if isinstance(thenOps[i], Scalar) else thenOps[i].data
-                result.data = np.where(condition.data, value,  # type: ignore[call-overload]
-                                       result.data)
+                result.data = np.where(
+                    condition.data, value, result.data  # type: ignore[call-overload]
+                )
 
             condition_mask_else = ~np.any([condition.data for condition in conditions], axis=0)
             else_value = elseOp.value if isinstance(elseOp, Scalar) else elseOp.data
-            result.data = pd.Series(np.where(condition_mask_else, else_value, result.data),
-                                    index=conditions[0].data.index)
+            result.data = pd.Series(
+                np.where(condition_mask_else, else_value, result.data),
+                index=conditions[0].data.index,
+            )
 
         if isinstance(result, Dataset):
             identifiers = result.get_identifiers_names()
             columns = [col for col in result.get_components_names() if col not in identifiers]
-            result.data = (conditions[0].data[identifiers] if conditions[0].data is not None
-                           else pd.DataFrame(columns=identifiers))
+            result.data = (
+                conditions[0].data[identifiers]
+                if conditions[0].data is not None
+                else pd.DataFrame(columns=identifiers)
+            )
 
             for i in range(len(conditions)):
                 condition = conditions[i]
@@ -336,28 +335,32 @@ class Case(Operator):
                 condition_mask = condition.data[bool_col]
 
                 result.data.loc[condition_mask, columns] = (
-                    thenOps[i].value if isinstance(thenOps[i], Scalar)
+                    thenOps[i].value
+                    if isinstance(thenOps[i], Scalar)
                     else thenOps[i].data.loc[condition_mask, columns]
                 )
 
-            condition_mask_else = ~np.logical_or.reduce([
-                condition.data[next(x.name for x in condition.get_measures() if
-                                    x.data_type == Boolean)].astype(bool) for
-                condition in conditions])
+            condition_mask_else = ~np.logical_or.reduce(
+                [
+                    condition.data[
+                        next(x.name for x in condition.get_measures() if x.data_type == Boolean)
+                    ].astype(bool)
+                    for condition in conditions
+                ]
+            )
 
             result.data.loc[condition_mask_else, columns] = (
-                elseOp.value if isinstance(elseOp, Scalar)
+                elseOp.value
+                if isinstance(elseOp, Scalar)
                 else elseOp.data.loc[condition_mask_else, columns]
             )
 
         return result
 
     @classmethod
-    def validate(cls,
-                 conditions: List[Any],
-                 thenOps: List[Any],
-                 elseOp: Any
-                 ) -> Union[Scalar, DataComponent, Dataset]:
+    def validate(
+        cls, conditions: List[Any], thenOps: List[Any], elseOp: Any
+    ) -> Union[Scalar, DataComponent, Dataset]:
 
         if len(set(map(type, conditions))) > 1:
             raise SemanticError("2-1-9-1", op=cls.op)
@@ -424,8 +427,4 @@ class Case(Operator):
             if isinstance(op, Dataset) and op.get_components_names() != comp_names:
                 raise SemanticError("2-1-9-7", op=cls.op)
 
-        return Dataset(
-            name="result",
-            components=components,
-            data=None
-        )
+        return Dataset(name="result", components=components, data=None)

vtlengine/Operators/General.py

@@ -1,11 +1,11 @@
-from typing import Dict, List, Any, Union
+import sqlite3
+from typing import Any, Dict, List, Union
 
 import pandas as pd
-import sqlite3
 
 from vtlengine.DataTypes import COMP_NAME_MAPPING
 from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Dataset, ExternalRoutine, Role, Component, DataComponent
+from vtlengine.Model import Component, DataComponent, Dataset, ExternalRoutine, Role
 from vtlengine.Operators import Binary, Unary
 
 
@@ -143,7 +143,7 @@ class Eval(Unary):
         df = cls._execute_query(
             external_routine.query, external_routine.dataset_names, empty_data_dict
         )
-        component_names = [name for name in df.columns]
+        component_names = df.columns.tolist()
         for comp_name in component_names:
             if comp_name not in output.components:
                 raise SemanticError(