vtlengine 1.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

vtlengine/Operators/Set.py

@@ -1,12 +1,12 @@
-import os
-from typing import List
+from typing import List, Any, Dict
 
 from vtlengine.Exceptions import SemanticError
 
-if os.environ.get("SPARK"):
-    import pyspark.pandas as pd
-else:
-    import pandas as pd
+# if os.environ.get("SPARK"):
+#     import pyspark.pandas as pd
+# else:
+#     import pandas as pd
+import pandas as pd
 
 from vtlengine.Model import Dataset
 from vtlengine.Operators import Operator
@@ -18,18 +18,22 @@ class Set(Operator):
     @classmethod
     def check_same_structure(cls, dataset_1: Dataset, dataset_2: Dataset) -> None:
         if len(dataset_1.components) != len(dataset_2.components):
-            raise SemanticError("1-1-17-1", op=cls.op, dataset_1=dataset_1.name,
-                                dataset_2=dataset_2.name)
+            raise SemanticError(
+                "1-1-17-1", op=cls.op, dataset_1=dataset_1.name, dataset_2=dataset_2.name
+            )
 
         for comp in dataset_1.components.values():
             if comp.name not in dataset_2.components:
                 raise Exception(f"Component {comp.name} not found in dataset {dataset_2.name}")
             second_comp = dataset_2.components[comp.name]
-            binary_implicit_promotion(comp.data_type, second_comp.data_type, cls.type_to_check,
-                                      cls.return_type)
+            binary_implicit_promotion(
+                comp.data_type, second_comp.data_type, cls.type_to_check, cls.return_type
+            )
             if comp.role != second_comp.role:
-                raise Exception(f"Component {comp.name} has different roles "
-                                f"in datasets {dataset_1.name} and {dataset_2.name}")
+                raise Exception(
+                    f"Component {comp.name} has different roles "
+                    f"in datasets {dataset_1.name} and {dataset_2.name}"
+                )
 
     @classmethod
     def validate(cls, operands: List[Dataset]) -> Dataset:
@@ -38,7 +42,7 @@ class Set(Operator):
         for operand in operands[1:]:
             cls.check_same_structure(base_operand, operand)
 
-        result_components = {}
+        result_components: Dict[str, Any] = {}
         for operand in operands:
             if len(result_components) == 0:
                 result_components = operand.components
@@ -46,7 +50,8 @@ class Set(Operator):
                 for comp_name, comp in operand.components.items():
                     current_comp = result_components[comp_name]
                     result_components[comp_name].data_type = binary_implicit_promotion(
-                        current_comp.data_type, comp.data_type)
+                        current_comp.data_type, comp.data_type
+                    )
                     result_components[comp_name].nullable = current_comp.nullable or comp.nullable
 
         result = Dataset(name="result", components=result_components, data=None)
@@ -58,10 +63,9 @@ class Union(Set):
     def evaluate(cls, operands: List[Dataset]) -> Dataset:
         result = cls.validate(operands)
         all_datapoints = [ds.data for ds in operands]
-        result.data = pd.concat(all_datapoints, sort=True,
-                                ignore_index=True)
+        result.data = pd.concat(all_datapoints, sort=True, ignore_index=True)
         identifiers_names = result.get_identifiers_names()
-        result.data = result.data.drop_duplicates(subset=identifiers_names, keep='first')
+        result.data = result.data.drop_duplicates(subset=identifiers_names, keep="first")
         result.data.reset_index(drop=True, inplace=True)
         return result
 
@@ -76,16 +80,22 @@ class Intersection(Set):
             if result.data is None:
                 result.data = data
             else:
-                result.data = result.data.merge(data, how='inner',
-                                                on=result.get_identifiers_names())
+                if data is None:
+                    result.data = pd.DataFrame(columns=result.get_identifiers_names())
+                    break
+                result.data = result.data.merge(
+                    data, how="inner", on=result.get_identifiers_names()
+                )
 
-                not_identifiers = [col for col in result.get_measures_names() +
-                                   result.get_attributes_names()]
+                not_identifiers = [
+                    col for col in result.get_measures_names() + result.get_attributes_names()
+                ]
 
                 for col in not_identifiers:
                     result.data[col] = result.data[col + "_x"]
                 result.data = result.data[result.get_identifiers_names() + not_identifiers]
-        result.data.reset_index(drop=True, inplace=True)
+        if result.data is not None:
+            result.data.reset_index(drop=True, inplace=True)
         return result
 
 
@@ -96,35 +106,46 @@ class Symdiff(Set):
         result = cls.validate(operands)
         all_datapoints = [ds.data for ds in operands]
         for data in all_datapoints:
+            if data is None:
+                data = pd.DataFrame(columns=result.get_identifiers_names())
             if result.data is None:
                 result.data = data
             else:
                 # Realiza la operación equivalente en pyspark.pandas
-                result.data = result.data.merge(data, how='outer',
-                                                on=result.get_identifiers_names(),
-                                                suffixes=('_x', '_y'))
+                result.data = result.data.merge(
+                    data, how="outer", on=result.get_identifiers_names(), suffixes=("_x", "_y")
+                )
 
                 for measure in result.get_measures_names():
-                    result.data['_merge'] = result.data.apply(
-                        lambda row: 'left_only' if pd.isnull(row[measure + '_y']) else (
-                            'right_only' if pd.isnull(row[measure + '_x']) else 'both'),
-                        axis=1
+                    result.data["_merge"] = result.data.apply(
+                        lambda row: (
+                            "left_only"
+                            if pd.isnull(row[measure + "_y"])
+                            else ("right_only" if pd.isnull(row[measure + "_x"]) else "both")
+                        ),
+                        axis=1,
                     )
 
                 not_identifiers = result.get_measures_names() + result.get_attributes_names()
                 for col in not_identifiers:
                     result.data[col] = result.data.apply(
-                        lambda x, c=col: x[c + '_x'] if x['_merge'] == 'left_only' else (
-                            x[c + '_y'] if x['_merge'] == 'right_only' else None), axis=1)
+                        lambda x, c=col: (
+                            x[c + "_x"]
+                            if x["_merge"] == "left_only"
+                            else (x[c + "_y"] if x["_merge"] == "right_only" else None)
+                        ),
+                        axis=1,
+                    )
                 result.data = result.data[result.get_identifiers_names() + not_identifiers].dropna()
-        result.data = result.data.reset_index(drop=True)
+        if result.data is not None:
+            result.data = result.data.reset_index(drop=True)
         return result
 
 
 class Setdiff(Set):
 
     @staticmethod
-    def has_null(row):
+    def has_null(row: Any) -> bool:
         return row.isnull().any()
 
     @classmethod
@@ -135,12 +156,15 @@ class Setdiff(Set):
             if result.data is None:
                 result.data = data
             else:
+                if data is None:
+                    data = pd.DataFrame(columns=result.get_identifiers_names())
                 result.data = result.data.merge(data, how="left", on=result.get_identifiers_names())
                 if len(result.data) > 0:
                     result.data = result.data[result.data.apply(cls.has_null, axis=1)]
 
-                not_identifiers = [col for col in result.get_measures_names() +
-                                   result.get_attributes_names()]
+                not_identifiers = [
+                    col for col in result.get_measures_names() + result.get_attributes_names()
+                ]
                 for col in not_identifiers:
                     if col + "_x" in result.data:
                         result.data[col] = result.data[col + "_x"]
@@ -148,5 +172,6 @@ class Setdiff(Set):
                     if col + "_y" in result.data:
                         del result.data[col + "_y"]
                 result.data = result.data[result.get_identifiers_names() + not_identifiers]
-        result.data.reset_index(drop=True, inplace=True)
+        if result.data is not None:
+            result.data.reset_index(drop=True, inplace=True)
         return result