vtlengine 1.0.3rc3__py3-none-any.whl → 1.1rc1__py3-none-any.whl

vtlengine/Operators/Time.py

@@ -33,7 +33,7 @@ from vtlengine.DataTypes import (
     unary_implicit_promotion,
 )
 from vtlengine.DataTypes.TimeHandling import (
-    DURATION_MAPPING,
+    PERIOD_IND_MAPPING,
     TimePeriodHandler,
     date_to_period,
     period_to_date,
@@ -98,7 +98,8 @@ class Time(Operators.Operator):
         months_deltas = differences.apply(lambda x: x.days // 30)
         days_deltas = differences.apply(lambda x: x.days)
         min_months = min(
-            (diff for diff in months_deltas if diff > 0 and diff % 12 != 0), default=None
+            (diff for diff in months_deltas if diff > 0 and diff % 12 != 0),
+            default=None,
         )
         min_days = min(
             (diff for diff in days_deltas if diff > 0 and diff % 365 != 0 and diff % 366 != 0),
@@ -118,7 +119,6 @@ class Time(Operators.Operator):
 
 
 class Unary(Time):
-
     @classmethod
     def validate(cls, operand: Any) -> Any:
         if not isinstance(operand, Dataset):
@@ -190,7 +190,10 @@ class Period_indicator(Unary):
                 if comp.role == Role.IDENTIFIER
             }
             result_components["duration_var"] = Component(
-                name="duration_var", data_type=Duration, role=Role.MEASURE, nullable=True
+                name="duration_var",
+                data_type=Duration,
+                role=Role.MEASURE,
+                nullable=True,
             )
             return Dataset(name="result", components=result_components, data=None)
         # DataComponent and Scalar validation
@@ -202,7 +205,7 @@ class Period_indicator(Unary):
 
     @classmethod
     def evaluate(
-            cls, operand: Union[Dataset, DataComponent, Scalar, str]
+        cls, operand: Union[Dataset, DataComponent, Scalar, str]
     ) -> Union[Dataset, DataComponent, Scalar, str]:
         result = cls.validate(operand)
         if isinstance(operand, str):
@@ -226,7 +229,6 @@ class Period_indicator(Unary):
 
 
 class Parametrized(Time):
-
     @classmethod
     def validate(cls, operand: Any, param: Any) -> Any:
         pass
@@ -237,14 +239,12 @@ class Parametrized(Time):
 
 
 class Flow_to_stock(Unary):
-
     @classmethod
     def py_op(cls, x: Any) -> Any:
         return x.cumsum().fillna(x)
 
 
 class Stock_to_flow(Unary):
-
     @classmethod
     def py_op(cls, x: Any) -> Any:
         return x.diff().fillna(x)
@@ -299,7 +299,6 @@ class Fill_time_series(Binary):
 
     @classmethod
     def max_min_from_period(cls, data: pd.DataFrame, mode: str = "all") -> Dict[str, Any]:
-
         result_dict: Dict[Any, Any] = {}
         data = data.assign(
             Periods_col=data[cls.time_id].apply(cls._get_period),
@@ -369,7 +368,10 @@ class Fill_time_series(Binary):
                 else:
                     if period in period_limits["min"] and period in period_limits["max"]:
                         vals = list(
-                            range(period_limits["min"][period], period_limits["max"][period] + 1)
+                            range(
+                                period_limits["min"][period],
+                                period_limits["max"][period] + 1,
+                            )
                         )
                         filled_data.extend(
                             cls.fill_periods_rows(group_df, period, years, vals=vals)
@@ -385,7 +387,11 @@ class Fill_time_series(Binary):
 
     @classmethod
     def fill_periods_rows(
-            cls, group_df: Any, period: str, years: List[int], vals: Optional[List[int]] = None
+        cls,
+        group_df: Any,
+        period: str,
+        years: List[int],
+        vals: Optional[List[int]] = None,
     ) -> List[Any]:
         rows = []
         for year in years:
@@ -398,7 +404,7 @@ class Fill_time_series(Binary):
 
     @classmethod
     def create_period_row(
-            cls, group_df: Any, period: str, year: int, val: Optional[int] = None
+        cls, group_df: Any, period: str, year: int, val: Optional[int] = None
     ) -> Any:
         row = group_df.iloc[0].copy()
         row[cls.time_id] = f"{year}" if period == "A" else f"{year}-{period}{val:d}"
@@ -436,9 +442,7 @@ class Fill_time_series(Binary):
         date_format = None
         filled_data = []
 
-        def create_filled_dates(
-                group: Any, min_max: Dict[str, Any]
-        ) -> (pd.DataFrame, str):  # type: ignore[syntax]
+        def create_filled_dates(group: Any, min_max: Dict[str, Any]) -> (pd.DataFrame, str):  # type: ignore[syntax]
             date_range = pd.date_range(start=min_max["min"], end=min_max["max"], freq=min_frequency)
             date_df = pd.DataFrame(date_range, columns=[cls.time_id])
             date_df[cls.other_ids] = group.iloc[0][cls.other_ids]
@@ -480,7 +484,7 @@ class Fill_time_series(Binary):
 
     @classmethod
     def fill_time_intervals(
-            cls, data: pd.DataFrame, fill_type: str, frequency: str
+        cls, data: pd.DataFrame, fill_type: str, frequency: str
     ) -> pd.DataFrame:
         result_data = cls.time_filler(data, fill_type, frequency)
         not_na = result_data[cls.measures].notna().any(axis=1)
@@ -588,7 +592,7 @@ class Time_Shift(Binary):
 
     @classmethod
     def shift_period(
-            cls, period_str: str, shift_value: int, frequency: Optional[int] = None
+        cls, period_str: str, shift_value: int, frequency: Optional[int] = None
     ) -> str:
         period_type = cls._get_period(period_str)
 
@@ -628,7 +632,7 @@ class Time_Aggregation(Time):
 
     @classmethod
     def _check_duration(cls, value: str) -> None:
-        if value not in DURATION_MAPPING:
+        if value not in PERIOD_IND_MAPPING:
             raise SemanticError("1-1-19-3", op=cls.op, param="duration")
 
     @classmethod
@@ -636,13 +640,13 @@ class Time_Aggregation(Time):
         cls._check_duration(period_to)
         if period_from is not None:
             cls._check_duration(period_from)
-            if DURATION_MAPPING[period_to] <= DURATION_MAPPING[period_from]:
+            if PERIOD_IND_MAPPING[period_to] <= PERIOD_IND_MAPPING[period_from]:
                 # OPERATORS_TIMEOPERATORS.19
                 raise SemanticError("1-1-19-4", op=cls.op, value_1=period_from, value_2=period_to)
 
     @classmethod
     def dataset_validation(
-            cls, operand: Dataset, period_from: Optional[str], period_to: str, conf: str
+        cls, operand: Dataset, period_from: Optional[str], period_to: str, conf: str
     ) -> Dataset:
         # TODO: Review with VTL TF as this makes no sense
 
@@ -661,7 +665,10 @@ class Time_Aggregation(Time):
                 count_time_types += 1
         if count_time_types != 1:
             raise SemanticError(
-                "1-1-19-9", op=cls.op, comp_type="dataset", param="single time identifier"
+                "1-1-19-9",
+                op=cls.op,
+                comp_type="dataset",
+                param="single time identifier",
             )
 
         if count_time_types != 1:
@@ -679,7 +686,11 @@ class Time_Aggregation(Time):
 
     @classmethod
     def component_validation(
-            cls, operand: DataComponent, period_from: Optional[str], period_to: str, conf: str
+        cls,
+        operand: DataComponent,
+        period_from: Optional[str],
+        period_to: str,
+        conf: str,
     ) -> DataComponent:
         if operand.data_type not in cls.TIME_DATA_TYPES:
             raise SemanticError("1-1-19-8", op=cls.op, comp_type="time component")
@@ -692,7 +703,7 @@ class Time_Aggregation(Time):
 
     @classmethod
     def scalar_validation(
-            cls, operand: Scalar, period_from: Optional[str], period_to: str, conf: str
+        cls, operand: Scalar, period_from: Optional[str], period_to: str, conf: str
     ) -> Scalar:
         if operand.data_type not in cls.TIME_DATA_TYPES:
             raise SemanticError("1-1-19-8", op=cls.op, comp_type="time scalar")
@@ -701,12 +712,12 @@ class Time_Aggregation(Time):
 
     @classmethod
     def _execute_time_aggregation(
-            cls,
-            value: str,
-            data_type: Type[ScalarType],
-            period_from: Optional[str],
-            period_to: str,
-            conf: str,
+        cls,
+        value: str,
+        data_type: Type[ScalarType],
+        period_from: Optional[str],
+        period_to: str,
+        conf: str,
     ) -> str:
         if data_type == TimePeriod:  # Time period
             return _time_period_access(value, period_to)
@@ -722,7 +733,7 @@ class Time_Aggregation(Time):
 
     @classmethod
     def dataset_evaluation(
-            cls, operand: Dataset, period_from: Optional[str], period_to: str, conf: str
+        cls, operand: Dataset, period_from: Optional[str], period_to: str, conf: str
     ) -> Dataset:
         result = cls.dataset_validation(operand, period_from, period_to, conf)
         result.data = operand.data.copy() if operand.data is not None else pd.DataFrame()
@@ -738,7 +749,11 @@ class Time_Aggregation(Time):
 
     @classmethod
     def component_evaluation(
-            cls, operand: DataComponent, period_from: Optional[str], period_to: str, conf: str
+        cls,
+        operand: DataComponent,
+        period_from: Optional[str],
+        period_to: str,
+        conf: str,
     ) -> DataComponent:
         result = cls.component_validation(operand, period_from, period_to, conf)
         if operand.data is not None:
@@ -752,7 +767,7 @@ class Time_Aggregation(Time):
 
     @classmethod
     def scalar_evaluation(
-            cls, operand: Scalar, period_from: Optional[str], period_to: str, conf: str
+        cls, operand: Scalar, period_from: Optional[str], period_to: str, conf: str
     ) -> Scalar:
         result = cls.scalar_validation(operand, period_from, period_to, conf)
         result.value = cls._execute_time_aggregation(
@@ -762,11 +777,11 @@ class Time_Aggregation(Time):
 
     @classmethod
     def validate(
-            cls,
-            operand: Union[Dataset, DataComponent, Scalar],
-            period_from: Optional[str],
-            period_to: str,
-            conf: str,
+        cls,
+        operand: Union[Dataset, DataComponent, Scalar],
+        period_from: Optional[str],
+        period_to: str,
+        conf: str,
     ) -> Union[Dataset, DataComponent, Scalar]:
         cls._check_params(period_from, period_to)
         if isinstance(operand, Dataset):
@@ -778,11 +793,11 @@ class Time_Aggregation(Time):
 
     @classmethod
     def evaluate(
-            cls,
-            operand: Union[Dataset, DataComponent, Scalar],
-            period_from: Optional[str],
-            period_to: str,
-            conf: str,
+        cls,
+        operand: Union[Dataset, DataComponent, Scalar],
+        period_from: Optional[str],
+        period_to: str,
+        conf: str,
     ) -> Union[Dataset, DataComponent, Scalar]:
         cls._check_params(period_from, period_to)
         if isinstance(operand, Dataset):
@@ -809,7 +824,6 @@ def _date_access(v: str, to_param: str, start: bool) -> Any:
 
 
 class Current_Date(Time):
-
     @classmethod
     def validate(cls) -> Scalar:
         return Scalar(name="current_date", data_type=Date, value=None)
@@ -830,26 +844,30 @@ class SimpleBinaryTime(Operators.Binary):
         if left == TimePeriod and right == Date:
             return False
 
-        return  not (left == TimePeriod and right == Date)
+        return not (left == TimePeriod and right == Date)
 
     @classmethod
     def validate(
-            cls, left_operand: Union[Dataset, DataComponent, Scalar],
-            right_operand: Union[Dataset, DataComponent, Scalar]
+        cls,
+        left_operand: Union[Dataset, DataComponent, Scalar],
+        right_operand: Union[Dataset, DataComponent, Scalar],
     ) -> Union[Dataset, DataComponent, Scalar]:
         if isinstance(left_operand, Dataset) or isinstance(right_operand, Dataset):
             raise SemanticError("1-1-19-8", op=cls.op, comp_type="time dataset")
         if not cls.validate_type_compatibility(left_operand.data_type, right_operand.data_type):
             raise SemanticError(
-                "1-1-1-2", type_1=left_operand.data_type, type_2=right_operand.data_type,
-                type_check=cls.type_to_check
+                "1-1-1-2",
+                type_1=left_operand.data_type,
+                type_2=right_operand.data_type,
+                type_check=cls.type_to_check,
             )
         return super().validate(left_operand, right_operand)
 
     @classmethod
     def evaluate(
-            cls, left_operand: Union[Dataset, DataComponent, Scalar],
-            right_operand: Union[Dataset, DataComponent, Scalar]
+        cls,
+        left_operand: Union[Dataset, DataComponent, Scalar],
+        right_operand: Union[Dataset, DataComponent, Scalar],
     ) -> Union[Dataset, DataComponent, Scalar]:
         if isinstance(left_operand, Dataset) or isinstance(right_operand, Dataset):
             raise SemanticError("1-1-19-8", op=cls.op, comp_type="time dataset")
@@ -869,12 +887,12 @@ class Date_Diff(SimpleBinaryTime):
             raise SemanticError("1-1-19-8", op=cls.op, comp_type="time dataset")
 
         if x.count("-") == 2:
-            fecha1 = datetime.strptime(x, '%Y-%m-%d').date()
+            fecha1 = datetime.strptime(x, "%Y-%m-%d").date()
         else:
             fecha1 = TimePeriodHandler(x).end_date(as_date=True)  # type: ignore[assignment]
 
         if y.count("-") == 2:
-            fecha2 = datetime.strptime(y, '%Y-%m-%d').date()
+            fecha2 = datetime.strptime(y, "%Y-%m-%d").date()
         else:
             fecha2 = TimePeriodHandler(y).end_date(as_date=True)  # type: ignore[assignment]
 
@@ -885,26 +903,31 @@ class Date_Add(Parametrized):
     op = DATE_ADD
 
     @classmethod
-    def validate(cls,
-                 operand: Union[Scalar, DataComponent, Dataset],
-                 param_list: List[Scalar]
-                 ) -> Union[Scalar, DataComponent, Dataset]:
-
+    def validate(
+        cls, operand: Union[Scalar, DataComponent, Dataset], param_list: List[Scalar]
+    ) -> Union[Scalar, DataComponent, Dataset]:
         expected_types = [Integer, String]
         for i, param in enumerate(param_list):
-            error = 12 if not isinstance(param, Scalar) else 13 if (  # type: ignore[redundant-expr]
-                    param.data_type != expected_types[i]) else None
+            error = (
+                12
+                if not isinstance(param, Scalar)  # type: ignore[redundant-expr]
+                else 13
+                if (param.data_type != expected_types[i])
+                else None
+            )
             if error is not None:
-                raise SemanticError(f"2-1-19-{error}",
-                                    op=cls.op,
-                                    type=param.__class__.__name__ if error == 12 else
-                                    param.data_type.__name__,
-                                    name="shiftNumber" if error == 12 else "periodInd",
-                                    expected="Scalar" if error == 12 else expected_types[i].__name__
-                                    )
-
-        if (isinstance(operand, (Scalar, DataComponent)) and
-                operand.data_type not in [Date, TimePeriod]):
+                raise SemanticError(
+                    f"2-1-19-{error}",
+                    op=cls.op,
+                    type=(param.__class__.__name__ if error == 12 else param.data_type.__name__),
+                    name="shiftNumber" if error == 12 else "periodInd",
+                    expected="Scalar" if error == 12 else expected_types[i].__name__,
+                )
+
+        if isinstance(operand, (Scalar, DataComponent)) and operand.data_type not in [
+            Date,
+            TimePeriod,
+        ]:
             unary_implicit_promotion(operand.data_type, Date)
 
         if isinstance(operand, Scalar):
@@ -914,31 +937,38 @@ class Date_Add(Parametrized):
 
         if all(comp.data_type not in [Date, TimePeriod] for comp in operand.components.values()):
             raise SemanticError("2-1-19-14", op=cls.op, name=operand.name)
-        return Dataset(name='result', components=operand.components.copy(), data=None)
+        return Dataset(name="result", components=operand.components.copy(), data=None)
 
     @classmethod
-    def evaluate(cls,
-                 operand: Union[Scalar, DataComponent, Dataset],
-                 param_list: List[Scalar]
-                 ) -> Union[Scalar, DataComponent, Dataset]:
+    def evaluate(
+        cls, operand: Union[Scalar, DataComponent, Dataset], param_list: List[Scalar]
+    ) -> Union[Scalar, DataComponent, Dataset]:
         result = cls.validate(operand, param_list)
         shift, period = param_list[0].value, param_list[1].value
         is_tp = isinstance(operand, (Scalar, DataComponent)) and operand.data_type == TimePeriod
 
         if isinstance(result, Scalar) and isinstance(operand, Scalar) and operand.value is not None:
             result.value = cls.py_op(operand.value, shift, period, is_tp)
-        elif (isinstance(result, DataComponent) and isinstance(operand, DataComponent) and
-              operand.data is not None):
-            result.data = operand.data.map(lambda x: cls.py_op(x, shift, period, is_tp),
-                                           na_action="ignore")
-        elif (isinstance(result, Dataset) and isinstance(operand, Dataset) and
-              operand.data is not None):
+        elif (
+            isinstance(result, DataComponent)
+            and isinstance(operand, DataComponent)
+            and operand.data is not None
+        ):
+            result.data = operand.data.map(
+                lambda x: cls.py_op(x, shift, period, is_tp), na_action="ignore"
+            )
+        elif (
+            isinstance(result, Dataset)
+            and isinstance(operand, Dataset)
+            and operand.data is not None
+        ):
             result.data = operand.data.copy()
             for measure in operand.get_measures():
                 if measure.data_type in [Date, TimePeriod]:
                     result.data[measure.name] = result.data[measure.name].map(
                         lambda x: cls.py_op(x, shift, period, measure.data_type == TimePeriod),
-                        na_action="ignore")
+                        na_action="ignore",
+                    )
                     measure.data_type = Date
 
         if isinstance(result, (Scalar, DataComponent)):
@@ -946,47 +976,47 @@ class Date_Add(Parametrized):
         return result
 
     @classmethod
-    def py_op(cls,
-              date_str: str,
-              shift: int, period: str,
-              is_tp: bool = False
-              ) -> str:
+    def py_op(cls, date_str: str, shift: int, period: str, is_tp: bool = False) -> str:
         if is_tp:
             tp_value = TimePeriodHandler(date_str)
             date = period_to_date(tp_value.year, tp_value.period_indicator, tp_value.period_number)
         else:
             date = datetime.strptime(date_str, "%Y-%m-%d")
 
-        if period in ['D', 'W']:
-            days_shift = shift * (7 if period == 'W' else 1)
+        if period in ["D", "W"]:
+            days_shift = shift * (7 if period == "W" else 1)
             return (date + timedelta(days=days_shift)).strftime("%Y-%m-%d")
 
-        month_shift = {'M': 1, 'Q': 3, 'S': 6, 'A': 12}[period] * shift
+        month_shift = {"M": 1, "Q": 3, "S": 6, "A": 12}[period] * shift
         new_year = date.year + (date.month - 1 + month_shift) // 12
         new_month = (date.month - 1 + month_shift) % 12 + 1
         last_day = (datetime(new_year, new_month % 12 + 1, 1) - timedelta(days=1)).day
-        return date.replace(year=new_year, month=new_month,
-                            day=min(date.day, last_day)).strftime("%Y-%m-%d")
+        return date.replace(year=new_year, month=new_month, day=min(date.day, last_day)).strftime(
+            "%Y-%m-%d"
+        )
 
 
 class SimpleUnaryTime(Operators.Unary):
-
     @classmethod
     def validate(
-            cls, operand: Union[Dataset, DataComponent, Scalar]
+        cls, operand: Union[Dataset, DataComponent, Scalar]
     ) -> Union[Dataset, DataComponent, Scalar]:
         if isinstance(operand, Dataset):
             raise SemanticError("1-1-19-8", op=cls.op, comp_type="time dataset")
 
         # Limit the operand to Date and TimePeriod (cannot be implemented with type_to_check)
-        if operand.data_type == TimeInterval or operand.data_type not in (Date, TimePeriod):
+        if operand.data_type == TimeInterval or operand.data_type not in (
+            Date,
+            TimePeriod,
+            Duration,
+        ):
             raise SemanticError("1-1-19-10", op=cls.op)
 
         return super().validate(operand)
 
     @classmethod
     def evaluate(
-            cls, operand: Union[Dataset, DataComponent, Scalar]
+        cls, operand: Union[Dataset, DataComponent, Scalar]
     ) -> Union[Dataset, DataComponent, Scalar]:
         cls.validate(operand)
         return super().evaluate(operand)
@@ -1040,19 +1070,21 @@ class Day_of_Year(SimpleUnaryTime):
 
         result = TimePeriodHandler(value).end_date(as_date=True)
         datetime_value = datetime(
-            year=result.year, month=result.month, day=result.day  # type: ignore[union-attr]
+            year=result.year,  # type: ignore[union-attr]
+            month=result.month,  # type: ignore[union-attr]
+            day=result.day,  # type: ignore[union-attr]
         )
         return datetime_value.timetuple().tm_yday
 
 
 class Day_to_Year(Operators.Unary):
     op = DAYTOYEAR
-    return_type = String
+    return_type = Duration
 
     @classmethod
     def py_op(cls, value: int) -> str:
         if value < 0:
-            raise SemanticError("2-1-19-17", op=cls.op)
+            raise SemanticError("2-1-19-16", op=cls.op)
         years = 0
         days_remaining = value
         if value >= 365:
@@ -1063,12 +1095,12 @@ class Day_to_Year(Operators.Unary):
 
 class Day_to_Month(Operators.Unary):
     op = DAYTOMONTH
-    return_type = String
+    return_type = Duration
 
     @classmethod
     def py_op(cls, value: int) -> str:
         if value < 0:
-            raise SemanticError("2-1-19-17", op=cls.op)
+            raise SemanticError("2-1-19-16", op=cls.op)
         months = 0
         days_remaining = value
         if value >= 30:
@@ -1083,14 +1115,8 @@ class Year_to_Day(Operators.Unary):
 
     @classmethod
     def py_op(cls, value: str) -> int:
-        if "/" in value:
-            raise SemanticError("2-1-19-11", op=cls.op)
-        if "Y" not in value:
-            raise SemanticError("2-1-19-15", op=cls.op)
-        index_y = value.index("Y")
-        years = int(value[1:index_y])
-        days = int(value[(index_y + 1): -1])
-        return years * 365 + days
+        days = Duration.to_days(value)
+        return days
 
 
 class Month_to_Day(Operators.Unary):
@@ -1099,11 +1125,5 @@ class Month_to_Day(Operators.Unary):
 
     @classmethod
     def py_op(cls, value: str) -> int:
-        if "/" in value:
-            raise SemanticError("2-1-19-11", op=cls.op)
-        if "M" not in value:
-            raise SemanticError("2-1-19-16", op=cls.op)
-        index_m = value.index("M")
-        months = int(value[1:index_m])
-        days = int(value[(index_m + 1): -1])
-        return months * 30 + days
+        days = Duration.to_days(value)
+        return days

vtlengine/Operators/Validation.py

@@ -4,7 +4,13 @@ from typing import Any, Dict, Optional
 import pandas as pd
 
 from vtlengine.AST.Grammar.tokens import CHECK, CHECK_HIERARCHY
-from vtlengine.DataTypes import Boolean, Integer, Number, String, check_unary_implicit_promotion
+from vtlengine.DataTypes import (
+    Boolean,
+    Integer,
+    Number,
+    String,
+    check_unary_implicit_promotion,
+)
 from vtlengine.Exceptions import SemanticError
 from vtlengine.Model import Component, Dataset, Role
 from vtlengine.Operators import Operator
@@ -103,7 +109,6 @@ class Check(Operator):
 
 # noinspection PyTypeChecker
 class Validation(Operator):
-
     @classmethod
     def _generate_result_data(cls, rule_info: Dict[str, Any]) -> pd.DataFrame:
         rule_list_df = []
@@ -171,7 +176,6 @@ class Validation(Operator):
         elif output == "all":
             result.data = result.data[result.get_identifiers_names() + validation_measures]
         else:  # output == 'all_measures'
-
             result.data = result.data[
                 result.get_identifiers_names()
                 + dataset_element.get_measures_names()
@@ -230,7 +234,9 @@ class Check_Hierarchy(Validation):
             )
         if dataset.components[component_name].role != Role.IDENTIFIER:
             raise SemanticError(
-                "1-3-20", name=component_name, role=dataset.components[component_name].role.value
+                "1-3-20",
+                name=component_name,
+                role=dataset.components[component_name].role.value,
             )
         # Remove attributes from dataset
         if len(dataset.get_attributes()) > 0: