vtlengine 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

vtlengine/Operators/__init__.py

@@ -1,31 +1,29 @@
 import os
 from copy import copy
-from typing import Any, Union, Optional
+from typing import Any, Optional, Union
 
+# if os.environ.get("SPARK", False):
+#     import pyspark.pandas as pd
+# else:
+#     import pandas as pd
+import pandas as pd
+
+from vtlengine.AST.Grammar.tokens import AND, CEIL, EQ, FLOOR, GT, GTE, LT, LTE, NEQ, OR, ROUND, XOR
 from vtlengine.DataTypes import (
     COMP_NAME_MAPPING,
+    SCALAR_TYPES_CLASS_REVERSE,
     binary_implicit_promotion,
     check_binary_implicit_promotion,
     check_unary_implicit_promotion,
     unary_implicit_promotion,
-    SCALAR_TYPES_CLASS_REVERSE,
 )
 from vtlengine.DataTypes.TimeHandling import (
+    DURATION_MAPPING,
     TimeIntervalHandler,
     TimePeriodHandler,
-    DURATION_MAPPING,
 )
-
-from vtlengine.AST.Grammar.tokens import CEIL, FLOOR, ROUND, EQ, NEQ, GT, GTE, LT, LTE, XOR, OR, AND
 from vtlengine.Exceptions import SemanticError
-
-# if os.environ.get("SPARK", False):
-#     import pyspark.pandas as pd
-# else:
-#     import pandas as pd
-import pandas as pd
-
-from vtlengine.Model import Component, Dataset, Role, Scalar, DataComponent, ScalarSet
+from vtlengine.Model import Component, DataComponent, Dataset, Role, Scalar, ScalarSet
 
 ALL_MODEL_DATA_TYPES = Union[Dataset, Scalar, DataComponent]
 
@@ -167,13 +165,12 @@ class Operator:
 
 
 def _id_type_promotion_join_keys(
-    c_left: Component,
-    c_right: Component,
-    join_key: str,
-    left_data: Optional[pd.DataFrame] = None,
-    right_data: Optional[pd.DataFrame] = None,
+        c_left: Component,
+        c_right: Component,
+        join_key: str,
+        left_data: Optional[pd.DataFrame] = None,
+        right_data: Optional[pd.DataFrame] = None,
 ) -> None:
-
     if left_data is None:
         left_data = pd.DataFrame()
     if right_data is None:
@@ -187,7 +184,7 @@ def _id_type_promotion_join_keys(
         right_data[join_key] = right_data[join_key].astype(object)
         return
     if (left_type_name == "Integer" and right_type_name == "Number") or (
-        left_type_name == "Number" and right_type_name == "Integer"
+            left_type_name == "Number" and right_type_name == "Integer"
     ):
         left_data[join_key] = left_data[join_key].map(lambda x: int(float(x)))
         right_data[join_key] = right_data[join_key].map(lambda x: int(float(x)))
@@ -200,7 +197,6 @@ def _id_type_promotion_join_keys(
 
 
 def _handle_str_number(x: Union[str, int, float]) -> Union[str, int, float]:
-
     if isinstance(x, int):
         return x
     try:
@@ -238,10 +234,10 @@ class Binary(Operator):
 
     @classmethod
     def apply_operation_series_scalar(
-        cls,
-        series: Any,
-        scalar: Scalar,
-        series_left: bool,
+            cls,
+            series: Any,
+            scalar: Scalar,
+            series_left: bool,
     ) -> Any:
 
         if scalar is None:
@@ -353,7 +349,8 @@ class Binary(Operator):
 
         if not cls.validate_type_compatibility(left_operand.data_type, right_operand.data_type):
             raise SemanticError(
-                "1-1-1-2",
+                "1-1-1-2", type_1=left_operand.data_type, type_2=right_operand.data_type,
+                type_check=cls.type_to_check
             )
         return Scalar(
             name="result",
@@ -363,7 +360,7 @@ class Binary(Operator):
 
     @classmethod
     def component_validation(
-        cls, left_operand: DataComponent, right_operand: DataComponent
+            cls, left_operand: DataComponent, right_operand: DataComponent
     ) -> DataComponent:
         """
         Validates the compatibility between the types of the components and the operator
@@ -415,7 +412,7 @@ class Binary(Operator):
 
     @classmethod
     def component_set_validation(
-        cls, component: DataComponent, scalar_set: ScalarSet
+            cls, component: DataComponent, scalar_set: ScalarSet
     ) -> DataComponent:
 
         cls.type_validation(component.data_type, scalar_set.data_type)
@@ -471,7 +468,7 @@ class Binary(Operator):
 
     @classmethod
     def apply_return_type_dataset(
-        cls, result_dataset: Dataset, left_operand: Any, right_operand: Any
+            cls, result_dataset: Dataset, left_operand: Any, right_operand: Any
     ) -> None:
         """
         Used in dataset's validation.
@@ -501,9 +498,9 @@ class Binary(Operator):
                 if result_dataset.data is not None:
                     result_dataset.data.rename(columns={measure.name: component.name}, inplace=True)
             elif (
-                changed_allowed is False
-                and is_mono_measure is False
-                and left_type.promotion_changed_type(result_data_type)
+                    changed_allowed is False
+                    and is_mono_measure is False
+                    and left_type.promotion_changed_type(result_data_type)
             ):
                 raise SemanticError("1-1-1-4", op=cls.op)
             else:
@@ -597,7 +594,7 @@ class Binary(Operator):
 
     @classmethod
     def dataset_scalar_evaluation(
-        cls, dataset: Dataset, scalar: Scalar, dataset_left: bool = True
+            cls, dataset: Dataset, scalar: Scalar, dataset_left: bool = True
     ) -> Dataset:
 
         result_dataset = cls.dataset_scalar_validation(dataset, scalar)
@@ -609,7 +606,7 @@ class Binary(Operator):
         for measure in dataset.get_measures():
             measure_data = cls.cast_time_types(measure.data_type, result_data[measure.name].copy())
             if measure.data_type.__name__.__str__() == "Duration" and not isinstance(
-                scalar_value, int
+                    scalar_value, int
             ):
                 scalar_value = DURATION_MAPPING[scalar_value]
             result_dataset.data[measure.name] = cls.apply_operation_series_scalar(
@@ -624,7 +621,7 @@ class Binary(Operator):
 
     @classmethod
     def component_evaluation(
-        cls, left_operand: DataComponent, right_operand: DataComponent
+            cls, left_operand: DataComponent, right_operand: DataComponent
     ) -> DataComponent:
 
         result_component = cls.component_validation(left_operand, right_operand)
@@ -641,7 +638,7 @@ class Binary(Operator):
 
     @classmethod
     def component_scalar_evaluation(
-        cls, component: DataComponent, scalar: Scalar, component_left: bool = True
+            cls, component: DataComponent, scalar: Scalar, component_left: bool = True
     ) -> DataComponent:
 
         result_component = cls.component_scalar_validation(component, scalar)
@@ -651,7 +648,7 @@ class Binary(Operator):
         )
         scalar_value = cls.cast_time_types_scalar(scalar.data_type, scalar.value)
         if component.data_type.__name__.__str__() == "Duration" and not isinstance(
-            scalar_value, int
+                scalar_value, int
         ):
             scalar_value = DURATION_MAPPING[scalar_value]
         result_component.data = cls.apply_operation_series_scalar(
@@ -679,7 +676,7 @@ class Binary(Operator):
 
     @classmethod
     def component_set_evaluation(
-        cls, component: DataComponent, scalar_set: ScalarSet
+            cls, component: DataComponent, scalar_set: ScalarSet
     ) -> DataComponent:
 
         result_component = cls.component_set_validation(component, scalar_set)
@@ -853,9 +850,9 @@ class Unary(Operator):
                 if result_dataset.data is not None:
                     result_dataset.data.rename(columns={measure.name: component.name}, inplace=True)
             elif (
-                changed_allowed is False
-                and is_mono_measure is False
-                and operand_type.promotion_changed_type(result_data_type)
+                    changed_allowed is False
+                    and is_mono_measure is False
+                    and operand_type.promotion_changed_type(result_data_type)
             ):
                 raise SemanticError("1-1-1-4", op=cls.op)
             else:

vtlengine/Utils/__init__.py

@@ -1,3 +1,101 @@
+from typing import Any, Dict
+
+from vtlengine.AST.Grammar.tokens import (
+    ABS,
+    AGGREGATE,
+    AND,
+    APPLY,
+    AS,
+    ATTRIBUTE,
+    AVG,
+    CALC,
+    CEIL,
+    CHARSET_MATCH,
+    CONCAT,
+    COUNT,
+    CROSS_JOIN,
+    DATE_ADD,
+    DATEDIFF,
+    DAYOFMONTH,
+    DAYOFYEAR,
+    DAYTOMONTH,
+    DAYTOYEAR,
+    DIV,
+    DROP,
+    EQ,
+    EXP,
+    FILL_TIME_SERIES,
+    FILTER,
+    FIRST_VALUE,
+    FLOOR,
+    FLOW_TO_STOCK,
+    FULL_JOIN,
+    GT,
+    GTE,
+    IDENTIFIER,
+    IN,
+    INNER_JOIN,
+    INTERSECT,
+    ISNULL,
+    KEEP,
+    LAG,
+    LAST_VALUE,
+    LCASE,
+    LEAD,
+    LEFT_JOIN,
+    LEN,
+    LN,
+    LOG,
+    LT,
+    LTE,
+    LTRIM,
+    MAX,
+    MEASURE,
+    MEDIAN,
+    MEMBERSHIP,
+    MIN,
+    MINUS,
+    MOD,
+    MONTH,
+    MONTHTODAY,
+    MULT,
+    NEQ,
+    NOT,
+    NOT_IN,
+    NVL,
+    OR,
+    PERIOD_INDICATOR,
+    PIVOT,
+    PLUS,
+    POWER,
+    RANDOM,
+    RANK,
+    RATIO_TO_REPORT,
+    RENAME,
+    REPLACE,
+    ROUND,
+    RTRIM,
+    SETDIFF,
+    SQRT,
+    STDDEV_POP,
+    STDDEV_SAMP,
+    STOCK_TO_FLOW,
+    SUBSPACE,
+    SUBSTR,
+    SUM,
+    SYMDIFF,
+    TIMESHIFT,
+    TRIM,
+    TRUNC,
+    UCASE,
+    UNION,
+    UNPIVOT,
+    VAR_POP,
+    VAR_SAMP,
+    XOR,
+    YEAR,
+    YEARTODAY,
+)
 from vtlengine.Operators.Aggregation import (
     Avg,
     Count,
@@ -12,20 +110,40 @@ from vtlengine.Operators.Aggregation import (
 )
 from vtlengine.Operators.Analytic import (
     Avg as AvgAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     Count as CountAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     FirstValue,
     Lag,
     LastValue,
     Lead,
+    Rank,
+    RatioToReport,
+)
+from vtlengine.Operators.Analytic import (
     Max as MaxAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     Median as MedianAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     Min as MinAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     PopulationStandardDeviation as PopulationStandardDeviationAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     PopulationVariance as PopulationVarianceAnalytic,
-    Rank,
-    RatioToReport,
+)
+from vtlengine.Operators.Analytic import (
     SampleStandardDeviation as SampleStandardDeviationAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     SampleVariance as SampleVarianceAnalytic,
+)
+from vtlengine.Operators.Analytic import (
     Sum as SumAnalytic,
 )
 from vtlengine.Operators.Boolean import And, Not, Or, Xor
@@ -48,22 +166,22 @@ from vtlengine.Operators.Comparison import (
     IsNull,
     Less,
     LessEqual,
+    Match,
     NotEqual,
     NotIn,
-    Match,
 )
 from vtlengine.Operators.Conditional import Nvl
 from vtlengine.Operators.General import Alias, Membership
 from vtlengine.Operators.HROperators import (
+    HRBinMinus,
+    HRBinPlus,
     HREqual,
     HRGreater,
     HRGreaterEqual,
     HRLess,
     HRLessEqual,
-    HRBinPlus,
-    HRBinMinus,
-    HRUnPlus,
     HRUnMinus,
+    HRUnPlus,
 )
 from vtlengine.Operators.Join import Apply, CrossJoin, FullJoin, InnerJoin, LeftJoin
 from vtlengine.Operators.Numeric import (
@@ -79,6 +197,7 @@ from vtlengine.Operators.Numeric import (
     Mult,
     NaturalLogarithm,
     Power,
+    Random,
     Round,
     SquareRoot,
     Trunc,
@@ -99,100 +218,24 @@ from vtlengine.Operators.String import (
     Upper,
 )
 from vtlengine.Operators.Time import (
+    Date_Add,
+    Date_Diff,
+    Day_of_Month,
+    Day_of_Year,
+    Day_to_Month,
+    Day_to_Year,
+    Fill_time_series,
     Flow_to_stock,
+    Month,
+    Month_to_Day,
     Period_indicator,
     Stock_to_flow,
-    Fill_time_series,
     Time_Shift,
+    Year,
+    Year_to_Day,
 )
 
-from vtlengine.AST.Grammar.tokens import (
-    MEMBERSHIP,
-    AND,
-    OR,
-    XOR,
-    EQ,
-    NEQ,
-    GT,
-    GTE,
-    LT,
-    LTE,
-    IN,
-    NOT_IN,
-    NVL,
-    PLUS,
-    MINUS,
-    MULT,
-    LOG,
-    MOD,
-    POWER,
-    DIV,
-    AS,
-    CONCAT,
-    TIMESHIFT,
-    CHARSET_MATCH,
-    NOT,
-    ABS,
-    EXP,
-    LN,
-    SQRT,
-    CEIL,
-    FLOOR,
-    ISNULL,
-    PERIOD_INDICATOR,
-    LEN,
-    LCASE,
-    LTRIM,
-    RTRIM,
-    TRIM,
-    UCASE,
-    FLOW_TO_STOCK,
-    STOCK_TO_FLOW,
-    ROUND,
-    TRUNC,
-    SUBSTR,
-    REPLACE,
-    FILL_TIME_SERIES,
-    IDENTIFIER,
-    ATTRIBUTE,
-    MEASURE,
-    CALC,
-    FILTER,
-    KEEP,
-    DROP,
-    RENAME,
-    PIVOT,
-    UNPIVOT,
-    SUBSPACE,
-    AGGREGATE,
-    APPLY,
-    UNION,
-    INTERSECT,
-    SYMDIFF,
-    SETDIFF,
-    MAX,
-    MIN,
-    SUM,
-    COUNT,
-    AVG,
-    MEDIAN,
-    STDDEV_POP,
-    STDDEV_SAMP,
-    VAR_POP,
-    VAR_SAMP,
-    LAG,
-    LEAD,
-    FIRST_VALUE,
-    LAST_VALUE,
-    RATIO_TO_REPORT,
-    RANK,
-    INNER_JOIN,
-    LEFT_JOIN,
-    FULL_JOIN,
-    CROSS_JOIN,
-)
-
-BINARY_MAPPING = {
+BINARY_MAPPING: Dict[Any, Any] = {
     # General
     MEMBERSHIP: Membership,
     # Boolean
@@ -218,6 +261,7 @@ BINARY_MAPPING = {
     MOD: Modulo,
     POWER: Power,
     DIV: Div,
+    RANDOM: Random,
     # General
     AS: Alias,
     # String
@@ -225,6 +269,7 @@ BINARY_MAPPING = {
     # Time
     TIMESHIFT: Time_Shift,
     CHARSET_MATCH: Match,
+    DATEDIFF: Date_Diff,
 }
 
 UNARY_MAPPING = {
@@ -253,6 +298,14 @@ UNARY_MAPPING = {
     PERIOD_INDICATOR: Period_indicator,
     FLOW_TO_STOCK: Flow_to_stock,
     STOCK_TO_FLOW: Stock_to_flow,
+    YEAR: Year,
+    MONTH: Month,
+    DAYOFMONTH: Day_of_Month,
+    DAYOFYEAR: Day_of_Year,
+    DAYTOYEAR: Day_to_Year,
+    DAYTOMONTH: Day_to_Month,
+    YEARTODAY: Year_to_Day,
+    MONTHTODAY: Month_to_Day,
 }
 
 PARAMETRIC_MAPPING = {
@@ -264,6 +317,7 @@ PARAMETRIC_MAPPING = {
     REPLACE: Replace,
     # Time
     FILL_TIME_SERIES: Fill_time_series,
+    DATE_ADD: Date_Add,
 }
 
 ROLE_SETTER_MAPPING = {
@@ -320,6 +374,7 @@ ANALYTIC_MAPPING = {
 }
 
 THEN_ELSE = {"then": "T", "else": "E"}
+
 JOIN_MAPPING = {
     INNER_JOIN: InnerJoin,
     LEFT_JOIN: LeftJoin,

vtlengine/__init__.py

@@ -1,3 +1,3 @@
-from vtlengine.API import semantic_analysis, run
+from vtlengine.API import run, semantic_analysis
 
 __all__ = ["semantic_analysis", "run"]

vtlengine/files/output/__init__.py

@@ -3,11 +3,11 @@ from typing import Optional, Union
 
 import pandas as pd
 
-from vtlengine.Model import Dataset
 from vtlengine.files.output._time_period_representation import (
-    format_time_period_external_representation,
     TimePeriodRepresentation,
+    format_time_period_external_representation,
 )
+from vtlengine.Model import Dataset
 
 
 def save_datapoints(

vtlengine/files/output/_time_period_representation.py

@@ -2,7 +2,6 @@ from enum import Enum
 
 from vtlengine.DataTypes import TimePeriod
 from vtlengine.DataTypes.TimeHandling import TimePeriodHandler
-
 from vtlengine.Model import Dataset, Scalar
 
 

vtlengine/files/parser/__init__.py

@@ -1,28 +1,27 @@
 import warnings
 from csv import DictReader
 from pathlib import Path
-
-from typing import Optional, Dict, Union, Any, Type, List
+from typing import Any, Dict, List, Optional, Type, Union
 
 import numpy as np
 import pandas as pd
+
 from vtlengine.DataTypes import (
+    SCALAR_TYPES_CLASS_REVERSE,
+    Boolean,
     Date,
-    TimePeriod,
-    TimeInterval,
+    Duration,
     Integer,
     Number,
-    Boolean,
-    Duration,
-    SCALAR_TYPES_CLASS_REVERSE,
     ScalarType,
+    TimeInterval,
+    TimePeriod,
 )
 from vtlengine.DataTypes.TimeHandling import DURATION_MAPPING
-from vtlengine.files.parser._rfc_dialect import register_rfc
-from vtlengine.files.parser._time_checking import check_date, check_time_period, check_time
-
 from vtlengine.Exceptions import InputValidationException, SemanticError
-from vtlengine.Model import Component, Role, Dataset
+from vtlengine.files.parser._rfc_dialect import register_rfc
+from vtlengine.files.parser._time_checking import check_date, check_time, check_time_period
+from vtlengine.Model import Component, Dataset, Role
 
 TIME_CHECKS_MAPPING: Dict[Type[ScalarType], Any] = {
     Date: check_date,
@@ -74,9 +73,9 @@ def _sanitize_pandas_columns(
     components: Dict[str, Component], csv_path: Union[str, Path], data: pd.DataFrame
 ) -> pd.DataFrame:
     # Fast loading from SDMX-CSV
-    if "DATAFLOW" in data.columns and data.columns[0] == "DATAFLOW":
-        if "DATAFLOW" not in components:
-            data.drop(columns=["DATAFLOW"], inplace=True)
+    if ("DATAFLOW" in data.columns and data.columns[0] == "DATAFLOW" and
+            "DATAFLOW" not in components):
+        data.drop(columns=["DATAFLOW"], inplace=True)
     if "STRUCTURE" in data.columns and data.columns[0] == "STRUCTURE":
         if "STRUCTURE" not in components:
             data.drop(columns=["STRUCTURE"], inplace=True)
@@ -133,9 +132,10 @@ def _pandas_load_s3_csv(components: Dict[str, Component], csv_path: str) -> pd.D
 
 
 def _parse_boolean(value: str) -> bool:
-    if value.lower() == "true" or value == "1":
-        return True
-    return False
+    if isinstance(value, bool):
+        return value
+    result = value.lower() == "true" or value == "1"
+    return result
 
 
 def _validate_pandas(
@@ -146,7 +146,7 @@ def _validate_pandas(
 
     id_names = [comp_name for comp_name, comp in components.items() if comp.role == Role.IDENTIFIER]
 
-    missing_columns = [name for name in components.keys() if name not in data.columns.tolist()]
+    missing_columns = [name for name in components if name not in data.columns.tolist()]
     if missing_columns:
         for name in missing_columns:
             if components[name].nullable is False:

vtlengine/files/parser/_time_checking.py

@@ -1,9 +1,8 @@
 import calendar
 import re
-from datetime import datetime, date
+from datetime import date, datetime
 
 from vtlengine.DataTypes.TimeHandling import TimePeriodHandler
-
 from vtlengine.Exceptions import InputValidationException
 
 
@@ -93,6 +92,8 @@ further_options_period_pattern = (
 
 
 def check_time_period(value: str) -> str:
+    if isinstance(value, int):
+        value = str(value)
     value = value.replace(" ", "")
     period_result = re.fullmatch(period_pattern, value)
     if period_result is not None: