vtlengine 1.1.1__py3-none-any.whl → 1.2.1rc1__py3-none-any.whl

vtlengine/API/_InternalApi.py

@@ -1,7 +1,7 @@
 import json
 import os
 from pathlib import Path
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 import jsonschema
 import pandas as pd
@@ -22,7 +22,11 @@ from vtlengine.__extras_check import __check_s3_extra
 from vtlengine.AST import Assignment, DPRuleset, HRuleset, Operator, PersistentAssignment, Start
 from vtlengine.AST.ASTString import ASTString
 from vtlengine.DataTypes import SCALAR_TYPES
-from vtlengine.Exceptions import InputValidationException, check_key
+from vtlengine.Exceptions import (
+    InputValidationException,
+    SemanticError,
+    check_key,
+)
 from vtlengine.files.parser import _fill_dataset_empty_data, _validate_pandas
 from vtlengine.Model import (
     Component as VTL_Component,
@@ -44,11 +48,14 @@ with open(schema_path / "json_schema_2.1.json", "r") as file:
     schema = json.load(file)
 
 
-def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
+def _load_dataset_from_structure(
+    structures: Dict[str, Any],
+) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     """
     Loads a dataset with the structure given.
     """
     datasets = {}
+    scalars = {}
 
     if "datasets" in structures:
         for dataset_json in structures["datasets"]:
@@ -110,8 +117,8 @@ def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
                 data_type=SCALAR_TYPES[scalar_json["type"]],
                 value=None,
             )
-            datasets[scalar_name] = scalar  # type: ignore[assignment]
-    return datasets
+            scalars[scalar_name] = scalar
+    return datasets, scalars
 
 
 def _load_single_datapoint(datapoint: Union[str, Path]) -> Dict[str, Any]:
@@ -159,7 +166,9 @@ def _load_datapoints_path(
     return _load_single_datapoint(datapoints)
 
 
-def _load_datastructure_single(data_structure: Union[Dict[str, Any], Path]) -> Dict[str, Dataset]:
+def _load_datastructure_single(
+    data_structure: Union[Dict[str, Any], Path],
+) -> Tuple[Dict[str, Dataset], Dict[str, Scalar]]:
     """
     Loads a single data structure.
     """
@@ -170,13 +179,15 @@ def _load_datastructure_single(data_structure: Union[Dict[str, Any], Path]) -> D
     if not data_structure.exists():
         raise Exception("Invalid datastructure. Input does not exist")
     if data_structure.is_dir():
-        datasets: Dict[str, Any] = {}
+        datasets: Dict[str, Dataset] = {}
+        scalars: Dict[str, Scalar] = {}
         for f in data_structure.iterdir():
             if f.suffix != ".json":
                 continue
-            dataset = _load_datastructure_single(f)
-            datasets = {**datasets, **dataset}
-        return datasets
+            ds, sc = _load_datastructure_single(f)
+            datasets = {**datasets, **ds}
+            scalars = {**scalars, **sc}
+        return datasets, scalars
     else:
         if data_structure.suffix != ".json":
             raise Exception("Invalid datastructure. Must have .json extension")
@@ -187,7 +198,7 @@ def _load_datastructure_single(data_structure: Union[Dict[str, Any], Path]) -> D
 
 def load_datasets(
     data_structure: Union[Dict[str, Any], Path, List[Dict[str, Any]], List[Path]],
-) -> Dict[str, Dataset]:
+) -> Tuple[Dict[str, Dataset], Dict[str, Scalar]]:
     """
     Loads multiple datasets.
 
@@ -205,21 +216,42 @@ def load_datasets(
     if isinstance(data_structure, dict):
         return _load_datastructure_single(data_structure)
     if isinstance(data_structure, list):
-        ds_structures: Dict[str, Any] = {}
+        ds_structures: Dict[str, Dataset] = {}
+        scalar_structures: Dict[str, Scalar] = {}
         for x in data_structure:
-            result = _load_datastructure_single(x)
-            ds_structures = {**ds_structures, **result}  # Overwrite ds_structures dict.
-        return ds_structures
+            ds, sc = _load_datastructure_single(x)
+            ds_structures = {**ds_structures, **ds}  # Overwrite ds_structures dict.
+            scalar_structures = {**scalar_structures, **sc}  # Overwrite scalar_structures dict.
+        return ds_structures, scalar_structures
     return _load_datastructure_single(data_structure)
 
 
-def load_datasets_with_data(data_structures: Any, datapoints: Optional[Any] = None) -> Any:
+def _handle_scalars_values(
+    scalars: Dict[str, Scalar],
+    scalar_values: Optional[Dict[str, Optional[Union[int, str, bool, float]]]] = None,
+) -> None:
+    if scalar_values is None:
+        return
+    # Handling scalar values with the scalar dict
+    for name, value in scalar_values.items():
+        if name not in scalars:
+            raise Exception(f"Not found scalar {name} in datastructures")
+        # Casting value to scalar data type
+        scalars[name].value = scalars[name].data_type.cast(value)
+
+
+def load_datasets_with_data(
+    data_structures: Any,
+    datapoints: Optional[Any] = None,
+    scalar_values: Optional[Dict[str, Optional[Union[int, str, bool, float]]]] = None,
+) -> Any:
     """
     Loads the dataset structures and fills them with the data contained in the datapoints.
 
     Args:
         data_structures: Dict, Path or a List of dicts or Paths.
         datapoints: Dict, Path or a List of Paths.
+        scalar_values: Dict with the scalar values.
 
     Returns:
         A dict with the structure and a pandas dataframe with the data.
@@ -227,17 +259,18 @@ def load_datasets_with_data(data_structures: Any, datapoints: Optional[Any] = No
     Raises:
         Exception: If the Path is wrong or the file is invalid.
     """
-    datasets = load_datasets(data_structures)
+    datasets, scalars = load_datasets(data_structures)
     if datapoints is None:
         for dataset in datasets.values():
             if isinstance(dataset, Dataset):
                 _fill_dataset_empty_data(dataset)
-        return datasets, None
+        _handle_scalars_values(scalars, scalar_values)
+        return datasets, scalars, None
     if isinstance(datapoints, dict):
         # Handling dictionary of Pandas Dataframes
         for dataset_name, data in datapoints.items():
             if dataset_name not in datasets:
-                raise Exception(f"Not found dataset {dataset_name}")
+                raise Exception(f"Not found dataset {dataset_name} in datastructures.")
             datasets[dataset_name].data = _validate_pandas(
                 datasets[dataset_name].components, data, dataset_name
             )
@@ -246,14 +279,17 @@ def load_datasets_with_data(data_structures: Any, datapoints: Optional[Any] = No
                 datasets[dataset_name].data = pd.DataFrame(
                     columns=list(datasets[dataset_name].components.keys())
                 )
-        return datasets, None
+        _handle_scalars_values(scalars, scalar_values)
+        return datasets, scalars, None
     # Handling dictionary of paths
     dict_datapoints = _load_datapoints_path(datapoints)
     for dataset_name, _ in dict_datapoints.items():
         if dataset_name not in datasets:
-            raise Exception(f"Not found dataset {dataset_name}")
+            raise Exception(f"Not found dataset {dataset_name} in datastructures.")
+
+    _handle_scalars_values(scalars, scalar_values)
 
-    return datasets, dict_datapoints
+    return datasets, scalars, dict_datapoints
 
 
 def load_vtl(input: Union[str, Path]) -> str:
@@ -362,8 +398,8 @@ def load_external_routines(input: Union[Dict[str, Any], Path, str]) -> Any:
 
 
 def _return_only_persistent_datasets(
-    datasets: Dict[str, Dataset], ast: Start
-) -> Dict[str, Dataset]:
+    datasets: Dict[str, Union[Dataset, Scalar]], ast: Start
+) -> Dict[str, Union[Dataset, Scalar]]:
     """
     Returns only the datasets with a persistent assignment.
     """
@@ -606,11 +642,9 @@ def _check_script(script: Union[str, TransformationScheme, Path]) -> str:
     Check if the TransformationScheme object is valid to generate a vtl script.
     """
     if not isinstance(script, (str, TransformationScheme, Path)):
-        raise Exception(
-            "Invalid script format. Input must be a string, TransformationScheme or Path object"
-        )
+        raise SemanticError("0-1-1-1", format_=type(script).__name__)
     if isinstance(script, TransformationScheme):
-        from pysdmx.toolkit.vtl.generate_vtl_script import (
+        from pysdmx.toolkit.vtl import (
             generate_vtl_script,
         )
 

vtlengine/API/__init__.py

@@ -35,7 +35,7 @@ from vtlengine.files.output._time_period_representation import (
     format_time_period_external_representation,
 )
 from vtlengine.Interpreter import InterpreterAnalyzer
-from vtlengine.Model import Dataset
+from vtlengine.Model import Dataset, Scalar
 
 pd.options.mode.chained_assignment = None
 
@@ -180,7 +180,7 @@ def semantic_analysis(
     ast = create_ast(vtl)
 
     # Loading datasets
-    structures = load_datasets(data_structures)
+    datasets, scalars = load_datasets(data_structures)
 
     # Handling of library items
     vd = None
@@ -192,9 +192,10 @@ def semantic_analysis(
 
     # Running the interpreter
     interpreter = InterpreterAnalyzer(
-        datasets=structures,
+        datasets=datasets,
         value_domains=vd,
         external_routines=ext_routines,
+        scalars=scalars,
         only_semantic=True,
     )
     result = interpreter.visit(ast)
@@ -210,7 +211,8 @@ def run(
     time_period_output_format: str = "vtl",
     return_only_persistent: bool = True,
     output_folder: Optional[Union[str, Path]] = None,
-) -> Dict[str, Dataset]:
+    scalar_values: Optional[Dict[str, Optional[Union[int, str, bool, float]]]] = None,
+) -> Dict[str, Union[Dataset, Scalar]]:
     """
     Run is the main function of the ``API``, which mission is to execute
     the vtl operation over the data.
@@ -276,6 +278,8 @@ def run(
 
         output_folder: Path or S3 URI to the output folder. (default: None)
 
+        scalar_values: Dict with the scalar values to be used in the VTL script. \
+
 
     Returns:
        The datasets are produced without data if the output folder is defined.
@@ -292,7 +296,9 @@ def run(
     ast = create_ast(vtl)
 
     # Loading datasets and datapoints
-    datasets, path_dict = load_datasets_with_data(data_structures, datapoints)
+    datasets, scalars, path_dict = load_datasets_with_data(
+        data_structures, datapoints, scalar_values
+    )
 
     # Handling of library items
     vd = None
@@ -322,13 +328,15 @@ def run(
         output_path=output_folder,
         time_period_representation=time_period_representation,
         return_only_persistent=return_only_persistent,
+        scalars=scalars,
     )
     result = interpreter.visit(ast)
 
     # Applying time period output format
     if output_folder is None:
-        for dataset in result.values():
-            format_time_period_external_representation(dataset, time_period_representation)
+        for obj in result.values():
+            if isinstance(obj, (Dataset, Scalar)):
+                format_time_period_external_representation(obj, time_period_representation)
 
     # Returning only persistent datasets
     if return_only_persistent:
@@ -345,7 +353,7 @@ def run_sdmx(  # noqa: C901
     time_period_output_format: str = "vtl",
     return_only_persistent: bool = True,
     output_folder: Optional[Union[str, Path]] = None,
-) -> Dict[str, Dataset]:
+) -> Dict[str, Union[Dataset, Scalar]]:
     """
     Executes a VTL script using a list of pysdmx `PandasDataset` objects.
 
@@ -403,8 +411,16 @@ def run_sdmx(  # noqa: C901
     mapping_dict = {}
     input_names = _extract_input_datasets(script)
 
-    # Mapping handling
+    if not isinstance(datasets, (list, set)) or any(
+        not isinstance(ds, PandasDataset) for ds in datasets
+    ):
+        type_ = type(datasets).__name__
+        if isinstance(datasets, (list, set)):
+            object_typing = {type(o).__name__ for o in datasets}
+            type_ = f"{type_}[{', '.join(object_typing)}]"
+        raise SemanticError("0-1-3-7", type_=type_)
 
+    # Mapping handling
     if mappings is None:
         if len(datasets) != 1:
             raise SemanticError("0-1-3-3")

vtlengine/AST/ASTConstructorModules/Expr.py

@@ -840,8 +840,8 @@ class Expr(VtlVisitor):
                 Parser.DayOfYearAtomContext,
                 Parser.DayToYearAtomContext,
                 Parser.DayToMonthAtomContext,
-                Parser.YearTodayAtomContext,
-                Parser.MonthTodayAtomContext,
+                Parser.YearToDayAtomContext,
+                Parser.MonthToDayAtomContext,
             ),
         ):
             return self.visitTimeUnaryAtom(ctx)
@@ -1901,7 +1901,10 @@ class Expr(VtlVisitor):
 
         left_node = Terminals().visitVarID(ctx_list[0])
         op_node = ctx_list[1].getSymbol().text
-        right_node = Terminals().visitScalarItem(ctx_list[2])
+        if isinstance(ctx_list[2], Parser.ScalarItemContext):
+            right_node = Terminals().visitScalarItem(ctx_list[2])
+        else:
+            right_node = Terminals().visitVarID(ctx_list[2])
         return BinOp(left=left_node, op=op_node, right=right_node, **extract_token_info(ctx))
 
     def visitOptionalExpr(self, ctx: Parser.OptionalExprContext):

vtlengine/AST/DAG/__init__.py

@@ -32,8 +32,8 @@ from vtlengine.AST import (
     VarID,
 )
 from vtlengine.AST.ASTTemplate import ASTTemplate
-from vtlengine.AST.DAG._words import DELETE, GLOBAL, INPUTS, INSERT, OUTPUTS, PERSISTENT
-from vtlengine.AST.Grammar.tokens import AS, MEMBERSHIP, TO
+from vtlengine.AST.DAG._words import DELETE, GLOBAL, INPUTS, INSERT, OUTPUTS, PERSISTENT, UNKNOWN
+from vtlengine.AST.Grammar.tokens import AS, DROP, KEEP, MEMBERSHIP, RENAME, TO
 from vtlengine.Exceptions import SemanticError
 
 
@@ -61,6 +61,8 @@ class DAGAnalyzer(ASTTemplate):
     inputs: Optional[list] = None
     outputs: Optional[list] = None
     persistent: Optional[list] = None
+    unknown_variables: Optional[list] = None
+    unknown_variables_statement: Optional[list] = None
 
     def __post_init__(self):
         self.dependencies = {}
@@ -72,6 +74,8 @@ class DAGAnalyzer(ASTTemplate):
         self.outputs = []
         self.persistent = []
         self.alias = []
+        self.unknown_variables = []
+        self.unknown_variables_statement = []
 
     @classmethod
     def ds_structure(cls, ast: AST):
@@ -176,7 +180,7 @@ class DAGAnalyzer(ASTTemplate):
         """ """
         # For each vertex
         for key, statement in self.dependencies.items():
-            output = statement[OUTPUTS] + statement[PERSISTENT]
+            output = statement[OUTPUTS] + statement[PERSISTENT] + statement[UNKNOWN]
             # If the statement has no := or -> symbol there is no vertex to add.
             if len(output) != 0:
                 self.vertex[key] = output[0]
@@ -245,12 +249,15 @@ class DAGAnalyzer(ASTTemplate):
         inputs = list(set(self.inputs))
         outputs = list(set(self.outputs))
         persistent = list(set(self.persistent))
+        unknown = list(set(self.unknown_variables_statement))
 
         # Remove inputs that are outputs of some statement.
         inputsF = [inputf for inputf in inputs if inputf not in outputs]
 
-        dict_ = {INPUTS: inputsF, OUTPUTS: outputs, PERSISTENT: persistent}
-
+        dict_ = {INPUTS: inputsF, OUTPUTS: outputs, PERSISTENT: persistent, UNKNOWN: unknown}
+        for variable in self.unknown_variables_statement:
+            if variable not in self.unknown_variables:
+                self.unknown_variables.append(variable)
         return dict_
 
     """______________________________________________________________________________________
@@ -293,6 +300,19 @@ class DAGAnalyzer(ASTTemplate):
                 self.inputs = []
                 self.outputs = []
                 self.persistent = []
+                self.unknown_variables_statement = []
+        aux = copy.copy(self.unknown_variables)
+        for variable in aux:
+            for _number_of_statement, dependency in self.dependencies.items():
+                if variable in dependency[OUTPUTS]:
+                    if variable in self.unknown_variables:
+                        self.unknown_variables.remove(variable)
+                    for _number_of_statement, dependency in self.dependencies.items():
+                        if variable in dependency[UNKNOWN]:
+                            dependency[UNKNOWN].remove(variable)
+                            dependency[INPUTS].append(variable)
+                        if variable not in self.inputs:
+                            self.inputs.append(variable)
 
     def visit_Assignment(self, node: Assignment) -> None:
         if self.isFirstAssignment:
@@ -310,6 +330,8 @@ class DAGAnalyzer(ASTTemplate):
 
     def visit_RegularAggregation(self, node: RegularAggregation) -> None:
         self.visit(node.dataset)
+        if node.op in [KEEP, DROP, RENAME]:
+            return
         for child in node.children:
             self.isFromRegularAggregation = True
             self.visit(child)
@@ -331,6 +353,13 @@ class DAGAnalyzer(ASTTemplate):
     def visit_VarID(self, node: VarID) -> None:
         if (not self.isFromRegularAggregation or self.isDataset) and node.value not in self.alias:
             self.inputs.append(node.value)
+        elif (
+            self.isFromRegularAggregation
+            and node.value not in self.alias
+            and not self.isDataset
+            and node.value not in self.unknown_variables_statement
+        ):
+            self.unknown_variables_statement.append(node.value)
 
     def visit_Identifier(self, node: Identifier) -> None:
         if node.kind == "DatasetID" and node.value not in self.alias:

vtlengine/AST/DAG/_words.py

@@ -7,3 +7,4 @@ INPUTS = "inputs"
 OUTPUTS = "outputs"
 PERSISTENT = "persistent"
 STATEMENT_ = "statement"
+UNKNOWN = "unknown_variables"

vtlengine/AST/Grammar/Vtl.g4

@@ -219,11 +219,11 @@ timeOperators:
     | YEAR_OP LPAREN expr RPAREN                                                # yearAtom
     | MONTH_OP LPAREN expr RPAREN                                               # monthAtom
     | DAYOFMONTH LPAREN expr RPAREN                                             # dayOfMonthAtom
-    | DAYOFYEAR LPAREN expr RPAREN                                              # datOfYearAtom
+    | DAYOFYEAR LPAREN expr RPAREN                                              # dayOfYearAtom
     | DAYTOYEAR LPAREN expr RPAREN                                              # dayToYearAtom
     | DAYTOMONTH LPAREN expr RPAREN                                             # dayToMonthAtom
-    | YEARTODAY LPAREN expr RPAREN                                              # yearTodayAtom
-    | MONTHTODAY LPAREN expr RPAREN                                             # monthTodayAtom
+    | YEARTODAY LPAREN expr RPAREN                                              # yearToDayAtom
+    | MONTHTODAY LPAREN expr RPAREN                                             # monthToDayAtom
 ;
 
 timeOperatorsComponent:
@@ -238,11 +238,11 @@ timeOperatorsComponent:
     | YEAR_OP LPAREN exprComponent RPAREN                                                # yearAtomComponent
     | MONTH_OP LPAREN exprComponent RPAREN                                               # monthAtomComponent
     | DAYOFMONTH LPAREN exprComponent RPAREN                                             # dayOfMonthAtomComponent
-    | DAYOFYEAR LPAREN exprComponent RPAREN                                              # datOfYearAtomComponent
+    | DAYOFYEAR LPAREN exprComponent RPAREN                                              # dayOfYearAtomComponent
     | DAYTOYEAR LPAREN exprComponent RPAREN                                              # dayToYearAtomComponent
     | DAYTOMONTH LPAREN exprComponent RPAREN                                             # dayToMonthAtomComponent
-    | YEARTODAY LPAREN exprComponent RPAREN                                              # yearTodayAtomComponent
-    | MONTHTODAY LPAREN exprComponent RPAREN                                             # monthTodayAtomComponent
+    | YEARTODAY LPAREN exprComponent RPAREN                                              # yearToDayAtomComponent
+    | MONTHTODAY LPAREN exprComponent RPAREN                                             # monthToDayAtomComponent
 ;
 
 setOperators:
@@ -363,7 +363,7 @@ calcClauseItem:
 
 /*SUBSPACE CLAUSE*/
 subspaceClauseItem:
-  componentID  EQ  scalarItem
+  componentID  EQ  (scalarItem | varID)
 ;
 
 scalarItem: