vtlengine 1.2.1rc1__py3-none-any.whl → 1.3.0rc1__py3-none-any.whl

vtlengine/API/_InternalApi.py

@@ -46,6 +46,10 @@ schema_path = base_path / "data" / "schema"
 sdmx_csv_path = base_path / "data" / "sdmx_csv"
 with open(schema_path / "json_schema_2.1.json", "r") as file:
     schema = json.load(file)
+with open(schema_path / "value_domain_schema.json", "r") as file:
+    vd_schema = json.load(file)
+with open(schema_path / "external_routines_schema.json", "r") as file:
+    external_routine_schema = json.load(file)
 
 
 def _load_dataset_from_structure(
@@ -112,6 +116,7 @@ def _load_dataset_from_structure(
     if "scalars" in structures:
         for scalar_json in structures["scalars"]:
             scalar_name = scalar_json["name"]
+            check_key("type", SCALAR_TYPES.keys(), scalar_json["type"])
             scalar = Scalar(
                 name=scalar_name,
                 data_type=SCALAR_TYPES[scalar_json["type"]],
@@ -235,8 +240,16 @@ def _handle_scalars_values(
     # Handling scalar values with the scalar dict
     for name, value in scalar_values.items():
         if name not in scalars:
-            raise Exception(f"Not found scalar {name} in datastructures")
+            raise InputValidationException(code="0-1-2-6", name=name)
         # Casting value to scalar data type
+        if not scalars[name].data_type.check(value):
+            raise InputValidationException(
+                code="0-1-2-7",
+                value=value,
+                type_=scalars[name].data_type.__name__,
+                op_type=type(scalars[name]).__name__,
+                name=name,
+            )
         scalars[name].value = scalars[name].data_type.cast(value)
 
 
@@ -321,11 +334,20 @@ def load_vtl(input: Union[str, Path]) -> str:
         return f.read()
 
 
+def _validate_json(data: Dict[str, Any], schema: Dict[str, Any]) -> None:
+    try:
+        jsonschema.validate(instance=data, schema=schema)
+    except jsonschema.ValidationError:
+        raise Exception("The given json does not follow the schema.")
+
+
 def _load_single_value_domain(input: Path) -> Dict[str, ValueDomain]:
     if input.suffix != ".json":
         raise Exception("Invalid Value Domain file. Must have .json extension")
     with open(input, "r") as f:
-        vd = ValueDomain.from_dict(json.load(f))
+        data = json.load(f)
+    _validate_json(data, vd_schema)
+    vd = ValueDomain.from_dict(data)
     return {vd.name: vd}
 
 
@@ -344,6 +366,7 @@ def load_value_domains(input: Union[Dict[str, Any], Path]) -> Dict[str, ValueDom
         or the value domains file does not exist.
     """
     if isinstance(input, dict):
+        _validate_json(input, vd_schema)
         vd = ValueDomain.from_dict(input)
         return {vd.name: vd}
     if not isinstance(input, Path):
@@ -377,12 +400,12 @@ def load_external_routines(input: Union[Dict[str, Any], Path, str]) -> Any:
     """
     external_routines = {}
     if isinstance(input, dict):
-        for name, query in input.items():
-            ext_routine = ExternalRoutine.from_sql_query(name, query)
-            external_routines[ext_routine.name] = ext_routine
+        _validate_json(input, external_routine_schema)
+        ext_routine = ExternalRoutine.from_sql_query(input["name"], input["query"])
+        external_routines[ext_routine.name] = ext_routine
         return external_routines
     if not isinstance(input, Path):
-        raise Exception("Input invalid. Input must be a sql file.")
+        raise Exception("Input invalid. Input must be a json file.")
     if not input.exists():
         raise Exception("Input invalid. Input does not exist")
     if input.is_dir():
@@ -411,17 +434,17 @@ def _return_only_persistent_datasets(
 
 
 def _load_single_external_routine_from_file(input: Path) -> Any:
-    """
-    Returns a single external routine.
-    """
     if not isinstance(input, Path):
         raise Exception("Input invalid")
     if not input.exists():
         raise Exception("Input does not exist")
-    if input.suffix != ".sql":
-        raise Exception("Input must be a sql file")
+    if input.suffix != ".json":
+        raise Exception("Input must be a json file")
+    routine_name = input.stem
     with open(input, "r") as f:
-        ext_rout = ExternalRoutine.from_sql_query(input.name.removesuffix(".sql"), f.read())
+        data = json.load(f)
+    _validate_json(data, external_routine_schema)
+    ext_rout = ExternalRoutine.from_sql_query(routine_name, data["query"])
     return ext_rout
 
 

vtlengine/API/__init__.py

@@ -163,8 +163,17 @@ def semantic_analysis(
         that holds the vtl script.
         data_structures: Dict or Path (file or folder), \
         or List of Dicts or Paths with the data structures JSON files.
-        value_domains: Dict or Path of the value domains JSON files. (default: None)
-        external_routines: String or Path of the external routines SQL files. (default: None)
+        value_domains: Dict or Path, or List of Dicts or Paths of the \
+        value domains JSON files. (default:None) It is passed as an object, that can be read from \
+        a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
+
+        external_routines: String or Path, or List of Strings or Paths of the \
+        external routines SQL files. (default: None) It is passed as an object, that can be read \
+        from a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
     Returns:
         The computed datasets.
@@ -206,8 +215,10 @@ def run(
     script: Union[str, TransformationScheme, Path],
     data_structures: Union[Dict[str, Any], Path, List[Dict[str, Any]], List[Path]],
     datapoints: Union[Dict[str, pd.DataFrame], str, Path, List[Dict[str, Any]], List[Path]],
-    value_domains: Optional[Union[Dict[str, Any], Path]] = None,
-    external_routines: Optional[Union[str, Path]] = None,
+    value_domains: Optional[Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]] = None,
+    external_routines: Optional[
+        Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]
+    ] = None,
     time_period_output_format: str = "vtl",
     return_only_persistent: bool = True,
     output_folder: Optional[Union[str, Path]] = None,
@@ -265,9 +276,17 @@ def run(
 
         datapoints: Dict, Path, S3 URI or List of S3 URIs or Paths with data.
 
-        value_domains: Dict or Path of the value domains JSON files. (default:None)
+        value_domains: Dict or Path, or List of Dicts or Paths of the \
+        value domains JSON files. (default:None) It is passed as an object, that can be read from \
+        a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
-        external_routines: String or Path of the external routines SQL files. (default: None)
+        external_routines: String or Path, or List of Strings or Paths of the \
+        external routines JSON files. (default: None) It is passed as an object, that can be read \
+        from a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
         time_period_output_format: String with the possible values \
         ("sdmx_gregorian", "sdmx_reporting", "vtl") for the representation of the \
@@ -303,10 +322,20 @@ def run(
     # Handling of library items
     vd = None
     if value_domains is not None:
-        vd = load_value_domains(value_domains)
+        if isinstance(value_domains, list):
+            vd = {}
+            for item in value_domains:
+                vd.update(load_value_domains(item))
+        else:
+            vd = load_value_domains(value_domains)
     ext_routines = None
     if external_routines is not None:
-        ext_routines = load_external_routines(external_routines)
+        if isinstance(external_routines, list):
+            ext_routines = {}
+            for item in external_routines:
+                ext_routines.update(load_external_routines(item))
+        else:
+            ext_routines = load_external_routines(external_routines)
 
     # Checking time period output format value
     time_period_representation = TimePeriodRepresentation.check_value(time_period_output_format)
@@ -348,8 +377,10 @@ def run_sdmx(  # noqa: C901
     script: Union[str, TransformationScheme, Path],
     datasets: Sequence[PandasDataset],
     mappings: Optional[Union[VtlDataflowMapping, Dict[str, str]]] = None,
-    value_domains: Optional[Union[Dict[str, Any], Path]] = None,
-    external_routines: Optional[Union[str, Path]] = None,
+    value_domains: Optional[Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]] = None,
+    external_routines: Optional[
+        Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]
+    ] = None,
     time_period_output_format: str = "vtl",
     return_only_persistent: bool = True,
     output_folder: Optional[Union[str, Path]] = None,
@@ -388,9 +419,17 @@ def run_sdmx(  # noqa: C901
 
         mappings: A dictionary or VtlDataflowMapping object that maps the dataset names.
 
-        value_domains: Dict or Path of the value domains JSON files. (default:None)
-
-        external_routines: String or Path of the external routines SQL files. (default: None)
+        value_domains: Dict or Path, or List of Dicts or Paths of the \
+        value domains JSON files. (default:None) It is passed as an object, that can be read from \
+        a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
+
+        external_routines: String or Path, or List of Strings or Paths of the \
+        external routines JSON files. (default: None) It is passed as an object, that can be read \
+        from a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
         time_period_output_format: String with the possible values \
         ("sdmx_gregorian", "sdmx_reporting", "vtl") for the representation of the \

vtlengine/API/data/schema/external_routines_schema.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "sqlQuery": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Identifier for the SQL query"
+        },
+        "query": {
+          "type": "string",
+          "minLength": 1,
+          "description": "SQL query statement"
+        }
+      },
+      "required": ["name", "query"],
+      "additionalProperties": false
+    }
+  },
+  "oneOf": [
+    {
+      "$ref": "#/definitions/sqlQuery"
+    },
+    {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/sqlQuery"
+      },
+      "minItems": 1
+    }
+  ]
+}

vtlengine/API/data/schema/value_domain_schema.json

@@ -0,0 +1,97 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "ValueDomain": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Name identifier for the Value Domain"
+        },
+        "setlist": {
+          "type": "array",
+          "minItems": 1,
+          "uniqueItems": true,
+          "description": "List of unique values of the Value Domain"
+        },
+        "type": {
+          "type": "string",
+          "enum": ["Integer", "Number", "String", "Boolean", "Date", "Time_Period", "Time", "Duration"],
+          "description": "Data type of the Value Domain"
+        }
+      },
+      "required": ["name", "setlist", "type"],
+      "additionalProperties": false,
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "type": {"const": "Integer"}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "integer"}
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {"const": "Number"}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "number"}
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {"const": "Boolean"}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "boolean"}
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {"enum": ["String", "Date", "Time_Period", "Time", "Duration"]}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "string"}
+              }
+            }
+          }
+        }
+      ]
+    }
+  },
+  "oneOf": [
+    {
+      "$ref": "#/definitions/ValueDomain"
+    },
+    {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/ValueDomain"
+      },
+      "minItems": 1
+    }
+  ]
+}

vtlengine/AST/ASTConstructorModules/Terminals.py

@@ -622,11 +622,7 @@ class Terminals(VtlVisitor):
         erLevel: ERRORLEVEL  constant;
         """
         ctx_list = list(ctx.getChildren())
-
-        try:
-            return int(self.visitConstant(ctx_list[1]).value)
-        except Exception:
-            raise Exception(f"Error level must be an integer, line {ctx_list[1].start.line}")
+        return self.visitConstant(ctx_list[1]).value
 
     def visitSignature(self, ctx: Parser.SignatureContext, kind="ComponentID"):
         """