vtlengine 1.2.1rc1__tar.gz → 1.3.0rc1__tar.gz

{vtlengine-1.2.1rc1 → vtlengine-1.3.0rc1}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: vtlengine
-Version: 1.2.1rc1
+Version: 1.3.0rc1
 Summary: Run and Validate VTL Scripts
-License: AGPL-3.0
+License-Expression: AGPL-3.0
+License-File: LICENSE.md
 Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
 Author: MeaningfulData
 Author-email: info@meaningfuldata.eu
@@ -16,8 +17,8 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Typing :: Typed
 Provides-Extra: all
 Provides-Extra: s3
-Requires-Dist: antlr4-python3-runtime (>=4.13.2,<4.14)
-Requires-Dist: duckdb (>=1.1,<1.2)
+Requires-Dist: antlr4-python3-runtime (>=4.9,<4.10)
+Requires-Dist: duckdb (>=1.4,<1.5)
 Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "all"
 Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "s3"
 Requires-Dist: jsonschema (>=3.2.0,<5.0)
@@ -25,7 +26,7 @@ Requires-Dist: networkx (>=2.8,<3.0)
 Requires-Dist: numpy (>=1.23.2,<2) ; python_version < "3.13"
 Requires-Dist: numpy (>=2.1.0) ; python_version >= "3.13"
 Requires-Dist: pandas (>=2.1.4,<3.0)
-Requires-Dist: pysdmx[xml] (>=1.4.0rc1,<2.0)
+Requires-Dist: pysdmx[xml] (>=1.5.2,<2.0)
 Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "all"
 Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "s3"
 Requires-Dist: sqlglot (>=22.2.0,<23.0)

{vtlengine-1.2.1rc1 → vtlengine-1.3.0rc1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vtlengine"
-version = "1.2.1rc1"
+version = "1.3.0rc1"
 description = "Run and Validate VTL Scripts"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -24,12 +24,12 @@ keywords = ['vtl', 'sdmx', 'vtlengine', 'Validation and Transformation Language'
 
 dependencies = [
     # PyPi dependencies
-    "duckdb>=1.1,<1.2",
-    "pysdmx[xml]>=1.4.0rc1,<2.0",
+    "duckdb>=1.4,<1.5",
+    "pysdmx[xml]>=1.5.2,<2.0",
     # APT-supported dependencies
     "jsonschema>=3.2.0,<5.0",
     "sqlglot>=22.2.0,<23.0",
-    "antlr4-python3-runtime>=4.13.2,<4.14",
+    "antlr4-python3-runtime>=4.9,<4.10",
     "pandas>=2.1.4,<3.0",
     "networkx>=2.8,<3.0",
     "numpy>=1.23.2,<2 ; python_version < '3.13'",
@@ -51,19 +51,19 @@ Authors = 'https://github.com/Meaningful-Data/vtlengine/graphs/contributors'
 python = ">=3.9,<4.0"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^8.4"
-pytest-cov = "^6.2.1"
-pytest-xdist = "^3.8.0"
-line-profiler-pycharm = "^1.2.0"
-mypy = "1.17.1"
-pandas-stubs = "2.2.2.240807"
-ruff = "^0.12.7"
-types-jsonschema = "4.25.0.20250720"
+pytest = ">=8.4,<9.0"
+pytest-cov = ">=7.0.0,<8.0"
+pytest-xdist = ">=3.8.0,<4.0"
+line-profiler-pycharm = ">=1.2.0,<2.0"
+mypy = ">=1.18,<2.0"
+pandas-stubs = ">=2.2.2,<3.0"
+ruff = ">=0.14,<1.0.0"
+types-jsonschema = ">=4.25.1,<5.0"
 
 [tool.poetry.group.docs.dependencies]
-sphinx = "^7.4.7"
-sphinx-rtd-theme = "^3.0.2"
-toml = "^0.10.2"
+sphinx = ">=7.4.7,<8.0"
+sphinx-rtd-theme = ">=3.0.2,<4.0"
+toml = ">=0.10.2,<0.11.0"
 
 
 
@@ -77,7 +77,7 @@ lint.select = [
 lint.ignore = ["B023", "B028", "B904", "C403", "D100", "D101", "D102", "D103", "D104", "D105",
     "D107", "D200", "D201", "D202", "D203", "D205", "D209", "D212", "D213", "D301",
     "D400", "D401", "D404", "D411", "D413", "D415", "D419", "E203", "S608"]
-lint.exclude = ["*/Grammar/*", "*/main.py"]
+lint.exclude = ["*/Grammar/*", "*/main.py", "*/dev.py"]
 
 [tool.ruff.lint.per-file-ignores]
 "tests/*" = ["S101", "PT006", "PT012", "PT013", "E501", "W605"]
@@ -85,7 +85,7 @@ lint.exclude = ["*/Grammar/*", "*/main.py"]
 
 [tool.mypy]
 files = "src"
-exclude = "src/vtlengine/AST/.*"
+exclude = "src/vtlengine/AST/.*|src/dev.py"
 disallow_untyped_defs = true
 disallow_untyped_calls = true
 ignore_errors = false

{vtlengine-1.2.1rc1 → vtlengine-1.3.0rc1}/src/vtlengine/API/_InternalApi.py

@@ -46,6 +46,10 @@ schema_path = base_path / "data" / "schema"
 sdmx_csv_path = base_path / "data" / "sdmx_csv"
 with open(schema_path / "json_schema_2.1.json", "r") as file:
     schema = json.load(file)
+with open(schema_path / "value_domain_schema.json", "r") as file:
+    vd_schema = json.load(file)
+with open(schema_path / "external_routines_schema.json", "r") as file:
+    external_routine_schema = json.load(file)
 
 
 def _load_dataset_from_structure(
@@ -112,6 +116,7 @@ def _load_dataset_from_structure(
     if "scalars" in structures:
         for scalar_json in structures["scalars"]:
             scalar_name = scalar_json["name"]
+            check_key("type", SCALAR_TYPES.keys(), scalar_json["type"])
             scalar = Scalar(
                 name=scalar_name,
                 data_type=SCALAR_TYPES[scalar_json["type"]],
@@ -235,8 +240,16 @@ def _handle_scalars_values(
     # Handling scalar values with the scalar dict
     for name, value in scalar_values.items():
         if name not in scalars:
-            raise Exception(f"Not found scalar {name} in datastructures")
+            raise InputValidationException(code="0-1-2-6", name=name)
         # Casting value to scalar data type
+        if not scalars[name].data_type.check(value):
+            raise InputValidationException(
+                code="0-1-2-7",
+                value=value,
+                type_=scalars[name].data_type.__name__,
+                op_type=type(scalars[name]).__name__,
+                name=name,
+            )
         scalars[name].value = scalars[name].data_type.cast(value)
 
 
@@ -321,11 +334,20 @@ def load_vtl(input: Union[str, Path]) -> str:
         return f.read()
 
 
+def _validate_json(data: Dict[str, Any], schema: Dict[str, Any]) -> None:
+    try:
+        jsonschema.validate(instance=data, schema=schema)
+    except jsonschema.ValidationError:
+        raise Exception("The given json does not follow the schema.")
+
+
 def _load_single_value_domain(input: Path) -> Dict[str, ValueDomain]:
     if input.suffix != ".json":
         raise Exception("Invalid Value Domain file. Must have .json extension")
     with open(input, "r") as f:
-        vd = ValueDomain.from_dict(json.load(f))
+        data = json.load(f)
+    _validate_json(data, vd_schema)
+    vd = ValueDomain.from_dict(data)
     return {vd.name: vd}
 
 
@@ -344,6 +366,7 @@ def load_value_domains(input: Union[Dict[str, Any], Path]) -> Dict[str, ValueDom
         or the value domains file does not exist.
     """
     if isinstance(input, dict):
+        _validate_json(input, vd_schema)
         vd = ValueDomain.from_dict(input)
         return {vd.name: vd}
     if not isinstance(input, Path):
@@ -377,12 +400,12 @@ def load_external_routines(input: Union[Dict[str, Any], Path, str]) -> Any:
     """
     external_routines = {}
     if isinstance(input, dict):
-        for name, query in input.items():
-            ext_routine = ExternalRoutine.from_sql_query(name, query)
-            external_routines[ext_routine.name] = ext_routine
+        _validate_json(input, external_routine_schema)
+        ext_routine = ExternalRoutine.from_sql_query(input["name"], input["query"])
+        external_routines[ext_routine.name] = ext_routine
         return external_routines
     if not isinstance(input, Path):
-        raise Exception("Input invalid. Input must be a sql file.")
+        raise Exception("Input invalid. Input must be a json file.")
     if not input.exists():
         raise Exception("Input invalid. Input does not exist")
     if input.is_dir():
@@ -411,17 +434,17 @@ def _return_only_persistent_datasets(
 
 
 def _load_single_external_routine_from_file(input: Path) -> Any:
-    """
-    Returns a single external routine.
-    """
     if not isinstance(input, Path):
         raise Exception("Input invalid")
     if not input.exists():
         raise Exception("Input does not exist")
-    if input.suffix != ".sql":
-        raise Exception("Input must be a sql file")
+    if input.suffix != ".json":
+        raise Exception("Input must be a json file")
+    routine_name = input.stem
     with open(input, "r") as f:
-        ext_rout = ExternalRoutine.from_sql_query(input.name.removesuffix(".sql"), f.read())
+        data = json.load(f)
+    _validate_json(data, external_routine_schema)
+    ext_rout = ExternalRoutine.from_sql_query(routine_name, data["query"])
     return ext_rout
 
 

{vtlengine-1.2.1rc1 → vtlengine-1.3.0rc1}/src/vtlengine/API/__init__.py

@@ -163,8 +163,17 @@ def semantic_analysis(
         that holds the vtl script.
         data_structures: Dict or Path (file or folder), \
         or List of Dicts or Paths with the data structures JSON files.
-        value_domains: Dict or Path of the value domains JSON files. (default: None)
-        external_routines: String or Path of the external routines SQL files. (default: None)
+        value_domains: Dict or Path, or List of Dicts or Paths of the \
+        value domains JSON files. (default:None) It is passed as an object, that can be read from \
+        a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
+
+        external_routines: String or Path, or List of Strings or Paths of the \
+        external routines SQL files. (default: None) It is passed as an object, that can be read \
+        from a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
     Returns:
         The computed datasets.
@@ -206,8 +215,10 @@ def run(
     script: Union[str, TransformationScheme, Path],
     data_structures: Union[Dict[str, Any], Path, List[Dict[str, Any]], List[Path]],
     datapoints: Union[Dict[str, pd.DataFrame], str, Path, List[Dict[str, Any]], List[Path]],
-    value_domains: Optional[Union[Dict[str, Any], Path]] = None,
-    external_routines: Optional[Union[str, Path]] = None,
+    value_domains: Optional[Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]] = None,
+    external_routines: Optional[
+        Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]
+    ] = None,
     time_period_output_format: str = "vtl",
     return_only_persistent: bool = True,
     output_folder: Optional[Union[str, Path]] = None,
@@ -265,9 +276,17 @@ def run(
 
         datapoints: Dict, Path, S3 URI or List of S3 URIs or Paths with data.
 
-        value_domains: Dict or Path of the value domains JSON files. (default:None)
+        value_domains: Dict or Path, or List of Dicts or Paths of the \
+        value domains JSON files. (default:None) It is passed as an object, that can be read from \
+        a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
-        external_routines: String or Path of the external routines SQL files. (default: None)
+        external_routines: String or Path, or List of Strings or Paths of the \
+        external routines JSON files. (default: None) It is passed as an object, that can be read \
+        from a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
         time_period_output_format: String with the possible values \
         ("sdmx_gregorian", "sdmx_reporting", "vtl") for the representation of the \
@@ -303,10 +322,20 @@ def run(
     # Handling of library items
     vd = None
     if value_domains is not None:
-        vd = load_value_domains(value_domains)
+        if isinstance(value_domains, list):
+            vd = {}
+            for item in value_domains:
+                vd.update(load_value_domains(item))
+        else:
+            vd = load_value_domains(value_domains)
     ext_routines = None
     if external_routines is not None:
-        ext_routines = load_external_routines(external_routines)
+        if isinstance(external_routines, list):
+            ext_routines = {}
+            for item in external_routines:
+                ext_routines.update(load_external_routines(item))
+        else:
+            ext_routines = load_external_routines(external_routines)
 
     # Checking time period output format value
     time_period_representation = TimePeriodRepresentation.check_value(time_period_output_format)
@@ -348,8 +377,10 @@ def run_sdmx(  # noqa: C901
     script: Union[str, TransformationScheme, Path],
     datasets: Sequence[PandasDataset],
     mappings: Optional[Union[VtlDataflowMapping, Dict[str, str]]] = None,
-    value_domains: Optional[Union[Dict[str, Any], Path]] = None,
-    external_routines: Optional[Union[str, Path]] = None,
+    value_domains: Optional[Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]] = None,
+    external_routines: Optional[
+        Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]
+    ] = None,
     time_period_output_format: str = "vtl",
     return_only_persistent: bool = True,
     output_folder: Optional[Union[str, Path]] = None,
@@ -388,9 +419,17 @@ def run_sdmx(  # noqa: C901
 
         mappings: A dictionary or VtlDataflowMapping object that maps the dataset names.
 
-        value_domains: Dict or Path of the value domains JSON files. (default:None)
-
-        external_routines: String or Path of the external routines SQL files. (default: None)
+        value_domains: Dict or Path, or List of Dicts or Paths of the \
+        value domains JSON files. (default:None) It is passed as an object, that can be read from \
+        a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
+
+        external_routines: String or Path, or List of Strings or Paths of the \
+        external routines JSON files. (default: None) It is passed as an object, that can be read \
+        from a Path or from a dictionary. Furthermore, a list of those objects can be passed. \
+        Check the following example: \
+        :ref:`Example 6 <example_6_run_with_multiple_value_domains_and_external_routines>`.
 
         time_period_output_format: String with the possible values \
         ("sdmx_gregorian", "sdmx_reporting", "vtl") for the representation of the \

vtlengine-1.3.0rc1/src/vtlengine/API/data/schema/external_routines_schema.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "sqlQuery": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Identifier for the SQL query"
+        },
+        "query": {
+          "type": "string",
+          "minLength": 1,
+          "description": "SQL query statement"
+        }
+      },
+      "required": ["name", "query"],
+      "additionalProperties": false
+    }
+  },
+  "oneOf": [
+    {
+      "$ref": "#/definitions/sqlQuery"
+    },
+    {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/sqlQuery"
+      },
+      "minItems": 1
+    }
+  ]
+}

vtlengine-1.3.0rc1/src/vtlengine/API/data/schema/value_domain_schema.json

@@ -0,0 +1,97 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "definitions": {
+    "ValueDomain": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Name identifier for the Value Domain"
+        },
+        "setlist": {
+          "type": "array",
+          "minItems": 1,
+          "uniqueItems": true,
+          "description": "List of unique values of the Value Domain"
+        },
+        "type": {
+          "type": "string",
+          "enum": ["Integer", "Number", "String", "Boolean", "Date", "Time_Period", "Time", "Duration"],
+          "description": "Data type of the Value Domain"
+        }
+      },
+      "required": ["name", "setlist", "type"],
+      "additionalProperties": false,
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "type": {"const": "Integer"}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "integer"}
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {"const": "Number"}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "number"}
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {"const": "Boolean"}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "boolean"}
+              }
+            }
+          }
+        },
+        {
+          "if": {
+            "properties": {
+              "type": {"enum": ["String", "Date", "Time_Period", "Time", "Duration"]}
+            }
+          },
+          "then": {
+            "properties": {
+              "setlist": {
+                "items": {"type": "string"}
+              }
+            }
+          }
+        }
+      ]
+    }
+  },
+  "oneOf": [
+    {
+      "$ref": "#/definitions/ValueDomain"
+    },
+    {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/ValueDomain"
+      },
+      "minItems": 1
+    }
+  ]
+}

{vtlengine-1.2.1rc1 → vtlengine-1.3.0rc1}/src/vtlengine/AST/ASTConstructorModules/Terminals.py

@@ -622,11 +622,7 @@ class Terminals(VtlVisitor):
         erLevel: ERRORLEVEL  constant;
         """
         ctx_list = list(ctx.getChildren())
-
-        try:
-            return int(self.visitConstant(ctx_list[1]).value)
-        except Exception:
-            raise Exception(f"Error level must be an integer, line {ctx_list[1].start.line}")
+        return self.visitConstant(ctx_list[1]).value
 
     def visitSignature(self, ctx: Parser.SignatureContext, kind="ComponentID"):
         """