vtlengine 1.0.3rc3__tar.gz → 1.1rc1__tar.gz

{vtlengine-1.0.3rc3 → vtlengine-1.1rc1}/PKG-INFO

@@ -1,31 +1,31 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: vtlengine
-Version: 1.0.3rc3
+Version: 1.1rc1
 Summary: Run and Validate VTL Scripts
 License: AGPL-3.0
 Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
 Author: MeaningfulData
 Author-email: info@meaningfuldata.eu
-Requires-Python: >=3.10,<4.0
+Maintainer: Francisco Javier Hernandez del Caño
+Maintainer-email: javier.hernandez@meaningfuldata.eu
+Requires-Python: >=3.9,<4
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: GNU Affero General Public License v3
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
-Requires-Dist: antlr4-python3-runtime (==4.9.2)
-Requires-Dist: bottleneck (>=1.3.4,<2.0.0)
-Requires-Dist: duckdb (>=1.1.1,<2.0.0)
-Requires-Dist: networkx (>=2.8.8,<3.0.0)
-Requires-Dist: numexpr (>=2.9.0,<3.0.0)
-Requires-Dist: pandas (>=2.1.4,<3.0.0)
-Requires-Dist: s3fs (>=2024.9.0,<2025.0.0)
-Requires-Dist: sqlglot (>=22.2.0,<23.0.0)
+Provides-Extra: all
+Provides-Extra: s3
+Requires-Dist: antlr4-python3-runtime (>=4.9.2,<4.10)
+Requires-Dist: duckdb (>=1.1,<1.2)
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: jsonschema (>=3.2.0,<5.0)
+Requires-Dist: networkx (>=2.8,<3.0)
+Requires-Dist: pandas (>=2.1,<3.0)
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: sqlglot (>=22.2.0,<23.0)
 Project-URL: Authors, https://github.com/Meaningful-Data/vtlengine/graphs/contributors
 Project-URL: Documentation, https://docs.vtlengine.meaningfuldata.eu
 Project-URL: IssueTracker, https://github.com/Meaningful-Data/vtlengine/issues

{vtlengine-1.0.3rc3 → vtlengine-1.1rc1}/pyproject.toml

@@ -1,11 +1,18 @@
-[tool.poetry]
+[project]
 name = "vtlengine"
-version = "1.0.3rc3"
+version = "1.1rc1"
 description = "Run and Validate VTL Scripts"
-authors = ["MeaningfulData <info@meaningfuldata.eu>"]
 license = "AGPL-3.0"
 readme = "README.md"
-
+requires-python = ">=3.9,<4"
+authors = [
+    {name = "MeaningfulData", email = "info@meaningfuldata.eu"},
+]
+maintainers = [
+    {name = "Francisco Javier Hernandez del Caño", email = "javier.hernandez@meaningfuldata.eu"},
+    {name = "Alberto Hernandez del Caño", email = "alberto.hernandez@meaningfuldata.eu"},
+    {name = "Mateo de Lorenzo Argeles", email = "mateo.delorenzo@meaningfuldata.eu"}
+]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
@@ -13,42 +20,44 @@ classifiers = [
     "Intended Audience :: Science/Research",
     "Typing :: Typed"
 ]
-
 keywords = ['vtl', 'sdmx', 'vtlengine', 'Validation and Transformation Language']
 
-[tool.poetry.urls]
+dependencies = [
+    # PyPi dependencies
+    "duckdb>=1.1,<1.2",
+
+    # APT-supported dependencies
+    "jsonschema>=3.2.0,<5.0",
+    "sqlglot>=22.2.0,<23.0",
+    "antlr4-python3-runtime>=4.9.2,<4.10",
+    "pandas>=2.1,<3.0",
+    "networkx>=2.8,<3.0",
+]
+
+[project.optional-dependencies]
+s3 = ["s3fs>=2022.11.0,<2023.0", "fsspec>=2022.11.0,<2023.0"]
+all = ["s3fs>=2022.11.0,<2023.0", "fsspec>=2022.11.0,<2023.0"]
+
+[project.urls]
 Repository = 'https://github.com/Meaningful-Data/vtlengine'
 Documentation = 'https://docs.vtlengine.meaningfuldata.eu'
 MeaningfulData = 'https://www.meaningfuldata.eu/'
 IssueTracker = 'https://github.com/Meaningful-Data/vtlengine/issues'
 Authors = 'https://github.com/Meaningful-Data/vtlengine/graphs/contributors'
 
-[tool.poetry.dependencies]
-python = "^3.10"
-# PyPi dependencies
-duckdb = "^1.1.1"
-#numba = "^0.60.0"
-s3fs = "^2024.9.0"
-
-# APT dependencies
-antlr4-python3-runtime = "4.9.2"
-networkx = "^2.8.8"
-numexpr = "^2.9.0"
-pandas = "^2.1.4"
-bottleneck = "^1.3.4"
-sqlglot = "^22.2.0"
-
-[tool.poetry.dev-dependencies]
-pytest = "^7.3"
-pytest-cov = "^5.0.0"
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3"
+pytest-cov = "^6.0.0"
 line-profiler-pycharm = "^1.2.0"
-sphinx = "^7.4"
-sphinx-rtd-theme = "^2.0.0"
-mypy = "^1.11.2"
-pandas-stubs = "^2.2.3.241009"
-stubs = "^1.0.0"
+mypy = "1.14.1"
+pandas-stubs = "^2.1.4.231227"
+ruff = "^0.9.4"
+types-jsonschema = "4.23.0.20241208"
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "^7.4.7"
+sphinx-rtd-theme = "^3.0.2"
 toml = "^0.10.2"
-ruff = "^0.7.1"
 
 [tool.ruff]
 line-length = 100
@@ -84,6 +93,12 @@ enable_error_code = [
 ]
 warn_return_any = false
 
+[tool.pytest.ini_options]
+addopts = "--strict-markers"
+markers = [
+    "input_path: directory where tests data files are stored"
+]
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

{vtlengine-1.0.3rc3 → vtlengine-1.1rc1}/src/vtlengine/API/_InternalApi.py

@@ -3,12 +3,13 @@ import os
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
+import jsonschema
 import pandas as pd
-from s3fs import S3FileSystem  # type: ignore[import-untyped]
 
+from vtlengine.__extras_check import __check_s3_extra
 from vtlengine.AST import PersistentAssignment, Start
 from vtlengine.DataTypes import SCALAR_TYPES
-from vtlengine.Exceptions import check_key
+from vtlengine.Exceptions import InputValidationException, check_key
 from vtlengine.files.parser import _fill_dataset_empty_data, _validate_pandas
 from vtlengine.Model import (
     Component,
@@ -21,13 +22,9 @@ from vtlengine.Model import (
 )
 
 base_path = Path(__file__).parent
-filepath_VTL = base_path / "data" / "vtl"
-filepath_ValueDomains = base_path / "data" / "ValueDomain"
-filepath_sql = base_path / "data" / "sql"
-filepath_json = base_path / "data" / "DataStructure" / "input"
-filepath_csv = base_path / "data" / "DataSet" / "input"
-filepath_out_json = base_path / "data" / "DataStructure" / "output"
-filepath_out_csv = base_path / "data" / "DataSet" / "output"
+schema_path = base_path / "data" / "schema"
+with open(schema_path / "json_schema_2.1.json", "r") as file:
+    schema = json.load(file)
 
 
 def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
@@ -41,22 +38,60 @@ def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
             dataset_name = dataset_json["name"]
             components = {}
 
-            for component in dataset_json["DataStructure"]:
-                check_key("data_type", SCALAR_TYPES.keys(), component["type"])
-                check_key("role", Role_keys, component["role"])
-                components[component["name"]] = Component(
-                    name=component["name"],
-                    data_type=SCALAR_TYPES[component["type"]],
-                    role=Role(component["role"]),
-                    nullable=component["nullable"],
-                )
+            if "structure" in dataset_json:
+                structure_name = dataset_json["structure"]
+                structure_json = None
+                for s in structures["structures"]:
+                    if s["name"] == structure_name:
+                        structure_json = s
+                if structure_json is None:
+                    raise InputValidationException(code="0-3-1-1", message="Structure not found.")
+                try:
+                    jsonschema.validate(instance=structure_json, schema=schema)
+                except jsonschema.exceptions.ValidationError as e:
+                    raise InputValidationException(code="0-3-1-1", message=e.message)
+
+                for component in structure_json["components"]:
+                    check_key("data_type", SCALAR_TYPES.keys(), component["data_type"])
+                    if component["role"] == "ViralAttribute":
+                        component["role"] = "Attribute"
+
+                    check_key("role", Role_keys, component["role"])
+
+                    if "nullable" not in component:
+                        if Role(component["role"]) == Role.IDENTIFIER:
+                            component["nullable"] = False
+                        elif Role(component["role"]) in (Role.MEASURE, Role.ATTRIBUTE):
+                            component["nullable"] = True
+                        else:
+                            component["nullable"] = False
+
+                    components[component["name"]] = Component(
+                        name=component["name"],
+                        data_type=SCALAR_TYPES[component["data_type"]],
+                        role=Role(component["role"]),
+                        nullable=component["nullable"],
+                    )
+
+            if "DataStructure" in dataset_json:
+                for component in dataset_json["DataStructure"]:
+                    check_key("data_type", SCALAR_TYPES.keys(), component["type"])
+                    check_key("role", Role_keys, component["role"])
+                    components[component["name"]] = Component(
+                        name=component["name"],
+                        data_type=SCALAR_TYPES[component["type"]],
+                        role=Role(component["role"]),
+                        nullable=component["nullable"],
+                    )
 
             datasets[dataset_name] = Dataset(name=dataset_name, components=components, data=None)
     if "scalars" in structures:
         for scalar_json in structures["scalars"]:
             scalar_name = scalar_json["name"]
             scalar = Scalar(
-                name=scalar_name, data_type=SCALAR_TYPES[scalar_json["type"]], value=None
+                name=scalar_name,
+                data_type=SCALAR_TYPES[scalar_json["type"]],
+                value=None,
             )
             datasets[scalar_name] = scalar  # type: ignore[assignment]
     return datasets
@@ -70,38 +105,16 @@ def _load_single_datapoint(datapoint: Union[str, Path]) -> Dict[str, Any]:
         raise Exception("Invalid datapoint. Input must be a Path or an S3 URI")
     if isinstance(datapoint, str):
         if "s3://" in datapoint:
-            # Handling S3 URI
-            s3fs_obj = S3FileSystem()
-
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(datapoint):
-                raise Exception(
-                    f"Invalid datapoint. S3 URI does not exist or it is not accessible: {datapoint}"
-                )
-
-            # Check if the S3 URI is a directory
-            if s3fs_obj.isdir(datapoint):
-                datapoints: Dict[str, Any] = {}
-                for f in s3fs_obj.ls(datapoint):
-                    if f.endswith(".csv"):
-                        dataset_name = f.split("/")[-1].removesuffix(".csv")
-                        dict_data = {dataset_name: f"s3://{f}"}
-                        datapoints = {**datapoints, **dict_data}
-                return datapoints
-
-            # Check if the S3 URI is a csv file
-            if s3fs_obj.isfile(datapoint) and not datapoint.endswith(".csv"):
-                raise Exception(f"Invalid datapoint. S3 URI must refer to a csv file: {datapoint}")
+            __check_s3_extra()
             dataset_name = datapoint.split("/")[-1].removesuffix(".csv")
             dict_data = {dataset_name: datapoint}
             return dict_data
-
         try:
             datapoint = Path(datapoint)
         except Exception:
             raise Exception("Invalid datapoint. Input must refer to a Path or an S3 URI")
     if datapoint.is_dir():
-        datapoints = {}
+        datapoints: Dict[str, Any] = {}
         for f in datapoint.iterdir():
             if f.suffix != ".csv":
                 continue
@@ -115,7 +128,7 @@ def _load_single_datapoint(datapoint: Union[str, Path]) -> Dict[str, Any]:
 
 
 def _load_datapoints_path(
-    datapoints: Union[Path, str, List[Union[str, Path]]]
+    datapoints: Union[Path, str, List[Union[str, Path]]],
 ) -> Dict[str, Dataset]:
     """
     Returns a dict with the data given from a Path.
@@ -156,7 +169,7 @@ def _load_datastructure_single(data_structure: Union[Dict[str, Any], Path]) -> D
 
 
 def load_datasets(
-    data_structure: Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]
+    data_structure: Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]],
 ) -> Dict[str, Dataset]:
     """
     Loads multiple datasets.
@@ -365,25 +378,18 @@ def _check_output_folder(output_folder: Union[str, Path]) -> None:
     """
     if isinstance(output_folder, str):
         if "s3://" in output_folder:
-            s3fs_obj = S3FileSystem()
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(output_folder):
-                try:
-                    s3fs_obj.mkdir(output_folder)
-                except Exception:
-                    raise Exception(
-                        f"Invalid output folder. S3 URI is invalid or "
-                        f"it is not accessible: {output_folder}"
-                    )
+            __check_s3_extra()
+            if not output_folder.endswith("/"):
+                raise ValueError("Output folder must be a Path or S3 URI to a directory")
             return
         try:
             output_folder = Path(output_folder)
         except Exception:
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
 
     if not isinstance(output_folder, Path):
-        raise Exception("Output folder must be a Path or S3 URI to a directory")
+        raise ValueError("Output folder must be a Path or S3 URI to a directory")
     if not output_folder.exists():
         if output_folder.suffix != "":
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
         os.mkdir(output_folder)

{vtlengine-1.0.3rc3 → vtlengine-1.1rc1}/src/vtlengine/API/__init__.py

@@ -32,7 +32,13 @@ class __VTLSingleErrorListener(ErrorListener):  # type: ignore[misc]
     """ """
 
     def syntaxError(
-        self, recognizer: Any, offendingSymbol: str, line: str, column: str, msg: str, e: Any
+        self,
+        recognizer: Any,
+        offendingSymbol: str,
+        line: str,
+        column: str,
+        msg: str,
+        e: Any,
     ) -> None:
         raise Exception(
             f"Not valid VTL Syntax \n "
@@ -150,7 +156,10 @@ def semantic_analysis(
 
     # Running the interpreter
     interpreter = InterpreterAnalyzer(
-        datasets=structures, value_domains=vd, external_routines=ext_routines, only_semantic=True
+        datasets=structures,
+        value_domains=vd,
+        external_routines=ext_routines,
+        only_semantic=True,
     )
     with pd.option_context("future.no_silent_downcasting", True):
         result = interpreter.visit(ast)

vtlengine-1.1rc1/src/vtlengine/API/data/schema/json_schema_2.1.json

@@ -0,0 +1,116 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "description": "VTL Metadata JSON serialization",
+    "$defs": {
+        "vtl-id": {
+            "type": "string",
+            "pattern": "^[a-zA-Z][a-zA-Z0-9_]*$|^'.*'$"
+        },
+        "set-type": {
+            "type": "array",
+            "uniqueItems": true,
+            "oneOf": [
+                { "items": { "oneOf": [ { "type": "string" }, { "type": "null" } ] } },
+                { "items": { "oneOf": [ { "type": "number" }, { "type": "null" } ] } }
+            ]
+        },
+        "identifiable": {
+            "type": "object",
+            "properties": {
+                "name": { "$ref": "#/$defs/vtl-id" },
+                "description": { "type": "string" }
+            },
+            "required": [ "name" ]
+        }
+    },
+    "type": "object",
+    "properties": {
+        "datasets": {
+            "type": "array",
+            "items": {
+                "allOf": [ { "$ref": "#/$defs/identifiable" } ],
+                "properties": {
+                    "source": { "type": "string" },
+                    "structure": { "$ref": "#/$defs/vtl-id" }
+                },
+                "required": [ "structure" ]
+            }
+        },
+        "structures": {
+            "type": "array",
+            "items": {
+                "allOf": [ { "$ref": "#/$defs/identifiable" } ],
+                "properties": {
+                    "components": {
+                        "type": "array",
+                        "items": {
+                            "allOf": [ { "$ref": "#/$defs/identifiable" } ],
+                            "properties": {
+                                "role": {
+                                    "type": "string",
+                                    "enum": [ "Identifier", "Measure", "Attribute", "Viral Attribute" ]
+                                },
+                                "subset": { "$ref": "#/$defs/vtl-id" },
+                                "nullable": { "type": "boolean" },
+                                "data_type": {
+                                    "type": "string",
+                                    "enum": [ "String", "Number", "Integer", "Boolean", "Time", "TimePeriod", "Date", "Duration" ]
+                                }
+                            },
+                            "required": [ "role" ]
+                        }
+                    }
+                },
+                "required": [ "components" ]
+            }
+        },
+        "variables": {
+            "type": "array",
+            "items": {
+                "allOf": [ { "$ref": "#/$defs/identifiable" } ],
+                "properties": {
+                    "domain": { "$ref": "#/$defs/vtl-id" }
+                },
+                "required": [ "domain" ]
+            }
+        },
+        "domains": {
+            "type": "array",
+            "items": {
+                "allOf": [ { "$ref": "#/$defs/identifiable" } ],
+                "unevaluatedProperties": false,
+                "oneOf": [
+                    {
+                        "properties": {
+                            "externalRef": { "type": "string" }
+                        },
+                        "required": [ "externalRef" ]
+                    }, {
+                        "properties": {
+                            "parent": { "$ref": "#/$defs/vtl-id" }
+                        },
+                        "required": [ "parent" ],
+                        "oneOf": [{
+                                "properties": {
+                                    "restriction": { "$ref": "#/$defs/set-type" }
+                                },
+                                "required": [ "restriction" ]
+                            }, {
+                                "properties": {
+                                    "enumerated": { "$ref": "#/$defs/set-type" }
+                                },
+                                "required": [ "enumerated" ]
+                            }, {
+                                "properties": {
+                                    "described": { "type": "string" }
+                                },
+                                "required": [ "described" ]
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+    }
+}
+

{vtlengine-1.0.3rc3 → vtlengine-1.1rc1}/src/vtlengine/AST/ASTConstructor.py

@@ -382,7 +382,8 @@ class ASTVisitor(VtlVisitor):
         if conditions:
             identifiers_list = [
                 DefIdentifier(
-                    value=elto.alias if getattr(elto, "alias", None) else elto.value, kind=kind
+                    value=elto.alias if getattr(elto, "alias", None) else elto.value,
+                    kind=kind,
                 )
                 for elto in conditions[0]
             ]
@@ -395,7 +396,7 @@ class ASTVisitor(VtlVisitor):
     def visitValueDomainSignature(self, ctx: Parser.ValueDomainSignatureContext):
         """
         valueDomainSignature: CONDITION IDENTIFIER (AS IDENTIFIER)? (',' IDENTIFIER (AS IDENTIFIER)?)* ;
-        """ # noqa E501
+        """  # noqa E501
         # AST_ASTCONSTRUCTOR.7
         ctx_list = list(ctx.getChildren())
         component_nodes = [
@@ -459,7 +460,7 @@ class ASTVisitor(VtlVisitor):
         codeItemRelation: ( WHEN expr THEN )? codeItemRef codeItemRelationClause (codeItemRelationClause)* ;
                         ( WHEN exprComponent THEN )? codetemRef=valueDomainValue comparisonOperand? codeItemRelationClause (codeItemRelationClause)*
 
-        """ # noqa E501
+        """  # noqa E501
 
         ctx_list = list(ctx.getChildren())
 
@@ -512,7 +513,7 @@ class ASTVisitor(VtlVisitor):
     def visitCodeItemRelationClause(self, ctx: Parser.CodeItemRelationClauseContext):
         """
         (opAdd=( PLUS | MINUS  ))? rightCodeItem=valueDomainValue ( QLPAREN  rightCondition=exprComponent  QRPAREN )?
-        """ # noqa E501
+        """  # noqa E501
         ctx_list = list(ctx.getChildren())
 
         expr = [expr for expr in ctx_list if isinstance(expr, Parser.ExprContext)]