vtlengine 1.0.3rc3__tar.gz → 1.1__tar.gz

{vtlengine-1.0.3rc3 → vtlengine-1.1}/PKG-INFO

@@ -1,31 +1,35 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: vtlengine
-Version: 1.0.3rc3
+Version: 1.1
 Summary: Run and Validate VTL Scripts
 License: AGPL-3.0
 Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
 Author: MeaningfulData
 Author-email: info@meaningfuldata.eu
-Requires-Python: >=3.10,<4.0
+Maintainer: Francisco Javier Hernandez del Caño
+Maintainer-email: javier.hernandez@meaningfuldata.eu
+Requires-Python: >=3.9
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: GNU Affero General Public License v3
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
-Requires-Dist: antlr4-python3-runtime (==4.9.2)
-Requires-Dist: bottleneck (>=1.3.4,<2.0.0)
-Requires-Dist: duckdb (>=1.1.1,<2.0.0)
-Requires-Dist: networkx (>=2.8.8,<3.0.0)
-Requires-Dist: numexpr (>=2.9.0,<3.0.0)
-Requires-Dist: pandas (>=2.1.4,<3.0.0)
-Requires-Dist: s3fs (>=2024.9.0,<2025.0.0)
-Requires-Dist: sqlglot (>=22.2.0,<23.0.0)
+Provides-Extra: all
+Provides-Extra: s3
+Requires-Dist: antlr4-python3-runtime (>=4.9.2,<4.10)
+Requires-Dist: duckdb (>=1.1,<1.2)
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: fsspec (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: jsonschema (>=3.2.0,<5.0)
+Requires-Dist: networkx (>=2.8,<3.0)
+Requires-Dist: numpy (>=1.23.2,<2) ; python_version < "3.13"
+Requires-Dist: numpy (>=2.1.0) ; python_version >= "3.13"
+Requires-Dist: pandas (>=2.1.4,<2.2) ; python_version < "3.13"
+Requires-Dist: pandas (>=2.2,<3.0) ; python_version >= "3.13"
+Requires-Dist: pysdmx[xml] (>=1.3.0,<2.0)
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "all"
+Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "s3"
+Requires-Dist: sqlglot (>=22.2.0,<23.0)
 Project-URL: Authors, https://github.com/Meaningful-Data/vtlengine/graphs/contributors
 Project-URL: Documentation, https://docs.vtlengine.meaningfuldata.eu
 Project-URL: IssueTracker, https://github.com/Meaningful-Data/vtlengine/issues

{vtlengine-1.0.3rc3 → vtlengine-1.1}/pyproject.toml

@@ -1,11 +1,18 @@
-[tool.poetry]
+[project]
 name = "vtlengine"
-version = "1.0.3rc3"
+version = "1.1"
 description = "Run and Validate VTL Scripts"
-authors = ["MeaningfulData <info@meaningfuldata.eu>"]
 license = "AGPL-3.0"
 readme = "README.md"
-
+requires-python = ">=3.9"
+authors = [
+    {name = "MeaningfulData", email = "info@meaningfuldata.eu"},
+]
+maintainers = [
+    {name = "Francisco Javier Hernandez del Caño", email = "javier.hernandez@meaningfuldata.eu"},
+    {name = "Alberto Hernandez del Caño", email = "alberto.hernandez@meaningfuldata.eu"},
+    {name = "Mateo de Lorenzo Argeles", email = "mateo.delorenzo@meaningfuldata.eu"}
+]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
@@ -13,10 +20,28 @@ classifiers = [
     "Intended Audience :: Science/Research",
     "Typing :: Typed"
 ]
-
 keywords = ['vtl', 'sdmx', 'vtlengine', 'Validation and Transformation Language']
 
-[tool.poetry.urls]
+dependencies = [
+    # PyPi dependencies
+    "duckdb>=1.1,<1.2",
+    "pysdmx[xml]>=1.3.0,<2.0",
+    # APT-supported dependencies
+    "jsonschema>=3.2.0,<5.0",
+    "sqlglot>=22.2.0,<23.0",
+    "antlr4-python3-runtime>=4.9.2,<4.10",
+    "pandas>=2.1.4,<2.2 ; python_version < '3.13'",
+    "pandas>=2.2,<3.0 ; python_version >= '3.13'",
+    "networkx>=2.8,<3.0",
+    "numpy>=1.23.2,<2 ; python_version < '3.13'",
+    "numpy>=2.1.0; python_version >= '3.13'",
+]
+
+[project.optional-dependencies]
+s3 = ["s3fs>=2022.11.0,<2023.0", "fsspec>=2022.11.0,<2023.0"]
+all = ["s3fs>=2022.11.0,<2023.0", "fsspec>=2022.11.0,<2023.0"]
+
+[project.urls]
 Repository = 'https://github.com/Meaningful-Data/vtlengine'
 Documentation = 'https://docs.vtlengine.meaningfuldata.eu'
 MeaningfulData = 'https://www.meaningfuldata.eu/'
@@ -24,31 +49,23 @@ IssueTracker = 'https://github.com/Meaningful-Data/vtlengine/issues'
 Authors = 'https://github.com/Meaningful-Data/vtlengine/graphs/contributors'
 
 [tool.poetry.dependencies]
-python = "^3.10"
-# PyPi dependencies
-duckdb = "^1.1.1"
-#numba = "^0.60.0"
-s3fs = "^2024.9.0"
+python = ">=3.9,<4.0"
 
-# APT dependencies
-antlr4-python3-runtime = "4.9.2"
-networkx = "^2.8.8"
-numexpr = "^2.9.0"
-pandas = "^2.1.4"
-bottleneck = "^1.3.4"
-sqlglot = "^22.2.0"
-
-[tool.poetry.dev-dependencies]
-pytest = "^7.3"
-pytest-cov = "^5.0.0"
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3"
+pytest-cov = "^6.1.1"
 line-profiler-pycharm = "^1.2.0"
-sphinx = "^7.4"
-sphinx-rtd-theme = "^2.0.0"
-mypy = "^1.11.2"
-pandas-stubs = "^2.2.3.241009"
-stubs = "^1.0.0"
+mypy = "1.15.0"
+pandas-stubs = "^2.1.4.231227"
+ruff = "^0.11.11"
+types-jsonschema = "4.23.0.20250516"
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "^7.4.7"
+sphinx-rtd-theme = "^3.0.2"
 toml = "^0.10.2"
-ruff = "^0.7.1"
+
+
 
 [tool.ruff]
 line-length = 100
@@ -84,6 +101,12 @@ enable_error_code = [
 ]
 warn_return_any = false
 
+[tool.pytest.ini_options]
+addopts = "--strict-markers"
+markers = [
+    "input_path: directory where tests data files are stored"
+]
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

{vtlengine-1.0.3rc3 → vtlengine-1.1}/src/vtlengine/API/_InternalApi.py

@@ -1,17 +1,33 @@
 import json
 import os
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
+import jsonschema
 import pandas as pd
-from s3fs import S3FileSystem  # type: ignore[import-untyped]
+from pysdmx.model.dataflow import Component as SDMXComponent
+from pysdmx.model.dataflow import DataStructureDefinition, Schema
+from pysdmx.model.dataflow import Role as SDMX_Role
+from pysdmx.model.vtl import (
+    Ruleset,
+    RulesetScheme,
+    Transformation,
+    TransformationScheme,
+    UserDefinedOperator,
+    UserDefinedOperatorScheme,
+)
 
-from vtlengine.AST import PersistentAssignment, Start
+from vtlengine import AST as AST
+from vtlengine.__extras_check import __check_s3_extra
+from vtlengine.AST import Assignment, DPRuleset, HRuleset, Operator, PersistentAssignment, Start
+from vtlengine.AST.ASTString import ASTString
 from vtlengine.DataTypes import SCALAR_TYPES
-from vtlengine.Exceptions import check_key
+from vtlengine.Exceptions import InputValidationException, check_key
 from vtlengine.files.parser import _fill_dataset_empty_data, _validate_pandas
 from vtlengine.Model import (
-    Component,
+    Component as VTL_Component,
+)
+from vtlengine.Model import (
     Dataset,
     ExternalRoutine,
     Role,
@@ -19,15 +35,13 @@ from vtlengine.Model import (
     Scalar,
     ValueDomain,
 )
+from vtlengine.Utils import VTL_DTYPES_MAPPING, VTL_ROLE_MAPPING
 
 base_path = Path(__file__).parent
-filepath_VTL = base_path / "data" / "vtl"
-filepath_ValueDomains = base_path / "data" / "ValueDomain"
-filepath_sql = base_path / "data" / "sql"
-filepath_json = base_path / "data" / "DataStructure" / "input"
-filepath_csv = base_path / "data" / "DataSet" / "input"
-filepath_out_json = base_path / "data" / "DataStructure" / "output"
-filepath_out_csv = base_path / "data" / "DataSet" / "output"
+schema_path = base_path / "data" / "schema"
+sdmx_csv_path = base_path / "data" / "sdmx_csv"
+with open(schema_path / "json_schema_2.1.json", "r") as file:
+    schema = json.load(file)
 
 
 def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
@@ -41,22 +55,60 @@ def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
             dataset_name = dataset_json["name"]
             components = {}
 
-            for component in dataset_json["DataStructure"]:
-                check_key("data_type", SCALAR_TYPES.keys(), component["type"])
-                check_key("role", Role_keys, component["role"])
-                components[component["name"]] = Component(
-                    name=component["name"],
-                    data_type=SCALAR_TYPES[component["type"]],
-                    role=Role(component["role"]),
-                    nullable=component["nullable"],
-                )
+            if "structure" in dataset_json:
+                structure_name = dataset_json["structure"]
+                structure_json = None
+                for s in structures["structures"]:
+                    if s["name"] == structure_name:
+                        structure_json = s
+                if structure_json is None:
+                    raise InputValidationException(code="0-3-1-1", message="Structure not found.")
+                try:
+                    jsonschema.validate(instance=structure_json, schema=schema)
+                except jsonschema.exceptions.ValidationError as e:
+                    raise InputValidationException(code="0-3-1-1", message=e.message)
+
+                for component in structure_json["components"]:
+                    check_key("data_type", SCALAR_TYPES.keys(), component["data_type"])
+                    if component["role"] == "ViralAttribute":
+                        component["role"] = "Attribute"
+
+                    check_key("role", Role_keys, component["role"])
+
+                    if "nullable" not in component:
+                        if Role(component["role"]) == Role.IDENTIFIER:
+                            component["nullable"] = False
+                        elif Role(component["role"]) in (Role.MEASURE, Role.ATTRIBUTE):
+                            component["nullable"] = True
+                        else:
+                            component["nullable"] = False
+
+                    components[component["name"]] = VTL_Component(
+                        name=component["name"],
+                        data_type=SCALAR_TYPES[component["data_type"]],
+                        role=Role(component["role"]),
+                        nullable=component["nullable"],
+                    )
+
+            if "DataStructure" in dataset_json:
+                for component in dataset_json["DataStructure"]:
+                    check_key("data_type", SCALAR_TYPES.keys(), component["type"])
+                    check_key("role", Role_keys, component["role"])
+                    components[component["name"]] = VTL_Component(
+                        name=component["name"],
+                        data_type=SCALAR_TYPES[component["type"]],
+                        role=Role(component["role"]),
+                        nullable=component["nullable"],
+                    )
 
             datasets[dataset_name] = Dataset(name=dataset_name, components=components, data=None)
     if "scalars" in structures:
         for scalar_json in structures["scalars"]:
             scalar_name = scalar_json["name"]
             scalar = Scalar(
-                name=scalar_name, data_type=SCALAR_TYPES[scalar_json["type"]], value=None
+                name=scalar_name,
+                data_type=SCALAR_TYPES[scalar_json["type"]],
+                value=None,
             )
             datasets[scalar_name] = scalar  # type: ignore[assignment]
     return datasets
@@ -70,38 +122,16 @@ def _load_single_datapoint(datapoint: Union[str, Path]) -> Dict[str, Any]:
         raise Exception("Invalid datapoint. Input must be a Path or an S3 URI")
     if isinstance(datapoint, str):
         if "s3://" in datapoint:
-            # Handling S3 URI
-            s3fs_obj = S3FileSystem()
-
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(datapoint):
-                raise Exception(
-                    f"Invalid datapoint. S3 URI does not exist or it is not accessible: {datapoint}"
-                )
-
-            # Check if the S3 URI is a directory
-            if s3fs_obj.isdir(datapoint):
-                datapoints: Dict[str, Any] = {}
-                for f in s3fs_obj.ls(datapoint):
-                    if f.endswith(".csv"):
-                        dataset_name = f.split("/")[-1].removesuffix(".csv")
-                        dict_data = {dataset_name: f"s3://{f}"}
-                        datapoints = {**datapoints, **dict_data}
-                return datapoints
-
-            # Check if the S3 URI is a csv file
-            if s3fs_obj.isfile(datapoint) and not datapoint.endswith(".csv"):
-                raise Exception(f"Invalid datapoint. S3 URI must refer to a csv file: {datapoint}")
+            __check_s3_extra()
             dataset_name = datapoint.split("/")[-1].removesuffix(".csv")
             dict_data = {dataset_name: datapoint}
             return dict_data
-
         try:
             datapoint = Path(datapoint)
         except Exception:
             raise Exception("Invalid datapoint. Input must refer to a Path or an S3 URI")
     if datapoint.is_dir():
-        datapoints = {}
+        datapoints: Dict[str, Any] = {}
         for f in datapoint.iterdir():
             if f.suffix != ".csv":
                 continue
@@ -115,7 +145,7 @@ def _load_single_datapoint(datapoint: Union[str, Path]) -> Dict[str, Any]:
 
 
 def _load_datapoints_path(
-    datapoints: Union[Path, str, List[Union[str, Path]]]
+    datapoints: Union[Path, str, List[Union[str, Path]]],
 ) -> Dict[str, Dataset]:
     """
     Returns a dict with the data given from a Path.
@@ -156,7 +186,7 @@ def _load_datastructure_single(data_structure: Union[Dict[str, Any], Path]) -> D
 
 
 def load_datasets(
-    data_structure: Union[Dict[str, Any], Path, List[Union[Dict[str, Any], Path]]]
+    data_structure: Union[Dict[str, Any], Path, List[Dict[str, Any]], List[Path]],
 ) -> Dict[str, Dataset]:
     """
     Loads multiple datasets.
@@ -365,25 +395,222 @@ def _check_output_folder(output_folder: Union[str, Path]) -> None:
     """
     if isinstance(output_folder, str):
         if "s3://" in output_folder:
-            s3fs_obj = S3FileSystem()
-            # Check if the S3 URI is valid
-            if not s3fs_obj.exists(output_folder):
-                try:
-                    s3fs_obj.mkdir(output_folder)
-                except Exception:
-                    raise Exception(
-                        f"Invalid output folder. S3 URI is invalid or "
-                        f"it is not accessible: {output_folder}"
-                    )
+            __check_s3_extra()
+            if not output_folder.endswith("/"):
+                raise ValueError("Output folder must be a Path or S3 URI to a directory")
             return
         try:
             output_folder = Path(output_folder)
         except Exception:
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
 
     if not isinstance(output_folder, Path):
-        raise Exception("Output folder must be a Path or S3 URI to a directory")
+        raise ValueError("Output folder must be a Path or S3 URI to a directory")
     if not output_folder.exists():
         if output_folder.suffix != "":
-            raise Exception("Output folder must be a Path or S3 URI to a directory")
+            raise ValueError("Output folder must be a Path or S3 URI to a directory")
         os.mkdir(output_folder)
+
+
+def to_vtl_json(dsd: Union[DataStructureDefinition, Schema], dataset_name: str) -> Dict[str, Any]:
+    """
+    Converts a pysdmx `DataStructureDefinition` or `Schema` into a VTL-compatible JSON
+    representation.
+
+    This function extracts and transforms the components (dimensions, measures, and attributes)
+    from the given SDMX data structure and maps them into a dictionary format that conforms
+    to the expected VTL data structure json schema.
+
+    Args:
+        dsd: An instance of `DataStructureDefinition` or `Schema` from the `pysdmx` model.
+        dataset_name: The name of the resulting VTL dataset.
+
+    Returns:
+            A dictionary representing the dataset in VTL format, with keys for dataset name and its
+            components, including their name, role, data type, and nullability.
+    """
+    components = []
+    NAME = "name"
+    ROLE = "role"
+    TYPE = "type"
+    NULLABLE = "nullable"
+
+    _components: List[SDMXComponent] = []
+    _components.extend(dsd.components.dimensions)
+    _components.extend(dsd.components.measures)
+    _components.extend(dsd.components.attributes)
+
+    for c in _components:
+        _type = VTL_DTYPES_MAPPING[c.dtype]
+        _nullability = c.role != SDMX_Role.DIMENSION
+        _role = VTL_ROLE_MAPPING[c.role]
+
+        component = {
+            NAME: c.id,
+            ROLE: _role,
+            TYPE: _type,
+            NULLABLE: _nullability,
+        }
+
+        components.append(component)
+
+    result = {"datasets": [{"name": dataset_name, "DataStructure": components}]}
+
+    return result
+
+
+def __generate_transformation(
+    child: Union[Assignment, PersistentAssignment], is_persistent: bool, count: int
+) -> Transformation:
+    expression = ASTString().render(ast=child.right)
+    result = child.left.value  # type: ignore[attr-defined]
+    return Transformation(
+        id=f"T{count}",
+        expression=expression,
+        is_persistent=is_persistent,
+        result=result,
+        name=f"Transformation {result}",
+    )
+
+
+def __generate_udo(child: Operator, count: int) -> UserDefinedOperator:
+    operator_definition = ASTString().render(ast=child)
+    return UserDefinedOperator(
+        id=f"UDO{count}",
+        operator_definition=operator_definition,
+        name=f"UserDefinedOperator {child.op}",
+    )
+
+
+def __generate_ruleset(child: Union[DPRuleset, HRuleset], count: int) -> Ruleset:
+    ruleset_definition = ASTString().render(ast=child)
+    ruleset_type: Literal["datapoint", "hierarchical"] = (
+        "datapoint" if isinstance(child, DPRuleset) else "hierarchical"
+    )
+    return Ruleset(
+        id=f"R{count}",
+        ruleset_definition=ruleset_definition,
+        ruleset_type=ruleset_type,
+        name=f"{ruleset_type.capitalize()} ruleset {child.name}",
+    )
+
+
+def ast_to_sdmx(ast: AST.Start, agency_id: str, id: str, version: str) -> TransformationScheme:
+    """
+    Converts a vtl AST into an SDMX compatible `TransformationScheme` object, following
+    the pysdmx model.
+
+    This function iterates over the child nodes of the given AST and categorizes each into one of
+    the following types:
+    - `PersistentAssignment`: Represents a persistent transformation. These are added to the
+    transformation list with a persistence flag.
+    - `Assignment`: Represents a temporary (non-persistent) transformation. These are added to the
+    transformation list without the persistence flag
+    - `DPRuleset` or `HRuleset`: Represent validation rule sets.
+    These are collected and wrapped into a `RulesetScheme` object.
+    - `Operator`: Defines user-defined operators. These are collected
+    into a `UserDefinedOperatorScheme` object.
+
+    After parsing all AST elements:
+    - If any rulesets were found, a `RulesetScheme` is created and added to the references.
+    - If any user-defined operators were found, a `UserDefinedOperatorScheme` is created and added
+    to the references.
+    - A `TransformationScheme` object is constructed with all collected transformations and any
+    additional references.
+
+    Args:
+        ast: The root node of the vtl ast representing the set of
+        vtl expressions.
+        agency_id: The identifier of the agency defining the SDMX structure as a string.
+        id: The identifier of the transformation scheme as a string.
+        version: The version of the transformation scheme given as a string.
+
+    Returns:
+        TransformationScheme: A fully constructed transformation scheme that includes
+        transformations, and optionally rule sets and user-defined operator schemes,
+        suitable for SDMX.
+
+    """
+    list_transformation = []
+    list_udos = []
+    list_rulesets = []
+    count_transformation = 0
+    count_udo = 0
+    count_ruleset = 0
+
+    for child in ast.children:
+        if isinstance(child, PersistentAssignment):
+            count_transformation += 1
+            list_transformation.append(
+                __generate_transformation(
+                    child=child, is_persistent=True, count=count_transformation
+                )
+            )
+        elif isinstance(child, Assignment):
+            count_transformation += 1
+            list_transformation.append(
+                __generate_transformation(
+                    child=child, is_persistent=False, count=count_transformation
+                )
+            )
+        elif isinstance(child, (DPRuleset, HRuleset)):
+            count_ruleset += 1
+            list_rulesets.append(__generate_ruleset(child=child, count=count_ruleset))
+        elif isinstance(child, Operator):
+            count_udo += 1
+            list_udos.append(__generate_udo(child=child, count=count_udo))
+
+    references: Any = {}
+    if list_rulesets:
+        references["ruleset_schemes"] = [
+            RulesetScheme(
+                items=list_rulesets,
+                agency=agency_id,
+                id="RS1",
+                vtl_version="2.1",
+                version=version,
+                name=f"RulesetScheme {id}-RS",
+            )
+        ]
+    if list_udos:
+        references["user_defined_operator_schemes"] = [
+            UserDefinedOperatorScheme(
+                items=list_udos,
+                agency=agency_id,
+                id="UDS1",
+                vtl_version="2.1",
+                version=version,
+                name=f"UserDefinedOperatorScheme {id}-UDS",
+            )
+        ]
+
+    transformation_scheme = TransformationScheme(
+        items=list_transformation,
+        agency=agency_id,
+        id="TS1",
+        vtl_version="2.1",
+        version=version,
+        name=f"TransformationScheme {id}",
+        **references,
+    )
+
+    return transformation_scheme
+
+
+def _check_script(script: Union[str, TransformationScheme, Path]) -> str:
+    """
+    Check if the TransformationScheme object is valid to generate a vtl script.
+    """
+    if not isinstance(script, (str, TransformationScheme, Path)):
+        raise Exception(
+            "Invalid script format. Input must be a string, TransformationScheme or Path object"
+        )
+    if isinstance(script, TransformationScheme):
+        from pysdmx.toolkit.vtl.generate_vtl_script import (
+            generate_vtl_script,
+        )
+
+        vtl_script = generate_vtl_script(script, model_validation=True)
+        return vtl_script
+    else:
+        return str(script)