vtlengine 1.0__tar.gz → 1.0.2__tar.gz

vtlengine-1.0.2/PKG-INFO

@@ -0,0 +1,245 @@
+Metadata-Version: 2.1
+Name: vtlengine
+Version: 1.0.2
+Summary: Run and Validate VTL Scripts
+Home-page: https://github.com/Meaningful-Data/vtlengine
+License: AGPL-3.0
+Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
+Author: MeaningfulData
+Author-email: info@meaningfuldata.eu
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Dist: antlr4-python3-runtime (==4.9.2)
+Requires-Dist: bottleneck (>=1.3.4,<2.0.0)
+Requires-Dist: duckdb (>=1.1.1,<2.0.0)
+Requires-Dist: networkx (>=2.8.8,<3.0.0)
+Requires-Dist: numexpr (>=2.9.0,<3.0.0)
+Requires-Dist: pandas (>=2.1.4,<3.0.0)
+Requires-Dist: s3fs (>=2024.9.0,<2025.0.0)
+Requires-Dist: sqlglot (>=22.2.0,<23.0.0)
+Project-URL: Repository, https://github.com/Meaningful-Data/vtlengine
+Description-Content-Type: text/markdown
+
+# VTL Engine
+
+|         |                                                                                                                                                                                |
+|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Testing | [![Testing](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml/badge.svg)](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml) |
+| Package | [![PyPI Latest Release](https://img.shields.io/pypi/v/vtlengine.svg)](https://pypi.org/project/vtlengine/)                                                                     |
+| License | [![License - AGPL 3.0](https://img.shields.io/pypi/l/vtlengine.svg)](https://github.com/Meaningful-Data/vtlengine/blob/main/LICENSE.md)                                        |
+
+## Introduction
+
+The VTL Engine is a Python library for validating and running VTL scripts.
+
+It is a Python-based library around the [VTL Language](http://sdmx.org/?page_id=5096).
+
+## Installation
+
+### Requirements
+
+The VTL Engine requires Python 3.10 or higher.
+
+### Install with pip
+
+To install the VTL Engine on any Operating System, you can use pip:
+
+```bash
+
+pip install vtlengine
+
+```
+
+*Note: it is recommended to install the VTL Engine in a virtual environment.*
+
+## Usage
+
+The VTL Engine API implements two basic methods:
+
+* **Semantic Analysis**: aimed at validating the correctness of a script and computing the data
+  structures of the data sets created in the script.
+* **Run**: aimed at executing the provided input on the provided input datasets.
+
+Any action with VTL requires the following elements as input:
+
+* **VTL Script**: Is the VTL to be executed, which includes the transformation scheme, as well as de
+  User Defined Operators, Hierarchical Rulesets and Datapoint Rulesets. It is provided as a string
+  or as a Path object to a vtl file.
+* **Data structures** : Provides the structure of the input artifacts of the VTL script, according
+  to
+  the VTL Information model. Given that the current version doesn't prescribe a standard format for
+  providing the information, the VTL Engine is implementing a JSON format that can be found here.
+  Data Structures can be provided as Dictionaries or as Paths to JSON files. It is possible to have
+* **External routines**: The VTL Engine allows using SQL (SQLite) with the eval operator. Can be
+  provided as a string with the SQL or as a path object to an SQL file. Its default value is `None`,
+  which shall be used if external routines are not applicable to the VTL script.
+* **Value domains**: Provides the value domains that are used in the VTL script, normally with an in
+  operator. Can be provided as a dictionary or as a path to a JSON file. Its default value
+  is `None`, which shall be used if value domains are not applicable to the VTL script.
+
+### Semantic Analysis
+
+The `semantic_analysis` method serves to validate the correctness of a VTL script, as well as to
+calculate the data structures of the datasets generated by the VTL script itself (that calculation
+is a pre-requisite for the semantic analysis).
+
+* If the VTL script is correct, the method returns a dictionary with the data structures of all the
+  datasets generated by the script.
+* If the VTL script is incorrect, raises a VTL Engine custom error Explaining the error.
+
+#### Example 1: Correct VTL
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```
+
+Returns:
+
+```
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=None)}
+```
+
+#### Example 2: Incorrect VTL
+
+Note that, as compared to Example 1, the only change is that Me_1 is of the String data type,
+instead of Number.
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'String',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```  
+
+Will raise the following Error:
+
+``` python
+raise SemanticError(code="1-1-1-2",
+vtlengine.Exceptions.SemanticError: ('Invalid implicit cast from String and Integer to Number.', '1-1-1-2')
+```
+
+### Run VTL Scripts
+
+The `run` method serves to execute a VTL script with input datapoints.
+
+Returns a dictionary with all the generated Datasets.
+When the output parameter is set, the engine will write the result of the computation to the output
+folder, else it will include the data in the dictionary of the computed datasets.
+
+Two validations are performed before running, which can raise errors:
+
+* Semantic analysis: Equivalent to running the `semantic_analysis` method
+* Data load analysis: Basic check of the data structure (names and types)
+
+#### Example 3: Simple run
+
+```python
+from vtlengine import run
+import pandas as pd
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+data_df = pd.DataFrame(
+    {"Id_1": [1, 2, 3],
+     "Me_1": [10, 20, 30]})
+
+datapoints = {"DS_1": data_df}
+
+run_result = run(script=script, data_structures=data_structures,
+                 datapoints=datapoints)
+
+print(run_result)
+```
+
+returns:
+
+``` python
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=  Id_1   Me_1
+0    1  100.0
+1    2  200.0
+2    3  300.0)}
+```
+
+For more information on usage, please refer to
+the [API documentation](https://docs.vtlengine.meaningfuldata.eu/api.html).

vtlengine-1.0.2/README.md

@@ -0,0 +1,213 @@
+# VTL Engine
+
+|         |                                                                                                                                                                                |
+|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Testing | [![Testing](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml/badge.svg)](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml) |
+| Package | [![PyPI Latest Release](https://img.shields.io/pypi/v/vtlengine.svg)](https://pypi.org/project/vtlengine/)                                                                     |
+| License | [![License - AGPL 3.0](https://img.shields.io/pypi/l/vtlengine.svg)](https://github.com/Meaningful-Data/vtlengine/blob/main/LICENSE.md)                                        |
+
+## Introduction
+
+The VTL Engine is a Python library for validating and running VTL scripts.
+
+It is a Python-based library around the [VTL Language](http://sdmx.org/?page_id=5096).
+
+## Installation
+
+### Requirements
+
+The VTL Engine requires Python 3.10 or higher.
+
+### Install with pip
+
+To install the VTL Engine on any Operating System, you can use pip:
+
+```bash
+
+pip install vtlengine
+
+```
+
+*Note: it is recommended to install the VTL Engine in a virtual environment.*
+
+## Usage
+
+The VTL Engine API implements two basic methods:
+
+* **Semantic Analysis**: aimed at validating the correctness of a script and computing the data
+  structures of the data sets created in the script.
+* **Run**: aimed at executing the provided input on the provided input datasets.
+
+Any action with VTL requires the following elements as input:
+
+* **VTL Script**: Is the VTL to be executed, which includes the transformation scheme, as well as de
+  User Defined Operators, Hierarchical Rulesets and Datapoint Rulesets. It is provided as a string
+  or as a Path object to a vtl file.
+* **Data structures** : Provides the structure of the input artifacts of the VTL script, according
+  to
+  the VTL Information model. Given that the current version doesn't prescribe a standard format for
+  providing the information, the VTL Engine is implementing a JSON format that can be found here.
+  Data Structures can be provided as Dictionaries or as Paths to JSON files. It is possible to have
+* **External routines**: The VTL Engine allows using SQL (SQLite) with the eval operator. Can be
+  provided as a string with the SQL or as a path object to an SQL file. Its default value is `None`,
+  which shall be used if external routines are not applicable to the VTL script.
+* **Value domains**: Provides the value domains that are used in the VTL script, normally with an in
+  operator. Can be provided as a dictionary or as a path to a JSON file. Its default value
+  is `None`, which shall be used if value domains are not applicable to the VTL script.
+
+### Semantic Analysis
+
+The `semantic_analysis` method serves to validate the correctness of a VTL script, as well as to
+calculate the data structures of the datasets generated by the VTL script itself (that calculation
+is a pre-requisite for the semantic analysis).
+
+* If the VTL script is correct, the method returns a dictionary with the data structures of all the
+  datasets generated by the script.
+* If the VTL script is incorrect, raises a VTL Engine custom error Explaining the error.
+
+#### Example 1: Correct VTL
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```
+
+Returns:
+
+```
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=None)}
+```
+
+#### Example 2: Incorrect VTL
+
+Note that, as compared to Example 1, the only change is that Me_1 is of the String data type,
+instead of Number.
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'String',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```  
+
+Will raise the following Error:
+
+``` python
+raise SemanticError(code="1-1-1-2",
+vtlengine.Exceptions.SemanticError: ('Invalid implicit cast from String and Integer to Number.', '1-1-1-2')
+```
+
+### Run VTL Scripts
+
+The `run` method serves to execute a VTL script with input datapoints.
+
+Returns a dictionary with all the generated Datasets.
+When the output parameter is set, the engine will write the result of the computation to the output
+folder, else it will include the data in the dictionary of the computed datasets.
+
+Two validations are performed before running, which can raise errors:
+
+* Semantic analysis: Equivalent to running the `semantic_analysis` method
+* Data load analysis: Basic check of the data structure (names and types)
+
+#### Example 3: Simple run
+
+```python
+from vtlengine import run
+import pandas as pd
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+data_df = pd.DataFrame(
+    {"Id_1": [1, 2, 3],
+     "Me_1": [10, 20, 30]})
+
+datapoints = {"DS_1": data_df}
+
+run_result = run(script=script, data_structures=data_structures,
+                 datapoints=datapoints)
+
+print(run_result)
+```
+
+returns:
+
+``` python
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=  Id_1   Me_1
+0    1  100.0
+1    2  200.0
+2    3  300.0)}
+```
+
+For more information on usage, please refer to
+the [API documentation](https://docs.vtlengine.meaningfuldata.eu/api.html).

vtlengine-1.0.2/pyproject.toml

@@ -0,0 +1,76 @@
+[tool.poetry]
+name = "vtlengine"
+version = "1.0.2"
+description = "Run and Validate VTL Scripts"
+authors = ["MeaningfulData <info@meaningfuldata.eu>"]
+license = "AGPL-3.0"
+readme = "README.md"
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: Science/Research",
+    "Typing :: Typed"
+]
+
+keywords = ['vtl', 'sdmx', 'vtlengine', 'Validation and Transformation Language']
+repository = 'https://github.com/Meaningful-Data/vtlengine'
+
+[project.urls]
+BugTracker = 'https://github.com/Meaningful-Data/vtlengine/issues'
+Documentation = 'https://docs.vtlengine.meaningfuldata.eu'
+SourceCode = 'https://github.com/Meaningful-Data/vtlengine'
+
+[tool.poetry.dependencies]
+python = "^3.10"
+# PyPi dependencies
+duckdb = "^1.1.1"
+#numba = "^0.60.0"
+s3fs = "^2024.9.0"
+
+# APT dependencies
+antlr4-python3-runtime = "4.9.2"
+networkx = "^2.8.8"
+numexpr = "^2.9.0"
+pandas = "^2.1.4"
+bottleneck = "^1.3.4"
+sqlglot = "^22.2.0"
+
+[tool.poetry.dev-dependencies]
+pytest = "^7.3"
+pytest-cov = "^5.0.0"
+line-profiler-pycharm = "^1.2.0"
+sphinx = "^7.4"
+sphinx-rtd-theme = "^2.0.0"
+flake8 = "^7.1.1"
+black = "^24.8.0"
+mypy = "^1.11.2"
+pandas-stubs = "^2.2.3.241009"
+stubs = "^1.0.0"
+toml = "^0.10.2"
+
+[tool.black]
+line_length = 100
+
+[tool.mypy]
+files = "src"
+exclude = "src/vtlengine/AST/.*"
+disallow_untyped_defs = true
+disallow_untyped_calls = true
+ignore_errors = false
+no_implicit_optional = true
+show_column_numbers = true
+strict_equality = true
+strict_optional = true
+strict = true
+enable_error_code = [
+    "ignore-without-code",
+    "redundant-expr",
+    "truthy-bool",
+]
+warn_return_any = false
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"