vtlengine 1.0.0__tar.gz → 1.0.1__tar.gz

vtlengine-1.0.1/PKG-INFO

@@ -0,0 +1,236 @@
+Metadata-Version: 2.1
+Name: vtlengine
+Version: 1.0.1
+Summary: Run and Validate VTL Scripts
+License: AGPL-3.0
+Author: MeaningfulData
+Author-email: info@meaningfuldata.eu
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: antlr4-python3-runtime (==4.9.2)
+Requires-Dist: bottleneck (>=1.3.4,<2.0.0)
+Requires-Dist: duckdb (>=1.1.1,<2.0.0)
+Requires-Dist: networkx (>=2.8.8,<3.0.0)
+Requires-Dist: numba (>=0.60.0,<0.61.0)
+Requires-Dist: numexpr (>=2.9.0,<3.0.0)
+Requires-Dist: pandas (>=2.1.4,<3.0.0)
+Requires-Dist: pyarrow (>=17.0.0,<18.0.0)
+Requires-Dist: s3fs (>=2024.9.0,<2025.0.0)
+Requires-Dist: sqlglot (>=22.2.0,<23.0.0)
+Description-Content-Type: text/markdown
+
+# VTL Engine
+
+## Introduction
+
+The VTL Engine is a Python library for validating and running VTL scripts.
+
+It is a Python-based library around the [VTL Language](http://sdmx.org/?page_id=5096).
+
+## Installation
+
+### Requirements
+
+The VTL Engine requires Python 3.10 or higher.
+
+### Install with pip
+
+To install the VTL Engine on any Operating System, you can use pip:
+
+```bash
+
+pip install vtlengine
+
+```
+
+*Note: it is recommended to install the VTL Engine in a virtual environment.*
+
+## Usage
+
+The VTL Engine API implements two basic methods:
+
+* **Semantic Analysis**: aimed at validating the correctness of a script and computing the data
+  structures of the data sets created in the script.
+* **Run**: aimed at executing the provided input on the provided input datasets.
+
+Any action with VTL requires the following elements as input:
+
+* **VTL Script**: Is the VTL to be executed, which includes the transformation scheme, as well as de
+  User Defined Operators, Hierarchical Rulesets and Datapoint Rulesets. It is provided as a string
+  or as a Path object to a vtl file.
+* **Data structures** : Provides the structure of the input artifacts of the VTL script, according to
+  the VTL Information model. Given that the current version doesn't prescribe a standard format for
+  providing the information, the VTL Engine is implementing a JSON format that can be found here.
+  Data Structures can be provided as Dictionaries or as Paths to JSON files. It is possible to have
+* **External routines**: The VTL Engine allows using SQL (SQLite) with the eval operator. Can be
+  provided as a string with the SQL or as a path object to an SQL file. Its default value is `None`,
+  which shall be used if external routines are not applicable to the VTL script.
+* **Value domains**: Provides the value domains that are used in the VTL script, normally with an in
+  operator. Can be provided as a dictionary or as a path to a JSON file. Its default value
+  is `None`, which shall be used if value domains are not applicable to the VTL script.
+
+### Semantic Analysis
+
+The `semantic_analysis` method serves to validate the correctness of a VTL script, as well as to
+calculate the data structures of the datasets generated by the VTL script itself (that calculation
+is a pre-requisite for the semantic analysis).
+
+* If the VTL script is correct, the method returns a dictionary with the data structures of all the
+  datasets generated by the script.
+* If the VTL script is incorrect, raises a VTL Engine custom error Explaining the error.
+
+#### Example 1: Correct VTL
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```
+
+Returns:
+
+```
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=None)}
+```
+
+#### Example 2: Incorrect VTL
+
+Note that, as compared to Example 1, the only change is that Me_1 is of the String data type,
+instead of Number.
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'String',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```  
+
+Will raise the following Error:
+
+``` python
+raise SemanticError(code="1-1-1-2",
+vtlengine.Exceptions.SemanticError: ('Invalid implicit cast from String and Integer to Number.', '1-1-1-2')
+```
+
+### Run VTL Scripts
+
+The `run` method serves to execute a VTL script with input datapoints.
+
+Returns a dictionary with all the generated Datasets.
+When the output parameter is set, the engine will write the result of the computation to the output
+folder, else it will include the data in the dictionary of the computed datasets.
+
+Two validations are performed before running, which can raise errors:
+
+* Semantic analysis: Equivalent to running the `semantic_analysis` method
+* Data load analysis: Basic check of the data structure (names and types)
+
+#### Example 3: Simple run
+
+```python
+from vtlengine import run
+import pandas as pd
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+data_df = pd.DataFrame(
+    {"Id_1": [1, 2, 3],
+     "Me_1": [10, 20, 30]})
+
+datapoints = {"DS_1": data_df}
+
+run_result = run(script=script, data_structures=data_structures,
+                 datapoints=datapoints)
+
+print(run_result)
+```
+
+returns:
+
+``` python
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=  Id_1   Me_1
+0    1  100.0
+1    2  200.0
+2    3  300.0)}
+```
+
+For more information on usage, please refer to
+the [API documentation](https://docs.vtlengine.meaningfuldata.eu/api.html).

vtlengine-1.0.1/README.md

@@ -0,0 +1,206 @@
+# VTL Engine
+
+## Introduction
+
+The VTL Engine is a Python library for validating and running VTL scripts.
+
+It is a Python-based library around the [VTL Language](http://sdmx.org/?page_id=5096).
+
+## Installation
+
+### Requirements
+
+The VTL Engine requires Python 3.10 or higher.
+
+### Install with pip
+
+To install the VTL Engine on any Operating System, you can use pip:
+
+```bash
+
+pip install vtlengine
+
+```
+
+*Note: it is recommended to install the VTL Engine in a virtual environment.*
+
+## Usage
+
+The VTL Engine API implements two basic methods:
+
+* **Semantic Analysis**: aimed at validating the correctness of a script and computing the data
+  structures of the data sets created in the script.
+* **Run**: aimed at executing the provided input on the provided input datasets.
+
+Any action with VTL requires the following elements as input:
+
+* **VTL Script**: Is the VTL to be executed, which includes the transformation scheme, as well as de
+  User Defined Operators, Hierarchical Rulesets and Datapoint Rulesets. It is provided as a string
+  or as a Path object to a vtl file.
+* **Data structures** : Provides the structure of the input artifacts of the VTL script, according to
+  the VTL Information model. Given that the current version doesn't prescribe a standard format for
+  providing the information, the VTL Engine is implementing a JSON format that can be found here.
+  Data Structures can be provided as Dictionaries or as Paths to JSON files. It is possible to have
+* **External routines**: The VTL Engine allows using SQL (SQLite) with the eval operator. Can be
+  provided as a string with the SQL or as a path object to an SQL file. Its default value is `None`,
+  which shall be used if external routines are not applicable to the VTL script.
+* **Value domains**: Provides the value domains that are used in the VTL script, normally with an in
+  operator. Can be provided as a dictionary or as a path to a JSON file. Its default value
+  is `None`, which shall be used if value domains are not applicable to the VTL script.
+
+### Semantic Analysis
+
+The `semantic_analysis` method serves to validate the correctness of a VTL script, as well as to
+calculate the data structures of the datasets generated by the VTL script itself (that calculation
+is a pre-requisite for the semantic analysis).
+
+* If the VTL script is correct, the method returns a dictionary with the data structures of all the
+  datasets generated by the script.
+* If the VTL script is incorrect, raises a VTL Engine custom error Explaining the error.
+
+#### Example 1: Correct VTL
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```
+
+Returns:
+
+```
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=None)}
+```
+
+#### Example 2: Incorrect VTL
+
+Note that, as compared to Example 1, the only change is that Me_1 is of the String data type,
+instead of Number.
+
+```python
+from vtlengine import semantic_analysis
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'String',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+sa_result = semantic_analysis(script=script, data_structures=data_structures)
+
+print(sa_result)
+
+```  
+
+Will raise the following Error:
+
+``` python
+raise SemanticError(code="1-1-1-2",
+vtlengine.Exceptions.SemanticError: ('Invalid implicit cast from String and Integer to Number.', '1-1-1-2')
+```
+
+### Run VTL Scripts
+
+The `run` method serves to execute a VTL script with input datapoints.
+
+Returns a dictionary with all the generated Datasets.
+When the output parameter is set, the engine will write the result of the computation to the output
+folder, else it will include the data in the dictionary of the computed datasets.
+
+Two validations are performed before running, which can raise errors:
+
+* Semantic analysis: Equivalent to running the `semantic_analysis` method
+* Data load analysis: Basic check of the data structure (names and types)
+
+#### Example 3: Simple run
+
+```python
+from vtlengine import run
+import pandas as pd
+
+script = """
+    DS_A := DS_1 * 10;
+"""
+
+data_structures = {
+    'datasets': [
+        {'name': 'DS_1',
+         'DataStructure': [
+             {'name': 'Id_1',
+              'type':
+                  'Integer',
+              'role': 'Identifier',
+              'nullable': False},
+             {'name': 'Me_1',
+              'type': 'Number',
+              'role': 'Measure',
+              'nullable': True}
+         ]
+         }
+    ]
+}
+
+data_df = pd.DataFrame(
+    {"Id_1": [1, 2, 3],
+     "Me_1": [10, 20, 30]})
+
+datapoints = {"DS_1": data_df}
+
+run_result = run(script=script, data_structures=data_structures,
+                 datapoints=datapoints)
+
+print(run_result)
+```
+
+returns:
+
+``` python
+{'DS_A': Dataset(name='DS_A', components={'Id_1': Component(name='Id_1', data_type=<class 'vtlengine.DataTypes.Integer'>, role=<Role.IDENTIFIER: 'Identifier'>, nullable=False), 'Me_1': Component(name='Me_1', data_type=<class 'vtlengine.DataTypes.Number'>, role=<Role.MEASURE: 'Measure'>, nullable=True)}, data=  Id_1   Me_1
+0    1  100.0
+1    2  200.0
+2    3  300.0)}
+```
+
+For more information on usage, please refer to
+the [API documentation](https://docs.vtlengine.meaningfuldata.eu/api.html).

{vtlengine-1.0 → vtlengine-1.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vtlengine"
-version = "1.0"
+version = "1.0.1"
 description = "Run and Validate VTL Scripts"
 authors = ["MeaningfulData <info@meaningfuldata.eu>"]
 license = "AGPL-3.0"
@@ -34,7 +34,32 @@ pytest-cov = "^5.0.0"
 line-profiler-pycharm = "^1.2.0"
 sphinx = "^7.4"
 sphinx-rtd-theme = "^2.0.0"
+flake8 = "^7.1.1"
+black = "^24.8.0"
+mypy = "^1.11.2"
+pandas-stubs = "^2.2.3.241009"
+stubs = "^1.0.0"
 
+[tool.black]
+line_length = 100
+
+[tool.mypy]
+files = "src"
+exclude = "src/vtlengine/AST/.*"
+disallow_untyped_defs = true
+disallow_untyped_calls = true
+ignore_errors = false
+no_implicit_optional = true
+show_column_numbers = true
+strict_equality = true
+strict_optional = true
+strict = true
+enable_error_code = [
+    "ignore-without-code",
+    "redundant-expr",
+    "truthy-bool",
+]
+warn_return_any = false
 
 [build-system]
 requires = ["poetry-core"]