vtlengine 1.0.1__tar.gz → 1.0.3__tar.gz

{vtlengine-1.0.1 → vtlengine-1.0.3}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.1
 Name: vtlengine
-Version: 1.0.1
+Version: 1.0.3
 Summary: Run and Validate VTL Scripts
 License: AGPL-3.0
+Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
 Author: MeaningfulData
 Author-email: info@meaningfuldata.eu
 Requires-Python: >=3.10,<4.0
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
@@ -16,20 +17,30 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
 Requires-Dist: antlr4-python3-runtime (==4.9.2)
 Requires-Dist: bottleneck (>=1.3.4,<2.0.0)
 Requires-Dist: duckdb (>=1.1.1,<2.0.0)
 Requires-Dist: networkx (>=2.8.8,<3.0.0)
-Requires-Dist: numba (>=0.60.0,<0.61.0)
 Requires-Dist: numexpr (>=2.9.0,<3.0.0)
 Requires-Dist: pandas (>=2.1.4,<3.0.0)
-Requires-Dist: pyarrow (>=17.0.0,<18.0.0)
 Requires-Dist: s3fs (>=2024.9.0,<2025.0.0)
 Requires-Dist: sqlglot (>=22.2.0,<23.0.0)
+Project-URL: Authors, https://github.com/Meaningful-Data/vtlengine/graphs/contributors
+Project-URL: Documentation, https://docs.vtlengine.meaningfuldata.eu
+Project-URL: IssueTracker, https://github.com/Meaningful-Data/vtlengine/issues
+Project-URL: MeaningfulData, https://www.meaningfuldata.eu/
+Project-URL: Repository, https://github.com/Meaningful-Data/vtlengine
 Description-Content-Type: text/markdown
 
 # VTL Engine
 
+|         |                                                                                                                                                                                |
+|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Testing | [![Testing](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml/badge.svg)](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml) |
+| Package | [![PyPI Latest Release](https://img.shields.io/pypi/v/vtlengine.svg)](https://pypi.org/project/vtlengine/)                                                                     |
+| License | [![License - AGPL 3.0](https://img.shields.io/pypi/l/vtlengine.svg)](https://github.com/Meaningful-Data/vtlengine/blob/main/LICENSE.md)                                        |
+
 ## Introduction
 
 The VTL Engine is a Python library for validating and running VTL scripts.
@@ -67,7 +78,8 @@ Any action with VTL requires the following elements as input:
 * **VTL Script**: Is the VTL to be executed, which includes the transformation scheme, as well as de
   User Defined Operators, Hierarchical Rulesets and Datapoint Rulesets. It is provided as a string
   or as a Path object to a vtl file.
-* **Data structures** : Provides the structure of the input artifacts of the VTL script, according to
+* **Data structures** : Provides the structure of the input artifacts of the VTL script, according
+  to
   the VTL Information model. Given that the current version doesn't prescribe a standard format for
   providing the information, the VTL Engine is implementing a JSON format that can be found here.
   Data Structures can be provided as Dictionaries or as Paths to JSON files. It is possible to have

{vtlengine-1.0.1 → vtlengine-1.0.3}/README.md

@@ -1,5 +1,11 @@
 # VTL Engine
 
+|         |                                                                                                                                                                                |
+|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Testing | [![Testing](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml/badge.svg)](https://github.com/Meaningful-Data/vtlengine/actions/workflows/testing.yml) |
+| Package | [![PyPI Latest Release](https://img.shields.io/pypi/v/vtlengine.svg)](https://pypi.org/project/vtlengine/)                                                                     |
+| License | [![License - AGPL 3.0](https://img.shields.io/pypi/l/vtlengine.svg)](https://github.com/Meaningful-Data/vtlengine/blob/main/LICENSE.md)                                        |
+
 ## Introduction
 
 The VTL Engine is a Python library for validating and running VTL scripts.
@@ -37,7 +43,8 @@ Any action with VTL requires the following elements as input:
 * **VTL Script**: Is the VTL to be executed, which includes the transformation scheme, as well as de
   User Defined Operators, Hierarchical Rulesets and Datapoint Rulesets. It is provided as a string
   or as a Path object to a vtl file.
-* **Data structures** : Provides the structure of the input artifacts of the VTL script, according to
+* **Data structures** : Provides the structure of the input artifacts of the VTL script, according
+  to
   the VTL Information model. Given that the current version doesn't prescribe a standard format for
   providing the information, the VTL Engine is implementing a JSON format that can be found here.
   Data Structures can be provided as Dictionaries or as Paths to JSON files. It is possible to have

vtlengine-1.0.3/pyproject.toml

@@ -0,0 +1,89 @@
+[tool.poetry]
+name = "vtlengine"
+version = "1.0.3"
+description = "Run and Validate VTL Scripts"
+authors = ["MeaningfulData <info@meaningfuldata.eu>"]
+license = "AGPL-3.0"
+readme = "README.md"
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: Science/Research",
+    "Typing :: Typed"
+]
+
+keywords = ['vtl', 'sdmx', 'vtlengine', 'Validation and Transformation Language']
+
+[tool.poetry.urls]
+Repository = 'https://github.com/Meaningful-Data/vtlengine'
+Documentation = 'https://docs.vtlengine.meaningfuldata.eu'
+MeaningfulData = 'https://www.meaningfuldata.eu/'
+IssueTracker = 'https://github.com/Meaningful-Data/vtlengine/issues'
+Authors = 'https://github.com/Meaningful-Data/vtlengine/graphs/contributors'
+
+[tool.poetry.dependencies]
+python = "^3.10"
+# PyPi dependencies
+duckdb = "^1.1.1"
+#numba = "^0.60.0"
+s3fs = "^2024.9.0"
+
+# APT dependencies
+antlr4-python3-runtime = "4.9.2"
+networkx = "^2.8.8"
+numexpr = "^2.9.0"
+pandas = "^2.1.4"
+bottleneck = "^1.3.4"
+sqlglot = "^22.2.0"
+
+[tool.poetry.dev-dependencies]
+pytest = "^7.3"
+pytest-cov = "^5.0.0"
+line-profiler-pycharm = "^1.2.0"
+sphinx = "^7.4"
+sphinx-rtd-theme = "^2.0.0"
+mypy = "^1.11.2"
+pandas-stubs = "^2.2.3.241009"
+stubs = "^1.0.0"
+toml = "^0.10.2"
+ruff = "^0.7.1"
+
+[tool.ruff]
+line-length = 100
+lint.mccabe.max-complexity = 20
+lint.select = [
+    "B", "B9", "C", "C4", "D", "E", "F", "I", "PT", "S", "SIM", "W"
+]
+# TODO: check S608 (duckdb querys)
+lint.ignore = ["B023", "B028", "B904", "C403", "D100", "D101", "D102", "D103", "D104", "D105",
+    "D107", "D200", "D201", "D202", "D203", "D205", "D209", "D212", "D213", "D301",
+    "D400", "D401", "D404", "D411", "D413", "D415", "D419", "E203", "S320", "S608"]
+lint.exclude = ["*/Grammar/*"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101", "PT006", "PT012", "PT013", "E501", "W605"]
+"src/vtlengine/*" = ["E712"]
+
+[tool.mypy]
+files = "src"
+exclude = "src/vtlengine/AST/.*"
+disallow_untyped_defs = true
+disallow_untyped_calls = true
+ignore_errors = false
+no_implicit_optional = true
+show_column_numbers = true
+strict_equality = true
+strict_optional = true
+strict = true
+enable_error_code = [
+    "ignore-without-code",
+    "redundant-expr",
+    "truthy-bool",
+]
+warn_return_any = false
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

{vtlengine-1.0.1 → vtlengine-1.0.3}/src/vtlengine/API/_InternalApi.py

@@ -1,15 +1,24 @@
 import json
 import os
 from pathlib import Path
-from typing import Union, Optional, Dict, List, Any
+from typing import Any, Dict, List, Optional, Union
 
 import pandas as pd
 from s3fs import S3FileSystem  # type: ignore[import-untyped]
 
 from vtlengine.AST import PersistentAssignment, Start
 from vtlengine.DataTypes import SCALAR_TYPES
-from vtlengine.Model import ValueDomain, Dataset, Scalar, Component, Role, ExternalRoutine
-from vtlengine.files.parser import _validate_pandas, _fill_dataset_empty_data
+from vtlengine.Exceptions import check_key
+from vtlengine.files.parser import _fill_dataset_empty_data, _validate_pandas
+from vtlengine.Model import (
+    Component,
+    Dataset,
+    ExternalRoutine,
+    Role,
+    Role_keys,
+    Scalar,
+    ValueDomain,
+)
 
 base_path = Path(__file__).parent
 filepath_VTL = base_path / "data" / "vtl"
@@ -30,15 +39,17 @@ def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
     if "datasets" in structures:
         for dataset_json in structures["datasets"]:
             dataset_name = dataset_json["name"]
-            components = {
-                component["name"]: Component(
+            components = {}
+
+            for component in dataset_json["DataStructure"]:
+                check_key("data_type", SCALAR_TYPES.keys(), component["type"])
+                check_key("role", Role_keys, component["role"])
+                components[component["name"]] = Component(
                     name=component["name"],
                     data_type=SCALAR_TYPES[component["type"]],
                     role=Role(component["role"]),
                     nullable=component["nullable"],
                 )
-                for component in dataset_json["DataStructure"]
-            }
 
             datasets[dataset_name] = Dataset(name=dataset_name, components=components, data=None)
     if "scalars" in structures:
@@ -208,7 +219,7 @@ def load_datasets_with_data(data_structures: Any, datapoints: Optional[Any] = No
         return datasets, None
     # Handling dictionary of paths
     dict_datapoints = _load_datapoints_path(datapoints)
-    for dataset_name, file_path in dict_datapoints.items():
+    for dataset_name, _ in dict_datapoints.items():
         if dataset_name not in datasets:
             raise Exception(f"Not found dataset {dataset_name}")
 

{vtlengine-1.0.1 → vtlengine-1.0.3}/src/vtlengine/API/__init__.py

@@ -1,29 +1,29 @@
 from pathlib import Path
-from typing import Any, Union, List, Optional, Dict
+from typing import Any, Dict, List, Optional, Union
 
 import pandas as pd
 from antlr4 import CommonTokenStream, InputStream  # type: ignore[import-untyped]
 from antlr4.error.ErrorListener import ErrorListener  # type: ignore[import-untyped]
 
 from vtlengine.API._InternalApi import (
-    load_vtl,
+    _check_output_folder,
+    _return_only_persistent_datasets,
     load_datasets,
-    load_value_domains,
-    load_external_routines,
     load_datasets_with_data,
-    _return_only_persistent_datasets,
-    _check_output_folder,
+    load_external_routines,
+    load_value_domains,
+    load_vtl,
 )
 from vtlengine.AST import Start
 from vtlengine.AST.ASTConstructor import ASTVisitor
 from vtlengine.AST.DAG import DAGAnalyzer
 from vtlengine.AST.Grammar.lexer import Lexer
 from vtlengine.AST.Grammar.parser import Parser
-from vtlengine.Interpreter import InterpreterAnalyzer
 from vtlengine.files.output._time_period_representation import (
-    format_time_period_external_representation,
     TimePeriodRepresentation,
+    format_time_period_external_representation,
 )
+from vtlengine.Interpreter import InterpreterAnalyzer
 
 pd.options.mode.chained_assignment = None
 
@@ -78,7 +78,7 @@ def create_ast(text: str) -> Start:
     stream = _lexer(text)
     cst = _parser(stream)
     visitor = ASTVisitor()
-    ast = visitor.visit(cst)
+    ast = visitor.visitStart(cst)
     DAGAnalyzer.createDAG(ast)
     return ast
 

{vtlengine-1.0.1 → vtlengine-1.0.3}/src/vtlengine/AST/ASTConstructor.py

@@ -10,18 +10,18 @@ Node Creator.
 from antlr4.tree.Tree import TerminalNodeImpl
 
 from vtlengine.AST import (
-    Start,
-    Assignment,
-    PersistentAssignment,
-    Operator,
     Argument,
+    Assignment,
+    DefIdentifier,
+    DPRule,
     DPRuleset,
     HRBinOp,
-    DPRule,
-    HRuleset,
-    DefIdentifier,
     HRule,
+    HRuleset,
     HRUnOp,
+    Operator,
+    PersistentAssignment,
+    Start,
 )
 from vtlengine.AST.ASTConstructorModules.Expr import Expr
 from vtlengine.AST.ASTConstructorModules.ExprComponents import ExprComp
@@ -30,8 +30,7 @@ from vtlengine.AST.ASTDataExchange import de_ruleset_elements
 from vtlengine.AST.Grammar.parser import Parser
 from vtlengine.AST.VtlVisitor import VtlVisitor
 from vtlengine.Exceptions import SemanticError
-from vtlengine.Model import Scalar, Component, Dataset
-
+from vtlengine.Model import Component, Dataset, Scalar
 
 # pylint: disable=unreachable,expression-not-assigned
 # pylint: disable=assignment-from-no-return
@@ -280,19 +279,13 @@ class ASTVisitor(VtlVisitor):
             for erCode_name in ctx_list
             if isinstance(erCode_name, Parser.ErCodeContext)
         ]
-        if len(er_code) == 0:
-            er_code = None
-        else:
-            er_code = er_code[0]
+        er_code = None if len(er_code) == 0 else er_code[0]
         er_level = [
             Terminals().visitErLevel(erLevel_name)
             for erLevel_name in ctx_list
             if isinstance(erLevel_name, Parser.ErLevelContext)
         ]
-        if len(er_level) == 0:
-            er_level = None
-        else:
-            er_level = er_level[0]
+        er_level = None if len(er_level) == 0 else er_level[0]
 
         return DPRule(name=rule_name, rule=rule_node, erCode=er_code, erLevel=er_level)
 
@@ -317,11 +310,7 @@ class ASTVisitor(VtlVisitor):
             for element in ctx_list
             if isinstance(element, Parser.ScalarItemContext)
         ]
-
-        if len(argument_default) == 0:
-            argument_default = None
-        else:
-            argument_default = argument_default[0]
+        argument_default = None if len(argument_default) == 0 else argument_default[0]
 
         if isinstance(argument_type, (Dataset, Component, Scalar)):
             argument_type.name = argument_name.value
@@ -375,10 +364,7 @@ class ASTVisitor(VtlVisitor):
             if isinstance(valueDomain, TerminalNodeImpl)
             and valueDomain.getSymbol().type == Parser.VALUE_DOMAIN
         ]
-        if len(value_domain) != 0:
-            kind = "ValuedomainID"
-        else:
-            kind = "DatasetID"
+        kind = "ValuedomainID" if len(value_domain) != 0 else "DatasetID"
 
         conditions = [
             self.visitValueDomainSignature(vtlsig)
@@ -408,8 +394,8 @@ class ASTVisitor(VtlVisitor):
     # TODO Support for valueDomainSignature.
     def visitValueDomainSignature(self, ctx: Parser.ValueDomainSignatureContext):
         """
-        valueDomainSignature: CONDITION IDENTIFIER (AS IDENTIFIER)? (',' IDENTIFIER (AS IDENTIFIER)?)* ; # noqa E501
-        """
+        valueDomainSignature: CONDITION IDENTIFIER (AS IDENTIFIER)? (',' IDENTIFIER (AS IDENTIFIER)?)* ;
+        """ # noqa E501
         # AST_ASTCONSTRUCTOR.7
         ctx_list = list(ctx.getChildren())
         component_nodes = [
@@ -458,28 +444,22 @@ class ASTVisitor(VtlVisitor):
             for erCode_name in ctx_list
             if isinstance(erCode_name, Parser.ErCodeContext)
         ]
-        if len(er_code) == 0:
-            er_code = None
-        else:
-            er_code = er_code[0]
+        er_code = None if len(er_code) == 0 else er_code[0]
         er_level = [
             Terminals().visitErLevel(erLevel_name)
             for erLevel_name in ctx_list
             if isinstance(erLevel_name, Parser.ErLevelContext)
         ]
-        if len(er_level) == 0:
-            er_level = None
-        else:
-            er_level = er_level[0]
+        er_level = None if len(er_level) == 0 else er_level[0]
 
         return HRule(name=rule_name, rule=rule_node, erCode=er_code, erLevel=er_level)
 
     def visitCodeItemRelation(self, ctx: Parser.CodeItemRelationContext):
         """
-        codeItemRelation: ( WHEN expr THEN )? codeItemRef codeItemRelationClause (codeItemRelationClause)* ; # noqa E501
-                        ( WHEN exprComponent THEN )? codetemRef=valueDomainValue comparisonOperand? codeItemRelationClause (codeItemRelationClause)* # noqa E501
+        codeItemRelation: ( WHEN expr THEN )? codeItemRef codeItemRelationClause (codeItemRelationClause)* ;
+                        ( WHEN exprComponent THEN )? codetemRef=valueDomainValue comparisonOperand? codeItemRelationClause (codeItemRelationClause)*
 
-        """
+        """ # noqa E501
 
         ctx_list = list(ctx.getChildren())
 
@@ -531,8 +511,8 @@ class ASTVisitor(VtlVisitor):
 
     def visitCodeItemRelationClause(self, ctx: Parser.CodeItemRelationClauseContext):
         """
-        (opAdd=( PLUS | MINUS  ))? rightCodeItem=valueDomainValue ( QLPAREN  rightCondition=exprComponent  QRPAREN )? # noqa E501
-        """
+        (opAdd=( PLUS | MINUS  ))? rightCodeItem=valueDomainValue ( QLPAREN  rightCondition=exprComponent  QRPAREN )?
+        """ # noqa E501
         ctx_list = list(ctx.getChildren())
 
         expr = [expr for expr in ctx_list if isinstance(expr, Parser.ExprContext)]
@@ -552,13 +532,13 @@ class ASTVisitor(VtlVisitor):
 
             code_item = DefIdentifier(value=value, kind="CodeItemID")
             if right_condition:
-                setattr(code_item, "_right_condition", right_condition[0])
+                code_item._right_condition = right_condition[0]
 
             return HRBinOp(left=None, op=op, right=code_item)
         else:
             value = Terminals().visitValueDomainValue(ctx_list[0])
             code_item = DefIdentifier(value=value, kind="CodeItemID")
             if right_condition:
-                setattr(code_item, "_right_condition", right_condition[0])
+                code_item._right_condition = right_condition[0]
 
             return code_item