vtlengine 1.0.3rc3__py3-none-any.whl → 1.1__py3-none-any.whl

vtlengine/AST/ASTConstructorModules/Terminals.py

@@ -12,6 +12,7 @@ from vtlengine.AST import (
     VarID,
     Windowing,
 )
+from vtlengine.AST.ASTConstructorModules import extract_token_info
 from vtlengine.AST.Grammar.parser import Parser
 from vtlengine.AST.VtlVisitor import VtlVisitor
 from vtlengine.DataTypes import (
@@ -37,26 +38,27 @@ def _remove_scaped_characters(text):
 class Terminals(VtlVisitor):
     def visitConstant(self, ctx: Parser.ConstantContext):
         token = ctx.children[0].getSymbol()
+        token_info = extract_token_info(token)
 
         if token.type == Parser.INTEGER_CONSTANT:
-            constant_node = Constant("INTEGER_CONSTANT", int(token.text))
+            constant_node = Constant(type_="INTEGER_CONSTANT", value=int(token.text), **token_info)
 
         elif token.type == Parser.NUMBER_CONSTANT:
-            constant_node = Constant("FLOAT_CONSTANT", float(token.text))
+            constant_node = Constant(type_="FLOAT_CONSTANT", value=float(token.text), **token_info)
 
         elif token.type == Parser.BOOLEAN_CONSTANT:
             if token.text == "true":
-                constant_node = Constant("BOOLEAN_CONSTANT", True)
+                constant_node = Constant(type_="BOOLEAN_CONSTANT", value=True, **token_info)
             elif token.text == "false":
-                constant_node = Constant("BOOLEAN_CONSTANT", False)
+                constant_node = Constant(type_="BOOLEAN_CONSTANT", value=False, **token_info)
             else:
                 raise NotImplementedError
 
         elif token.type == Parser.STRING_CONSTANT:
-            constant_node = Constant("STRING_CONSTANT", token.text[1:-1])
+            constant_node = Constant(type_="STRING_CONSTANT", value=token.text[1:-1], **token_info)
 
         elif token.type == Parser.NULL_CONSTANT:
-            constant_node = Constant("NULL_CONSTANT", None)
+            constant_node = Constant(type_="NULL_CONSTANT", value=None, **token_info)
 
         else:
             raise NotImplementedError
@@ -66,18 +68,19 @@ class Terminals(VtlVisitor):
     def visitVarID(self, ctx: Parser.VarIDContext):
         token = ctx.children[0].getSymbol()
         token.text = _remove_scaped_characters(token.text)
-        var_id_node = VarID(token.text)
+        token_info = extract_token_info(token)
+        var_id_node = VarID(value=token.text, **token_info)
         return var_id_node
 
     def visitVarIdExpr(self, ctx: Parser.VarIdExprContext):
-
         if isinstance(ctx.children[0], Parser.VarIDContext):
             return self.visitVarID(ctx.children[0])
 
         token = ctx.children[0].getSymbol()
         # check token text
         token.text = _remove_scaped_characters(token.text)
-        var_id_node = VarID(token.text)
+        token_info = extract_token_info(token)
+        var_id_node = VarID(value=token.text, **token_info)
         return var_id_node
 
     def visitSimpleComponentId(self, ctx: Parser.SimpleComponentIdContext):
@@ -88,7 +91,7 @@ class Terminals(VtlVisitor):
         # check token text
         token.text = _remove_scaped_characters(token.text)
 
-        return Identifier(token.text, "ComponentID")
+        return Identifier(value=token.text, kind="ComponentID", **extract_token_info(ctx))
 
     def visitComponentID(self, ctx: Parser.ComponentIDContext):
         ctx_list = list(ctx.getChildren())
@@ -99,7 +102,11 @@ class Terminals(VtlVisitor):
                 "'"
             ):  # The component could be imbalance, errorcode or errorlevel
                 component_name = component_name[1:-1]
-            return Identifier(component_name, "ComponentID")
+            return Identifier(
+                value=component_name,
+                kind="ComponentID",
+                **extract_token_info(ctx_list[0].getSymbol()),
+            )
         else:
             component_name = ctx_list[2].getSymbol().text
             if component_name.startswith("'") and component_name.endswith(
@@ -108,9 +115,18 @@ class Terminals(VtlVisitor):
                 component_name = component_name[1:-1]
             op_node = ctx_list[1].getSymbol().text
             return BinOp(
-                left=Identifier(ctx_list[0].getSymbol().text, "DatasetID"),
+                left=Identifier(
+                    value=ctx_list[0].getSymbol().text,
+                    kind="DatasetID",
+                    **extract_token_info(ctx_list[0].getSymbol()),
+                ),
                 op=op_node,
-                right=Identifier(component_name, "ComponentID"),
+                right=Identifier(
+                    value=component_name,
+                    kind="ComponentID",
+                    **extract_token_info(ctx_list[1].getSymbol()),
+                ),
+                **extract_token_info(ctx),
             )
 
     def visitOperatorID(self, ctx: Parser.OperatorIDContext):
@@ -127,7 +143,11 @@ class Terminals(VtlVisitor):
         valueDomainID: IDENTIFIER ;
         """
         return Collection(
-            name=ctx.children[0].getSymbol().text, children=[], kind="ValueDomain", type=""
+            name=ctx.children[0].getSymbol().text,
+            children=[],
+            kind="ValueDomain",
+            type="",
+            **extract_token_info(ctx),
         )
 
     def visitRulesetID(self, ctx: Parser.RulesetIDContext):
@@ -252,7 +272,9 @@ class Terminals(VtlVisitor):
         for scalar_with_cast in scalars_with_cast:
             scalar_nodes.append(self.visitScalarWithCast(scalar_with_cast))
 
-        return Collection("Set", None, scalar_nodes)
+        return Collection(
+            name="List", type="Lists", children=scalar_nodes, **extract_token_info(ctx)
+        )
 
     def visitMultModifier(self, ctx: Parser.MultModifierContext):
         """
@@ -381,7 +403,7 @@ class Terminals(VtlVisitor):
             | DATAPOINT_ON_VD  (GLPAREN  valueDomainName (MUL valueDomainName)*  GRPAREN )?         # dataPointVd
             | DATAPOINT_ON_VAR  (GLPAREN  varID (MUL varID)*  GRPAREN )?                            # dataPointVar
         ;
-        """ # noqa E501
+        """  # noqa E501
         # AST_ASTCONSTRUCTOR.54
         raise NotImplementedError
 
@@ -391,7 +413,7 @@ class Terminals(VtlVisitor):
             | HIERARCHICAL_ON_VD ( GLPAREN  vdName=IDENTIFIER (LPAREN valueDomainName (MUL valueDomainName)* RPAREN)?  GRPAREN )?   # hrRulesetVdType
             | HIERARCHICAL_ON_VAR ( GLPAREN  varName=varID (LPAREN  varID (MUL varID)* RPAREN)?  GRPAREN )?                         # hrRulesetVarType
         ;
-        """ # noqa E501
+        """  # noqa E501
         # AST_ASTCONSTRUCTOR.55
         raise NotImplementedError
 
@@ -483,7 +505,6 @@ class Terminals(VtlVisitor):
             raise NotImplementedError
 
     def visitScalarItem(self, ctx: Parser.ScalarItemContext):
-
         ctx_list = list(ctx.getChildren())
         c = ctx_list[0]
 
@@ -497,7 +518,7 @@ class Terminals(VtlVisitor):
     def visitScalarWithCast(self, ctx: Parser.ScalarWithCastContext):
         """
         |  CAST LPAREN constant COMMA (basicScalarType) (COMMA STRING_CONSTANT)? RPAREN    #scalarWithCast  # noqa E501
-        """ # noqa E501
+        """  # noqa E501
         ctx_list = list(ctx.getChildren())
         c = ctx_list[0]
 
@@ -507,12 +528,22 @@ class Terminals(VtlVisitor):
         const_node = self.visitConstant(ctx_list[2])
         basic_scalar_type = [self.visitBasicScalarType(ctx_list[4])]
 
-        param_node = [ParamConstant("PARAM_CAST", ctx_list[6])] if len(ctx_list) > 6 else []
+        param_node = (
+            [
+                ParamConstant(
+                    type_="PARAM_CAST", value=ctx_list[6], **extract_token_info(ctx_list[6])
+                )
+            ]
+            if len(ctx_list) > 6
+            else []
+        )
 
         if len(basic_scalar_type) == 1:
             children_nodes = [const_node, basic_scalar_type[0]]
 
-            return ParamOp(op=op, children=children_nodes, params=param_node)
+            return ParamOp(
+                op=op, children=children_nodes, params=param_node, **extract_token_info(ctx)
+            )
 
         else:
             # AST_ASTCONSTRUCTOR.14
@@ -535,14 +566,20 @@ class Terminals(VtlVisitor):
 
         if token.type == Parser.BOOLEAN_CONSTANT:
             if token.text == "true":
-                param_constant_node = Constant("BOOLEAN_CONSTANT", True)
+                param_constant_node = Constant(
+                    type_="BOOLEAN_CONSTANT", value=True, **extract_token_info(token)
+                )
             elif token.text == "false":
-                param_constant_node = Constant("BOOLEAN_CONSTANT", False)
+                param_constant_node = Constant(
+                    type_="BOOLEAN_CONSTANT", value=False, **extract_token_info(token)
+                )
             else:
                 raise NotImplementedError
 
         elif token.type == Parser.ALL:
-            param_constant_node = ParamConstant("PARAM_CONSTANT", token.text)
+            param_constant_node = ParamConstant(
+                type_="PARAM_CONSTANT", value=token.text, **extract_token_info(token)
+            )
 
         else:
             raise NotImplementedError
@@ -564,7 +601,7 @@ class Terminals(VtlVisitor):
         return ctx.children[0].getSymbol().text
 
     def visitSignedInteger(self, ctx: Parser.SignedIntegerContext):
-        return ctx.children[0].getSymbol().text
+        return int(ctx.children[0].getSymbol().text)
 
     def visitComparisonOperand(self, ctx: Parser.ComparisonOperandContext):
         return ctx.children[0].getSymbol().text
@@ -593,8 +630,9 @@ class Terminals(VtlVisitor):
 
     def visitSignature(self, ctx: Parser.SignatureContext, kind="ComponentID"):
         """
-        varID (AS alias)?
+        VarID (AS alias)?
         """
+        token_info = extract_token_info(ctx)
 
         ctx_list = list(ctx.getChildren())
         c = ctx_list[0]
@@ -604,10 +642,10 @@ class Terminals(VtlVisitor):
         alias_name = None
 
         if len(ctx_list) == 1:
-            return DPRIdentifier(value=node_name, kind=kind, alias=alias_name)
+            return DPRIdentifier(value=node_name, kind=kind, alias=alias_name, **token_info)
 
         alias_name = self.visitAlias(ctx_list[2])
-        return DPRIdentifier(value=node_name, kind=kind, alias=alias_name)
+        return DPRIdentifier(value=node_name, kind=kind, alias=alias_name, **token_info)
 
     """
         From Hierarchical
@@ -662,6 +700,8 @@ class Terminals(VtlVisitor):
 
         win_mode = ctx_list[0].getSymbol().text  # Windowing mode (data points | range )
 
+        token_info = extract_token_info(ctx)
+
         if win_mode == "data":
             num_rows_1, mode_1 = self.visitLimitClauseItem(ctx_list[3])
             num_rows_2, mode_2 = self.visitLimitClauseItem(ctx_list[5])
@@ -672,8 +712,12 @@ class Terminals(VtlVisitor):
         first = num_rows_1  # unbounded (default value)
         second = num_rows_2  # current data point (default value)
 
-        if (mode_2 == "preceding" and mode_1 == "preceding" and num_rows_1 == -1
-                and num_rows_2 == -1):  # preceding and preceding (error)
+        if (
+            mode_2 == "preceding"
+            and mode_1 == "preceding"
+            and num_rows_1 == -1
+            and num_rows_2 == -1
+        ):  # preceding and preceding (error)
             raise Exception(
                 f"Cannot have 2 preceding clauses with unbounded in analytic clause, "
                 f"line {ctx_list[3].start.line}"
@@ -689,30 +733,36 @@ class Terminals(VtlVisitor):
 
         if mode_1 == mode_2:
             if mode_1 == "preceding" and first != -1 and second > first:  # 3 and 1: must be [-3:-1]
-                return create_windowing(win_mode, [second, first], [mode_2, mode_1])
+                return create_windowing(win_mode, [second, first], [mode_2, mode_1], token_info)
             if mode_1 == "preceding" and second == -1:
-                return create_windowing(win_mode, [second, first], [mode_2, mode_1])
+                return create_windowing(win_mode, [second, first], [mode_2, mode_1], token_info)
             if mode_1 == "following" and second != -1 and second < first:  # 3 and 1: must be [1:3]
-                return create_windowing(win_mode, [second, first], [mode_2, mode_1])
+                return create_windowing(win_mode, [second, first], [mode_2, mode_1], token_info)
             if mode_1 == "following" and first == -1:
-                return create_windowing(win_mode, [second, first], [mode_2, mode_1])
+                return create_windowing(win_mode, [second, first], [mode_2, mode_1], token_info)
 
-        return create_windowing(win_mode, [first, second], [mode_1, mode_2])
+        return create_windowing(win_mode, [first, second], [mode_1, mode_2], token_info)
 
     def visitOrderByItem(self, ctx: Parser.OrderByItemContext):
         ctx_list = list(ctx.getChildren())
 
+        token_info = extract_token_info(ctx)
+
         if len(ctx_list) == 1:
-            return OrderBy(component=self.visitComponentID(ctx_list[0]).value, order="asc")
+            return OrderBy(
+                component=self.visitComponentID(ctx_list[0]).value, order="asc", **token_info
+            )
 
         return OrderBy(
-            component=self.visitComponentID(ctx_list[0]).value, order=ctx_list[1].getSymbol().text
+            component=self.visitComponentID(ctx_list[0]).value,
+            order=ctx_list[1].getSymbol().text,
+            **token_info,
         )
 
     def visitLimitClauseItem(self, ctx: Parser.LimitClauseItemContext):
         ctx_list = list(ctx.getChildren())
         c = ctx_list[0]
-        if c.getSymbol().text == "unbounded":
+        if c.getSymbol().text.lower() == "unbounded":
             result = -1
         elif c.getSymbol().text == "current":
             result = 0
@@ -727,13 +777,18 @@ class Terminals(VtlVisitor):
         return result, ctx_list[1].getSymbol().text
 
 
-def create_windowing(win_mode, values, modes):
+def create_windowing(win_mode, values, modes, token_info):
     for e in range(0, 2):
         if values[e] == -1:
-            values[e] = "UNBOUNDED"
+            values[e] = "unbounded"
         elif values[e] == 0:
-            values[e] = "CURRENT ROW"
+            values[e] = "current row"
 
     return Windowing(
-        type_=win_mode, start=values[0], stop=values[1], start_mode=modes[0], stop_mode=modes[1]
+        type_=win_mode,
+        start=values[0],
+        stop=values[1],
+        start_mode=modes[0],
+        stop_mode=modes[1],
+        **token_info,
     )

vtlengine/AST/ASTConstructorModules/__init__.py

@@ -0,0 +1,50 @@
+from typing import Dict, Union
+
+from antlr4.ParserRuleContext import ParserRuleContext
+from antlr4.Token import CommonToken
+
+from vtlengine.AST.Grammar.lexer import Lexer
+
+
+def extract_token_info(token: Union[CommonToken, ParserRuleContext]) -> Dict[str, int]:
+    """
+    Extracts the token information from a token or ParserRuleContext.
+
+    The Token information includes:
+    - column_start: The starting column of the token.
+    - column_stop: The stopping column of the token.
+    - line_start: The starting line number of the token.
+    - line_stop: The stopping line number of the token.
+
+    The overall idea is to provide the information from which line and column,
+    and to which line and column, the text is referenced by the AST object, including children.
+
+    Important Note: the keys of the dict are the same as the class attributes of the AST Object.
+
+    Args:
+        token (Union[CommonToken, ParserRuleContext]): The token or ParserRuleContext to extract
+        information from.
+
+    Returns:
+        Dict[str, int]: A dictionary containing the token information.
+    """
+
+    if isinstance(token, ParserRuleContext):
+        return {
+            "column_start": token.start.column,
+            "column_stop": token.stop.column + len(token.stop.text),
+            "line_start": token.start.line,
+            "line_stop": token.stop.line,
+        }
+    line_start = token.line
+    line_stop = token.line
+    # For block comments, we need to add the lines inside the block, marked by \n, to the stop line.
+    # The ML_COMMENT does not take into account the final \n in its grammar.
+    if token.type == Lexer.ML_COMMENT:
+        line_stop = token.line + token.text.count("\n")
+    return {
+        "column_start": token.column,
+        "column_stop": token.column + len(token.text),
+        "line_start": line_start,
+        "line_stop": line_stop,
+    }

vtlengine/AST/ASTEncoders.py

@@ -1,12 +1,16 @@
 import json
 
-import AST
+from vtlengine import AST
+from vtlengine.Model import Dataset
 
 
 class ComplexEncoder(json.JSONEncoder):
     def default(self, obj):
         if hasattr(obj, "toJSON"):
             return obj.toJSON()
+        # Makes a circular reference error if we do not check for this
+        elif isinstance(obj, Dataset):
+            return "dataset"
         else:
             return json.__dict__