vtlengine 1.2.1__py3-none-any.whl → 1.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of vtlengine might be problematic. Click here for more details.
- vtlengine/AST/ASTConstructorModules/Terminals.py +1 -5
- vtlengine/AST/__init__.py +3 -3
- vtlengine/DataTypes/TimeHandling.py +8 -7
- vtlengine/Exceptions/messages.py +7 -0
- vtlengine/Interpreter/__init__.py +60 -16
- vtlengine/Operators/Aggregation.py +4 -0
- vtlengine/Operators/Conditional.py +51 -34
- vtlengine/Operators/Validation.py +33 -5
- vtlengine/__init__.py +1 -1
- vtlengine/files/parser/__init__.py +10 -0
- vtlengine/files/parser/_time_checking.py +5 -0
- {vtlengine-1.2.1.dist-info → vtlengine-1.2.2.dist-info}/METADATA +5 -4
- {vtlengine-1.2.1.dist-info → vtlengine-1.2.2.dist-info}/RECORD +15 -15
- {vtlengine-1.2.1.dist-info → vtlengine-1.2.2.dist-info}/WHEEL +1 -1
- {vtlengine-1.2.1.dist-info → vtlengine-1.2.2.dist-info/licenses}/LICENSE.md +0 -0
|
@@ -622,11 +622,7 @@ class Terminals(VtlVisitor):
|
|
|
622
622
|
erLevel: ERRORLEVEL constant;
|
|
623
623
|
"""
|
|
624
624
|
ctx_list = list(ctx.getChildren())
|
|
625
|
-
|
|
626
|
-
try:
|
|
627
|
-
return int(self.visitConstant(ctx_list[1]).value)
|
|
628
|
-
except Exception:
|
|
629
|
-
raise Exception(f"Error level must be an integer, line {ctx_list[1].start.line}")
|
|
625
|
+
return self.visitConstant(ctx_list[1]).value
|
|
630
626
|
|
|
631
627
|
def visitSignature(self, ctx: Parser.SignatureContext, kind="ComponentID"):
|
|
632
628
|
"""
|
vtlengine/AST/__init__.py
CHANGED
|
@@ -443,7 +443,7 @@ class Validation(AST):
|
|
|
443
443
|
op: str
|
|
444
444
|
validation: str
|
|
445
445
|
error_code: Optional[str]
|
|
446
|
-
error_level: Optional[int]
|
|
446
|
+
error_level: Optional[Union[int, str]]
|
|
447
447
|
imbalance: Optional[AST]
|
|
448
448
|
invalid: bool
|
|
449
449
|
|
|
@@ -590,7 +590,7 @@ class HRule(AST):
|
|
|
590
590
|
name: Optional[str]
|
|
591
591
|
rule: HRBinOp
|
|
592
592
|
erCode: Optional[str]
|
|
593
|
-
erLevel: Optional[int]
|
|
593
|
+
erLevel: Optional[Union[int, str]]
|
|
594
594
|
|
|
595
595
|
__eq__ = AST.ast_equality
|
|
596
596
|
|
|
@@ -604,7 +604,7 @@ class DPRule(AST):
|
|
|
604
604
|
name: Optional[str]
|
|
605
605
|
rule: HRBinOp
|
|
606
606
|
erCode: Optional[str]
|
|
607
|
-
erLevel: Optional[int]
|
|
607
|
+
erLevel: Optional[Union[int, str]]
|
|
608
608
|
|
|
609
609
|
__eq__ = AST.ast_equality
|
|
610
610
|
|
|
@@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Union
|
|
|
7
7
|
|
|
8
8
|
import pandas as pd
|
|
9
9
|
|
|
10
|
+
from vtlengine.AST.Grammar.tokens import GT, GTE, LT, LTE
|
|
10
11
|
from vtlengine.Exceptions import SemanticError
|
|
11
12
|
|
|
12
13
|
PERIOD_IND_MAPPING = {"A": 6, "S": 5, "Q": 4, "M": 3, "W": 2, "D": 1}
|
|
@@ -180,7 +181,7 @@ class TimePeriodHandler:
|
|
|
180
181
|
|
|
181
182
|
@staticmethod
|
|
182
183
|
def _check_year(year: int) -> None:
|
|
183
|
-
if year <
|
|
184
|
+
if year < 0 or year > 9999:
|
|
184
185
|
raise SemanticError("2-1-19-10", year=year)
|
|
185
186
|
# raise ValueError(f'Invalid year {year}, must be between 1900 and 9999.')
|
|
186
187
|
|
|
@@ -407,22 +408,22 @@ class TimeIntervalHandler:
|
|
|
407
408
|
return py_op(self.length, other.length)
|
|
408
409
|
|
|
409
410
|
def __eq__(self, other: Any) -> Optional[bool]: # type: ignore[override]
|
|
410
|
-
return self
|
|
411
|
+
return str(self) == str(other) if other is not None else None
|
|
411
412
|
|
|
412
413
|
def __ne__(self, other: Any) -> Optional[bool]: # type: ignore[override]
|
|
413
|
-
return self
|
|
414
|
+
return str(self) != str(other) if other is not None else None
|
|
414
415
|
|
|
415
416
|
def __lt__(self, other: Any) -> Optional[bool]:
|
|
416
|
-
|
|
417
|
+
raise SemanticError("2-1-19-17", op=LT, type="Time")
|
|
417
418
|
|
|
418
419
|
def __le__(self, other: Any) -> Optional[bool]:
|
|
419
|
-
|
|
420
|
+
raise SemanticError("2-1-19-17", op=LTE, type="Time")
|
|
420
421
|
|
|
421
422
|
def __gt__(self, other: Any) -> Optional[bool]:
|
|
422
|
-
|
|
423
|
+
raise SemanticError("2-1-19-17", op=GT, type="Time")
|
|
423
424
|
|
|
424
425
|
def __ge__(self, other: Any) -> Optional[bool]:
|
|
425
|
-
|
|
426
|
+
raise SemanticError("2-1-19-17", op=GTE, type="Time")
|
|
426
427
|
|
|
427
428
|
@classmethod
|
|
428
429
|
def from_time_period(cls, value: TimePeriodHandler) -> "TimeIntervalHandler":
|
vtlengine/Exceptions/messages.py
CHANGED
|
@@ -44,6 +44,8 @@ centralised_messages = {
|
|
|
44
44
|
"0-1-1-12": "On Dataset {name} loading: not possible to cast column {column} to {type}.",
|
|
45
45
|
"0-1-1-13": "Invalid key on {field} field: {key}{closest_key}.",
|
|
46
46
|
"0-1-1-14": "Empty datasets {dataset1} and {dataset2} shape missmatch.",
|
|
47
|
+
"0-1-1-15": "On Dataset {name} loading: Duplicated identifiers are not allowed, "
|
|
48
|
+
"found on row {row_index}",
|
|
47
49
|
"0-1-0-1": " Trying to redefine input datasets {dataset}.", # Semantic Error
|
|
48
50
|
# ------------Operators-------------
|
|
49
51
|
# General Semantic errors
|
|
@@ -51,6 +53,7 @@ centralised_messages = {
|
|
|
51
53
|
"1-1-1-2": "Invalid implicit cast from {type_1} and {type_2} to {type_check}.",
|
|
52
54
|
"1-1-1-3": "At op {op}: {entity} {name} cannot be promoted to {target_type}.",
|
|
53
55
|
"1-1-1-4": "At op {op}: Operation not allowed for multimeasure datasets.",
|
|
56
|
+
"1-1-1-5": "At op {op}: Invalid type {type}.",
|
|
54
57
|
"1-1-1-8": "At op {op}: Invalid Dataset {name}, no measures defined.",
|
|
55
58
|
"1-1-1-9": "At op {op}: Invalid Dataset {name}, all measures must have the same type: {type}.",
|
|
56
59
|
"1-1-1-10": "Component {comp_name} not found in Dataset {dataset_name}.",
|
|
@@ -240,6 +243,10 @@ centralised_messages = {
|
|
|
240
243
|
"measure.",
|
|
241
244
|
"2-1-19-15": "{op} can only be applied according to the iso 8601 format mask",
|
|
242
245
|
"2-1-19-16": "{op} can only be positive numbers",
|
|
246
|
+
"2-1-19-17": "At op {op}: Time operators comparison are only support "
|
|
247
|
+
"= and <> comparison operations",
|
|
248
|
+
"2-1-19-18": "At op {op}: Time operators do not support < and > comparison operations, "
|
|
249
|
+
"so its not possible to use get the max or min between two time operators",
|
|
243
250
|
# ----------- Interpreter Common ------
|
|
244
251
|
"2-3-1": "{comp_type} {comp_name} not found.",
|
|
245
252
|
"2-3-2": "{op_type} cannot be used with {node_op} operators.",
|
|
@@ -151,6 +151,8 @@ class InterpreterAnalyzer(ASTTemplate):
|
|
|
151
151
|
dprs: Optional[Dict[str, Optional[Dict[str, Any]]]] = None
|
|
152
152
|
udos: Optional[Dict[str, Optional[Dict[str, Any]]]] = None
|
|
153
153
|
hrs: Optional[Dict[str, Optional[Dict[str, Any]]]] = None
|
|
154
|
+
is_from_case_then: bool = False
|
|
155
|
+
signature_values: Optional[Dict[str, Any]] = None
|
|
154
156
|
|
|
155
157
|
# **********************************
|
|
156
158
|
# * *
|
|
@@ -1078,15 +1080,43 @@ class InterpreterAnalyzer(ASTTemplate):
|
|
|
1078
1080
|
|
|
1079
1081
|
if self.condition_stack is None:
|
|
1080
1082
|
self.condition_stack = []
|
|
1083
|
+
if self.then_condition_dataset is None:
|
|
1084
|
+
self.then_condition_dataset = []
|
|
1085
|
+
if self.else_condition_dataset is None:
|
|
1086
|
+
self.else_condition_dataset = []
|
|
1081
1087
|
|
|
1082
|
-
|
|
1083
|
-
case = node.cases.pop(0)
|
|
1088
|
+
for case in node.cases:
|
|
1084
1089
|
self.is_from_condition = True
|
|
1085
|
-
|
|
1090
|
+
cond = self.visit(case.condition)
|
|
1086
1091
|
self.is_from_condition = False
|
|
1087
|
-
thenOps.append(self.visit(case.thenOp))
|
|
1088
1092
|
|
|
1089
|
-
|
|
1093
|
+
conditions.append(cond)
|
|
1094
|
+
if isinstance(cond, Scalar):
|
|
1095
|
+
then_result = self.visit(case.thenOp)
|
|
1096
|
+
thenOps.append(then_result)
|
|
1097
|
+
continue
|
|
1098
|
+
|
|
1099
|
+
self.generate_then_else_datasets(copy(cond))
|
|
1100
|
+
|
|
1101
|
+
self.condition_stack.append(THEN_ELSE["then"])
|
|
1102
|
+
self.is_from_if = True
|
|
1103
|
+
self.is_from_case_then = True
|
|
1104
|
+
|
|
1105
|
+
then_result = self.visit(case.thenOp)
|
|
1106
|
+
thenOps.append(then_result)
|
|
1107
|
+
|
|
1108
|
+
self.is_from_case_then = False
|
|
1109
|
+
self.is_from_if = False
|
|
1110
|
+
if len(self.condition_stack) > 0:
|
|
1111
|
+
self.condition_stack.pop()
|
|
1112
|
+
if len(self.then_condition_dataset) > 0:
|
|
1113
|
+
self.then_condition_dataset.pop()
|
|
1114
|
+
if len(self.else_condition_dataset) > 0:
|
|
1115
|
+
self.else_condition_dataset.pop()
|
|
1116
|
+
|
|
1117
|
+
elseOp = self.visit(node.elseOp)
|
|
1118
|
+
|
|
1119
|
+
return Case.analyze(conditions, thenOps, elseOp)
|
|
1090
1120
|
|
|
1091
1121
|
def visit_RenameNode(self, node: AST.RenameNode) -> Any:
|
|
1092
1122
|
if self.udo_params is not None:
|
|
@@ -1575,11 +1605,10 @@ class InterpreterAnalyzer(ASTTemplate):
|
|
|
1575
1605
|
if self.else_condition_dataset is None:
|
|
1576
1606
|
self.else_condition_dataset = []
|
|
1577
1607
|
if isinstance(condition, Dataset):
|
|
1578
|
-
if (
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
raise ValueError("Only one boolean measure is allowed on condition dataset")
|
|
1608
|
+
if len(condition.get_measures()) != 1:
|
|
1609
|
+
raise SemanticError("1-1-1-4", op="condition")
|
|
1610
|
+
if condition.get_measures()[0].data_type != BASIC_TYPES[bool]:
|
|
1611
|
+
raise SemanticError("2-1-9-5", op="condition", name=condition.name)
|
|
1583
1612
|
name = condition.get_measures_names()[0]
|
|
1584
1613
|
if condition.data is None or condition.data.empty:
|
|
1585
1614
|
data = None
|
|
@@ -1589,7 +1618,7 @@ class InterpreterAnalyzer(ASTTemplate):
|
|
|
1589
1618
|
|
|
1590
1619
|
else:
|
|
1591
1620
|
if condition.data_type != BASIC_TYPES[bool]:
|
|
1592
|
-
raise
|
|
1621
|
+
raise SemanticError("2-1-9-4", op="condition", name=condition.name)
|
|
1593
1622
|
name = condition.name
|
|
1594
1623
|
data = None if condition.data is None else condition.data
|
|
1595
1624
|
|
|
@@ -1667,11 +1696,18 @@ class InterpreterAnalyzer(ASTTemplate):
|
|
|
1667
1696
|
):
|
|
1668
1697
|
return left_operand, right_operand
|
|
1669
1698
|
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1699
|
+
if self.is_from_case_then:
|
|
1700
|
+
merge_dataset = (
|
|
1701
|
+
self.then_condition_dataset[-1]
|
|
1702
|
+
if self.condition_stack[-1] == THEN_ELSE["then"]
|
|
1703
|
+
else self.else_condition_dataset[-1]
|
|
1704
|
+
)
|
|
1705
|
+
else:
|
|
1706
|
+
merge_dataset = (
|
|
1707
|
+
self.then_condition_dataset.pop()
|
|
1708
|
+
if self.condition_stack.pop() == THEN_ELSE["then"]
|
|
1709
|
+
else (self.else_condition_dataset.pop())
|
|
1710
|
+
)
|
|
1675
1711
|
|
|
1676
1712
|
merge_index = merge_dataset.data[merge_dataset.get_measures_names()[0]].to_list()
|
|
1677
1713
|
ids = merge_dataset.get_identifiers_names()
|
|
@@ -1826,6 +1862,8 @@ class InterpreterAnalyzer(ASTTemplate):
|
|
|
1826
1862
|
raise SemanticError("2-3-10", comp_type="User Defined Operators")
|
|
1827
1863
|
elif node.op not in self.udos:
|
|
1828
1864
|
raise SemanticError("1-3-5", node_op=node.op, op_type="User Defined Operator")
|
|
1865
|
+
if self.signature_values is None:
|
|
1866
|
+
self.signature_values = {}
|
|
1829
1867
|
|
|
1830
1868
|
operator = self.udos[node.op]
|
|
1831
1869
|
signature_values = {}
|
|
@@ -1919,6 +1957,12 @@ class InterpreterAnalyzer(ASTTemplate):
|
|
|
1919
1957
|
self.udo_params = []
|
|
1920
1958
|
|
|
1921
1959
|
# Adding parameters to the stack
|
|
1960
|
+
for k, v in signature_values.items():
|
|
1961
|
+
if hasattr(v, "name"):
|
|
1962
|
+
v = v.name # type: ignore[assignment]
|
|
1963
|
+
if v in self.signature_values:
|
|
1964
|
+
signature_values[k] = self.signature_values[v] # type: ignore[index]
|
|
1965
|
+
self.signature_values.update(signature_values)
|
|
1922
1966
|
self.udo_params.append(signature_values)
|
|
1923
1967
|
|
|
1924
1968
|
# Calling the UDO AST, we use deepcopy to avoid changing the original UDO AST
|
|
@@ -259,6 +259,10 @@ class Aggregation(Operator.Unary):
|
|
|
259
259
|
result_df = result_df[grouping_keys + measure_names]
|
|
260
260
|
if cls.op == COUNT:
|
|
261
261
|
result_df = result_df.dropna(subset=measure_names, how="any")
|
|
262
|
+
if cls.op in [MAX, MIN]:
|
|
263
|
+
for measure in operand.get_measures():
|
|
264
|
+
if measure.data_type == TimeInterval:
|
|
265
|
+
raise SemanticError("2-1-19-18", op=cls.op)
|
|
262
266
|
cls._handle_data_types(result_df, operand.get_measures(), "input")
|
|
263
267
|
result_df = cls._agg_func(result_df, grouping_keys, measure_names, having_expr)
|
|
264
268
|
|
|
@@ -316,6 +316,7 @@ class Case(Operator):
|
|
|
316
316
|
cls, conditions: List[Any], thenOps: List[Any], elseOp: Any
|
|
317
317
|
) -> Union[Scalar, DataComponent, Dataset]:
|
|
318
318
|
result = cls.validate(conditions, thenOps, elseOp)
|
|
319
|
+
|
|
319
320
|
for condition in conditions:
|
|
320
321
|
if isinstance(condition, Dataset) and condition.data is not None:
|
|
321
322
|
condition.data.fillna(False, inplace=True)
|
|
@@ -344,57 +345,73 @@ class Case(Operator):
|
|
|
344
345
|
result.value = thenOps[i].value
|
|
345
346
|
|
|
346
347
|
if isinstance(result, DataComponent):
|
|
347
|
-
|
|
348
|
+
full_index = conditions[0].data.index
|
|
349
|
+
result.data = pd.Series(None, index=full_index)
|
|
348
350
|
|
|
349
351
|
for i, condition in enumerate(conditions):
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
)
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
352
|
+
if isinstance(thenOps[i], Scalar):
|
|
353
|
+
value_series = pd.Series(thenOps[i].value, index=full_index)
|
|
354
|
+
else:
|
|
355
|
+
value_series = thenOps[i].data.reindex(full_index)
|
|
356
|
+
cond_series = condition.data.reindex(full_index)
|
|
357
|
+
cond_mask = cond_series.notna() & cond_series == True
|
|
358
|
+
result_data = result.data.copy()
|
|
359
|
+
result_data[cond_mask] = value_series[cond_mask]
|
|
360
|
+
result.data = result_data
|
|
361
|
+
|
|
362
|
+
conditions_stack = [c.data.reindex(full_index).fillna(False) for c in conditions]
|
|
363
|
+
else_cond_mask = (
|
|
364
|
+
~np.logical_or.reduce(conditions_stack)
|
|
365
|
+
if conditions_stack
|
|
366
|
+
else pd.Series(True, index=full_index)
|
|
362
367
|
)
|
|
368
|
+
if isinstance(elseOp, Scalar):
|
|
369
|
+
else_series = pd.Series(elseOp.value, index=full_index)
|
|
370
|
+
else:
|
|
371
|
+
else_series = elseOp.data.reindex(full_index)
|
|
372
|
+
result.data[else_cond_mask] = else_series[else_cond_mask]
|
|
363
373
|
|
|
364
|
-
|
|
374
|
+
elif isinstance(result, Dataset):
|
|
365
375
|
identifiers = result.get_identifiers_names()
|
|
366
376
|
columns = [col for col in result.get_components_names() if col not in identifiers]
|
|
367
377
|
result.data = (
|
|
368
378
|
conditions[0].data[identifiers]
|
|
369
379
|
if conditions[0].data is not None
|
|
370
380
|
else pd.DataFrame(columns=identifiers)
|
|
371
|
-
)
|
|
381
|
+
).copy()
|
|
372
382
|
|
|
383
|
+
full_index = result.data.index
|
|
373
384
|
for i in range(len(conditions)):
|
|
374
385
|
condition = conditions[i]
|
|
375
386
|
bool_col = next(x.name for x in condition.get_measures() if x.data_type == Boolean)
|
|
376
|
-
|
|
387
|
+
cond_mask = condition.data[bool_col].reindex(full_index).astype(bool)
|
|
377
388
|
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
389
|
+
if isinstance(thenOps[i], Scalar):
|
|
390
|
+
for col in columns:
|
|
391
|
+
result.data.loc[cond_mask, col] = thenOps[i].value
|
|
392
|
+
else:
|
|
393
|
+
cond_df = thenOps[i].data.reindex(full_index)
|
|
394
|
+
result.data.loc[cond_mask, columns] = cond_df.loc[cond_mask, columns]
|
|
395
|
+
|
|
396
|
+
then_cond_masks = [
|
|
397
|
+
c.data[next(x.name for x in c.get_measures() if x.data_type == Boolean)]
|
|
398
|
+
.reindex(full_index)
|
|
399
|
+
.fillna(False)
|
|
400
|
+
.astype(bool)
|
|
401
|
+
for c in conditions
|
|
402
|
+
]
|
|
403
|
+
else_cond_mask = (
|
|
404
|
+
~np.logical_or.reduce(then_cond_masks)
|
|
405
|
+
if then_cond_masks
|
|
406
|
+
else pd.Series(True, index=full_index)
|
|
391
407
|
)
|
|
392
408
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
409
|
+
if isinstance(elseOp, Scalar):
|
|
410
|
+
for col in columns:
|
|
411
|
+
result.data.loc[else_cond_mask, col] = elseOp.value
|
|
412
|
+
else:
|
|
413
|
+
else_df = elseOp.data.reindex(full_index)
|
|
414
|
+
result.data.loc[else_cond_mask, columns] = else_df.loc[else_cond_mask, columns]
|
|
398
415
|
|
|
399
416
|
return result
|
|
400
417
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from copy import copy
|
|
2
|
-
from typing import Any, Dict, Optional
|
|
2
|
+
from typing import Any, Dict, Optional, Union
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
|
|
@@ -27,7 +27,7 @@ class Check(Operator):
|
|
|
27
27
|
validation_element: Dataset,
|
|
28
28
|
imbalance_element: Optional[Dataset],
|
|
29
29
|
error_code: Optional[str],
|
|
30
|
-
error_level: Optional[int],
|
|
30
|
+
error_level: Optional[Union[int, str]],
|
|
31
31
|
invalid: bool,
|
|
32
32
|
) -> Dataset:
|
|
33
33
|
dataset_name = VirtualCounter._new_ds_name()
|
|
@@ -36,6 +36,13 @@ class Check(Operator):
|
|
|
36
36
|
measure = validation_element.get_measures()[0]
|
|
37
37
|
if measure.data_type != Boolean:
|
|
38
38
|
raise SemanticError("1-1-10-1", op=cls.op, op_type="validation", me_type="Boolean")
|
|
39
|
+
error_level_type = None
|
|
40
|
+
if error_level is None or isinstance(error_level, int):
|
|
41
|
+
error_level_type = Integer
|
|
42
|
+
elif isinstance(error_level, str):
|
|
43
|
+
error_level_type = String # type: ignore[assignment]
|
|
44
|
+
else:
|
|
45
|
+
error_level_type = String
|
|
39
46
|
|
|
40
47
|
imbalance_measure = None
|
|
41
48
|
if imbalance_element is not None:
|
|
@@ -69,8 +76,12 @@ class Check(Operator):
|
|
|
69
76
|
result_components["errorcode"] = Component(
|
|
70
77
|
name="errorcode", data_type=String, role=Role.MEASURE, nullable=True
|
|
71
78
|
)
|
|
79
|
+
|
|
72
80
|
result_components["errorlevel"] = Component(
|
|
73
|
-
name="errorlevel",
|
|
81
|
+
name="errorlevel",
|
|
82
|
+
data_type=error_level_type, # type: ignore[arg-type]
|
|
83
|
+
role=Role.MEASURE,
|
|
84
|
+
nullable=True,
|
|
74
85
|
)
|
|
75
86
|
|
|
76
87
|
return Dataset(name=dataset_name, components=result_components, data=None)
|
|
@@ -81,7 +92,7 @@ class Check(Operator):
|
|
|
81
92
|
validation_element: Dataset,
|
|
82
93
|
imbalance_element: Optional[Dataset],
|
|
83
94
|
error_code: Optional[str],
|
|
84
|
-
error_level: Optional[int],
|
|
95
|
+
error_level: Optional[Union[int, str]],
|
|
85
96
|
invalid: bool,
|
|
86
97
|
) -> Dataset:
|
|
87
98
|
result = cls.validate(
|
|
@@ -128,6 +139,20 @@ class Validation(Operator):
|
|
|
128
139
|
|
|
129
140
|
@classmethod
|
|
130
141
|
def validate(cls, dataset_element: Dataset, rule_info: Dict[str, Any], output: str) -> Dataset:
|
|
142
|
+
error_level_type = None
|
|
143
|
+
error_levels = [
|
|
144
|
+
rule_data.get("errorlevel")
|
|
145
|
+
for rule_data in rule_info.values()
|
|
146
|
+
if "errorlevel" in rule_data
|
|
147
|
+
]
|
|
148
|
+
non_null_levels = [el for el in error_levels if el is not None]
|
|
149
|
+
|
|
150
|
+
if len(non_null_levels) == 0 or all(isinstance(el, int) for el in non_null_levels):
|
|
151
|
+
error_level_type = Number
|
|
152
|
+
elif all(isinstance(el, str) for el in non_null_levels):
|
|
153
|
+
error_level_type = String # type: ignore[assignment]
|
|
154
|
+
else:
|
|
155
|
+
error_level_type = String # type: ignore[assignment]
|
|
131
156
|
dataset_name = VirtualCounter._new_ds_name()
|
|
132
157
|
result_components = {comp.name: comp for comp in dataset_element.get_identifiers()}
|
|
133
158
|
result_components["ruleid"] = Component(
|
|
@@ -154,7 +179,10 @@ class Validation(Operator):
|
|
|
154
179
|
name="errorcode", data_type=String, role=Role.MEASURE, nullable=True
|
|
155
180
|
)
|
|
156
181
|
result_components["errorlevel"] = Component(
|
|
157
|
-
name="errorlevel",
|
|
182
|
+
name="errorlevel",
|
|
183
|
+
data_type=error_level_type, # type: ignore[arg-type]
|
|
184
|
+
role=Role.MEASURE,
|
|
185
|
+
nullable=True,
|
|
158
186
|
)
|
|
159
187
|
|
|
160
188
|
return Dataset(name=dataset_name, components=result_components, data=None)
|
vtlengine/__init__.py
CHANGED
|
@@ -203,9 +203,19 @@ def _validate_pandas(
|
|
|
203
203
|
str_comp = SCALAR_TYPES_CLASS_REVERSE[comp.data_type] if comp else "Null"
|
|
204
204
|
raise SemanticError("0-1-1-12", name=dataset_name, column=comp_name, type=str_comp)
|
|
205
205
|
|
|
206
|
+
if id_names:
|
|
207
|
+
check_identifiers_duplicity(data, id_names, dataset_name)
|
|
208
|
+
|
|
206
209
|
return data
|
|
207
210
|
|
|
208
211
|
|
|
212
|
+
def check_identifiers_duplicity(data: pd.DataFrame, identifiers: List[str], name: str) -> None:
|
|
213
|
+
dup_id_row = data.duplicated(subset=identifiers, keep=False)
|
|
214
|
+
if dup_id_row.any():
|
|
215
|
+
row_index = int(dup_id_row.idxmax()) + 1
|
|
216
|
+
raise SemanticError("0-1-1-15", name=name, row_index=row_index)
|
|
217
|
+
|
|
218
|
+
|
|
209
219
|
def load_datapoints(
|
|
210
220
|
components: Dict[str, Component],
|
|
211
221
|
dataset_name: str,
|
|
@@ -95,6 +95,11 @@ def check_time_period(value: str) -> str:
|
|
|
95
95
|
if isinstance(value, int):
|
|
96
96
|
value = str(value)
|
|
97
97
|
value = value.replace(" ", "")
|
|
98
|
+
|
|
99
|
+
match = re.fullmatch(r"^(\d{4})-(\d{2})$", value)
|
|
100
|
+
if match:
|
|
101
|
+
value = f"{match.group(1)}-M{match.group(2)}"
|
|
102
|
+
|
|
98
103
|
period_result = re.fullmatch(period_pattern, value)
|
|
99
104
|
if period_result is not None:
|
|
100
105
|
result = TimePeriodHandler(value)
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: vtlengine
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.2
|
|
4
4
|
Summary: Run and Validate VTL Scripts
|
|
5
|
-
License: AGPL-3.0
|
|
5
|
+
License-Expression: AGPL-3.0
|
|
6
|
+
License-File: LICENSE.md
|
|
6
7
|
Keywords: vtl,sdmx,vtlengine,Validation and Transformation Language
|
|
7
8
|
Author: MeaningfulData
|
|
8
9
|
Author-email: info@meaningfuldata.eu
|
|
@@ -25,7 +26,7 @@ Requires-Dist: networkx (>=2.8,<3.0)
|
|
|
25
26
|
Requires-Dist: numpy (>=1.23.2,<2) ; python_version < "3.13"
|
|
26
27
|
Requires-Dist: numpy (>=2.1.0) ; python_version >= "3.13"
|
|
27
28
|
Requires-Dist: pandas (>=2.1.4,<3.0)
|
|
28
|
-
Requires-Dist: pysdmx[xml] (>=1.
|
|
29
|
+
Requires-Dist: pysdmx[xml] (>=1.5.2,<2.0)
|
|
29
30
|
Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "all"
|
|
30
31
|
Requires-Dist: s3fs (>=2022.11.0,<2023.0) ; extra == "s3"
|
|
31
32
|
Requires-Dist: sqlglot (>=22.2.0,<23.0)
|
|
@@ -5,7 +5,7 @@ vtlengine/AST/ASTComment.py,sha256=bAJW7aaqBXU2LqMtRvL_XOttdl1AFZufa15vmQdvNlY,1
|
|
|
5
5
|
vtlengine/AST/ASTConstructor.py,sha256=X55I98BKG1ItyGIDObF9ALVfCcWnU-0wwCWJsiPILkg,21488
|
|
6
6
|
vtlengine/AST/ASTConstructorModules/Expr.py,sha256=PdI66D3dwA4ymxgqqcChkctsWMRgBSfuyUtgH-KOkss,70207
|
|
7
7
|
vtlengine/AST/ASTConstructorModules/ExprComponents.py,sha256=2Ft4e5w2NtbfaqSNW8I9qSpG9iUaPIfdug7yYWo2gqE,38553
|
|
8
|
-
vtlengine/AST/ASTConstructorModules/Terminals.py,sha256=
|
|
8
|
+
vtlengine/AST/ASTConstructorModules/Terminals.py,sha256=0-5XZbGTXSQxAAWeF-xBBszyaGEb02lD1Ar82ljxk28,26911
|
|
9
9
|
vtlengine/AST/ASTConstructorModules/__init__.py,sha256=J6g6NhJD8j0Ek1YmpethxRiFdjhLxUTM0mc3NHRFLlM,1879
|
|
10
10
|
vtlengine/AST/ASTDataExchange.py,sha256=kPSz21DGbEv-2bZowObseqf2d2_iQj1VnrqWuD9ZwtA,140
|
|
11
11
|
vtlengine/AST/ASTEncoders.py,sha256=-Ar6a0GqMdJZK4CtZ1pUpIeGv57oSdN5qy3-aF0Zt9c,948
|
|
@@ -21,21 +21,21 @@ vtlengine/AST/Grammar/lexer.py,sha256=66cH0cJi83Sxd8XPrPRYkBwdz4NGhPaadUnq5p0GYU
|
|
|
21
21
|
vtlengine/AST/Grammar/parser.py,sha256=fWJaGcXvUCwN2pvjJBU4l5apoQnkhQbUv_mSlKKiDXc,712465
|
|
22
22
|
vtlengine/AST/Grammar/tokens.py,sha256=YF7tO0nF2zYC-VaBAJLyc6VitM72CvYfFQpoPDGCMzo,3139
|
|
23
23
|
vtlengine/AST/VtlVisitor.py,sha256=NJfXJVP6wNmasJmPLlojFqm9R5VSamOAKg_w7BMrhac,35332
|
|
24
|
-
vtlengine/AST/__init__.py,sha256=
|
|
25
|
-
vtlengine/DataTypes/TimeHandling.py,sha256=
|
|
24
|
+
vtlengine/AST/__init__.py,sha256=zTrSDHd3AFaHvvqdPYT8ZthqN2anHfI9Ul1QomA4rNo,11708
|
|
25
|
+
vtlengine/DataTypes/TimeHandling.py,sha256=K8769_b3Q4kPXGC2XVQ4oN7VsGnhxmJtgdhC_ryXjho,20250
|
|
26
26
|
vtlengine/DataTypes/__init__.py,sha256=LYXrde68bYm7MLeMLmr4haeOTSE4Fnpq9G2Ewy7DiaU,23084
|
|
27
27
|
vtlengine/Exceptions/__init__.py,sha256=rSSskV_qCBFzg_W67Q1QBAL7Lnq88D7yi2BDYo1hytw,4727
|
|
28
|
-
vtlengine/Exceptions/messages.py,sha256=
|
|
29
|
-
vtlengine/Interpreter/__init__.py,sha256=
|
|
28
|
+
vtlengine/Exceptions/messages.py,sha256=V68Al8_TGvb_kY9PHEtSRp26rF_PjhjHdDMRtB-IuTI,20113
|
|
29
|
+
vtlengine/Interpreter/__init__.py,sha256=6Ffl5bJRL1KSF335xSxfA8a5y_pV8ZNQUYM9BmYN6hg,87256
|
|
30
30
|
vtlengine/Model/__init__.py,sha256=xWrwhdUOj8Y-5x38zP5XnmFPw8IkBVBBG2bPsUBGLA8,15869
|
|
31
|
-
vtlengine/Operators/Aggregation.py,sha256=
|
|
31
|
+
vtlengine/Operators/Aggregation.py,sha256=K9u0PE77ZAqEwKTCRyRx9G9uvpyVVZQBB_E4648B4V8,12158
|
|
32
32
|
vtlengine/Operators/Analytic.py,sha256=adm8y4mTeen4iVMsQvcvxM9U5f6Xj9UNjdCQI2OBINE,12934
|
|
33
33
|
vtlengine/Operators/Assignment.py,sha256=xyJgGPoFYbq6mzX06gz7Q7L8jXJxpUkgzdY3Lrne2hw,793
|
|
34
34
|
vtlengine/Operators/Boolean.py,sha256=3U5lHkxW5d7QQdGDNxXeXqejlPfFrXKG8_TqknrC8Ls,2856
|
|
35
35
|
vtlengine/Operators/CastOperator.py,sha256=pXTSs0UYBeR5hS3J2HWUyaHmoZoifl2EFch6ol_Taok,17115
|
|
36
36
|
vtlengine/Operators/Clause.py,sha256=Lu6zjcUBkShN6kQmjEZu_7ytaFGwfH-yB4ROoCSkLGI,15505
|
|
37
37
|
vtlengine/Operators/Comparison.py,sha256=CRMvs9qXVXUW32pxAnCua8b7ZHpJy0-Egvs691ekOCk,17403
|
|
38
|
-
vtlengine/Operators/Conditional.py,sha256=
|
|
38
|
+
vtlengine/Operators/Conditional.py,sha256=Py4QwCgCi4t4-FG75-RMe7M5sDRgUZlRGyuFt4RwwJ4,21182
|
|
39
39
|
vtlengine/Operators/General.py,sha256=ltRK8Sw686sb4rC5ji2OX-GYVxaK_PpL0Lev8P5OFHI,6828
|
|
40
40
|
vtlengine/Operators/HROperators.py,sha256=YybwD70906AA00c0k4IP6sjeta0pg7hqb2EUVsFqdmA,8979
|
|
41
41
|
vtlengine/Operators/Join.py,sha256=lYmC_jGlJ4RRmn2vplB13Ysrxgv6O8sNFEHQYZzun5s,18393
|
|
@@ -44,20 +44,20 @@ vtlengine/Operators/RoleSetter.py,sha256=mHZIdcHC3wflj81ekLbioDG1f8yHZXYDQFymV-K
|
|
|
44
44
|
vtlengine/Operators/Set.py,sha256=f1uLeY4XZF0cWEwpXRB_CczgbXr6s33DYPuFt39HlEg,7084
|
|
45
45
|
vtlengine/Operators/String.py,sha256=ghWtYl6oUEAAzynY1a9Hg4yqRA9Sa7uk2B6iF9uuSqQ,20230
|
|
46
46
|
vtlengine/Operators/Time.py,sha256=ESn6ldPg73bdZxOXZYJuIwCLDQnXDGTqR1y7ckQmV1M,43025
|
|
47
|
-
vtlengine/Operators/Validation.py,sha256=
|
|
47
|
+
vtlengine/Operators/Validation.py,sha256=ot-og75Ce690DddQM-xILrY3PoRf8Z0M3aIovFK-wMY,11245
|
|
48
48
|
vtlengine/Operators/__init__.py,sha256=N1zi9RFC_l0qggRm5IPLOkPFtFS4CGAg-r1taHOrbTI,37667
|
|
49
49
|
vtlengine/Utils/__Virtual_Assets.py,sha256=0jPXysZrBr0hYVzqFoyg9La8ZcZoZ01Ql245X5vrth4,862
|
|
50
50
|
vtlengine/Utils/__init__.py,sha256=zhGPJA8MjHmtEEwMS4CxEFYL0tk2L5F0YPn7bitdRzM,8954
|
|
51
51
|
vtlengine/__extras_check.py,sha256=Wr-lxGZhXJZEacVV5cUkvKt7XM-mry0kYAe3VxNrVcY,614
|
|
52
|
-
vtlengine/__init__.py,sha256=
|
|
52
|
+
vtlengine/__init__.py,sha256=2ylv7tLHXE_OKN9-zlvcdqenn_WPaMklmkjAt-EJ16A,188
|
|
53
53
|
vtlengine/files/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
54
54
|
vtlengine/files/output/__init__.py,sha256=4tmf-p1Y1u5Ohrwt3clQA-FMGaijKI3HC_iwn3H9J8c,1250
|
|
55
55
|
vtlengine/files/output/_time_period_representation.py,sha256=D5XCSXyEuX_aBzTvBV3sZxACcgwXz2Uu_YH3loMP8q0,1610
|
|
56
|
-
vtlengine/files/parser/__init__.py,sha256=
|
|
56
|
+
vtlengine/files/parser/__init__.py,sha256=WdvToMTIeWgkkuUWSDvtACQlguEZN3plz5LPvyCvdt0,9030
|
|
57
57
|
vtlengine/files/parser/_rfc_dialect.py,sha256=Y8kAYBxH_t9AieN_tYg7QRh5A4DgvabKarx9Ko3QeCQ,462
|
|
58
|
-
vtlengine/files/parser/_time_checking.py,sha256
|
|
58
|
+
vtlengine/files/parser/_time_checking.py,sha256=-MsfAJdIHpVwPulaiklxvyPfWAVocwvf43WSqgusryc,4849
|
|
59
59
|
vtlengine/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
60
|
-
vtlengine-1.2.
|
|
61
|
-
vtlengine-1.2.
|
|
62
|
-
vtlengine-1.2.
|
|
63
|
-
vtlengine-1.2.
|
|
60
|
+
vtlengine-1.2.2.dist-info/METADATA,sha256=BzikHNHbu6YDewYFLYeYD5tAkwoEg2sfI2ySJBBcnR8,4165
|
|
61
|
+
vtlengine-1.2.2.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
62
|
+
vtlengine-1.2.2.dist-info/licenses/LICENSE.md,sha256=2xqHuoHohba7gpcZZKtOICRjzeKsQANXG8WoV9V35KM,33893
|
|
63
|
+
vtlengine-1.2.2.dist-info/RECORD,,
|
|
File without changes
|