vtlengine 1.0.3rc3__py3-none-any.whl → 1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of vtlengine might be problematic. Click here for more details.
- vtlengine/API/_InternalApi.py +288 -61
- vtlengine/API/__init__.py +269 -71
- vtlengine/API/data/schema/json_schema_2.1.json +116 -0
- vtlengine/AST/ASTComment.py +56 -0
- vtlengine/AST/ASTConstructor.py +76 -22
- vtlengine/AST/ASTConstructorModules/Expr.py +238 -120
- vtlengine/AST/ASTConstructorModules/ExprComponents.py +126 -61
- vtlengine/AST/ASTConstructorModules/Terminals.py +97 -42
- vtlengine/AST/ASTConstructorModules/__init__.py +50 -0
- vtlengine/AST/ASTEncoders.py +5 -1
- vtlengine/AST/ASTString.py +608 -0
- vtlengine/AST/ASTTemplate.py +28 -2
- vtlengine/AST/DAG/__init__.py +10 -4
- vtlengine/AST/Grammar/lexer.py +0 -1
- vtlengine/AST/Grammar/parser.py +185 -440
- vtlengine/AST/VtlVisitor.py +0 -1
- vtlengine/AST/__init__.py +127 -14
- vtlengine/DataTypes/TimeHandling.py +50 -15
- vtlengine/DataTypes/__init__.py +79 -7
- vtlengine/Exceptions/__init__.py +3 -5
- vtlengine/Exceptions/messages.py +74 -105
- vtlengine/Interpreter/__init__.py +136 -46
- vtlengine/Model/__init__.py +14 -11
- vtlengine/Operators/Aggregation.py +17 -9
- vtlengine/Operators/Analytic.py +64 -20
- vtlengine/Operators/Assignment.py +0 -1
- vtlengine/Operators/CastOperator.py +44 -44
- vtlengine/Operators/Clause.py +16 -10
- vtlengine/Operators/Comparison.py +20 -12
- vtlengine/Operators/Conditional.py +47 -15
- vtlengine/Operators/General.py +9 -4
- vtlengine/Operators/HROperators.py +4 -14
- vtlengine/Operators/Join.py +15 -14
- vtlengine/Operators/Numeric.py +32 -26
- vtlengine/Operators/RoleSetter.py +6 -2
- vtlengine/Operators/Set.py +12 -8
- vtlengine/Operators/String.py +9 -9
- vtlengine/Operators/Time.py +145 -124
- vtlengine/Operators/Validation.py +10 -4
- vtlengine/Operators/__init__.py +56 -69
- vtlengine/Utils/__init__.py +55 -1
- vtlengine/__extras_check.py +17 -0
- vtlengine/__init__.py +2 -2
- vtlengine/files/output/__init__.py +2 -1
- vtlengine/files/output/_time_period_representation.py +2 -1
- vtlengine/files/parser/__init__.py +52 -46
- vtlengine/files/parser/_time_checking.py +4 -4
- {vtlengine-1.0.3rc3.dist-info → vtlengine-1.1.dist-info}/METADATA +21 -17
- vtlengine-1.1.dist-info/RECORD +61 -0
- {vtlengine-1.0.3rc3.dist-info → vtlengine-1.1.dist-info}/WHEEL +1 -1
- vtlengine/DataTypes/NumericTypesHandling.py +0 -38
- vtlengine-1.0.3rc3.dist-info/RECORD +0 -58
- {vtlengine-1.0.3rc3.dist-info → vtlengine-1.1.dist-info}/LICENSE.md +0 -0
|
@@ -39,7 +39,7 @@ class If(Operator):
|
|
|
39
39
|
validate: Class method that has two branches so datacomponent and datasets can be validated. With datacomponent,
|
|
40
40
|
the code reviews if it is actually a Measure and if it is a binary operation. Dataset branch reviews if the
|
|
41
41
|
identifiers are the same in 'if', 'then' and 'else'.
|
|
42
|
-
"""
|
|
42
|
+
""" # noqa E501
|
|
43
43
|
|
|
44
44
|
@classmethod
|
|
45
45
|
def evaluate(cls, condition: Any, true_branch: Any, false_branch: Any) -> Any:
|
|
@@ -66,7 +66,8 @@ class If(Operator):
|
|
|
66
66
|
else:
|
|
67
67
|
false_data = false_branch.data.reindex(condition.data.index)
|
|
68
68
|
result = np.where(condition.data, true_data, false_data)
|
|
69
|
-
|
|
69
|
+
|
|
70
|
+
return pd.Series(result, index=condition.data.index) # type: ignore[union-attr]
|
|
70
71
|
|
|
71
72
|
@classmethod
|
|
72
73
|
def dataset_level_evaluation(
|
|
@@ -80,7 +81,11 @@ class If(Operator):
|
|
|
80
81
|
if isinstance(true_branch, Dataset):
|
|
81
82
|
if len(true_data) > 0 and true_branch.data is not None:
|
|
82
83
|
true_data = pd.merge(
|
|
83
|
-
true_data,
|
|
84
|
+
true_data,
|
|
85
|
+
true_branch.data,
|
|
86
|
+
on=ids,
|
|
87
|
+
how="right",
|
|
88
|
+
suffixes=("_condition", ""),
|
|
84
89
|
)
|
|
85
90
|
else:
|
|
86
91
|
true_data = pd.DataFrame(columns=true_branch.get_components_names())
|
|
@@ -91,7 +96,11 @@ class If(Operator):
|
|
|
91
96
|
if isinstance(false_branch, Dataset):
|
|
92
97
|
if len(false_data) > 0 and false_branch.data is not None:
|
|
93
98
|
false_data = pd.merge(
|
|
94
|
-
false_data,
|
|
99
|
+
false_data,
|
|
100
|
+
false_branch.data,
|
|
101
|
+
on=ids,
|
|
102
|
+
how="right",
|
|
103
|
+
suffixes=("_condition", ""),
|
|
95
104
|
)
|
|
96
105
|
else:
|
|
97
106
|
false_data = pd.DataFrame(columns=false_branch.get_components_names())
|
|
@@ -141,7 +150,9 @@ class If(Operator):
|
|
|
141
150
|
if isinstance(condition, DataComponent):
|
|
142
151
|
if not condition.data_type == Boolean:
|
|
143
152
|
raise SemanticError(
|
|
144
|
-
"1-1-9-11",
|
|
153
|
+
"1-1-9-11",
|
|
154
|
+
op=cls.op,
|
|
155
|
+
type=SCALAR_TYPES_CLASS_REVERSE[condition.data_type],
|
|
145
156
|
)
|
|
146
157
|
if not isinstance(left, Scalar) or not isinstance(right, Scalar):
|
|
147
158
|
nullable = condition.nullable
|
|
@@ -191,7 +202,8 @@ class If(Operator):
|
|
|
191
202
|
if component.data_type != right.components[component.name].data_type:
|
|
192
203
|
component.data_type = right.components[component.name].data_type = (
|
|
193
204
|
binary_implicit_promotion(
|
|
194
|
-
component.data_type,
|
|
205
|
+
component.data_type,
|
|
206
|
+
right.components[component.name].data_type,
|
|
195
207
|
)
|
|
196
208
|
)
|
|
197
209
|
if isinstance(condition, Dataset):
|
|
@@ -219,14 +231,14 @@ class Nvl(Binary):
|
|
|
219
231
|
Validate: Class method that validates if the operation at scalar,
|
|
220
232
|
datacomponent or dataset level can be performed.
|
|
221
233
|
Evaluate: Evaluates the actual operation, returning the result.
|
|
222
|
-
"""
|
|
234
|
+
""" # noqa E501
|
|
223
235
|
|
|
224
236
|
@classmethod
|
|
225
237
|
def evaluate(cls, left: Any, right: Any) -> Union[Scalar, DataComponent, Dataset]:
|
|
226
238
|
result = cls.validate(left, right)
|
|
227
239
|
|
|
228
240
|
if isinstance(left, Scalar) and isinstance(result, Scalar):
|
|
229
|
-
if
|
|
241
|
+
if left.data_type is Null:
|
|
230
242
|
result.value = right.value
|
|
231
243
|
else:
|
|
232
244
|
result.value = left.value
|
|
@@ -290,13 +302,31 @@ class Nvl(Binary):
|
|
|
290
302
|
|
|
291
303
|
|
|
292
304
|
class Case(Operator):
|
|
293
|
-
|
|
294
305
|
@classmethod
|
|
295
306
|
def evaluate(
|
|
296
307
|
cls, conditions: List[Any], thenOps: List[Any], elseOp: Any
|
|
297
308
|
) -> Union[Scalar, DataComponent, Dataset]:
|
|
298
|
-
|
|
299
309
|
result = cls.validate(conditions, thenOps, elseOp)
|
|
310
|
+
for condition in conditions:
|
|
311
|
+
if isinstance(condition, Dataset) and condition.data is not None:
|
|
312
|
+
condition.data.fillna(False, inplace=True)
|
|
313
|
+
condition_measure = condition.get_measures_names()[0]
|
|
314
|
+
if condition.data[condition_measure].dtype != bool:
|
|
315
|
+
condition.data[condition_measure] = condition.data[condition_measure].astype(
|
|
316
|
+
bool
|
|
317
|
+
)
|
|
318
|
+
elif (
|
|
319
|
+
isinstance(
|
|
320
|
+
condition,
|
|
321
|
+
DataComponent,
|
|
322
|
+
)
|
|
323
|
+
and condition.data is not None
|
|
324
|
+
):
|
|
325
|
+
condition.data.fillna(False, inplace=True)
|
|
326
|
+
if condition.data.dtype != bool:
|
|
327
|
+
condition.data = condition.data.astype(bool)
|
|
328
|
+
elif isinstance(condition, Scalar) and condition.value is None:
|
|
329
|
+
condition.value = False
|
|
300
330
|
|
|
301
331
|
if isinstance(result, Scalar):
|
|
302
332
|
result.value = elseOp.value
|
|
@@ -309,8 +339,10 @@ class Case(Operator):
|
|
|
309
339
|
|
|
310
340
|
for i, condition in enumerate(conditions):
|
|
311
341
|
value = thenOps[i].value if isinstance(thenOps[i], Scalar) else thenOps[i].data
|
|
312
|
-
result.data = np.where(
|
|
313
|
-
condition.data,
|
|
342
|
+
result.data = np.where( # type: ignore[call-overload]
|
|
343
|
+
condition.data.notna(),
|
|
344
|
+
np.where(condition.data, value, result.data), # type: ignore[call-overload]
|
|
345
|
+
result.data,
|
|
314
346
|
)
|
|
315
347
|
|
|
316
348
|
condition_mask_else = ~np.any([condition.data for condition in conditions], axis=0)
|
|
@@ -349,7 +381,7 @@ class Case(Operator):
|
|
|
349
381
|
]
|
|
350
382
|
)
|
|
351
383
|
|
|
352
|
-
result.data.loc[condition_mask_else, columns] = (
|
|
384
|
+
result.data.loc[condition_mask_else, columns] = ( # type: ignore[index, unused-ignore]
|
|
353
385
|
elseOp.value
|
|
354
386
|
if isinstance(elseOp, Scalar)
|
|
355
387
|
else elseOp.data.loc[condition_mask_else, columns]
|
|
@@ -361,7 +393,6 @@ class Case(Operator):
|
|
|
361
393
|
def validate(
|
|
362
394
|
cls, conditions: List[Any], thenOps: List[Any], elseOp: Any
|
|
363
395
|
) -> Union[Scalar, DataComponent, Dataset]:
|
|
364
|
-
|
|
365
396
|
if len(set(map(type, conditions))) > 1:
|
|
366
397
|
raise SemanticError("2-1-9-1", op=cls.op)
|
|
367
398
|
|
|
@@ -395,9 +426,10 @@ class Case(Operator):
|
|
|
395
426
|
raise SemanticError("2-1-9-4", op=cls.op, name=condition.name)
|
|
396
427
|
|
|
397
428
|
nullable = any(
|
|
398
|
-
thenOp.nullable if isinstance(thenOp, DataComponent) else thenOp.data_type == Null
|
|
429
|
+
(thenOp.nullable if isinstance(thenOp, DataComponent) else thenOp.data_type == Null)
|
|
399
430
|
for thenOp in ops
|
|
400
431
|
)
|
|
432
|
+
nullable |= any(condition.nullable for condition in conditions)
|
|
401
433
|
|
|
402
434
|
data_type = ops[0].data_type
|
|
403
435
|
for op in ops[1:]:
|
vtlengine/Operators/General.py
CHANGED
|
@@ -23,7 +23,10 @@ class Membership(Binary):
|
|
|
23
23
|
def validate(cls, left_operand: Any, right_operand: Any) -> Dataset:
|
|
24
24
|
if right_operand not in left_operand.components:
|
|
25
25
|
raise SemanticError(
|
|
26
|
-
"1-1-1-10",
|
|
26
|
+
"1-1-1-10",
|
|
27
|
+
op=cls.op,
|
|
28
|
+
comp_name=right_operand,
|
|
29
|
+
dataset_name=left_operand.name,
|
|
27
30
|
)
|
|
28
31
|
|
|
29
32
|
component = left_operand.components[right_operand]
|
|
@@ -48,7 +51,10 @@ class Membership(Binary):
|
|
|
48
51
|
|
|
49
52
|
@classmethod
|
|
50
53
|
def evaluate(
|
|
51
|
-
cls,
|
|
54
|
+
cls,
|
|
55
|
+
left_operand: Dataset,
|
|
56
|
+
right_operand: str,
|
|
57
|
+
is_from_component_assignment: bool = False,
|
|
52
58
|
) -> Union[DataComponent, Dataset]:
|
|
53
59
|
result_dataset = cls.validate(left_operand, right_operand)
|
|
54
60
|
if left_operand.data is not None:
|
|
@@ -128,12 +134,11 @@ class Eval(Unary):
|
|
|
128
134
|
external_routine: ExternalRoutine,
|
|
129
135
|
output: Dataset,
|
|
130
136
|
) -> Dataset:
|
|
131
|
-
|
|
132
137
|
empty_data_dict = {}
|
|
133
138
|
for ds_name in external_routine.dataset_names:
|
|
134
139
|
if ds_name not in operands:
|
|
135
140
|
raise ValueError(
|
|
136
|
-
f"External Routine dataset {ds_name}
|
|
141
|
+
f"External Routine dataset {ds_name} is not present in Eval operands"
|
|
137
142
|
)
|
|
138
143
|
empty_data = pd.DataFrame(
|
|
139
144
|
columns=[comp.name for comp in operands[ds_name].components.values()]
|
|
@@ -24,12 +24,9 @@ def get_measure_from_dataset(dataset: Dataset, code_item: str) -> DataComponent:
|
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
class HRComparison(Operators.Binary):
|
|
27
|
-
|
|
28
27
|
@classmethod
|
|
29
28
|
def imbalance_func(cls, x: Any, y: Any) -> Any:
|
|
30
|
-
if pd.isnull(x) or pd.isnull(y)
|
|
31
|
-
return None
|
|
32
|
-
return x - y
|
|
29
|
+
return None if pd.isnull(x) or pd.isnull(y) else x - y
|
|
33
30
|
|
|
34
31
|
@staticmethod
|
|
35
32
|
def hr_func(left_series: Any, right_series: Any, hr_mode: str) -> Any:
|
|
@@ -44,8 +41,8 @@ class HRComparison(Operators.Binary):
|
|
|
44
41
|
result[mask_remove] = "REMOVE_VALUE"
|
|
45
42
|
result[mask_null] = None
|
|
46
43
|
elif hr_mode == "non_null":
|
|
47
|
-
|
|
48
|
-
|
|
44
|
+
mask_remove = left_series.isnull() | right_series.isnull()
|
|
45
|
+
result[mask_remove] = "REMOVE_VALUE"
|
|
49
46
|
elif hr_mode == "non_zero":
|
|
50
47
|
mask_remove = (left_series == 0) & (right_series == 0)
|
|
51
48
|
result[mask_remove] = "REMOVE_VALUE"
|
|
@@ -66,11 +63,7 @@ class HRComparison(Operators.Binary):
|
|
|
66
63
|
return result
|
|
67
64
|
|
|
68
65
|
@classmethod
|
|
69
|
-
def validate(cls,
|
|
70
|
-
left_operand: Dataset,
|
|
71
|
-
right_operand: DataComponent,
|
|
72
|
-
hr_mode: str
|
|
73
|
-
) -> Dataset:
|
|
66
|
+
def validate(cls, left_operand: Dataset, right_operand: DataComponent, hr_mode: str) -> Dataset:
|
|
74
67
|
result_components = {
|
|
75
68
|
comp_name: copy(comp)
|
|
76
69
|
for comp_name, comp in left_operand.components.items()
|
|
@@ -136,7 +129,6 @@ class HRLessEqual(HRComparison):
|
|
|
136
129
|
|
|
137
130
|
|
|
138
131
|
class HRBinNumeric(Operators.Binary):
|
|
139
|
-
|
|
140
132
|
@classmethod
|
|
141
133
|
def op_func(cls, x: Any, y: Any) -> Any:
|
|
142
134
|
if not pd.isnull(x) and x == "REMOVE_VALUE":
|
|
@@ -166,7 +158,6 @@ class HRBinMinus(HRBinNumeric):
|
|
|
166
158
|
|
|
167
159
|
|
|
168
160
|
class HRUnNumeric(Operators.Unary):
|
|
169
|
-
|
|
170
161
|
@classmethod
|
|
171
162
|
def evaluate(cls, operand: DataComponent) -> DataComponent: # type: ignore[override]
|
|
172
163
|
result_data = cls.apply_operation_component(operand.data)
|
|
@@ -190,7 +181,6 @@ class HRUnMinus(HRUnNumeric):
|
|
|
190
181
|
|
|
191
182
|
|
|
192
183
|
class HAAssignment(Operators.Binary):
|
|
193
|
-
|
|
194
184
|
@classmethod
|
|
195
185
|
def validate(cls, left: Dataset, right: DataComponent, hr_mode: str) -> Dataset:
|
|
196
186
|
result_components = {comp_name: copy(comp) for comp_name, comp in left.components.items()}
|
vtlengine/Operators/Join.py
CHANGED
|
@@ -70,7 +70,9 @@ class Join(Operator):
|
|
|
70
70
|
comp.role = (
|
|
71
71
|
Role.IDENTIFIER
|
|
72
72
|
if is_identifier
|
|
73
|
-
else Role.MEASURE
|
|
73
|
+
else Role.MEASURE
|
|
74
|
+
if comp.role == Role.IDENTIFIER
|
|
75
|
+
else comp.role
|
|
74
76
|
)
|
|
75
77
|
if comp.name not in nullability:
|
|
76
78
|
nullability[comp.name] = copy(comp.nullable)
|
|
@@ -107,7 +109,8 @@ class Join(Operator):
|
|
|
107
109
|
else:
|
|
108
110
|
if component_name in using and component_name in merged_components:
|
|
109
111
|
data_type = binary_implicit_promotion(
|
|
110
|
-
merged_components[component_name].data_type,
|
|
112
|
+
merged_components[component_name].data_type,
|
|
113
|
+
component.data_type,
|
|
111
114
|
)
|
|
112
115
|
component.data_type = data_type
|
|
113
116
|
merged_components[component_name] = component
|
|
@@ -216,7 +219,6 @@ class Join(Operator):
|
|
|
216
219
|
|
|
217
220
|
@classmethod
|
|
218
221
|
def identifiers_validation(cls, operands: List[Dataset], using: Optional[List[str]]) -> None:
|
|
219
|
-
|
|
220
222
|
# (Case A)
|
|
221
223
|
info = {op.name: op.get_identifiers_names() for op in operands}
|
|
222
224
|
for op_name, identifiers in info.items():
|
|
@@ -224,11 +226,12 @@ class Join(Operator):
|
|
|
224
226
|
raise SemanticError("1-1-13-14", op=cls.op, name=op_name)
|
|
225
227
|
|
|
226
228
|
for op_name, identifiers in info.items():
|
|
227
|
-
if (
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
229
|
+
if (
|
|
230
|
+
using is None
|
|
231
|
+
and op_name != cls.reference_dataset.name
|
|
232
|
+
and not set(identifiers).issubset(set(info[cls.reference_dataset.name]))
|
|
233
|
+
):
|
|
234
|
+
missing_components = list(set(identifiers) - set(info[cls.reference_dataset.name]))
|
|
232
235
|
raise SemanticError(
|
|
233
236
|
"1-1-13-11",
|
|
234
237
|
op=cls.op,
|
|
@@ -277,7 +280,6 @@ class InnerJoin(Join):
|
|
|
277
280
|
def generate_result_components(
|
|
278
281
|
cls, operands: List[Dataset], using: Optional[List[str]] = None
|
|
279
282
|
) -> Dict[str, Component]:
|
|
280
|
-
|
|
281
283
|
if using is None:
|
|
282
284
|
return super().generate_result_components(operands, using)
|
|
283
285
|
|
|
@@ -334,7 +336,9 @@ class CrossJoin(Join):
|
|
|
334
336
|
else:
|
|
335
337
|
if result.data is not None:
|
|
336
338
|
result.data = pd.merge(
|
|
337
|
-
result.data,
|
|
339
|
+
result.data,
|
|
340
|
+
op.data,
|
|
341
|
+
how=cls.how, # type: ignore[arg-type]
|
|
338
342
|
)
|
|
339
343
|
if result.data is not None:
|
|
340
344
|
result.data = result.data.rename(
|
|
@@ -357,7 +361,6 @@ class CrossJoin(Join):
|
|
|
357
361
|
|
|
358
362
|
|
|
359
363
|
class Apply(Operator):
|
|
360
|
-
|
|
361
364
|
@classmethod
|
|
362
365
|
def evaluate(cls, dataset: Dataset, expression: Any, op_map: Dict[str, Any]) -> Dataset:
|
|
363
366
|
for child in expression:
|
|
@@ -424,9 +427,7 @@ class Apply(Operator):
|
|
|
424
427
|
return Dataset(name=name, components=components, data=data)
|
|
425
428
|
|
|
426
429
|
@classmethod
|
|
427
|
-
def get_common_components(
|
|
428
|
-
cls, left: Dataset, right: Dataset
|
|
429
|
-
) -> (Dataset, Dataset): # type: ignore[syntax]
|
|
430
|
+
def get_common_components(cls, left: Dataset, right: Dataset) -> (Dataset, Dataset): # type: ignore[syntax]
|
|
430
431
|
common = set(left.get_components_names()) & set(right.get_components_names())
|
|
431
432
|
left.components = {
|
|
432
433
|
comp.name: comp for comp in left.components.values() if comp.name in common
|
vtlengine/Operators/Numeric.py
CHANGED
|
@@ -73,7 +73,7 @@ class Binary(Operator.Binary):
|
|
|
73
73
|
class UnPlus(Unary):
|
|
74
74
|
"""
|
|
75
75
|
`Plus <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=94&zoom=100,72,142> `_ unary operator
|
|
76
|
-
"""
|
|
76
|
+
""" # noqa E501
|
|
77
77
|
|
|
78
78
|
op = PLUS
|
|
79
79
|
py_op = operator.pos
|
|
@@ -86,7 +86,7 @@ class UnPlus(Unary):
|
|
|
86
86
|
class UnMinus(Unary):
|
|
87
87
|
"""
|
|
88
88
|
`Minus <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=95&zoom=100,72,414> `_unary operator
|
|
89
|
-
"""
|
|
89
|
+
""" # noqa E501
|
|
90
90
|
|
|
91
91
|
op = MINUS
|
|
92
92
|
py_op = operator.neg
|
|
@@ -95,7 +95,7 @@ class UnMinus(Unary):
|
|
|
95
95
|
class AbsoluteValue(Unary):
|
|
96
96
|
"""
|
|
97
97
|
`Absolute <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=112&zoom=100,72,801> `_ unary operator
|
|
98
|
-
"""
|
|
98
|
+
""" # noqa E501
|
|
99
99
|
|
|
100
100
|
op = ABS
|
|
101
101
|
py_op = operator.abs
|
|
@@ -104,7 +104,7 @@ class AbsoluteValue(Unary):
|
|
|
104
104
|
class Exponential(Unary):
|
|
105
105
|
"""
|
|
106
106
|
`Exponential <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=114&zoom=100,72,94>`_ unary operator
|
|
107
|
-
"""
|
|
107
|
+
""" # noqa E501
|
|
108
108
|
|
|
109
109
|
op = EXP
|
|
110
110
|
py_op = math.exp
|
|
@@ -115,7 +115,7 @@ class NaturalLogarithm(Unary):
|
|
|
115
115
|
"""
|
|
116
116
|
`Natural logarithm <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=115&zoom=100,72,394> `_
|
|
117
117
|
unary operator
|
|
118
|
-
"""
|
|
118
|
+
""" # noqa E501
|
|
119
119
|
|
|
120
120
|
op = LN
|
|
121
121
|
py_op = math.log
|
|
@@ -126,7 +126,7 @@ class SquareRoot(Unary):
|
|
|
126
126
|
"""
|
|
127
127
|
`Square Root <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=119&zoom=100,72,556> '_
|
|
128
128
|
unary operator
|
|
129
|
-
"""
|
|
129
|
+
""" # noqa E501
|
|
130
130
|
|
|
131
131
|
op = SQRT
|
|
132
132
|
py_op = math.sqrt
|
|
@@ -136,7 +136,7 @@ class SquareRoot(Unary):
|
|
|
136
136
|
class Ceil(Unary):
|
|
137
137
|
"""
|
|
138
138
|
`Ceilling <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=110&zoom=100,72,94> `_ unary operator
|
|
139
|
-
"""
|
|
139
|
+
""" # noqa E501
|
|
140
140
|
|
|
141
141
|
op = CEIL
|
|
142
142
|
py_op = math.ceil
|
|
@@ -146,7 +146,7 @@ class Ceil(Unary):
|
|
|
146
146
|
class Floor(Unary):
|
|
147
147
|
"""
|
|
148
148
|
`Floor <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=111&zoom=100,72,442> `_ unary operator
|
|
149
|
-
"""
|
|
149
|
+
""" # noqa E501
|
|
150
150
|
|
|
151
151
|
op = FLOOR
|
|
152
152
|
py_op = math.floor
|
|
@@ -156,7 +156,7 @@ class Floor(Unary):
|
|
|
156
156
|
class BinPlus(Binary):
|
|
157
157
|
"""
|
|
158
158
|
`Addition <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=96&zoom=100,72,692> `_ binary operator
|
|
159
|
-
"""
|
|
159
|
+
""" # noqa E501
|
|
160
160
|
|
|
161
161
|
op = PLUS
|
|
162
162
|
py_op = operator.add
|
|
@@ -166,7 +166,7 @@ class BinPlus(Binary):
|
|
|
166
166
|
class BinMinus(Binary):
|
|
167
167
|
"""
|
|
168
168
|
`Subtraction <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=98&zoom=100,72,448> `_ binary operator
|
|
169
|
-
"""
|
|
169
|
+
""" # noqa E501
|
|
170
170
|
|
|
171
171
|
op = MINUS
|
|
172
172
|
py_op = operator.sub
|
|
@@ -177,7 +177,7 @@ class Mult(Binary):
|
|
|
177
177
|
"""
|
|
178
178
|
`Multiplication <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=100&zoom=100,72,254>`_
|
|
179
179
|
binary operator
|
|
180
|
-
"""
|
|
180
|
+
""" # noqa E501
|
|
181
181
|
|
|
182
182
|
op = MULT
|
|
183
183
|
py_op = operator.mul
|
|
@@ -187,7 +187,7 @@ class Div(Binary):
|
|
|
187
187
|
"""
|
|
188
188
|
`Division <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=102&zoom=100,72,94>`_
|
|
189
189
|
binary operator
|
|
190
|
-
"""
|
|
190
|
+
""" # noqa E501
|
|
191
191
|
|
|
192
192
|
op = DIV
|
|
193
193
|
py_op = operator.truediv
|
|
@@ -197,7 +197,7 @@ class Div(Binary):
|
|
|
197
197
|
class Logarithm(Binary):
|
|
198
198
|
"""
|
|
199
199
|
`Logarithm <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=118&zoom=100,72,228>`_ operator
|
|
200
|
-
"""
|
|
200
|
+
""" # noqa E501
|
|
201
201
|
|
|
202
202
|
op = LOG
|
|
203
203
|
return_type = Number
|
|
@@ -215,7 +215,7 @@ class Logarithm(Binary):
|
|
|
215
215
|
class Modulo(Binary):
|
|
216
216
|
"""
|
|
217
217
|
`Module <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=104&zoom=100,72,94>`_ operator
|
|
218
|
-
"""
|
|
218
|
+
""" # noqa E501
|
|
219
219
|
|
|
220
220
|
op = MOD
|
|
221
221
|
py_op = operator.mod
|
|
@@ -224,7 +224,7 @@ class Modulo(Binary):
|
|
|
224
224
|
class Power(Binary):
|
|
225
225
|
"""
|
|
226
226
|
`Power <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=116&zoom=100,72,693>`_ operator
|
|
227
|
-
"""
|
|
227
|
+
""" # noqa E501
|
|
228
228
|
|
|
229
229
|
op = POWER
|
|
230
230
|
return_type = Number
|
|
@@ -248,14 +248,15 @@ class Parameterized(Unary):
|
|
|
248
248
|
operand: Operator.ALL_MODEL_DATA_TYPES,
|
|
249
249
|
param: Optional[Union[DataComponent, Scalar]] = None,
|
|
250
250
|
) -> Any:
|
|
251
|
-
|
|
252
251
|
if param is not None:
|
|
253
252
|
if isinstance(param, Dataset):
|
|
254
253
|
raise SemanticError("1-1-15-8", op=cls.op, comp_type="Dataset")
|
|
255
254
|
if isinstance(param, DataComponent):
|
|
256
255
|
if isinstance(operand, Scalar):
|
|
257
256
|
raise SemanticError(
|
|
258
|
-
"1-1-15-8",
|
|
257
|
+
"1-1-15-8",
|
|
258
|
+
op=cls.op,
|
|
259
|
+
comp_type="DataComponent and an Scalar operand",
|
|
259
260
|
)
|
|
260
261
|
cls.validate_type_compatibility(param.data_type)
|
|
261
262
|
else:
|
|
@@ -298,14 +299,19 @@ class Parameterized(Unary):
|
|
|
298
299
|
)
|
|
299
300
|
except ValueError:
|
|
300
301
|
raise SemanticError(
|
|
301
|
-
"2-1-15-1",
|
|
302
|
+
"2-1-15-1",
|
|
303
|
+
op=cls.op,
|
|
304
|
+
comp_name=measure_name,
|
|
305
|
+
dataset_name=operand.name,
|
|
302
306
|
) from None
|
|
303
307
|
result.data = result.data[result.get_components_names()]
|
|
304
308
|
return result
|
|
305
309
|
|
|
306
310
|
@classmethod
|
|
307
311
|
def component_evaluation(
|
|
308
|
-
cls,
|
|
312
|
+
cls,
|
|
313
|
+
operand: DataComponent,
|
|
314
|
+
param: Optional[Union[DataComponent, Scalar]] = None,
|
|
309
315
|
) -> DataComponent:
|
|
310
316
|
result = cls.validate(operand, param)
|
|
311
317
|
if operand.data is None:
|
|
@@ -327,7 +333,9 @@ class Parameterized(Unary):
|
|
|
327
333
|
|
|
328
334
|
@classmethod
|
|
329
335
|
def evaluate(
|
|
330
|
-
cls,
|
|
336
|
+
cls,
|
|
337
|
+
operand: ALL_MODEL_DATA_TYPES,
|
|
338
|
+
param: Optional[Union[DataComponent, Scalar]] = None,
|
|
331
339
|
) -> Union[DataComponent, Dataset, Scalar]:
|
|
332
340
|
if isinstance(operand, Dataset):
|
|
333
341
|
return cls.dataset_evaluation(operand, param)
|
|
@@ -340,7 +348,7 @@ class Parameterized(Unary):
|
|
|
340
348
|
class Round(Parameterized):
|
|
341
349
|
"""
|
|
342
350
|
`Round <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=106&zoom=100,72,94>`_ operator
|
|
343
|
-
"""
|
|
351
|
+
""" # noqa E501
|
|
344
352
|
|
|
345
353
|
op = ROUND
|
|
346
354
|
return_type = Integer
|
|
@@ -365,14 +373,14 @@ class Round(Parameterized):
|
|
|
365
373
|
class Trunc(Parameterized):
|
|
366
374
|
"""
|
|
367
375
|
`Trunc <https://sdmx.org/wp-content/uploads/VTL-2.1-Reference-Manual.pdf#page=108&zoom=100,72,94>`_ operator.
|
|
368
|
-
"""
|
|
376
|
+
""" # noqa E501
|
|
369
377
|
|
|
370
378
|
op = TRUNC
|
|
371
379
|
|
|
372
380
|
@classmethod
|
|
373
381
|
def py_op(cls, x: float, param: Optional[float]) -> Any:
|
|
374
382
|
multiplier = 1.0
|
|
375
|
-
if not pd.isnull(param):
|
|
383
|
+
if not pd.isnull(param) and param is not None:
|
|
376
384
|
multiplier = 10**param
|
|
377
385
|
|
|
378
386
|
truncated_value = int(x * multiplier) / multiplier
|
|
@@ -384,14 +392,12 @@ class Trunc(Parameterized):
|
|
|
384
392
|
|
|
385
393
|
|
|
386
394
|
class PseudoRandom(_random.Random):
|
|
387
|
-
|
|
388
395
|
def __init__(self, seed: Union[int, float]) -> None:
|
|
389
396
|
super().__init__()
|
|
390
397
|
self.seed(seed)
|
|
391
398
|
|
|
392
399
|
|
|
393
400
|
class Random(Parameterized):
|
|
394
|
-
|
|
395
401
|
op = RANDOM
|
|
396
402
|
return_type = Number
|
|
397
403
|
|
|
@@ -403,7 +409,7 @@ class Random(Parameterized):
|
|
|
403
409
|
raise SemanticError("2-1-15-2", op=cls.op, value=index)
|
|
404
410
|
if index.value > 10000:
|
|
405
411
|
warnings.warn(
|
|
406
|
-
"Random: The value of 'index' is very big. This can affect
|
|
412
|
+
"Random: The value of 'index' is very big. This can affect performance.",
|
|
407
413
|
UserWarning,
|
|
408
414
|
)
|
|
409
415
|
return super().validate(seed, index)
|
|
@@ -35,8 +35,12 @@ class RoleSetter(Unary):
|
|
|
35
35
|
|
|
36
36
|
@classmethod
|
|
37
37
|
def evaluate(cls, operand: Any, data_size: int = 0) -> DataComponent:
|
|
38
|
-
if (
|
|
39
|
-
|
|
38
|
+
if (
|
|
39
|
+
isinstance(operand, DataComponent)
|
|
40
|
+
and operand.data is not None
|
|
41
|
+
and not operand.nullable
|
|
42
|
+
and any(operand.data.isnull())
|
|
43
|
+
):
|
|
40
44
|
raise SemanticError("1-1-1-16")
|
|
41
45
|
result = cls.validate(operand, data_size)
|
|
42
46
|
if isinstance(operand, Scalar):
|
vtlengine/Operators/Set.py
CHANGED
|
@@ -13,12 +13,14 @@ from vtlengine.Operators import Operator
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class Set(Operator):
|
|
16
|
-
|
|
17
16
|
@classmethod
|
|
18
17
|
def check_same_structure(cls, dataset_1: Dataset, dataset_2: Dataset) -> None:
|
|
19
18
|
if len(dataset_1.components) != len(dataset_2.components):
|
|
20
19
|
raise SemanticError(
|
|
21
|
-
"1-1-17-1",
|
|
20
|
+
"1-1-17-1",
|
|
21
|
+
op=cls.op,
|
|
22
|
+
dataset_1=dataset_1.name,
|
|
23
|
+
dataset_2=dataset_2.name,
|
|
22
24
|
)
|
|
23
25
|
|
|
24
26
|
for comp in dataset_1.components.values():
|
|
@@ -26,7 +28,10 @@ class Set(Operator):
|
|
|
26
28
|
raise Exception(f"Component {comp.name} not found in dataset {dataset_2.name}")
|
|
27
29
|
second_comp = dataset_2.components[comp.name]
|
|
28
30
|
binary_implicit_promotion(
|
|
29
|
-
comp.data_type,
|
|
31
|
+
comp.data_type,
|
|
32
|
+
second_comp.data_type,
|
|
33
|
+
cls.type_to_check,
|
|
34
|
+
cls.return_type,
|
|
30
35
|
)
|
|
31
36
|
if comp.role != second_comp.role:
|
|
32
37
|
raise Exception(
|
|
@@ -36,7 +41,6 @@ class Set(Operator):
|
|
|
36
41
|
|
|
37
42
|
@classmethod
|
|
38
43
|
def validate(cls, operands: List[Dataset]) -> Dataset:
|
|
39
|
-
|
|
40
44
|
base_operand = operands[0]
|
|
41
45
|
for operand in operands[1:]:
|
|
42
46
|
cls.check_same_structure(base_operand, operand)
|
|
@@ -70,7 +74,6 @@ class Union(Set):
|
|
|
70
74
|
|
|
71
75
|
|
|
72
76
|
class Intersection(Set):
|
|
73
|
-
|
|
74
77
|
@classmethod
|
|
75
78
|
def evaluate(cls, operands: List[Dataset]) -> Dataset:
|
|
76
79
|
result = cls.validate(operands)
|
|
@@ -97,7 +100,6 @@ class Intersection(Set):
|
|
|
97
100
|
|
|
98
101
|
|
|
99
102
|
class Symdiff(Set):
|
|
100
|
-
|
|
101
103
|
@classmethod
|
|
102
104
|
def evaluate(cls, operands: List[Dataset]) -> Dataset:
|
|
103
105
|
result = cls.validate(operands)
|
|
@@ -110,7 +112,10 @@ class Symdiff(Set):
|
|
|
110
112
|
else:
|
|
111
113
|
# Realiza la operación equivalente en pyspark.pandas
|
|
112
114
|
result.data = result.data.merge(
|
|
113
|
-
data,
|
|
115
|
+
data,
|
|
116
|
+
how="outer",
|
|
117
|
+
on=result.get_identifiers_names(),
|
|
118
|
+
suffixes=("_x", "_y"),
|
|
114
119
|
)
|
|
115
120
|
|
|
116
121
|
for measure in result.get_measures_names():
|
|
@@ -140,7 +145,6 @@ class Symdiff(Set):
|
|
|
140
145
|
|
|
141
146
|
|
|
142
147
|
class Setdiff(Set):
|
|
143
|
-
|
|
144
148
|
@staticmethod
|
|
145
149
|
def has_null(row: Any) -> bool:
|
|
146
150
|
return row.isnull().any()
|