deltafi 2.0rc1720728217472__py3-none-any.whl → 2.0rc1720817063181__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltafi/action.py +21 -123
- deltafi/actiontype.py +2 -6
- deltafi/domain.py +66 -62
- deltafi/exception.py +0 -10
- deltafi/input.py +1 -165
- deltafi/plugin.py +9 -17
- deltafi/result.py +32 -245
- deltafi/test_kit/framework.py +69 -66
- deltafi/test_kit/transform.py +41 -13
- {deltafi-2.0rc1720728217472.dist-info → deltafi-2.0rc1720817063181.dist-info}/METADATA +5 -5
- deltafi-2.0rc1720817063181.dist-info/RECORD +23 -0
- deltafi/test_kit/domain.py +0 -59
- deltafi/test_kit/enrich.py +0 -70
- deltafi/test_kit/format.py +0 -105
- deltafi/test_kit/load.py +0 -128
- deltafi/test_kit/validate.py +0 -54
- deltafi-2.0rc1720728217472.dist-info/RECORD +0 -28
- {deltafi-2.0rc1720728217472.dist-info → deltafi-2.0rc1720817063181.dist-info}/WHEEL +0 -0
deltafi/test_kit/transform.py
CHANGED
|
@@ -18,24 +18,34 @@
|
|
|
18
18
|
|
|
19
19
|
from typing import List
|
|
20
20
|
|
|
21
|
-
from deltafi.result import TransformResult
|
|
21
|
+
from deltafi.result import TransformResult, TransformResults
|
|
22
22
|
|
|
23
23
|
from .assertions import *
|
|
24
|
-
from .framework import TestCaseBase, ActionTest
|
|
24
|
+
from .framework import TestCaseBase, ActionTest, IOContent
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class TransformTestCase(TestCaseBase):
|
|
28
28
|
def __init__(self, fields: Dict):
|
|
29
29
|
super().__init__(fields)
|
|
30
|
-
self.
|
|
31
|
-
self.delete_metadata_keys = []
|
|
32
|
-
self.annotations = {}
|
|
30
|
+
self.results = []
|
|
33
31
|
|
|
34
|
-
def expect_transform_result(self
|
|
32
|
+
def expect_transform_result(self):
|
|
35
33
|
self.expected_result_type = TransformResult
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
self.
|
|
34
|
+
|
|
35
|
+
def expect_transform_results(self):
|
|
36
|
+
self.expected_result_type = TransformResults
|
|
37
|
+
|
|
38
|
+
def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
|
|
39
|
+
annotations: Dict, name: str = None):
|
|
40
|
+
self.results.append(
|
|
41
|
+
{
|
|
42
|
+
'content': content,
|
|
43
|
+
'metadata': metadata,
|
|
44
|
+
'delete_metadata_keys': delete_metadata_keys,
|
|
45
|
+
'annotations': annotations,
|
|
46
|
+
'name': name
|
|
47
|
+
}
|
|
48
|
+
)
|
|
39
49
|
|
|
40
50
|
|
|
41
51
|
class TransformActionTest(ActionTest):
|
|
@@ -50,6 +60,8 @@ class TransformActionTest(ActionTest):
|
|
|
50
60
|
def transform(self, test_case: TransformTestCase):
|
|
51
61
|
if test_case.expected_result_type == TransformResult:
|
|
52
62
|
self.expect_transform_result(test_case)
|
|
63
|
+
elif test_case.expected_result_type == TransformResults:
|
|
64
|
+
self.expect_transform_results(test_case)
|
|
53
65
|
else:
|
|
54
66
|
super().execute(test_case)
|
|
55
67
|
|
|
@@ -57,19 +69,35 @@ class TransformActionTest(ActionTest):
|
|
|
57
69
|
result = super().run_and_check_result_type(test_case, TransformResult)
|
|
58
70
|
self.assert_transform_result(test_case, result)
|
|
59
71
|
|
|
72
|
+
def expect_transform_results(self, test_case: TransformTestCase):
|
|
73
|
+
result = super().run_and_check_result_type(test_case, TransformResults)
|
|
74
|
+
self.assert_transform_results(test_case, result)
|
|
75
|
+
|
|
76
|
+
def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
|
|
77
|
+
assert_equal_len(test_case.results, result.named_results)
|
|
78
|
+
for index, named_result in enumerate(result.named_results):
|
|
79
|
+
self.compare_one_transform_result(test_case, named_result.result, index)
|
|
80
|
+
expected = test_case.results[index]
|
|
81
|
+
if 'name' in expected:
|
|
82
|
+
assert_equal_with_label(expected["name"], named_result.name, f"name[{index}]")
|
|
83
|
+
|
|
60
84
|
def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
|
|
61
85
|
# Check metrics
|
|
62
86
|
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
87
|
+
self.compare_one_transform_result(test_case, result, 0)
|
|
88
|
+
|
|
89
|
+
def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
|
|
90
|
+
expected = test_case.results[index]
|
|
63
91
|
|
|
64
92
|
# Check output
|
|
65
|
-
self.
|
|
93
|
+
self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
|
|
66
94
|
|
|
67
95
|
# Check metadata
|
|
68
|
-
assert_keys_and_values(
|
|
96
|
+
assert_keys_and_values(expected['metadata'], result.metadata)
|
|
69
97
|
|
|
70
98
|
# Check deleted metadata
|
|
71
|
-
for key in
|
|
99
|
+
for key in expected['delete_metadata_keys']:
|
|
72
100
|
assert_key_in(key, result.delete_metadata_keys)
|
|
73
101
|
|
|
74
102
|
# Check annotations
|
|
75
|
-
assert_keys_and_values(
|
|
103
|
+
assert_keys_and_values(expected['annotations'], result.annotations)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: deltafi
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.0rc1720817063181
|
|
4
4
|
Summary: SDK for DeltaFi plugins and actions
|
|
5
5
|
License: Apache License, Version 2.0
|
|
6
6
|
Keywords: deltafi
|
|
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
20
20
|
Classifier: Topic :: Software Development
|
|
21
21
|
Requires-Dist: deepdiff (>=6.7.1)
|
|
22
22
|
Requires-Dist: json-logging (>=1.3.0)
|
|
23
|
-
Requires-Dist: minio (>=7.2.
|
|
24
|
-
Requires-Dist: pydantic (>=2.
|
|
25
|
-
Requires-Dist: redis (>=5.0.
|
|
23
|
+
Requires-Dist: minio (>=7.2.5)
|
|
24
|
+
Requires-Dist: pydantic (>=2.7.1)
|
|
25
|
+
Requires-Dist: redis (>=5.0.4)
|
|
26
26
|
Requires-Dist: requests (>=2.31.0)
|
|
27
|
-
Requires-Dist: urllib3 (>=2.1
|
|
27
|
+
Requires-Dist: urllib3 (>=2.2.1)
|
|
28
28
|
Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
|
|
29
29
|
Project-URL: Documentation, https://docs.deltafi.org/#/
|
|
30
30
|
Project-URL: Source Code, https://gitlab.com/deltafi/deltafi
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
deltafi/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
|
|
2
|
+
deltafi/action.py,sha256=KmQgYhKey37EoKLa8LOHQxHItppMeGIKYLXPIwQIUBk,5328
|
|
3
|
+
deltafi/actioneventqueue.py,sha256=mCRE1PFXy_KvaeezTeMze5N400CO1V06zEF8FFD6xZk,2847
|
|
4
|
+
deltafi/actiontype.py,sha256=V7JbTgHQIaWQ2N36pG-XV7kAXhq8qvj954z71iHw_wI,913
|
|
5
|
+
deltafi/domain.py,sha256=98295Cpm-y6BlbI2b7LvHj4OPD6LXOw1bqK9E8Pm5no,12030
|
|
6
|
+
deltafi/exception.py,sha256=Z8LwxxTiTyuG3v7FAsjfz6cAqRAJFKK2q_2TbEDrIJE,943
|
|
7
|
+
deltafi/genericmodel.py,sha256=8VXWgFMjIwG4o3rkcLOa3bQH0Nf-T1Kw621QtMw-4q0,1090
|
|
8
|
+
deltafi/input.py,sha256=2qQcOWqz7lOXZIZipTz5XtcGESpjdnUAsZ7b9YqHw7M,1656
|
|
9
|
+
deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
|
|
10
|
+
deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
|
|
11
|
+
deltafi/plugin.py,sha256=k2VO_5OJUqTGn6yN00KY-DoOv6Jhe2QoMpy9hr9vUi4,12407
|
|
12
|
+
deltafi/result.py,sha256=AMjGygjvix9PdUbQvSa_gZEkTn9BJh2p-BdirqdjJpM,8212
|
|
13
|
+
deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
|
|
14
|
+
deltafi/test_kit/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
|
|
15
|
+
deltafi/test_kit/assertions.py,sha256=2eahqmbedhnHXMguiJHdaV-6sx4_jvCbxH4HSzx6PV8,1378
|
|
16
|
+
deltafi/test_kit/compare_helpers.py,sha256=Q0e14q4XQ4rlMUbPt3aMHThAXHFRXlue3-M4oAayyJE,12957
|
|
17
|
+
deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI,833
|
|
18
|
+
deltafi/test_kit/egress.py,sha256=nF7YYak-_X35m7h2rSIcju86IJBf8cVaFHs7xw7-_b8,1899
|
|
19
|
+
deltafi/test_kit/framework.py,sha256=mgKR1Yd75p76wIqWO5XpP6eMgC8fvz9MAfS01QV9WuU,14910
|
|
20
|
+
deltafi/test_kit/transform.py,sha256=q9zQdZswTKwU5Om8E9Lwg0G5JI8LEXuRZjT9oam-Zoo,4085
|
|
21
|
+
deltafi-2.0rc1720817063181.dist-info/METADATA,sha256=sbwZK-ebu8dRSNcJK11favXyKyRgimOiVodv2YOYok8,1446
|
|
22
|
+
deltafi-2.0rc1720817063181.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
23
|
+
deltafi-2.0rc1720817063181.dist-info/RECORD,,
|
deltafi/test_kit/domain.py
DELETED
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
from deltafi.result import DomainResult
|
|
20
|
-
|
|
21
|
-
from .assertions import *
|
|
22
|
-
from .framework import TestCaseBase, ActionTest
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class DomainTestCase(TestCaseBase):
|
|
26
|
-
def __init__(self, fields: Dict):
|
|
27
|
-
super().__init__(fields)
|
|
28
|
-
self.annotations = {}
|
|
29
|
-
|
|
30
|
-
def expect_domain_result(self, annotations: Dict):
|
|
31
|
-
self.expected_result_type = DomainResult
|
|
32
|
-
self.annotations = annotations
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class DomainActionTest(ActionTest):
|
|
36
|
-
def __init__(self, package_name: str):
|
|
37
|
-
"""
|
|
38
|
-
Provides structure for testing DeltaFi Domain action
|
|
39
|
-
Args:
|
|
40
|
-
package_name: name of the actions package for finding resources
|
|
41
|
-
"""
|
|
42
|
-
super().__init__(package_name)
|
|
43
|
-
|
|
44
|
-
def domain(self, test_case: DomainTestCase):
|
|
45
|
-
if test_case.expected_result_type == DomainResult:
|
|
46
|
-
self.expect_domain_result(test_case)
|
|
47
|
-
else:
|
|
48
|
-
super().execute(test_case)
|
|
49
|
-
|
|
50
|
-
def expect_domain_result(self, test_case: DomainTestCase):
|
|
51
|
-
result = super().run_and_check_result_type(test_case, DomainResult)
|
|
52
|
-
self.assert_domain_result(test_case, result)
|
|
53
|
-
|
|
54
|
-
def assert_domain_result(self, test_case: DomainTestCase, result: DomainResult):
|
|
55
|
-
# Check metrics
|
|
56
|
-
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
57
|
-
|
|
58
|
-
# Check annotations
|
|
59
|
-
assert_keys_and_values(test_case.annotations, result.annotations)
|
deltafi/test_kit/enrich.py
DELETED
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
from deltafi.result import EnrichResult
|
|
20
|
-
|
|
21
|
-
from .assertions import *
|
|
22
|
-
from .framework import TestCaseBase, ActionTest
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class EnrichTestCase(TestCaseBase):
|
|
26
|
-
def __init__(self, fields: Dict):
|
|
27
|
-
super().__init__(fields)
|
|
28
|
-
self.enrichments = []
|
|
29
|
-
self.annotations = {}
|
|
30
|
-
|
|
31
|
-
def expect_enrich_result(self, annotations: Dict):
|
|
32
|
-
self.expected_result_type = EnrichResult
|
|
33
|
-
self.annotations = annotations
|
|
34
|
-
|
|
35
|
-
def add_enrichment(self, name: str, value: str, media_type: str):
|
|
36
|
-
self.enrichments.append({'name': name, 'value': value, 'mediaType': media_type})
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
class EnrichActionTest(ActionTest):
|
|
40
|
-
def __init__(self, package_name: str):
|
|
41
|
-
"""
|
|
42
|
-
Provides structure for testing DeltaFi Enrich action
|
|
43
|
-
Args:
|
|
44
|
-
package_name: name of the actions package for finding resources
|
|
45
|
-
"""
|
|
46
|
-
super().__init__(package_name)
|
|
47
|
-
|
|
48
|
-
def enrich(self, test_case: EnrichTestCase):
|
|
49
|
-
if test_case.expected_result_type == EnrichResult:
|
|
50
|
-
self.expect_enrich_result(test_case)
|
|
51
|
-
else:
|
|
52
|
-
super().execute(test_case)
|
|
53
|
-
|
|
54
|
-
def expect_enrich_result(self, test_case: EnrichTestCase):
|
|
55
|
-
result = super().run_and_check_result_type(test_case, EnrichResult)
|
|
56
|
-
self.assert_enrich_result(test_case, result)
|
|
57
|
-
|
|
58
|
-
def assert_enrich_result(self, test_case: EnrichTestCase, result: EnrichResult):
|
|
59
|
-
# Check metrics
|
|
60
|
-
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
61
|
-
|
|
62
|
-
# Check annotations
|
|
63
|
-
assert_keys_and_values(test_case.annotations, result.annotations)
|
|
64
|
-
|
|
65
|
-
# Check enrichments
|
|
66
|
-
assert_equal_len(test_case.enrichments, result.enrichments)
|
|
67
|
-
if len(test_case.enrichments) > 0:
|
|
68
|
-
for index, expected in enumerate(test_case.enrichments):
|
|
69
|
-
actual = result.enrichments[index]
|
|
70
|
-
assert_equal(expected, actual)
|
deltafi/test_kit/format.py
DELETED
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
from typing import List
|
|
20
|
-
|
|
21
|
-
from deltafi.result import FormatResult, FormatManyResult
|
|
22
|
-
|
|
23
|
-
from .assertions import *
|
|
24
|
-
from .framework import TestCaseBase, ActionTest
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class FormatTestCase(TestCaseBase):
|
|
28
|
-
def __init__(self, fields: Dict):
|
|
29
|
-
super().__init__(fields)
|
|
30
|
-
self.metadata = {}
|
|
31
|
-
self.delete_metadata_keys = []
|
|
32
|
-
self.expected_format_many_result = []
|
|
33
|
-
|
|
34
|
-
def expect_format_result(self, metadata: Dict, delete_metadata_keys: List[str]):
|
|
35
|
-
self.expected_result_type = FormatResult
|
|
36
|
-
self.metadata = metadata
|
|
37
|
-
self.delete_metadata_keys = delete_metadata_keys
|
|
38
|
-
|
|
39
|
-
def add_format_many_result(self, metadata: Dict, delete_metadata_keys: List):
|
|
40
|
-
self.expected_result_type = FormatManyResult
|
|
41
|
-
self.expected_format_many_result.append(
|
|
42
|
-
{
|
|
43
|
-
"metadata": metadata,
|
|
44
|
-
"delete_metadata_keys": delete_metadata_keys
|
|
45
|
-
}
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
class FormatActionTest(ActionTest):
|
|
50
|
-
def __init__(self, package_name: str):
|
|
51
|
-
"""
|
|
52
|
-
Provides structure for testing DeltaFi Format action
|
|
53
|
-
Args:
|
|
54
|
-
package_name: name of the actions package for finding resources
|
|
55
|
-
"""
|
|
56
|
-
super().__init__(package_name)
|
|
57
|
-
|
|
58
|
-
def format(self, test_case: FormatTestCase):
|
|
59
|
-
if test_case.expected_result_type == FormatManyResult:
|
|
60
|
-
self.expect_format_many_result(test_case)
|
|
61
|
-
elif test_case.expected_result_type == FormatResult:
|
|
62
|
-
self.expect_format_result(test_case)
|
|
63
|
-
else:
|
|
64
|
-
super().execute(test_case)
|
|
65
|
-
|
|
66
|
-
def expect_format_result(self, test_case: FormatTestCase):
|
|
67
|
-
result = super().run_and_check_result_type(test_case, FormatResult)
|
|
68
|
-
self.assert_format_result(test_case, result)
|
|
69
|
-
|
|
70
|
-
def expect_format_many_result(self, test_case: FormatTestCase):
|
|
71
|
-
result = super().run_and_check_result_type(test_case, FormatManyResult)
|
|
72
|
-
self.assert_format_many_result(test_case, result)
|
|
73
|
-
|
|
74
|
-
def assert_format_result(self, test_case: FormatTestCase, result: FormatResult):
|
|
75
|
-
# Check metrics
|
|
76
|
-
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
77
|
-
|
|
78
|
-
# Check output
|
|
79
|
-
if result.content is None:
|
|
80
|
-
self.compare_all_output(test_case.compare_tool, [])
|
|
81
|
-
else:
|
|
82
|
-
self.compare_all_output(test_case.compare_tool, [result.content])
|
|
83
|
-
|
|
84
|
-
# Check metadata
|
|
85
|
-
assert_keys_and_values(test_case.metadata, result.metadata)
|
|
86
|
-
|
|
87
|
-
# Check deleted metadata
|
|
88
|
-
for key in test_case.delete_metadata_keys:
|
|
89
|
-
assert_key_in(key, result.delete_metadata_keys)
|
|
90
|
-
|
|
91
|
-
def assert_format_many_result(self, test_case: FormatTestCase, actual: FormatManyResult):
|
|
92
|
-
# Check metrics
|
|
93
|
-
self.compare_metrics(test_case.expected_metrics, actual.metrics)
|
|
94
|
-
|
|
95
|
-
assert_equal_len(test_case.expected_format_many_result, actual.format_results)
|
|
96
|
-
for index, expected_child_result in enumerate(test_case.expected_format_many_result):
|
|
97
|
-
actual_child = actual.format_results[index]
|
|
98
|
-
self.compare_one_content(
|
|
99
|
-
test_case.compare_tool,
|
|
100
|
-
self.expected_outputs[index],
|
|
101
|
-
actual_child.format_result.content, index)
|
|
102
|
-
|
|
103
|
-
assert_keys_and_values(expected_child_result['metadata'], actual_child.format_result.metadata)
|
|
104
|
-
for key in expected_child_result['delete_metadata_keys']:
|
|
105
|
-
assert_key_in(key, actual_child.format_result.delete_metadata_keys)
|
deltafi/test_kit/load.py
DELETED
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
from typing import List
|
|
20
|
-
|
|
21
|
-
from deltafi.result import LoadResult, ReinjectResult
|
|
22
|
-
|
|
23
|
-
from .assertions import *
|
|
24
|
-
from .compare_helpers import GenericCompareHelper, CompareHelper
|
|
25
|
-
from .framework import TestCaseBase, ActionTest, IOContent
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class LoadTestCase(TestCaseBase):
|
|
29
|
-
def __init__(self, fields: Dict):
|
|
30
|
-
super().__init__(fields)
|
|
31
|
-
self.metadata = {}
|
|
32
|
-
self.delete_metadata_keys = []
|
|
33
|
-
self.annotations = {}
|
|
34
|
-
self.domains = []
|
|
35
|
-
self.domain_cmp_tool = GenericCompareHelper()
|
|
36
|
-
self.children = []
|
|
37
|
-
|
|
38
|
-
def set_domain_compare_tool(self, helper: CompareHelper):
|
|
39
|
-
self.domain_cmp_tool = helper
|
|
40
|
-
|
|
41
|
-
def expect_load_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict, domains: List):
|
|
42
|
-
self.expected_result_type = LoadResult
|
|
43
|
-
self.metadata = metadata
|
|
44
|
-
self.delete_metadata_keys = delete_metadata_keys
|
|
45
|
-
self.annotations = annotations
|
|
46
|
-
self.domains = domains
|
|
47
|
-
|
|
48
|
-
def add_reinject_child(self, filename: str, flow: str, content: IOContent, metadata: Dict):
|
|
49
|
-
self.expected_result_type = ReinjectResult
|
|
50
|
-
self.children.append(
|
|
51
|
-
{
|
|
52
|
-
"filename": filename,
|
|
53
|
-
"flow": flow,
|
|
54
|
-
"content": content,
|
|
55
|
-
"metadata": metadata
|
|
56
|
-
}
|
|
57
|
-
)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
class LoadActionTest(ActionTest):
|
|
61
|
-
def __init__(self, package_name: str):
|
|
62
|
-
"""
|
|
63
|
-
Provides structure for testing DeltaFi Load action
|
|
64
|
-
Args:
|
|
65
|
-
package_name: name of the actions package for finding resources
|
|
66
|
-
"""
|
|
67
|
-
super().__init__(package_name)
|
|
68
|
-
|
|
69
|
-
def load(self, test_case: LoadTestCase):
|
|
70
|
-
if test_case.expected_result_type == ReinjectResult:
|
|
71
|
-
self.expect_reinject_result(test_case)
|
|
72
|
-
elif test_case.expected_result_type == LoadResult:
|
|
73
|
-
self.expect_load_result(test_case)
|
|
74
|
-
else:
|
|
75
|
-
super().execute(test_case)
|
|
76
|
-
|
|
77
|
-
def expect_load_result(self, test_case: LoadTestCase):
|
|
78
|
-
result = super().run_and_check_result_type(test_case, LoadResult)
|
|
79
|
-
self.assert_load_result(test_case, result)
|
|
80
|
-
|
|
81
|
-
def expect_reinject_result(self, test_case: LoadTestCase):
|
|
82
|
-
result = super().run_and_check_result_type(test_case, ReinjectResult)
|
|
83
|
-
self.assert_reinject_result(test_case, result)
|
|
84
|
-
|
|
85
|
-
def assert_load_result(self, test_case: LoadTestCase, result: LoadResult):
|
|
86
|
-
# Check metrics
|
|
87
|
-
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
88
|
-
|
|
89
|
-
# Check output
|
|
90
|
-
self.compare_all_output(test_case.compare_tool, result.content)
|
|
91
|
-
|
|
92
|
-
# Check metadata
|
|
93
|
-
assert_keys_and_values(test_case.metadata, result.metadata)
|
|
94
|
-
|
|
95
|
-
# Check deleted metadata
|
|
96
|
-
for key in test_case.delete_metadata_keys:
|
|
97
|
-
assert_key_in(key, result.delete_metadata_keys)
|
|
98
|
-
|
|
99
|
-
# Check annotations
|
|
100
|
-
assert_keys_and_values(test_case.annotations, result.annotations)
|
|
101
|
-
|
|
102
|
-
# Check domains
|
|
103
|
-
self.compare_domains(test_case.domain_cmp_tool, test_case.domains, result.domains)
|
|
104
|
-
|
|
105
|
-
def assert_reinject_result(self, test_case: LoadTestCase, actual: ReinjectResult):
|
|
106
|
-
# Check metrics
|
|
107
|
-
self.compare_metrics(test_case.expected_metrics, actual.metrics)
|
|
108
|
-
|
|
109
|
-
# Check reinject
|
|
110
|
-
assert_equal_len(test_case.children, actual.children)
|
|
111
|
-
for index, expected in enumerate(test_case.children):
|
|
112
|
-
reinject_child = actual.children[index]
|
|
113
|
-
assert_equal(expected['filename'], reinject_child.filename)
|
|
114
|
-
assert_equal(expected['flow'], reinject_child.flow)
|
|
115
|
-
assert_keys_and_values(expected['metadata'], reinject_child.metadata)
|
|
116
|
-
|
|
117
|
-
expected_value = expected['content']
|
|
118
|
-
child_content = reinject_child.content[0]
|
|
119
|
-
seg_id = child_content.segments[0].uuid
|
|
120
|
-
actual_content = self.content_service.get_output(seg_id)
|
|
121
|
-
|
|
122
|
-
if type(expected_value) == str:
|
|
123
|
-
test_case.domain_cmp_tool.compare(expected_value, actual_content, f"RI_child[{index}]")
|
|
124
|
-
elif type(expected_value) == IOContent:
|
|
125
|
-
expected_data = self.load_file(expected_value)
|
|
126
|
-
test_case.domain_cmp_tool.compare(expected_data, actual_content, f"RI_child[{index}]")
|
|
127
|
-
else:
|
|
128
|
-
raise ValueError(f"unknown expected_value type: {type(expected_value)}")
|
deltafi/test_kit/validate.py
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
from deltafi.result import ValidateResult
|
|
20
|
-
|
|
21
|
-
from .assertions import *
|
|
22
|
-
from .framework import TestCaseBase, ActionTest
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class ValidateTestCase(TestCaseBase):
|
|
26
|
-
def __init__(self, fields: Dict):
|
|
27
|
-
super().__init__(fields)
|
|
28
|
-
|
|
29
|
-
def expect_validate_result(self):
|
|
30
|
-
self.expected_result_type = ValidateResult
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
class ValidateActionTest(ActionTest):
|
|
34
|
-
def __init__(self, package_name: str):
|
|
35
|
-
"""
|
|
36
|
-
Provides structure for testing DeltaFi Validate action
|
|
37
|
-
Args:
|
|
38
|
-
package_name: name of the actions package for finding resources
|
|
39
|
-
"""
|
|
40
|
-
super().__init__(package_name)
|
|
41
|
-
|
|
42
|
-
def validate(self, test_case: ValidateTestCase):
|
|
43
|
-
if test_case.expected_result_type == ValidateResult:
|
|
44
|
-
self.expect_validate_result(test_case)
|
|
45
|
-
else:
|
|
46
|
-
super().execute(test_case)
|
|
47
|
-
|
|
48
|
-
def expect_validate_result(self, test_case: ValidateTestCase):
|
|
49
|
-
result = super().run_and_check_result_type(test_case, ValidateResult)
|
|
50
|
-
self.assert_validate_result(test_case, result)
|
|
51
|
-
|
|
52
|
-
def assert_validate_result(self, test_case: ValidateTestCase, result: ValidateResult):
|
|
53
|
-
# Check metrics
|
|
54
|
-
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
deltafi/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
|
|
2
|
-
deltafi/action.py,sha256=TM8HwfC12GgfHMBWPmO88N7J9xvwGO8BMwxKTNqwg9A,10498
|
|
3
|
-
deltafi/actioneventqueue.py,sha256=mCRE1PFXy_KvaeezTeMze5N400CO1V06zEF8FFD6xZk,2847
|
|
4
|
-
deltafi/actiontype.py,sha256=SgCxs5Kn_6BuzuKDk5h93ZLtfdbzV0sWLcEaaB0mqlU,997
|
|
5
|
-
deltafi/domain.py,sha256=M97_n6KsMmIxCh3h622jNj1n6c5iMEAOn-SFH36oLis,12161
|
|
6
|
-
deltafi/exception.py,sha256=qQ2TY2QPtMU9t5RH8jmFo_cX98AJ-zOFWP_Wfm5U6qQ,1149
|
|
7
|
-
deltafi/genericmodel.py,sha256=8VXWgFMjIwG4o3rkcLOa3bQH0Nf-T1Kw621QtMw-4q0,1090
|
|
8
|
-
deltafi/input.py,sha256=DuY280ZglZxEUkhjTtyr0g9Ohu2drn16UVhASH244nA,6290
|
|
9
|
-
deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
|
|
10
|
-
deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
|
|
11
|
-
deltafi/plugin.py,sha256=GkDq9oc5O0QYUZ7CdYcP1f9zRT38ufvzudZ4addHVS8,13076
|
|
12
|
-
deltafi/result.py,sha256=KkFKewPxcP8_Slp0D2COJ43h5_fF5Vti9ocWtp8pjpc,14911
|
|
13
|
-
deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
|
|
14
|
-
deltafi/test_kit/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
|
|
15
|
-
deltafi/test_kit/assertions.py,sha256=2eahqmbedhnHXMguiJHdaV-6sx4_jvCbxH4HSzx6PV8,1378
|
|
16
|
-
deltafi/test_kit/compare_helpers.py,sha256=Q0e14q4XQ4rlMUbPt3aMHThAXHFRXlue3-M4oAayyJE,12957
|
|
17
|
-
deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI,833
|
|
18
|
-
deltafi/test_kit/domain.py,sha256=uqpDravlb6E3Ddm4QpRaJTdiVuQ2MMdZnzc4Oi9EZJA,2090
|
|
19
|
-
deltafi/test_kit/egress.py,sha256=nF7YYak-_X35m7h2rSIcju86IJBf8cVaFHs7xw7-_b8,1899
|
|
20
|
-
deltafi/test_kit/enrich.py,sha256=M682cqInVS4aipI0jOrR25x7m0ErrpjGpebXGZDPwxo,2587
|
|
21
|
-
deltafi/test_kit/format.py,sha256=j5spoimv4fIB7wrPSUS-aLcYvDJS0ntu4oSJjQ1INsw,4170
|
|
22
|
-
deltafi/test_kit/framework.py,sha256=5PTWoKu8eMU5GaVLmyoghoOrp6G65efXwqZKlUlyEaM,14733
|
|
23
|
-
deltafi/test_kit/load.py,sha256=dzXZDjb5BKxM9F5E2B06ArnGy0HvE1kwPizL8VybsOQ,5055
|
|
24
|
-
deltafi/test_kit/transform.py,sha256=AZ5oamhaOm4yaR07_gCoTlDiHkd4KwiAWWNj3mnYp8Q,2706
|
|
25
|
-
deltafi/test_kit/validate.py,sha256=NnXD5amOE_9a4Zr-v9DKYIv5f_HvwtmlI-IdFtVp7ko,1933
|
|
26
|
-
deltafi-2.0rc1720728217472.dist-info/METADATA,sha256=lhhN66ZvCacThjEosjup0r8FOzyPPvEs-Lt0LWQnFQc,1446
|
|
27
|
-
deltafi-2.0rc1720728217472.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
28
|
-
deltafi-2.0rc1720728217472.dist-info/RECORD,,
|
|
File without changes
|