bigtesty 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigtesty/__init__.py +0 -0
- bigtesty/arguments.py +7 -0
- bigtesty/dataset_helper.py +5 -0
- bigtesty/definition_test_config_helper.py +51 -0
- bigtesty/definition_test_file_exception.py +2 -0
- bigtesty/files_loader_helper.py +12 -0
- bigtesty/given/__init__.py +0 -0
- bigtesty/given/insertion_test_data_bigquery.py +40 -0
- bigtesty/infra/__init__.py +0 -0
- bigtesty/infra/create_iac_for_datasets_with_tables.py +49 -0
- bigtesty/infra/datasets_with_tables_config_file_loader.py +28 -0
- bigtesty/infra/datasets_with_tables_factory.py +55 -0
- bigtesty/infra/launch_tests_ephemeral_infra.py +107 -0
- bigtesty/lambda_functions.py +1 -0
- bigtesty/then/__init__.py +0 -0
- bigtesty/then/assertion_and_tests_reports_result.py +129 -0
- bigtesty/then/failure_test_exception.py +2 -0
- bigtesty-0.1.0a1.dist-info/LICENSE +21 -0
- bigtesty-0.1.0a1.dist-info/METADATA +49 -0
- bigtesty-0.1.0a1.dist-info/RECORD +23 -0
- bigtesty-0.1.0a1.dist-info/WHEEL +5 -0
- bigtesty-0.1.0a1.dist-info/entry_points.txt +4 -0
- bigtesty-0.1.0a1.dist-info/top_level.txt +1 -0
bigtesty/__init__.py
ADDED
|
File without changes
|
bigtesty/arguments.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
|
|
3
|
+
parser = argparse.ArgumentParser()
|
|
4
|
+
parser.add_argument('--root_folder', dest='root_folder', type=str, help='Root folder')
|
|
5
|
+
parser.add_argument('--project_id', dest='project_id', type=str, help='GCP Project ID')
|
|
6
|
+
parser.add_argument('--datasets_hash', dest='datasets_hash', type=str, help='Unique hash to isolate datasets')
|
|
7
|
+
args = parser.parse_args()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import glob
|
|
2
|
+
import json
|
|
3
|
+
from typing import List, Dict
|
|
4
|
+
|
|
5
|
+
from bigtesty.definition_test_file_exception import DefinitionTestFileException
|
|
6
|
+
from bigtesty.lambda_functions import flat_map
|
|
7
|
+
|
|
8
|
+
DEFINITION_TEST_FILE_PATTERN = "**/definition_*.json"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def get_definition_test_dicts_from_path(data_testing_root_path: str) -> List[Dict]:
|
|
12
|
+
"""
|
|
13
|
+
Gets all the test definition files existing in the given folder path and then map them to dicts.
|
|
14
|
+
The result is a list of dict, and each dict represents a test definition.
|
|
15
|
+
If there is no definition files in the given folder, a :class:`~DefinitionTestFileException` is raised.
|
|
16
|
+
:param data_testing_root_path: the root folder path containing test definition files
|
|
17
|
+
:return: a list of test definition dict
|
|
18
|
+
"""
|
|
19
|
+
print("#############ROOT DIR")
|
|
20
|
+
print(data_testing_root_path)
|
|
21
|
+
|
|
22
|
+
definition_test_file_paths: List[str] = get_definition_tests_file_paths(data_testing_root_path)
|
|
23
|
+
|
|
24
|
+
if not definition_test_file_paths:
|
|
25
|
+
raise DefinitionTestFileException(
|
|
26
|
+
f'There is no definition files in the given folder path {data_testing_root_path}'
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
return list(map(lambda path: to_definition_test_dict(path), definition_test_file_paths))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def get_scenarios(data_testing_root_path: str):
|
|
33
|
+
return list(
|
|
34
|
+
flat_map(lambda deftest: deftest['scenarios'], get_definition_test_dicts_from_path(data_testing_root_path))
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_definition_tests_file_paths(data_testing_root_path: str) -> List[str]:
|
|
39
|
+
"""
|
|
40
|
+
Gets all the test definitions file paths existing in the given folder path.
|
|
41
|
+
The search of files is recursive.
|
|
42
|
+
"""
|
|
43
|
+
return glob.glob(
|
|
44
|
+
pathname=f'{data_testing_root_path}/{DEFINITION_TEST_FILE_PATTERN}',
|
|
45
|
+
recursive=True
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def to_definition_test_dict(definition_test_file_path: str):
|
|
50
|
+
with open(definition_test_file_path) as definition_test_file:
|
|
51
|
+
return json.load(definition_test_file)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import List, Dict
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def load_file_as_dicts(file_path: str) -> List[Dict]:
|
|
6
|
+
with open(file_path) as file:
|
|
7
|
+
return json.load(file)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def load_file_as_string(file_path: str) -> str:
|
|
11
|
+
with open(file_path, 'r') as file:
|
|
12
|
+
return file.read().rstrip()
|
|
File without changes
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from typing import List, Dict
|
|
2
|
+
|
|
3
|
+
from google.cloud import bigquery
|
|
4
|
+
|
|
5
|
+
from bigtesty.dataset_helper import build_unique_dataset_id_for_scenario
|
|
6
|
+
from bigtesty.files_loader_helper import load_file_as_dicts
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def insert_test_data_to_bq_tables(
|
|
10
|
+
project_id: str,
|
|
11
|
+
root_test_folder: str,
|
|
12
|
+
datasets_hash: str,
|
|
13
|
+
scenarios: List[Dict]) -> None:
|
|
14
|
+
client = bigquery.Client(project=project_id)
|
|
15
|
+
|
|
16
|
+
for scenario in scenarios:
|
|
17
|
+
for given in scenario['given']:
|
|
18
|
+
print("###### GIVEN")
|
|
19
|
+
print(given)
|
|
20
|
+
|
|
21
|
+
input_file_path = f"{root_test_folder}/{given['input_file_path']}"
|
|
22
|
+
input = given.get('input')
|
|
23
|
+
input_as_dicts: List[Dict] = input if input else load_file_as_dicts(input_file_path)
|
|
24
|
+
|
|
25
|
+
print("####### INPUT AS DICT")
|
|
26
|
+
print(input_as_dicts)
|
|
27
|
+
|
|
28
|
+
dataset_id_with_hash = build_unique_dataset_id_for_scenario(
|
|
29
|
+
dataset_id=given['destination_dataset'],
|
|
30
|
+
scenario_id=scenario["id"],
|
|
31
|
+
datasets_hash=datasets_hash
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
results = client.insert_rows_json(
|
|
35
|
+
f"{dataset_id_with_hash}.{given['destination_table']}",
|
|
36
|
+
input_as_dicts
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
print("###### RESULTS")
|
|
40
|
+
print(results)
|
|
File without changes
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from typing import Dict, List
|
|
2
|
+
|
|
3
|
+
from pulumi_gcp.bigquery import Dataset
|
|
4
|
+
|
|
5
|
+
from bigtesty.dataset_helper import build_unique_dataset_id_for_scenario
|
|
6
|
+
from bigtesty.infra.datasets_with_tables_factory import get_dataset, get_table_with_partitioning, get_table
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def create_datasets_and_tables(root_tables_folder: str,
|
|
10
|
+
datasets_with_tables_config: List[Dict],
|
|
11
|
+
datasets_hash: str,
|
|
12
|
+
scenarios: List[Dict]) -> None:
|
|
13
|
+
for dataset_config in datasets_with_tables_config:
|
|
14
|
+
for scenario in scenarios:
|
|
15
|
+
dataset_id_with_hash = build_unique_dataset_id_for_scenario(
|
|
16
|
+
dataset_id=dataset_config["datasetId"],
|
|
17
|
+
scenario_id=scenario["id"],
|
|
18
|
+
datasets_hash=datasets_hash
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
bq_dataset = get_dataset(dataset_id_with_hash, dataset_config)
|
|
22
|
+
|
|
23
|
+
create_tables(
|
|
24
|
+
root_tables_folder=root_tables_folder,
|
|
25
|
+
dataset_config=dataset_config,
|
|
26
|
+
dataset=bq_dataset,
|
|
27
|
+
dataset_id_with_hash=dataset_id_with_hash
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def create_tables(root_tables_folder: str,
|
|
32
|
+
dataset_config: Dict,
|
|
33
|
+
dataset: Dataset,
|
|
34
|
+
dataset_id_with_hash: str) -> None:
|
|
35
|
+
for table_config in dataset_config["tables"]:
|
|
36
|
+
(
|
|
37
|
+
get_table_with_partitioning(
|
|
38
|
+
root_tables_folder=root_tables_folder,
|
|
39
|
+
dataset_id_with_hash=dataset_id_with_hash,
|
|
40
|
+
dataset=dataset,
|
|
41
|
+
table=table_config
|
|
42
|
+
) if table_config.get("partitionType") is not None
|
|
43
|
+
else get_table(
|
|
44
|
+
root_tables_folder=root_tables_folder,
|
|
45
|
+
dataset_id_with_hash=dataset_id_with_hash,
|
|
46
|
+
dataset=dataset,
|
|
47
|
+
table=table_config
|
|
48
|
+
)
|
|
49
|
+
)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import random
|
|
3
|
+
import string
|
|
4
|
+
from typing import List, Dict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _load_file_as_dicts(file_path: str) -> List[Dict]:
|
|
8
|
+
with open(file_path) as file:
|
|
9
|
+
return json.load(file)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _load_file_as_string(file_path: str) -> str:
|
|
13
|
+
with open(file_path) as file:
|
|
14
|
+
return file.read()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_table_schema(root_tables_folder: str, table: Dict) -> str:
|
|
18
|
+
datasets_tables_config_file_path = f'{root_tables_folder}/{table["tableSchemaPath"]}'
|
|
19
|
+
|
|
20
|
+
return _load_file_as_string(datasets_tables_config_file_path)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_datasets_hash(length) -> str:
|
|
24
|
+
letters = string.ascii_lowercase
|
|
25
|
+
datasets_hash = ''.join(random.choice(letters) for _ in range(length))
|
|
26
|
+
print(f'The generated datasets hash is : {datasets_hash}')
|
|
27
|
+
|
|
28
|
+
return datasets_hash
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
from typing import Dict
|
|
2
|
+
|
|
3
|
+
import pulumi_gcp as gcp
|
|
4
|
+
from pulumi import ResourceOptions
|
|
5
|
+
from pulumi_gcp.bigquery import Dataset
|
|
6
|
+
|
|
7
|
+
from bigtesty.infra.datasets_with_tables_config_file_loader import get_table_schema
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_table(root_tables_folder: str,
|
|
11
|
+
dataset_id_with_hash: str,
|
|
12
|
+
dataset: Dataset,
|
|
13
|
+
table: Dict):
|
|
14
|
+
return gcp.bigquery.Table(
|
|
15
|
+
get_table_resource_name(dataset_id_with_hash, table['tableId']),
|
|
16
|
+
deletion_protection=False,
|
|
17
|
+
dataset_id=dataset.dataset_id,
|
|
18
|
+
table_id=table["tableId"],
|
|
19
|
+
clusterings=table.get("clustering"),
|
|
20
|
+
schema=get_table_schema(root_tables_folder, table),
|
|
21
|
+
opts=ResourceOptions(depends_on=[dataset])
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_table_with_partitioning(root_tables_folder: str,
|
|
26
|
+
dataset_id_with_hash: str,
|
|
27
|
+
dataset: Dataset,
|
|
28
|
+
table: Dict):
|
|
29
|
+
return gcp.bigquery.Table(
|
|
30
|
+
get_table_resource_name(dataset_id_with_hash, table['tableId']),
|
|
31
|
+
deletion_protection=False,
|
|
32
|
+
dataset_id=dataset.dataset_id,
|
|
33
|
+
table_id=table["tableId"],
|
|
34
|
+
clusterings=table.get("clustering"),
|
|
35
|
+
time_partitioning=gcp.bigquery.TableTimePartitioningArgs(
|
|
36
|
+
type=table["partitionType"],
|
|
37
|
+
field=table["partitionField"],
|
|
38
|
+
),
|
|
39
|
+
schema=get_table_schema(root_tables_folder, table),
|
|
40
|
+
opts=ResourceOptions(depends_on=[dataset])
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def get_table_resource_name(dataset_id: str, table_id: str) -> str:
|
|
45
|
+
return f'{dataset_id}_{table_id}'
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def get_dataset(dataset_id_with_hash: str, dataset: Dict):
|
|
49
|
+
return gcp.bigquery.Dataset(
|
|
50
|
+
dataset_id_with_hash,
|
|
51
|
+
dataset_id=dataset_id_with_hash,
|
|
52
|
+
friendly_name=dataset["datasetFriendlyName"],
|
|
53
|
+
description=dataset["datasetDescription"],
|
|
54
|
+
location=dataset["datasetRegion"]
|
|
55
|
+
)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
from typing import List, Dict
|
|
4
|
+
|
|
5
|
+
from pulumi import automation as auto
|
|
6
|
+
|
|
7
|
+
from bigtesty.definition_test_config_helper import get_scenarios
|
|
8
|
+
from bigtesty.given.insertion_test_data_bigquery import insert_test_data_to_bq_tables
|
|
9
|
+
from bigtesty.infra.create_iac_for_datasets_with_tables import create_datasets_and_tables
|
|
10
|
+
from bigtesty.infra.datasets_with_tables_config_file_loader import get_datasets_hash, \
|
|
11
|
+
_load_file_as_dicts
|
|
12
|
+
from bigtesty.then.assertion_and_tests_reports_result import execute_query_and_build_reports_result, \
|
|
13
|
+
check_any_failed_test_in_reports
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def launch_tests_ephemeral_infra(root_test_folder: str,
|
|
17
|
+
root_tables_folder: str,
|
|
18
|
+
tables_config_file_path: str):
|
|
19
|
+
project_id = os.environ["GOOGLE_PROJECT"]
|
|
20
|
+
region = os.environ["GOOGLE_REGION"]
|
|
21
|
+
stack_name = os.environ["BIGTESTY_STACK_NAME"]
|
|
22
|
+
|
|
23
|
+
datasets_hash = get_datasets_hash(5)
|
|
24
|
+
|
|
25
|
+
scenarios = get_scenarios(root_test_folder)
|
|
26
|
+
|
|
27
|
+
stack = auto.create_or_select_stack(
|
|
28
|
+
stack_name=stack_name,
|
|
29
|
+
project_name=project_id,
|
|
30
|
+
program=lambda: pulumi_program(scenarios,
|
|
31
|
+
datasets_hash,
|
|
32
|
+
root_tables_folder,
|
|
33
|
+
tables_config_file_path)
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
print("Successfully initialized stack")
|
|
37
|
+
|
|
38
|
+
# for inline programs, we must manage plugins ourselves
|
|
39
|
+
print("Installing plugins...")
|
|
40
|
+
stack.workspace.install_plugin("gcp", "v6.67.0")
|
|
41
|
+
print("Plugins installed")
|
|
42
|
+
|
|
43
|
+
print("Setting up config")
|
|
44
|
+
stack.set_config("gcp:project", auto.ConfigValue(value=project_id))
|
|
45
|
+
stack.set_config("gcp:region", auto.ConfigValue(value=region))
|
|
46
|
+
print("Config set")
|
|
47
|
+
|
|
48
|
+
print("Refreshing stack...")
|
|
49
|
+
stack.refresh(on_output=print)
|
|
50
|
+
print("Refresh complete")
|
|
51
|
+
|
|
52
|
+
print("#################### Creating the ephemeral infra...")
|
|
53
|
+
up_res = stack.up(
|
|
54
|
+
on_output=print,
|
|
55
|
+
color="always",
|
|
56
|
+
show_secrets=False,
|
|
57
|
+
diff=True
|
|
58
|
+
)
|
|
59
|
+
print(f"Update summary: \n{json.dumps(up_res.summary.resource_changes, indent=4)}")
|
|
60
|
+
|
|
61
|
+
print("#################### Inserting Test data to Tables...")
|
|
62
|
+
insert_test_data_to_bq_tables(
|
|
63
|
+
project_id=project_id,
|
|
64
|
+
root_test_folder=root_test_folder,
|
|
65
|
+
datasets_hash=datasets_hash,
|
|
66
|
+
scenarios=scenarios
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
print("#################### Execute SQL queries and generate reports result...")
|
|
70
|
+
reports_result: List[Dict] = execute_query_and_build_reports_result(
|
|
71
|
+
project_id=project_id,
|
|
72
|
+
root_test_folder=root_test_folder,
|
|
73
|
+
datasets_hash=datasets_hash,
|
|
74
|
+
scenarios=scenarios
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
print("################## The reports result is #################")
|
|
78
|
+
print(reports_result)
|
|
79
|
+
print("#################")
|
|
80
|
+
|
|
81
|
+
print("################### Destroying the ephemeral infra and tests assertions...")
|
|
82
|
+
stack.destroy(
|
|
83
|
+
on_output=print,
|
|
84
|
+
color="always",
|
|
85
|
+
show_secrets=False,
|
|
86
|
+
log_flow=True,
|
|
87
|
+
log_verbosity=3
|
|
88
|
+
)
|
|
89
|
+
print("############### Destroy ephemeral infra complete")
|
|
90
|
+
|
|
91
|
+
print("############### After destroying the ephemeral infra, checking if there is any failed test...")
|
|
92
|
+
check_any_failed_test_in_reports(reports_result)
|
|
93
|
+
print("############### Tests assertions finished")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def pulumi_program(scenarios: List[Dict],
|
|
97
|
+
datasets_hash: str,
|
|
98
|
+
root_tables_folder: str,
|
|
99
|
+
tables_config_file_path: str):
|
|
100
|
+
datasets_with_tables_config = _load_file_as_dicts(tables_config_file_path)
|
|
101
|
+
|
|
102
|
+
create_datasets_and_tables(
|
|
103
|
+
root_tables_folder=root_tables_folder,
|
|
104
|
+
datasets_with_tables_config=datasets_with_tables_config,
|
|
105
|
+
datasets_hash=datasets_hash,
|
|
106
|
+
scenarios=scenarios
|
|
107
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
flat_map = lambda f, xs: (y for ys in xs for y in f(ys))
|
|
File without changes
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from typing import List, Dict
|
|
3
|
+
|
|
4
|
+
import deepdiff
|
|
5
|
+
from google.cloud import bigquery
|
|
6
|
+
from google.cloud.bigquery import Client
|
|
7
|
+
from toolz.curried import pipe, map
|
|
8
|
+
|
|
9
|
+
from bigtesty.dataset_helper import build_unique_dataset_id_for_scenario
|
|
10
|
+
from bigtesty.files_loader_helper import load_file_as_string, load_file_as_dicts
|
|
11
|
+
from bigtesty.then.failure_test_exception import FailureTestException
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def execute_query_and_build_reports_result(
|
|
15
|
+
project_id: str,
|
|
16
|
+
root_test_folder: str,
|
|
17
|
+
datasets_hash: str,
|
|
18
|
+
scenarios: List[Dict]) -> List[Dict]:
|
|
19
|
+
client = bigquery.Client(project=project_id)
|
|
20
|
+
|
|
21
|
+
return list(pipe(
|
|
22
|
+
scenarios,
|
|
23
|
+
map(lambda scenario: _execute_query_and_build_report_result(
|
|
24
|
+
bigquery_python_client=client,
|
|
25
|
+
scenario=scenario,
|
|
26
|
+
root_test_folder=root_test_folder,
|
|
27
|
+
datasets_hash=datasets_hash
|
|
28
|
+
))
|
|
29
|
+
))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def check_any_failed_test_in_reports(reports_result: List[Dict]) -> bool:
|
|
33
|
+
any_failed_test: bool = any(report['result'] is False for report in reports_result)
|
|
34
|
+
|
|
35
|
+
if any_failed_test:
|
|
36
|
+
print("There is any failed test in the reports result, the pipeline is in failure status !!")
|
|
37
|
+
raise FailureTestException('Error : there is any failed test !!!')
|
|
38
|
+
|
|
39
|
+
print("There is no failed test in the reports result, the pipeline is in success status !!")
|
|
40
|
+
return any_failed_test
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _execute_query_and_build_report_result(bigquery_python_client: Client,
|
|
44
|
+
scenario: Dict,
|
|
45
|
+
root_test_folder: str,
|
|
46
|
+
datasets_hash: str) -> Dict:
|
|
47
|
+
given_list: List[Dict] = scenario["given"]
|
|
48
|
+
|
|
49
|
+
for then in scenario["then"]:
|
|
50
|
+
print("######THEN")
|
|
51
|
+
print(then)
|
|
52
|
+
|
|
53
|
+
query = _build_query(
|
|
54
|
+
scenario_id=scenario["id"],
|
|
55
|
+
datasets_hash=datasets_hash,
|
|
56
|
+
root_test_folder=root_test_folder,
|
|
57
|
+
given_list=given_list,
|
|
58
|
+
then=then
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
print("#########SQL QUERY")
|
|
62
|
+
print(query)
|
|
63
|
+
|
|
64
|
+
query_job = bigquery_python_client.query(query)
|
|
65
|
+
actual_list: List[Dict] = list(map(lambda r: dict(r), query_job.result()))
|
|
66
|
+
print("#########ACTUAL")
|
|
67
|
+
print(actual_list)
|
|
68
|
+
|
|
69
|
+
expected_list = _get_expected_list(root_test_folder, then)
|
|
70
|
+
print("#########EXPECTED")
|
|
71
|
+
print(expected_list)
|
|
72
|
+
|
|
73
|
+
fields_to_ignore_compiled_regex = list(
|
|
74
|
+
map(lambda field_regex: re.compile(field_regex), then['fields_to_ignore'])
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
result: Dict = deepdiff.DeepDiff(actual_list,
|
|
78
|
+
expected_list,
|
|
79
|
+
ignore_order=True,
|
|
80
|
+
exclude_regex_paths=fields_to_ignore_compiled_regex,
|
|
81
|
+
view='tree')
|
|
82
|
+
print("#########TEST RESULT")
|
|
83
|
+
print(result)
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
'scenario': scenario,
|
|
87
|
+
'actual': actual_list,
|
|
88
|
+
'expected': expected_list,
|
|
89
|
+
'comparison_info': result,
|
|
90
|
+
'result': result == {}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _build_query(scenario_id: str,
|
|
95
|
+
datasets_hash: str,
|
|
96
|
+
root_test_folder: str,
|
|
97
|
+
given_list: List[Dict],
|
|
98
|
+
then: Dict) -> str:
|
|
99
|
+
actual = then.get('actual')
|
|
100
|
+
sql_query = actual if actual else load_file_as_string(f"{root_test_folder}/{then['actual_file_path']}")
|
|
101
|
+
|
|
102
|
+
sql_query_result = ""
|
|
103
|
+
for given in given_list:
|
|
104
|
+
sql_query_result = _replace_current_dataset_by_unique_dataset_for_scenario(
|
|
105
|
+
sql_query=sql_query,
|
|
106
|
+
current_dataset=given["destination_dataset"],
|
|
107
|
+
scenario_id=scenario_id,
|
|
108
|
+
datasets_hash=datasets_hash
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
return sql_query_result
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _replace_current_dataset_by_unique_dataset_for_scenario(sql_query: str,
|
|
115
|
+
current_dataset: str,
|
|
116
|
+
scenario_id: str,
|
|
117
|
+
datasets_hash: str) -> str:
|
|
118
|
+
unique_dataset_with_for_scenario = build_unique_dataset_id_for_scenario(
|
|
119
|
+
dataset_id=current_dataset,
|
|
120
|
+
scenario_id=scenario_id,
|
|
121
|
+
datasets_hash=datasets_hash
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
return sql_query.replace(f"{current_dataset}.", f"{unique_dataset_with_for_scenario}.")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _get_expected_list(root_test_folder: str, then: Dict) -> List[Dict]:
|
|
128
|
+
expected = then.get('expected')
|
|
129
|
+
return expected if expected else load_file_as_dicts(f"{root_test_folder}/{then['expected_file_path']}")
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Mazlum Tosun
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: bigtesty
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: BigTesty is an integration testing framework for BigQuery
|
|
5
|
+
Home-page: https://github.com/tosun-si/bigtesty
|
|
6
|
+
Author: Mazlum TOSUN
|
|
7
|
+
Author-email: mazlum.tosun@gmail.com
|
|
8
|
+
License: UNKNOWN
|
|
9
|
+
Platform: UNKNOWN
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.9.18
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: pulumi-gcp ==6.67.0
|
|
17
|
+
Requires-Dist: google-cloud-bigquery ==3.7.0
|
|
18
|
+
Requires-Dist: typer[all] ==0.9.0
|
|
19
|
+
Requires-Dist: pytest ==7.2.1
|
|
20
|
+
Requires-Dist: deepdiff ==6.3.0
|
|
21
|
+
Requires-Dist: toolz ==0.12.0
|
|
22
|
+
|
|
23
|
+
# bigtesty
|
|
24
|
+
|
|
25
|
+
BigTesty is a framework that allows to create Integration Tests with BigQuery on a real and short-lived Infrastructure.
|
|
26
|
+
|
|
27
|
+
## Run integration tests with BigTesty
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
docker run -it \
|
|
31
|
+
-e GOOGLE_PROJECT=$PROJECT_ID \
|
|
32
|
+
-e SA_EMAIL=$SA_EMAIL \
|
|
33
|
+
-e GOOGLE_REGION=$LOCATION \
|
|
34
|
+
-e PULUMI_BACKEND_URL=$IAC_BACKEND_URL \
|
|
35
|
+
-e ROOT_TEST_FOLDER=$ROOT_TEST_FOLDER \
|
|
36
|
+
-e BIGTESTY_STACK_NAME=bigtesty \
|
|
37
|
+
-e PULUMI_CONFIG_PASSPHRASE=gcp_fake_passphrase \
|
|
38
|
+
-v $(pwd)/tests:/app/tests \
|
|
39
|
+
-v $(pwd)/tests/tables:/app/bigtesty/infra/resource/tables \
|
|
40
|
+
-v /var/run/docker.sock:/var/run/docker.sock \
|
|
41
|
+
-v $HOME/.config/gcloud:/root/.config/gcloud \
|
|
42
|
+
bigtesty-pulumi
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
docker build -f Dockerfile -t bigtesty --progress plain .
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
bigtesty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
bigtesty/arguments.py,sha256=JwwMPFkRSSrWILzO6mY9Xlbbajk4cs6ddFbC4zyrQs0,365
|
|
3
|
+
bigtesty/dataset_helper.py,sha256=mk7Klu-Nb8t17yyNie948nkYHGIY9rz-Atxr2cuGdGw,286
|
|
4
|
+
bigtesty/definition_test_config_helper.py,sha256=41xakzxtZAGxoj8ZvsuTgyYf9PQYU_8WoJbyp1zxpf0,1921
|
|
5
|
+
bigtesty/definition_test_file_exception.py,sha256=2ojAbfQ_FSN4Gq-f82ARynQvFYDBaAOy4bAZ4tqUZB0,55
|
|
6
|
+
bigtesty/files_loader_helper.py,sha256=hUBQ8jVuIULc0auAd0g2Bw2HEHOckg2WKo89VlamWe4,288
|
|
7
|
+
bigtesty/lambda_functions.py,sha256=Kha-s1_P_BU5d-PAY2HOW11tnZNa0EiGVSFyn9HquYI,57
|
|
8
|
+
bigtesty/given/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
bigtesty/given/insertion_test_data_bigquery.py,sha256=lLNM9KP2zf-X3X5yFbkDCYCiWhR_Hlodrvoqvs37ank,1296
|
|
10
|
+
bigtesty/infra/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
bigtesty/infra/create_iac_for_datasets_with_tables.py,sha256=oWX7tuK1xzLpASARQ0Qpp0-vLLcwtKwBTYJLw8mjeAo,1887
|
|
12
|
+
bigtesty/infra/datasets_with_tables_config_file_loader.py,sha256=zIsQkq-_ygZ8ffYWyPo1z31v_wlBTYrDxvNwafzMZfQ,771
|
|
13
|
+
bigtesty/infra/datasets_with_tables_factory.py,sha256=xxKn7H6e5ufxtneG_svYvDwUutZm1UZcXnrlrMGYmQ0,1903
|
|
14
|
+
bigtesty/infra/launch_tests_ephemeral_infra.py,sha256=8Tbx70TsP63pUGpBq2Ujab9d6GpDtDJL5O9XyNHCVDQ,3801
|
|
15
|
+
bigtesty/then/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
+
bigtesty/then/assertion_and_tests_reports_result.py,sha256=Elx0NOhgwxFVDWN4f3vFreQU3L4WIvAfFiCqhYbX9ik,4659
|
|
17
|
+
bigtesty/then/failure_test_exception.py,sha256=XGdZ75LXL3TpNV7zaZvSj-6S21pUBRkh-J_ygaQDIIE,48
|
|
18
|
+
bigtesty-0.1.0a1.dist-info/LICENSE,sha256=aBCOCCCvY_F67UK2dRXZ9vJccUsM1hyU34OTSo_CgJI,1069
|
|
19
|
+
bigtesty-0.1.0a1.dist-info/METADATA,sha256=_wpLt4dwhmkDzMKxHtIhvG-rOp0xlcj9TSlLo7EcXYg,1475
|
|
20
|
+
bigtesty-0.1.0a1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
21
|
+
bigtesty-0.1.0a1.dist-info/entry_points.txt,sha256=kjIHnEfQrvmIoEceW6l5xLvzqQNjbVJ2almYITl9RFw,70
|
|
22
|
+
bigtesty-0.1.0a1.dist-info/top_level.txt,sha256=iAitepHcGGQQNekCFAr2NrkMPd8rvWichurwR-mrxXw,9
|
|
23
|
+
bigtesty-0.1.0a1.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
bigtesty
|