bigtesty 0.1.0__py3-none-any.whl

bigtesty/arguments.py

@@ -0,0 +1,7 @@
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--root_folder', dest='root_folder', type=str, help='Root folder')
+parser.add_argument('--project_id', dest='project_id', type=str, help='GCP Project ID')
+parser.add_argument('--datasets_hash', dest='datasets_hash', type=str, help='Unique hash to isolate datasets')
+args = parser.parse_args()

bigtesty/cli/cli_env_vars.py

@@ -0,0 +1,13 @@
+GOOGLE_PROJECT_KEY = "GOOGLE_PROJECT"
+GOOGLE_REGION_KEY = "GOOGLE_REGION"
+IAC_BACKEND_URL_KEY = "IAC_BACKEND_URL"
+ROOT_TEST_FOLDER_KEY = "ROOT_TEST_FOLDER"
+ROOT_TABLES_FOLDER_KEY = "ROOT_TABLES_FOLDER"
+TABLES_CONFIG_FILE_KEY = "TABLES_CONFIG_FILE"
+KEEP_INFRA_KEY = "KEEP_INFRA"
+BIGTESTY_STACK_NAME_KEY = "BIGTESTY_STACK_NAME"
+PULUMI_CONFIG_PASSPHRASE_KEY = "PULUMI_CONFIG_PASSPHRASE"
+PULUMI_BACKEND_URL_KEY = "PULUMI_BACKEND_URL"
+
+BIGTESTY_STACK_NAME_VALUE = "bigtesty"
+PULUMI_CONFIG_PASSPHRASE_VALUE = "gcp_fake_passphrase"

bigtesty/cli/main.py

@@ -0,0 +1,102 @@
+import os
+
+import typer
+from typing_extensions import Annotated
+
+from bigtesty.cli.cli_env_vars import ROOT_TEST_FOLDER_KEY, ROOT_TABLES_FOLDER_KEY, TABLES_CONFIG_FILE_KEY, \
+    KEEP_INFRA_KEY, BIGTESTY_STACK_NAME_KEY, PULUMI_CONFIG_PASSPHRASE_KEY, PULUMI_BACKEND_URL_KEY, IAC_BACKEND_URL_KEY, \
+    BIGTESTY_STACK_NAME_VALUE, PULUMI_CONFIG_PASSPHRASE_VALUE, GOOGLE_PROJECT_KEY, GOOGLE_REGION_KEY
+from bigtesty.infra.launch_tests_ephemeral_infra import launch_tests_ephemeral_infra
+
+app = typer.Typer()
+
+
+@app.command("test", help="Run tests")
+def run_tests(
+        project: Annotated[
+            str,
+            typer.Option(
+                "--project",
+                envvar=GOOGLE_PROJECT_KEY,
+                help="GCP project ID."
+            )
+        ],
+        region: Annotated[
+            str,
+            typer.Option(
+                "--region",
+                envvar=GOOGLE_REGION_KEY,
+                help="GCP region."
+            )
+        ],
+        iac_backend_url: Annotated[
+            str,
+            typer.Option(
+                "--iac-backend-url",
+                envvar=IAC_BACKEND_URL_KEY,
+                help="IaC backend URL for the ephemeral infra used for the tests."
+            )
+        ],
+        root_test_folder: Annotated[
+            str,
+            typer.Option(
+                "--root-test-folder",
+                envvar=ROOT_TEST_FOLDER_KEY,
+                help="Directory that contains test scenarios."
+            )
+        ],
+        root_tables_folder: Annotated[
+            str,
+            typer.Option(
+                "--root-tables-folder",
+                envvar=ROOT_TABLES_FOLDER_KEY,
+                help="Directory that contains schema files."
+            )
+        ],
+        tables_config_file_path: Annotated[
+            str,
+            typer.Option(
+                "--tables-config-file",
+                envvar=TABLES_CONFIG_FILE_KEY,
+                help="Path to configuration file."
+            )
+        ],
+        keep_infra: Annotated[
+            bool,
+            typer.Option(
+                "--keep-infra",
+                envvar=KEEP_INFRA_KEY,
+                help="Keep infrastructure after exit."
+            )
+        ] = False):
+    print(f"####################### The CLI is invoked with params : ")
+
+    print(f"project : {project}")
+    print(f"region : {region}")
+    print(f"iac-backend-url : {iac_backend_url}")
+    print(f"root-test-folder : {root_test_folder}")
+    print(f"root-tables-folder : {root_tables_folder}")
+    print(f"tables-config-file : {tables_config_file_path}")
+    print(f"keep-infra : {keep_infra}")
+
+    os.environ[GOOGLE_PROJECT_KEY] = project
+    os.environ[GOOGLE_REGION_KEY] = region
+    os.environ[PULUMI_BACKEND_URL_KEY] = iac_backend_url
+    os.environ[PULUMI_CONFIG_PASSPHRASE_KEY] = PULUMI_CONFIG_PASSPHRASE_VALUE
+    os.environ[BIGTESTY_STACK_NAME_KEY] = BIGTESTY_STACK_NAME_VALUE
+
+    launch_tests_ephemeral_infra(
+        root_test_folder=root_test_folder,
+        root_tables_folder=root_tables_folder,
+        tables_config_file_path=tables_config_file_path,
+        keep_infra=keep_infra
+    )
+
+
+@app.command("help")
+def help():
+    print("BigTesty is an integration testing framework for BigQuery")
+
+
+def run():
+    app()

bigtesty/dataset_helper.py

@@ -0,0 +1,4 @@
+def build_unique_dataset_id_for_scenario(dataset_id: str,
+                                         scenario_hash: str,
+                                         datasets_hash: str) -> str:
+    return f'{dataset_id}_{scenario_hash}_{datasets_hash}'

bigtesty/definition_test_config_helper.py

@@ -0,0 +1,57 @@
+import glob
+import json
+from typing import List, Dict
+
+from deepdiff import DeepHash
+
+from bigtesty.definition_test_file_exception import DefinitionTestFileException
+from bigtesty.lambda_functions import flat_map
+
+DEFINITION_TEST_FILE_PATTERN = "**/definition_*.json"
+
+
+def get_definition_test_dicts_from_path(data_testing_root_path: str) -> List[Dict]:
+    """
+     Gets all the test definition files existing in the given folder path and then map them to dicts.
+     The result is a list of dict, and each dict represents a test definition.
+     If there is no definition files in the given folder, a :class:`~DefinitionTestFileException` is raised.
+    :param data_testing_root_path: the root folder path containing test definition files
+    :return: a list of test definition dict
+    """
+    print("#############ROOT DIR")
+    print(data_testing_root_path)
+
+    definition_test_file_paths: List[str] = get_definition_tests_file_paths(data_testing_root_path)
+
+    if not definition_test_file_paths:
+        raise DefinitionTestFileException(
+            f'There is no definition files in the given folder path {data_testing_root_path}'
+        )
+
+    return list(map(lambda path: to_definition_test_dict(path), definition_test_file_paths))
+
+
+def get_scenarios(data_testing_root_path: str):
+    return list(
+        flat_map(lambda deftest: deftest['scenarios'], get_definition_test_dicts_from_path(data_testing_root_path))
+    )
+
+
+def get_definition_tests_file_paths(data_testing_root_path: str) -> List[str]:
+    """
+    Gets all the test definitions file paths existing in the given folder path.
+    The search of files is recursive.
+    """
+    return glob.glob(
+        pathname=f'{data_testing_root_path}/{DEFINITION_TEST_FILE_PATTERN}',
+        recursive=True
+    )
+
+
+def to_definition_test_dict(definition_test_file_path: str):
+    with open(definition_test_file_path) as definition_test_file:
+        return json.load(definition_test_file)
+
+
+def get_scenario_hash(scenario_dict: Dict) -> str:
+    return DeepHash(scenario_dict)[scenario_dict]

bigtesty/definition_test_file_exception.py

@@ -0,0 +1,2 @@
+class DefinitionTestFileException(Exception):
+    pass

bigtesty/files_loader_helper.py

@@ -0,0 +1,12 @@
+import json
+from typing import List, Dict
+
+
+def load_file_as_dicts(file_path: str) -> List[Dict]:
+    with open(file_path) as file:
+        return json.load(file)
+
+
+def load_file_as_string(file_path: str) -> str:
+    with open(file_path, 'r') as file:
+        return file.read().rstrip()

bigtesty/given/insertion_test_data_bigquery.py

@@ -0,0 +1,43 @@
+from typing import List, Dict
+
+from google.cloud import bigquery
+
+from bigtesty.dataset_helper import build_unique_dataset_id_for_scenario
+from bigtesty.definition_test_config_helper import get_scenario_hash
+from bigtesty.files_loader_helper import load_file_as_dicts
+
+
+def insert_test_data_to_bq_tables(
+        project_id: str,
+        root_test_folder: str,
+        datasets_hash: str,
+        scenarios: List[Dict]) -> None:
+    client = bigquery.Client(project=project_id)
+
+    for scenario in scenarios:
+        for given in scenario['given']:
+            print("###### GIVEN")
+            print(given)
+
+            input = given.get('input')
+            input_as_dicts: List[Dict] = (
+                input if input
+                else load_file_as_dicts(f"{root_test_folder}/{given['input_file_path']}")
+            )
+
+            print("####### INPUT AS DICT")
+            print(input_as_dicts)
+
+            dataset_id_with_hash = build_unique_dataset_id_for_scenario(
+                dataset_id=given['destination_dataset'],
+                scenario_hash=get_scenario_hash(scenario),
+                datasets_hash=datasets_hash
+            )
+
+            results = client.insert_rows_json(
+                f"{dataset_id_with_hash}.{given['destination_table']}",
+                input_as_dicts
+            )
+
+            print("###### RESULTS")
+            print(results)

bigtesty/infra/create_iac_for_datasets_with_tables.py

@@ -0,0 +1,50 @@
+from typing import Dict, List
+
+from pulumi_gcp.bigquery import Dataset
+
+from bigtesty.dataset_helper import build_unique_dataset_id_for_scenario
+from bigtesty.definition_test_config_helper import get_scenario_hash
+from bigtesty.infra.datasets_with_tables_factory import get_dataset, get_table_with_partitioning, get_table
+
+
+def create_datasets_and_tables(root_tables_folder: str,
+                               datasets_with_tables_config: List[Dict],
+                               datasets_hash: str,
+                               scenarios: List[Dict]) -> None:
+    for dataset_config in datasets_with_tables_config:
+        for scenario in scenarios:
+            dataset_id_with_hash = build_unique_dataset_id_for_scenario(
+                dataset_id=dataset_config["datasetId"],
+                scenario_hash=get_scenario_hash(scenario),
+                datasets_hash=datasets_hash
+            )
+
+            bq_dataset = get_dataset(dataset_id_with_hash, dataset_config)
+
+            create_tables(
+                root_tables_folder=root_tables_folder,
+                dataset_config=dataset_config,
+                dataset=bq_dataset,
+                dataset_id_with_hash=dataset_id_with_hash
+            )
+
+
+def create_tables(root_tables_folder: str,
+                  dataset_config: Dict,
+                  dataset: Dataset,
+                  dataset_id_with_hash: str) -> None:
+    for table_config in dataset_config["tables"]:
+        (
+            get_table_with_partitioning(
+                root_tables_folder=root_tables_folder,
+                dataset_id_with_hash=dataset_id_with_hash,
+                dataset=dataset,
+                table=table_config
+            ) if table_config.get("partitionType") is not None
+            else get_table(
+                root_tables_folder=root_tables_folder,
+                dataset_id_with_hash=dataset_id_with_hash,
+                dataset=dataset,
+                table=table_config
+            )
+        )

bigtesty/infra/datasets_with_tables_config_file_loader.py

@@ -0,0 +1,28 @@
+import json
+import random
+import string
+from typing import List, Dict
+
+
+def _load_file_as_dicts(file_path: str) -> List[Dict]:
+    with open(file_path) as file:
+        return json.load(file)
+
+
+def _load_file_as_string(file_path: str) -> str:
+    with open(file_path) as file:
+        return file.read()
+
+
+def get_table_schema(root_tables_folder: str, table: Dict) -> str:
+    datasets_tables_config_file_path = f'{root_tables_folder}/{table["tableSchemaPath"]}'
+
+    return _load_file_as_string(datasets_tables_config_file_path)
+
+
+def get_datasets_hash(length) -> str:
+    letters = string.ascii_lowercase
+    datasets_hash = ''.join(random.choice(letters) for _ in range(length))
+    print(f'The generated datasets hash is : {datasets_hash}')
+
+    return datasets_hash

bigtesty/infra/datasets_with_tables_factory.py

@@ -0,0 +1,55 @@
+from typing import Dict
+
+import pulumi_gcp as gcp
+from pulumi import ResourceOptions
+from pulumi_gcp.bigquery import Dataset
+
+from bigtesty.infra.datasets_with_tables_config_file_loader import get_table_schema
+
+
+def get_table(root_tables_folder: str,
+              dataset_id_with_hash: str,
+              dataset: Dataset,
+              table: Dict):
+    return gcp.bigquery.Table(
+        get_table_resource_name(dataset_id_with_hash, table['tableId']),
+        deletion_protection=False,
+        dataset_id=dataset.dataset_id,
+        table_id=table["tableId"],
+        clusterings=table.get("clustering"),
+        schema=get_table_schema(root_tables_folder, table),
+        opts=ResourceOptions(depends_on=[dataset])
+    )
+
+
+def get_table_with_partitioning(root_tables_folder: str,
+                                dataset_id_with_hash: str,
+                                dataset: Dataset,
+                                table: Dict):
+    return gcp.bigquery.Table(
+        get_table_resource_name(dataset_id_with_hash, table['tableId']),
+        deletion_protection=False,
+        dataset_id=dataset.dataset_id,
+        table_id=table["tableId"],
+        clusterings=table.get("clustering"),
+        time_partitioning=gcp.bigquery.TableTimePartitioningArgs(
+            type=table["partitionType"],
+            field=table["partitionField"],
+        ),
+        schema=get_table_schema(root_tables_folder, table),
+        opts=ResourceOptions(depends_on=[dataset])
+    )
+
+
+def get_table_resource_name(dataset_id: str, table_id: str) -> str:
+    return f'{dataset_id}_{table_id}'
+
+
+def get_dataset(dataset_id_with_hash: str, dataset: Dict):
+    return gcp.bigquery.Dataset(
+        dataset_id_with_hash,
+        dataset_id=dataset_id_with_hash,
+        friendly_name=dataset["datasetFriendlyName"],
+        description=dataset["datasetDescription"],
+        location=dataset["datasetRegion"]
+    )

bigtesty/infra/launch_tests_ephemeral_infra.py

@@ -0,0 +1,134 @@
+import json
+import os
+import sys
+import traceback
+from typing import List, Dict
+
+from pulumi import automation as auto
+
+from bigtesty.definition_test_config_helper import get_scenarios
+from bigtesty.given.insertion_test_data_bigquery import insert_test_data_to_bq_tables
+from bigtesty.infra.create_iac_for_datasets_with_tables import create_datasets_and_tables
+from bigtesty.infra.datasets_with_tables_config_file_loader import get_datasets_hash, \
+    _load_file_as_dicts
+from bigtesty.then.assertion_and_tests_reports_result import execute_query_and_build_reports_result, \
+    check_any_failed_test_in_reports
+
+
+def launch_tests_ephemeral_infra(root_test_folder: str,
+                                 root_tables_folder: str,
+                                 tables_config_file_path: str,
+                                 keep_infra: bool):
+    project_id = os.environ["GOOGLE_PROJECT"]
+    region = os.environ["GOOGLE_REGION"]
+    stack_name = os.environ["BIGTESTY_STACK_NAME"]
+
+    datasets_hash = get_datasets_hash(5)
+
+    scenarios = get_scenarios(root_test_folder)
+
+    stack = auto.create_or_select_stack(
+        stack_name=stack_name,
+        project_name=project_id,
+        program=lambda: pulumi_program(scenarios,
+                                       datasets_hash,
+                                       root_tables_folder,
+                                       tables_config_file_path)
+    )
+
+    print("Successfully initialized stack")
+
+    # for inline programs, we must manage plugins ourselves
+    print("Installing plugins...")
+    stack.workspace.install_plugin("gcp", "v7.24.0")
+    print("Plugins installed")
+
+    print("Setting up config")
+    stack.set_config("gcp:project", auto.ConfigValue(value=project_id))
+    stack.set_config("gcp:region", auto.ConfigValue(value=region))
+    print("Config set")
+
+    print("Refreshing stack...")
+    stack.refresh(on_output=print)
+    print("Refresh complete")
+
+    print("#################### Creating the ephemeral infra...")
+    up_res = stack.up(
+        on_output=print,
+        color="always",
+        show_secrets=False,
+        diff=True
+    )
+    print(f"Update summary: \n{json.dumps(up_res.summary.resource_changes, indent=4)}")
+
+    reports_result: List[Dict]
+    try:
+        print("#################### Inserting Test data to Tables...")
+        insert_test_data_to_bq_tables(
+            project_id=project_id,
+            root_test_folder=root_test_folder,
+            datasets_hash=datasets_hash,
+            scenarios=scenarios
+        )
+
+        print("#################### Execute SQL queries and generate reports result...")
+        reports_result = execute_query_and_build_reports_result(
+            project_id=project_id,
+            root_test_folder=root_test_folder,
+            datasets_hash=datasets_hash,
+            scenarios=scenarios
+        )
+
+        print("################## The reports result is #################")
+        print(reports_result)
+        print("#################")
+
+    except Exception as e:
+        print(
+            "################# Oops! Error in the pipeline, we will rollback the created ephemeral infra :",
+            file=sys.stderr
+        )
+
+        exception_traceback = ''.join(traceback.format_tb(e.__traceback__))
+        print(exception_traceback, file=sys.stderr)
+
+        stack.destroy(
+            on_output=print,
+            color="always",
+            show_secrets=False,
+            log_flow=True,
+            log_verbosity=3
+        )
+
+        sys.exit(-1)
+
+    print(f"################### Parameter to keep the infra (true/false) : {keep_infra}")
+
+    if not keep_infra:
+        print("################### Destroying the ephemeral infra and tests assertions...")
+        stack.destroy(
+            on_output=print,
+            color="always",
+            show_secrets=False,
+            log_flow=True,
+            log_verbosity=3
+        )
+        print("############### Destroy ephemeral infra complete")
+
+    print("############### Checking if there is any failed test...")
+    check_any_failed_test_in_reports(reports_result)
+    print("############### Tests assertions finished")
+
+
+def pulumi_program(scenarios: List[Dict],
+                   datasets_hash: str,
+                   root_tables_folder: str,
+                   tables_config_file_path: str):
+    datasets_with_tables_config = _load_file_as_dicts(tables_config_file_path)
+
+    create_datasets_and_tables(
+        root_tables_folder=root_tables_folder,
+        datasets_with_tables_config=datasets_with_tables_config,
+        datasets_hash=datasets_hash,
+        scenarios=scenarios
+    )

bigtesty/lambda_functions.py

@@ -0,0 +1 @@
+flat_map = lambda f, xs: (y for ys in xs for y in f(ys))

bigtesty/then/assertion_and_tests_reports_result.py

@@ -0,0 +1,227 @@
+import importlib.util
+import re
+import traceback
+from typing import List, Dict
+
+import deepdiff
+from google.cloud import bigquery
+from google.cloud.bigquery import Client
+from toolz.curried import pipe, map
+
+from bigtesty.dataset_helper import build_unique_dataset_id_for_scenario
+from bigtesty.definition_test_config_helper import get_scenario_hash
+from bigtesty.files_loader_helper import load_file_as_string, load_file_as_dicts
+from bigtesty.then.assertion_types import AssertionType
+from bigtesty.then.failure_test_exception import FailureTestException
+
+
+def execute_query_and_build_reports_result(
+        project_id: str,
+        root_test_folder: str,
+        datasets_hash: str,
+        scenarios: List[Dict]) -> List[Dict]:
+    client = bigquery.Client(project=project_id)
+
+    return list(pipe(
+        scenarios,
+        map(lambda scenario: _execute_query_and_build_report_result(
+            bigquery_python_client=client,
+            scenario=scenario,
+            root_test_folder=root_test_folder,
+            datasets_hash=datasets_hash
+        ))
+    ))
+
+
+def check_any_failed_test_in_reports(reports_result: List[Dict]) -> bool:
+    any_failed_test: bool = any(report['result'] is False for report in reports_result)
+
+    if any_failed_test:
+        print("There is any failed test in the reports result, the pipeline is in failure status !!")
+        raise FailureTestException('Error : there is any failed test !!!')
+
+    print("There is no failed test in the reports result, the pipeline is in success status !!")
+    return any_failed_test
+
+
+def _execute_query_and_build_report_result(bigquery_python_client: Client,
+                                           scenario: Dict,
+                                           root_test_folder: str,
+                                           datasets_hash: str) -> Dict:
+    given_list: List[Dict] = scenario["given"]
+
+    print(f"######################################## treating the scenario {scenario['description']}... ##########")
+
+    for then in scenario["then"]:
+        query = _build_query(
+            scenario=scenario,
+            datasets_hash=datasets_hash,
+            root_test_folder=root_test_folder,
+            given_list=given_list,
+            then=then
+        )
+
+        print("#########SQL QUERY")
+        print(query)
+
+        query_job = bigquery_python_client.query(query)
+        actual_list: List[Dict] = list(map(lambda r: dict(r), query_job.result()))
+        print("#########ACTUAL")
+        print(actual_list)
+
+        expected_list = _get_expected_list(root_test_folder, then)
+        print("#########EXPECTED")
+        print(expected_list)
+
+        assertion_type: str = then['assertion_type']
+        print(f"######### The assertion type is: {assertion_type}")
+
+        if assertion_type == AssertionType.ROW_MATCH.value:
+            return _build_report_row_match_assertion(
+                scenario=scenario,
+                then=then,
+                actual_list=actual_list,
+                expected_list=expected_list
+            )
+        elif assertion_type == AssertionType.FUNCTION_ASSERTION.value:
+            return _build_report_functions_assertion(
+                root_test_folder=root_test_folder,
+                scenario=scenario,
+                then=then,
+                actual_list=actual_list,
+                expected_list=expected_list
+            )
+        else:
+            assertion_types = [e.value for e in AssertionType]
+            raise ValueError(
+                f"The then bloc doesn't contains a known assertion_type field.The known types are {assertion_types}"
+            )
+
+
+def _build_report_row_match_assertion(scenario: Dict,
+                                      then: Dict,
+                                      actual_list: List[Dict],
+                                      expected_list: List[Dict]) -> Dict:
+    fields_to_ignore_compiled_regex = list(
+        map(lambda field_regex: re.compile(field_regex), then['fields_to_ignore'])
+    )
+
+    result: Dict = deepdiff.DeepDiff(actual_list,
+                                     expected_list,
+                                     ignore_order=True,
+                                     exclude_regex_paths=fields_to_ignore_compiled_regex,
+                                     view='tree')
+    print(f"#########TEST RESULT for scenario: {scenario['description']}")
+    print(result)
+
+    return {
+        'scenario': scenario,
+        'actual': actual_list,
+        'expected': expected_list,
+        'comparison_info': result,
+        'result': result == {}
+    }
+
+
+def _build_report_functions_assertion(root_test_folder: str,
+                                      scenario: Dict,
+                                      then: Dict,
+                                      actual_list: List[Dict],
+                                      expected_list: List[Dict]) -> Dict:
+    assertions_result: List[Dict] = list(pipe(
+        then['expected_functions'],
+        map(
+            lambda fn: _execute_assertions_functions_and_get_result(
+                root_test_folder=root_test_folder,
+                actual_list=actual_list,
+                expected_list=expected_list,
+                expected_function=fn)
+        )
+    ))
+
+    any_assertion_failed: bool = any(result['assertion_error'] is True for result in assertions_result)
+
+    print(f"#########TEST RESULT for scenario: {scenario['description']}")
+
+    return {
+        'scenario': scenario,
+        'actual': actual_list,
+        'expected': expected_list,
+        'comparison_info': assertions_result,
+        'result': not any_assertion_failed
+    }
+
+
+def _execute_assertions_functions_and_get_result(root_test_folder: str,
+                                                 actual_list: List[Dict],
+                                                 expected_list: List[Dict],
+                                                 expected_function: Dict) -> Dict:
+    assertion_module = expected_function["module"]
+    assertion_function = expected_function["function"]
+
+    module_spec = importlib.util.spec_from_file_location(
+        "assertion_functions", f'{root_test_folder}/{assertion_module}'
+    )
+    gfg_module = importlib.util.module_from_spec(module_spec)
+    module_spec.loader.exec_module(gfg_module)
+
+    current_function = getattr(gfg_module, assertion_function)
+
+    assertion_error = False
+    assertion_error_message = ""
+    try:
+        print(f"Executing Assertion Function {assertion_function} from module {assertion_module}...")
+        current_function(actual_list, expected_list)
+        print(f"The Assertion Function {assertion_function} from module {assertion_module} is in success status...")
+    except AssertionError as e:
+        assertion_error = True
+        assertion_error_message = ''.join(traceback.format_tb(e.__traceback__))
+        print(
+            f"The Assertion Function {assertion_function} from module {assertion_module} "
+            f"has the following error {assertion_error_message}"
+        )
+
+    return {
+        'assertion_module': assertion_module,
+        'assertion_function': assertion_function,
+        'assertion_error': assertion_error,
+        'assertion_error_message': assertion_error_message
+    }
+
+
+def _build_query(scenario: Dict,
+                 datasets_hash: str,
+                 root_test_folder: str,
+                 given_list: List[Dict],
+                 then: Dict) -> str:
+    actual = then.get('actual')
+    sql_query = actual if actual else load_file_as_string(f"{root_test_folder}/{then['actual_file_path']}")
+
+    sql_query_result = ""
+    for given in given_list:
+        sql_query_result = _replace_current_dataset_by_unique_dataset_for_scenario(
+            sql_query=sql_query,
+            current_dataset=given["destination_dataset"],
+            scenario_hash=get_scenario_hash(scenario),
+            datasets_hash=datasets_hash
+        )
+
+    return sql_query_result
+
+
+def _replace_current_dataset_by_unique_dataset_for_scenario(sql_query: str,
+                                                            current_dataset: str,
+                                                            scenario_hash: str,
+                                                            datasets_hash: str) -> str:
+    unique_dataset_with_for_scenario = build_unique_dataset_id_for_scenario(
+        dataset_id=current_dataset,
+        scenario_hash=scenario_hash,
+        datasets_hash=datasets_hash
+    )
+
+    return sql_query.replace(f"{current_dataset}.", f"{unique_dataset_with_for_scenario}.")
+
+
+def _get_expected_list(root_test_folder: str, then: Dict) -> List[Dict]:
+    expected = then.get('expected')
+    return expected if expected else load_file_as_dicts(f"{root_test_folder}/{then['expected_file_path']}")

bigtesty/then/assertion_types.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class AssertionType(Enum):
+    ROW_MATCH = 'row_match'
+    FUNCTION_ASSERTION = 'function_assertion'

bigtesty/then/failure_test_exception.py

@@ -0,0 +1,2 @@
+class FailureTestException(Exception):
+    pass

bigtesty-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Mazlum Tosun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bigtesty-0.1.0.dist-info/METADATA

@@ -0,0 +1,226 @@
+Metadata-Version: 2.1
+Name: bigtesty
+Version: 0.1.0
+Summary: BigTesty is an integration testing framework for BigQuery
+Home-page: https://github.com/tosun-si/bigtesty
+Author: Mazlum TOSUN
+Author-email: mazlum.tosun@gmail.com
+License: UNKNOWN
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9.19
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pulumi-gcp ==7.24.0
+Requires-Dist: google-cloud-bigquery ==3.23.1
+Requires-Dist: typer[all] ==0.12.3
+Requires-Dist: pytest ==8.2.1
+Requires-Dist: deepdiff ==7.0.1
+Requires-Dist: toolz ==0.12.1
+
+# BigTesty
+
+BigTesty is a framework that allows to create Integration Tests with BigQuery on a real and short-lived Infrastructure.
+
+Integration and End-to-End tests are a robust way to validate if SQL queries work as expected.\
+There is no an emulator in this case and the queries are executed directly in the BigQuery Engine.
+
+BigTesty isolates the tests for each execution to prevent collisions.\
+Multiples developers or CI CD pipelines can execute tests at the same time.
+
+The infrastructure proposed for the tests is ephemeral by default, but we can keep it if needed, to analyse the\
+result in BigQuery.
+
+After each test, a report result is returned to indicate the good and failure cases.
+
+![bigtesty_pulumi_automation.png](diagram%2Fbigtesty_pulumi_automation.png)
+
+## Getting started
+
+There is a `Python` package for BigTesty and it can be installed from `PiPy`:
+
+```bash
+pip install bigtesty
+```
+
+## How to run tests
+
+**You need to be authenticated with Google Cloud Platform before running command.**
+
+We recommend to be authenticated with [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials)
+
+```bash
+gcloud auth application-default login
+```
+
+### Example of code structure
+
+![bigtesty_examples_code_structure.png](bigtesty-doc%2Fstatic%2Fimg%2Fbigtesty_examples_code_structure.png)
+
+**The root test folder**
+
+This folder contains all the testing definition files and the tests scenarios. The format is Json.
+
+Example of a scenarios with a nominal case `definition_spec_failure_by_feature_name_no_error.json`:
+
+```json
+{
+  "description": "Test of monitoring data",
+  "scenarios": [
+    {
+      "description": "Nominal case find failure by feature name",
+      "given": [
+        {
+          "input_file_path": "monitoring/given/input_failures_feature_name.json",
+          "destination_dataset": "monitoring",
+          "destination_table": "job_failure"
+        }
+      ],
+      "then": [
+        {
+          "fields_to_ignore": [
+            "\\[\\d+\\]\\['dwhCreationDate']"
+          ],
+          "actual_file_path": "monitoring/when/find_failures_by_feature_name.sql",
+          "expected_file_path": "monitoring/then/expected_failures_feature_name.json"
+        }
+      ]
+    }
+  ]
+}
+```
+
+In this example, there is only one scenario with 3 blocs:
+- `given`: a list of input test data to ingest to the BigQuery tables. The input data can be proposed in a separate `Json` file or directly embedded. `input_file_path` for a separate file and `input` for an embedded object.
+- `then`: a list of objects contains the SQL query to test and execute and the expected data. `actual/actual_file_path` => SQL query | `expected/expected_file_path` => expected data
+
+**The root tables folder**
+
+This folder contains the resources concerning the `BigQuery` `datasets` and `tables` to create.\
+For example, all the `BigQuery` schemas are proposed in this folder.
+
+**The tables config file**
+
+The config file that lists all the `BigQuery` `datasets` and `tables` to create in a `Json` format.
+
+Example:
+
+```json
+[
+  {
+    "datasetId": "monitoring",
+    "datasetRegion": "EU",
+    "datasetFriendlyName": "Monitoring Dataset",
+    "datasetDescription": "Monitoring Dataset description",
+    "tables": [
+      {
+        "tableId": "job_failure",
+        "autodetect": false,
+        "tableSchemaPath": "schema/monitoring/job_failure.json",
+        "partitionType": "DAY",
+        "partitionField": "dwhCreationDate",
+        "clustering": [
+          "featureName",
+          "jobName",
+          "componentType"
+        ]
+      }
+    ]
+  }
+]
+```
+
+In this example, we have a dataset called `monitoring` with the metadata.\
+This dataset contains a table called `job_failure` with the metadata. Some fields can target on the `BigQuery` schemas proposed in the `root tables folder`.
+
+### Run with CLI
+
+We need to pass the 3 parameters indicated in the previous section, in the command line to launch the tests:
+
+- root-test-folder: the root folder containing all the testing files
+- root-tables-folder: the root folder containing all the needed files to create the datasets and tables in BigQuery (Json schema...)
+- tables-config-file: the Json configuration file that lists all the datasets and tables to create in BigQuery
+
+Also, common GCP parameters like:
+- project: the GCP project ID
+- region: the GCP region
+
+`BigTesty` uses an ephemeral infra internally via the concept of Infra As Code and the backend to host the state must be a `cloud Storage` bucket.\
+We need to pass the backend URL via parameter in the CLI:
+- iac-backend-url
+
+The tests can be executed with the following command line:
+
+```bash
+bigtesty test \
+  --project $PROJECT_ID \
+  --region $LOCATION \
+  --iac-backend-url gs://$IAC_BUCKET_STATE/bigtesty \
+  --root-test-folder $(pwd)/examples/tests \
+  --root-tables-folder $(pwd)/examples/tests/tables \
+  --tables-config-file $(pwd)/examples/tests/tables/tables.json
+```
+
+**All the testing files showed in the documentation are accessible from the `examples` folder proposed at the root of the `BigTesty` repo.**
+
+### Run with Docker
+
+Instead of pass the arguments by the CLI, we can also pass them with environment variables.
+
+```bash
+export PROJECT_ID={{project_id}}
+export LOCATION={{region}}
+export IAC_BACKEND_URL=gs://{{gcs_state_bucket}}/bigtesty
+export ROOT_TEST_FOLDER=/opt/bigtesty/tests
+export ROOT_TABLES_FOLDER=/opt/bigtesty/tests/tables
+export TABLES_CONFIG_FILE_PATH=/opt/bigtesty/tests/tables/tables.json
+
+docker run -it \
+   -e GOOGLE_PROJECT=$PROJECT_ID \
+   -e GOOGLE_REGION=$LOCATION \
+   -e IAC_BACKEND_URL=$IAC_BACKEND_URL \
+   -e TABLES_CONFIG_FILE="$TABLES_CONFIG_FILE_PATH" \
+   -e ROOT_TEST_FOLDER=$ROOT_TEST_FOLDER \
+   -e ROOT_TABLES_FOLDER="$ROOT_TABLES_FOLDER" \
+   -v $(pwd)/examples/tests:/opt/bigtesty/tests \
+   -v $(pwd)/examples/tests/tables:/opt/bigtesty/tests/tables \
+   -v $HOME/.config/gcloud:/opt/bigtesty/.config/gcloud \
+   groupbees/bigtesty test
+```
+
+Some explanations:
+
+All the parameters are passed as environment variables.\
+We need also to mount as volumes:
+- the tests root folder : `-v $(pwd)/examples/tests:/opt/bigtesty/tests`
+- the tables root folder: `-v $(pwd)/examples/tests/tables:/opt/bigtesty/tests/tables`
+- the `gcloud` configuration: `-v $HOME/.config/gcloud:/opt/bigtesty/.config/gcloud`
+
+When the authentication is done with Applications Default Credentials via the following command `gcloud auth application-default login`,\
+a short-lived credential is generated in the local `gcloud` configuration: `$HOME/.config/gcloud`
+
+To prevent the use of a long-lived SA token key, we can share and mount as volume the local `gcloud` configuration with the `Docker` container: `-v $HOME/.config/gcloud:/opt/bigtesty/.config/gcloud`\
+With this technic, the container will be authenticated in Google Cloud securely, with your current user in the Shell session.
+
+### Run with Cloud Build
+
+```bash
+export PROJECT_ID={{project_id}}
+export LOCATION={{region}}
+export IAC_BACKEND_URL=gs://{{gcs_state_bucket}}/bigtesty
+
+gcloud builds submit \
+   --project=$PROJECT_ID \
+   --region=$LOCATION \
+   --config examples/ci/cloud_build/run-tests-cloud-build.yaml \
+   --substitutions _IAC_BACKEND_URL=$IAC_BACKEND_URL \
+   --verbosity="debug" .
+```
+
+## Contributing
+
+cf. [CONTRIBUTING](CONTRIBUTING.md).
+
+

bigtesty-0.1.0.dist-info/RECORD

@@ -0,0 +1,27 @@
+bigtesty/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigtesty/arguments.py,sha256=JwwMPFkRSSrWILzO6mY9Xlbbajk4cs6ddFbC4zyrQs0,365
+bigtesty/dataset_helper.py,sha256=BrX1ZUv4IDvS9lCMwidlsT7DKrfKyhtXoq06N4-38w4,247
+bigtesty/definition_test_config_helper.py,sha256=WaqpouC7tvIwC8LcKvaHXlyGJt9IcOJ8-PXH0ifea8E,2055
+bigtesty/definition_test_file_exception.py,sha256=2ojAbfQ_FSN4Gq-f82ARynQvFYDBaAOy4bAZ4tqUZB0,55
+bigtesty/files_loader_helper.py,sha256=hUBQ8jVuIULc0auAd0g2Bw2HEHOckg2WKo89VlamWe4,288
+bigtesty/lambda_functions.py,sha256=Kha-s1_P_BU5d-PAY2HOW11tnZNa0EiGVSFyn9HquYI,57
+bigtesty/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigtesty/cli/cli_env_vars.py,sha256=gBW38CwQOKrHlCKt9A_1n63jdA0N5bJ1d02HXCYY0CM,525
+bigtesty/cli/main.py,sha256=M-apIOyZZOM44uMSCp0yyaYHp-hS5vFvRz5sUlWNP28,3263
+bigtesty/given/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigtesty/given/insertion_test_data_bigquery.py,sha256=-lobytt7Pwvyw1RRPoX9AI5T_7VABDuofmgtu0zXXRo,1382
+bigtesty/infra/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigtesty/infra/create_iac_for_datasets_with_tables.py,sha256=ESi8WuAxO8PIj2FdCTccKpyR92U4dkreefkvWB40nZc,1971
+bigtesty/infra/datasets_with_tables_config_file_loader.py,sha256=zIsQkq-_ygZ8ffYWyPo1z31v_wlBTYrDxvNwafzMZfQ,771
+bigtesty/infra/datasets_with_tables_factory.py,sha256=xxKn7H6e5ufxtneG_svYvDwUutZm1UZcXnrlrMGYmQ0,1903
+bigtesty/infra/launch_tests_ephemeral_infra.py,sha256=qzSjtcDMBCaVIFNlDhNIWoUpDcj_fmmtZMoSqYaKuns,4607
+bigtesty/then/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigtesty/then/assertion_and_tests_reports_result.py,sha256=L5pCb_n4bhli5gzPq4J3ygdgGDOalMKFJoCMkYeLSoI,8749
+bigtesty/then/assertion_types.py,sha256=DIrxmA5utBGUIUHy29exenMdWr1wa9ATqvJryMPKQLQ,125
+bigtesty/then/failure_test_exception.py,sha256=XGdZ75LXL3TpNV7zaZvSj-6S21pUBRkh-J_ygaQDIIE,48
+bigtesty-0.1.0.dist-info/LICENSE,sha256=aBCOCCCvY_F67UK2dRXZ9vJccUsM1hyU34OTSo_CgJI,1069
+bigtesty-0.1.0.dist-info/METADATA,sha256=qBdmU4oWuSYgaOrmOL9YxLJluMhk_xM6EHME1aUOCEQ,8033
+bigtesty-0.1.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+bigtesty-0.1.0.dist-info/entry_points.txt,sha256=kjIHnEfQrvmIoEceW6l5xLvzqQNjbVJ2almYITl9RFw,70
+bigtesty-0.1.0.dist-info/top_level.txt,sha256=iAitepHcGGQQNekCFAr2NrkMPd8rvWichurwR-mrxXw,9
+bigtesty-0.1.0.dist-info/RECORD,,

bigtesty-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

bigtesty-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+
+        [console_scripts]
+        bigtesty=bigtesty.cli.main:run
+    

bigtesty-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+bigtesty