google-pso-data-validator 7.3.0__tar.gz → 7.5.0__tar.gz

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: google-pso-data-validator
-Version: 7.3.0
+Version: 7.5.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -51,6 +51,7 @@ Dynamic: author-email
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
@@ -651,6 +652,8 @@ data-validation query
           The connection name to be queried
   --query or -q QUERY
           The raw query to run against the supplied connection
+  [--format or -f {minimal,python}]
+          Format for query output (default: python)
 ```
 
 ### Building Matched Table Lists

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.5.0}/README.md

@@ -593,6 +593,8 @@ data-validation query
           The connection name to be queried
   --query or -q QUERY
           The raw query to run against the supplied connection
+  [--format or -f {minimal,python}]
+          Format for query output (default: python)
 ```
 
 ### Building Matched Table Lists

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.5.0}/data_validation/__main__.py

@@ -16,9 +16,9 @@ import json
 import logging
 import os
 import sys
+from typing import TYPE_CHECKING
 
 from yaml import Dumper, dump
-from argparse import Namespace
 from typing import List
 from data_validation import (
     cli_tools,
@@ -34,6 +34,9 @@ from data_validation.data_validation import DataValidation
 from data_validation.find_tables import find_tables_using_string_matching
 from data_validation.partition_builder import PartitionBuilder
 
+if TYPE_CHECKING:
+    from argparse import Namespace
+
 # by default yaml dumps lists as pointers. This disables that feature
 Dumper.ignore_aliases = lambda *args: True
 
@@ -246,7 +249,7 @@ def _get_comparison_config(
     return config_manager.build_config_comparison_fields(comparison_fields)
 
 
-def build_config_from_args(args: Namespace, config_manager: ConfigManager):
+def build_config_from_args(args: "Namespace", config_manager: ConfigManager):
     """This function is used to append build configs to the config manager for all validation commands and generate-table-partitions.
     Instead of having two separate commands, e.g. validate row and validate custom-query row, generate-table-partitions
     uses implicit choice of table or custom-query. A user can specify either tables or source/target query/file,
@@ -337,7 +340,7 @@ def build_config_from_args(args: Namespace, config_manager: ConfigManager):
 
 
 def build_config_managers_from_args(
-    args: Namespace, validate_cmd: str = None
+    args: "Namespace", validate_cmd: str = None
 ) -> List[ConfigManager]:
     """Return a list of config managers ready to execute."""
 
@@ -596,7 +599,7 @@ def store_json_config_file(args, config_managers):
     cli_tools.store_validation(config_file_path, json_config)
 
 
-def partition_and_store_config_files(args: Namespace) -> None:
+def partition_and_store_config_files(args: "Namespace") -> None:
     """Build multiple YAML Config files using user specified partition logic
 
     Args:
@@ -691,7 +694,7 @@ def main():
         print(find_tables_using_string_matching(args))
     elif args.command == "query":
         raw_query.print_raw_query_output(
-            raw_query.run_raw_query_against_connection(args)
+            raw_query.run_raw_query_against_connection(args), format=args.output_format
         )
     elif args.command == "validate":
         validate(args)

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.5.0}/data_validation/cli_tools.py

@@ -52,8 +52,7 @@ import sys
 import uuid
 import os
 import math
-from argparse import Namespace
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, TYPE_CHECKING
 from yaml import Dumper, Loader, dump, load
 
 from data_validation import (
@@ -66,6 +65,9 @@ from data_validation import (
 )
 from data_validation.validation_builder import list_to_sublists
 
+if TYPE_CHECKING:
+    from argparse import Namespace
+
 
 CONNECTION_SOURCE_FIELDS = {
     "BigQuery": [
@@ -191,7 +193,7 @@ VALIDATE_SCHEMA_HELP_TEXT = "Run a schema validation"
 VALIDATE_CUSTOM_QUERY_HELP_TEXT = "Run a custom query validation"
 
 
-def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: Namespace):
+def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: "Namespace"):
     # This is where we make additional checks if the arguments provided are what we expect
     # For example, only one of -tbls and custom query options can be provided
     if hasattr(parsed_args, "tables_list") and hasattr(
@@ -223,7 +225,7 @@ def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: Names
         return  # old format - only one of them is present
 
 
-def get_parsed_args() -> Namespace:
+def get_parsed_args() -> "Namespace":
     """Return ArgParser with configured CLI arguments."""
     parser = configure_arg_parser()
     args = ["--help"] if len(sys.argv) == 1 else None
@@ -393,6 +395,14 @@ def _configure_raw_query(subparsers):
     )
     query_parser.add_argument("--conn", "-c", help="Connection name to query")
     query_parser.add_argument("--query", "-q", help="Raw query to execute")
+    query_parser.add_argument(
+        "--format",
+        "-f",
+        dest="output_format",
+        choices=consts.RAW_QUERY_FORMAT_TYPES,
+        default=consts.FORMAT_TYPE_PYTHON,
+        help=f"Format for query output (default: {consts.FORMAT_TYPE_PYTHON})",
+    )
 
 
 def _configure_validation_config_parser(subparsers):
@@ -1027,7 +1037,7 @@ def _add_common_arguments(
     optional_arguments.add_argument(
         "--format",
         "-fmt",
-        default="table",
+        default=consts.FORMAT_TYPE_TABLE,
         help="Set the format for printing command output, Supported formats are (text, csv, json, table). Defaults "
         "to table",
     )
@@ -1443,7 +1453,7 @@ def _concat_column_count_configs(
     return return_list
 
 
-def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
+def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
     """Return a dict of configurations to build ConfigManager object"""
 
     def cols_from_arg(concat_arg: str, client, table_obj: dict, query_str: str) -> list:
@@ -1503,7 +1513,7 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
     target_client = clients.get_data_client(mgr.get_connection_config(args.target_conn))
 
     # Get format: text, csv, json, table. Default is table
-    format = args.format if args.format else "table"
+    format = args.format if args.format else consts.FORMAT_TYPE_TABLE
 
     # Get random row arguments. Only in row validations these attributes can be present.
     # Bad coding here, but keeping it so as not to introduce a breaking change. See

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.5.0}/data_validation/combiner.py

@@ -22,10 +22,12 @@ import datetime
 import functools
 import json
 import logging
+from typing import TYPE_CHECKING
+
 import ibis
 import ibis.expr.datatypes as dt
+import pandas
 
-from typing import TYPE_CHECKING
 from data_validation import consts
 
 if TYPE_CHECKING:
@@ -34,22 +36,109 @@ if TYPE_CHECKING:
     from data_validation.metadata import RunMetadata, ValidationMetadata
 
 
+# At around 140 columns we hit RecursionError when unioning Ibis subqueries.
+# This constant is a threshold at which we slice up the input Dataframes
+# and stitch them back together again after Ibis processing.
+COMBINER_COLUMN_SLICE_WIDTH = 120
+
+COMBINER_GET_SUMMARY_EXC_TEXT = (
+    "Error while generating summary report of row validation results"
+)
+
+
 def generate_report(
-    client,
     run_metadata: "RunMetadata",
-    source: "IbisTable",
-    target: "IbisTable",
+    source_df: "DataFrame",
+    target_df: "DataFrame",
     join_on_fields=(),
     is_value_comparison=False,
     verbose=False,
-):
+) -> "DataFrame":
+    """Combine results into a report.
+
+    This function is a wrapper around _generate_report_slice(). _generate_report_slice() does the main work, this
+    wrapper simply manages the input columns and stitches the results back together.
+    This is because validations of > 140(ish) columns trigger a RecursionError when unioning Ibis subqueries.
+    In this method we pass in column slices of the incoming Dataframes and combine the results.
+
+    It is a bit of a hack but I cannot find a way to optimize the Ibis processing. It appears to be
+    inefficient in that we create a subquery for each validation (column) in _calculate_differences() and
+    then union them all. We then do the same on the source/target table expressions to join it all back
+    together again. I (nj1973) spent a singificant amount of time trying to understand/optimize the Ibis
+    processing but fell back on this simpler (less risky) workaround.
+
+    Returns:
+        pandas.DataFrame:
+            A pandas DataFrame with the results of the validation in the same
+            schema as the report table.
+    """
+    _check_schema_names(source_df, target_df)
+
+    join_on_fields = tuple(join_on_fields)
+
+    validation_columns = run_metadata.validations.keys()
+    # slice_thresholds is a list of points at which we should break up the Dataframe by column.
+    # e.g. [10, 20, 30] would mean process columns 0-9, 10-19 and 20-the max column.
+    # 1. len(...) / COMBINER_COLUMN_SLICE_WIDTH: Divides total columns by the slice width to get the number of slices.
+    # 2. int(...) + 1: int()+1 is effectively ceil() which is what we want to get the actual whole number of slices
+    # 3. _ * COMBINER_COLUMN_SLICE_WIDTH: Multiplies each number by the slice width to get actual column counts for each slice.
+    slice_thresholds = [
+        (_ * COMBINER_COLUMN_SLICE_WIDTH)
+        for _ in range(int(len(validation_columns) / COMBINER_COLUMN_SLICE_WIDTH) + 1)
+    ]
+
+    result_df = None
+    # Process the input Dataframes in slices of columns to avoid "RecursionError"s.
+    for slice_start in slice_thresholds:
+        columns_in_vertical_slice = list(validation_columns)[
+            slice_start : slice_start + COMBINER_COLUMN_SLICE_WIDTH
+        ]
+        # Ensure any join columns are included in the column slice.
+        columns_in_vertical_slice.extend(
+            set(join_on_fields) - set(columns_in_vertical_slice)
+        )
+        interim_result_df = _generate_report_slice(
+            run_metadata,
+            source_df[columns_in_vertical_slice],
+            target_df[columns_in_vertical_slice],
+            join_on_fields=join_on_fields,
+            is_value_comparison=is_value_comparison,
+            verbose=verbose,
+        )
+        if result_df is None:
+            result_df = interim_result_df
+        else:
+            result_df = pandas.concat([result_df, interim_result_df])
+
+    # Get the first validation metadata object to fill source and/or target empty table names.
+    first = run_metadata.validations[next(iter(run_metadata.validations))]
+    if first.validation_type != consts.CUSTOM_QUERY:
+        result_df.source_table_name.fillna(
+            first.get_table_name(consts.RESULT_TYPE_SOURCE), inplace=True
+        )
+        result_df.target_table_name.fillna(
+            first.get_table_name(consts.RESULT_TYPE_TARGET), inplace=True
+        )
+
+    _get_summary(run_metadata, result_df, source_df, target_df)
+
+    return result_df
+
+
+def _generate_report_slice(
+    run_metadata: "RunMetadata",
+    source_df: "DataFrame",
+    target_df: "DataFrame",
+    join_on_fields=(),
+    is_value_comparison=False,
+    verbose=False,
+) -> "DataFrame":
     """Combine results into a report.
 
     Args:
-        client (ibis.client.Client): Ibis client used to combine results.
         run_metadata: Metadata about the run and validations.
-        source (ibis.QUERY): Ibis query / table object.
-        target (ibis.QUERY): Ibis query / table object.
+        source_df: Dataframe contains results of source query.
+        target_df: Dataframe contains results of target query.
         join_on_fields (Sequence[str]):
             A collection of column names to use to join source and target.
             These are the columns that both the source and target queries
@@ -63,16 +152,15 @@ def generate_report(
             A pandas DataFrame with the results of the validation in the same
             schema as the report table.
     """
-    join_on_fields = tuple(join_on_fields)
-
-    source_names = source.schema().names
-    target_names = target.schema().names
+    client = ibis.pandas.connect(
+        {
+            consts.RESULT_TYPE_SOURCE: source_df,
+            consts.RESULT_TYPE_TARGET: target_df,
+        }
+    )
+    source = client.table(consts.RESULT_TYPE_SOURCE)
+    target = client.table(consts.RESULT_TYPE_TARGET)
 
-    if source_names != target_names:
-        raise ValueError(
-            "Expected source and target to have same schema, got "
-            f"{consts.RESULT_TYPE_SOURCE}: {source_names}; {consts.RESULT_TYPE_TARGET}: {target_names}"
-        )
     differences_pivot = _calculate_differences(
         source, target, join_on_fields, run_metadata.validations, is_value_comparison
     )
@@ -81,18 +169,18 @@ def generate_report(
     source_pivot = _pivot_result(
         source, join_on_fields, run_metadata.validations, consts.RESULT_TYPE_SOURCE
     )
-    source_df = client.execute(source_pivot)
+    source_pivot_df = client.execute(source_pivot)
 
     target_pivot = _pivot_result(
         target, join_on_fields, run_metadata.validations, consts.RESULT_TYPE_TARGET
     )
-    target_df = client.execute(target_pivot)
+    target_pivot_df = client.execute(target_pivot)
 
     con = ibis.pandas.connect(
         {
-            consts.RESULT_TYPE_SOURCE: source_df,
-            "differences": differences_df,
-            consts.RESULT_TYPE_TARGET: target_df,
+            consts.RESULT_TYPE_SOURCE: source_pivot_df,
+            consts.RESULT_TYPE_DIFFERENCES: differences_df,
+            consts.RESULT_TYPE_TARGET: target_pivot_df,
         }
     )
     joined = _join_pivots(
@@ -107,19 +195,6 @@ def generate_report(
 
     result_df = client.execute(documented)
     result_df.validation_status.fillna(consts.VALIDATION_STATUS_FAIL, inplace=True)
-
-    # get the first validation metadata object to fill source and/or target empty table names
-    first = run_metadata.validations[next(iter(run_metadata.validations))]
-    if first.validation_type != consts.CUSTOM_QUERY:
-        result_df.source_table_name.fillna(
-            first.get_table_name(consts.RESULT_TYPE_SOURCE), inplace=True
-        )
-        result_df.target_table_name.fillna(
-            first.get_table_name(consts.RESULT_TYPE_TARGET), inplace=True
-        )
-
-    _get_summary(run_metadata, result_df, source_df, target_df)
-
     return result_df
 
 
@@ -279,6 +354,18 @@ def _calculate_differences(
     return differences_pivot
 
 
+def _check_schema_names(source_df, target_df):
+    """Check that the two input Dataframes have matching column names."""
+    source_names = tuple(source_df.columns)
+    target_names = tuple(target_df.columns)
+
+    if source_names != target_names:
+        raise ValueError(
+            "Expected source and target to have same schema, got "
+            f"{consts.RESULT_TYPE_SOURCE}: {source_names}; {consts.RESULT_TYPE_TARGET}: {target_names}"
+        )
+
+
 def _pivot_result(
     result: "IbisTable",
     join_on_fields: tuple,
@@ -441,12 +528,19 @@ def _get_summary(
 ):
     """Logs a summary report/stats of row validation results."""
     try:
-        if result_df.loc[0, consts.VALIDATION_TYPE] == consts.ROW_VALIDATION:
-            # Vectorized calculations for all counts
+        if result_df.empty:
+            return
+
+        if (result_df.loc[0, consts.VALIDATION_TYPE] == consts.ROW_VALIDATION) or (
+            # Check for custom-query row validation which always should have primary keys (not null)
+            result_df.loc[0, consts.VALIDATION_TYPE] == consts.CUSTOM_QUERY
+            and result_df.loc[0, consts.CONFIG_PRIMARY_KEYS]
+        ):
+            # Vectorized calculations for all counts.
             success_condition = (
                 result_df[consts.VALIDATION_STATUS] == consts.VALIDATION_STATUS_SUCCESS
             )
-            fail_condition = ~success_condition  # Invert success for fail condition
+            fail_condition = ~success_condition  # Invert success for fail condition.
 
             source_not_in_target = (
                 result_df[consts.SOURCE_AGG_VALUE].notnull()
@@ -462,33 +556,32 @@ def _get_summary(
             )
 
             logging.info(
-                {
-                    consts.CONFIG_RUN_ID: run_metadata.run_id,
-                    consts.CONFIG_START_TIME: run_metadata.start_time.strftime(
-                        "%Y-%m-%d %H:%M:%S %Z"
-                    ),
-                    consts.CONFIG_END_TIME: run_metadata.end_time.strftime(
-                        "%Y-%m-%d %H:%M:%S %Z"
-                    ),
-                    consts.TOTAL_SOURCE_ROWS: source_df.shape[0],
-                    consts.TOTAL_TARGET_ROWS: target_df.shape[0],
-                    consts.TOTAL_ROWS_VALIDATED: result_df.shape[0],
-                    # Using .sum() on boolean Series for much faster counting
-                    consts.TOTAL_ROWS_SUCCESS: success_condition.sum(),
-                    consts.TOTAL_ROWS_FAIL: fail_condition.sum(),
-                    consts.FAILED_SOURCE_NOT_IN_TARGET: (
-                        fail_condition & source_not_in_target
-                    ).sum(),
-                    consts.FAILED_TARGET_NOT_IN_SOURCE: (
-                        fail_condition & target_not_in_source
-                    ).sum(),
-                    consts.FAILED_PRESENT_IN_BOTH_TABLES: (
-                        fail_condition & present_in_both_tables
-                    ).sum(),
-                }
+                json.dumps(
+                    {
+                        consts.CONFIG_RUN_ID: run_metadata.run_id,
+                        consts.CONFIG_START_TIME: run_metadata.start_time.isoformat(),
+                        consts.CONFIG_END_TIME: run_metadata.end_time.isoformat(),
+                        # Explicit conversion of numpy's int64 values to int for JSON serializability
+                        consts.TOTAL_SOURCE_ROWS: int(source_df.shape[0]),
+                        consts.TOTAL_TARGET_ROWS: int(target_df.shape[0]),
+                        consts.TOTAL_ROWS_VALIDATED: int(result_df.shape[0]),
+                        # Using .sum() on boolean Series for much faster counting
+                        consts.TOTAL_ROWS_SUCCESS: int(success_condition.sum()),
+                        consts.TOTAL_ROWS_FAIL: int(fail_condition.sum()),
+                        consts.FAILED_SOURCE_NOT_IN_TARGET: int(
+                            (fail_condition & source_not_in_target).sum()
+                        ),
+                        consts.FAILED_TARGET_NOT_IN_SOURCE: int(
+                            (fail_condition & target_not_in_source).sum()
+                        ),
+                        consts.FAILED_PRESENT_IN_BOTH_TABLES: int(
+                            (fail_condition & present_in_both_tables).sum()
+                        ),
+                    }
+                )
             )
     except Exception as e:
         logging.warning(
-            f"Error while generating summary report of row validation results: {e}",
+            f"{COMBINER_GET_SUMMARY_EXC_TEXT}: {e}",
             exc_info=True,
         )

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.5.0}/data_validation/config_manager.py

@@ -16,7 +16,7 @@ import copy
 import logging
 import string
 import random
-from typing import TYPE_CHECKING, Optional, Union, List, Dict
+from typing import TYPE_CHECKING, Dict, List, Optional, Union, Tuple
 
 import google.oauth2.service_account
 import ibis.expr.datatypes as dt
@@ -66,6 +66,9 @@ class ConfigManager(object):
         if self.validation_type not in consts.CONFIG_TYPES:
             raise ValueError(f"Unknown Configuration Type: {self.validation_type}")
         self._comparison_max_col_length = None
+        # For some engines we need to know the actual raw data type rather than the Ibis canonical type.
+        self._source_raw_data_types = None
+        self._target_raw_data_types = None
 
     @property
     def config(self):
@@ -94,6 +97,50 @@ class ConfigManager(object):
 
         return self._target_conn
 
+    def get_source_raw_data_types(self) -> Dict[str, Tuple]:
+        """Return raw data type information from source system.
+
+        The raw data type is the source/target engine type, for example it might
+        be "NCLOB" or "char" when the Ibis type simply states "string".
+        The data is cached in state when fetched for the first time.
+        The retuen value is keyed on the casefolded column name and the tuple is
+        the remaining 6 elements of the DB API cursor description specification."""
+        if self._source_raw_data_types is None:
+            if hasattr(self.source_client, "raw_column_metadata"):
+                raw_data_types = self.source_client.raw_column_metadata(
+                    database=self.source_schema,
+                    table=self.source_table,
+                    query=self.source_query,
+                )
+                self._source_raw_data_types = {
+                    _[0].casefold(): _[1:] for _ in raw_data_types
+                }
+            else:
+                self._source_raw_data_types = {}
+        return self._source_raw_data_types
+
+    def get_target_raw_data_types(self) -> Dict[str, Tuple]:
+        """Return raw data type information from target system.
+
+        The raw data type is the source/target engine type, for example it might
+        be "NCLOB" or "char" when the Ibis type simply states "string".
+        The data is cached in state when fetched for the first time.
+        The retuen value is keyed on the casefolded column name and the tuple is
+        the remaining 6 elements of the DB API cursor description specification."""
+        if self._target_raw_data_types is None:
+            if hasattr(self.target_client, "raw_column_metadata"):
+                raw_data_types = self.target_client.raw_column_metadata(
+                    database=self.target_schema,
+                    table=self.target_table,
+                    query=self.target_query,
+                )
+                self._target_raw_data_types = {
+                    _[0].casefold(): _[1:] for _ in raw_data_types
+                }
+            else:
+                self._target_raw_data_types = {}
+        return self._target_raw_data_types
+
     def close_client_connections(self):
         """Attempt to clean up any source/target connections, based on the client types.
 
@@ -138,10 +185,6 @@ class ConfigManager(object):
         """Return if the validation should perform a case insensitive match."""
         return self._config.get(consts.CONFIG_CASE_INSENSITIVE_MATCH) or False
 
-    def process_in_memory(self):
-        """Return whether to process in memory or on a remote platform."""
-        return True
-
     @property
     def max_recursive_query_size(self):
         """Return Aggregates from Config"""
@@ -459,7 +502,7 @@ class ConfigManager(object):
                 cols_filter_list = consts.COLUMN_FILTER_LIST
             # handler that display results either to output or in a file
             return TextResultHandler(
-                self._config.get(consts.CONFIG_FORMAT, "table"),
+                self._config.get(consts.CONFIG_FORMAT, consts.FORMAT_TYPE_TABLE),
                 self.filter_status,
                 cols_filter_list,
             )
@@ -486,7 +529,9 @@ class ConfigManager(object):
                 table_id=table_id,
                 credentials=credentials,
                 api_endpoint=api_endpoint,
-                text_format=self._config.get(consts.CONFIG_FORMAT, "table"),
+                text_format=self._config.get(
+                    consts.CONFIG_FORMAT, consts.FORMAT_TYPE_TABLE
+                ),
             )
         else:
             raise ValueError(f"Unknown ResultHandler Class: {result_type}")
@@ -874,7 +919,7 @@ class ConfigManager(object):
         target_column_ibis_type: dt.DataType,
         margin: int = 0,
     ) -> bool:
-        """Identifies Decimal columns that will cause problems in a Pandas Dataframe.
+        """Identifies numeric columns that will cause problems in a Pandas Dataframe.
 
         i.e. are of greater precision than a 64bit int/real can hold.
 
@@ -883,17 +928,23 @@ class ConfigManager(object):
         """
         return bool(
             (
-                isinstance(source_column_ibis_type, dt.Decimal)
-                and (
-                    source_column_ibis_type.precision is None
-                    or source_column_ibis_type.precision > (18 - margin)
+                (isinstance(source_column_ibis_type, dt.Int64) and margin > 0)
+                or (
+                    isinstance(source_column_ibis_type, dt.Decimal)
+                    and (
+                        source_column_ibis_type.precision is None
+                        or source_column_ibis_type.precision > (18 - margin)
+                    )
                 )
             )
             and (
-                isinstance(target_column_ibis_type, dt.Decimal)
-                and (
-                    target_column_ibis_type.precision is None
-                    or target_column_ibis_type.precision > (18 - margin)
+                (isinstance(target_column_ibis_type, dt.Int64) and margin > 0)
+                or (
+                    isinstance(target_column_ibis_type, dt.Decimal)
+                    and (
+                        target_column_ibis_type.precision is None
+                        or target_column_ibis_type.precision > (18 - margin)
+                    )
                 )
             )
         )

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.5.0}/data_validation/consts.py

@@ -166,8 +166,16 @@ COUNT_STAR = "{count_star}"
 # Validation metadata
 RESULT_TYPE_SOURCE = "source"
 RESULT_TYPE_TARGET = "target"
-
-FORMAT_TYPES = ["csv", "json", "table", "text"]
+RESULT_TYPE_DIFFERENCES = "differences"
+
+FORMAT_TYPE_CSV = "csv"
+FORMAT_TYPE_JSON = "json"
+FORMAT_TYPE_MINIMAL = "minimal"
+FORMAT_TYPE_PYTHON = "python"
+FORMAT_TYPE_TABLE = "table"
+FORMAT_TYPE_TEXT = "text"
+FORMAT_TYPES = [FORMAT_TYPE_CSV, FORMAT_TYPE_JSON, FORMAT_TYPE_TABLE, FORMAT_TYPE_TEXT]
+RAW_QUERY_FORMAT_TYPES = [FORMAT_TYPE_MINIMAL, FORMAT_TYPE_PYTHON]
 
 # Text Result Handler column filter list
 COLUMN_FILTER_LIST = [