google-pso-data-validator 7.3.0__tar.gz → 7.4.0__tar.gz

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.3.0
+Version: 7.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -651,6 +651,8 @@ data-validation query
           The connection name to be queried
   --query or -q QUERY
           The raw query to run against the supplied connection
+  [--format or -f {minimal,python}]
+          Format for query output (default: python)
 ```
 
 ### Building Matched Table Lists

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/README.md

@@ -593,6 +593,8 @@ data-validation query
           The connection name to be queried
   --query or -q QUERY
           The raw query to run against the supplied connection
+  [--format or -f {minimal,python}]
+          Format for query output (default: python)
 ```
 
 ### Building Matched Table Lists

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/data_validation/__main__.py

@@ -16,9 +16,9 @@ import json
 import logging
 import os
 import sys
+from typing import TYPE_CHECKING
 
 from yaml import Dumper, dump
-from argparse import Namespace
 from typing import List
 from data_validation import (
     cli_tools,
@@ -34,6 +34,9 @@ from data_validation.data_validation import DataValidation
 from data_validation.find_tables import find_tables_using_string_matching
 from data_validation.partition_builder import PartitionBuilder
 
+if TYPE_CHECKING:
+    from argparse import Namespace
+
 # by default yaml dumps lists as pointers. This disables that feature
 Dumper.ignore_aliases = lambda *args: True
 
@@ -246,7 +249,7 @@ def _get_comparison_config(
     return config_manager.build_config_comparison_fields(comparison_fields)
 
 
-def build_config_from_args(args: Namespace, config_manager: ConfigManager):
+def build_config_from_args(args: "Namespace", config_manager: ConfigManager):
     """This function is used to append build configs to the config manager for all validation commands and generate-table-partitions.
     Instead of having two separate commands, e.g. validate row and validate custom-query row, generate-table-partitions
     uses implicit choice of table or custom-query. A user can specify either tables or source/target query/file,
@@ -337,7 +340,7 @@ def build_config_from_args(args: Namespace, config_manager: ConfigManager):
 
 
 def build_config_managers_from_args(
-    args: Namespace, validate_cmd: str = None
+    args: "Namespace", validate_cmd: str = None
 ) -> List[ConfigManager]:
     """Return a list of config managers ready to execute."""
 
@@ -596,7 +599,7 @@ def store_json_config_file(args, config_managers):
     cli_tools.store_validation(config_file_path, json_config)
 
 
-def partition_and_store_config_files(args: Namespace) -> None:
+def partition_and_store_config_files(args: "Namespace") -> None:
     """Build multiple YAML Config files using user specified partition logic
 
     Args:
@@ -691,7 +694,7 @@ def main():
         print(find_tables_using_string_matching(args))
     elif args.command == "query":
         raw_query.print_raw_query_output(
-            raw_query.run_raw_query_against_connection(args)
+            raw_query.run_raw_query_against_connection(args), format=args.output_format
         )
     elif args.command == "validate":
         validate(args)

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/data_validation/cli_tools.py

@@ -52,8 +52,7 @@ import sys
 import uuid
 import os
 import math
-from argparse import Namespace
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, TYPE_CHECKING
 from yaml import Dumper, Loader, dump, load
 
 from data_validation import (
@@ -66,6 +65,9 @@ from data_validation import (
 )
 from data_validation.validation_builder import list_to_sublists
 
+if TYPE_CHECKING:
+    from argparse import Namespace
+
 
 CONNECTION_SOURCE_FIELDS = {
     "BigQuery": [
@@ -191,7 +193,7 @@ VALIDATE_SCHEMA_HELP_TEXT = "Run a schema validation"
 VALIDATE_CUSTOM_QUERY_HELP_TEXT = "Run a custom query validation"
 
 
-def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: Namespace):
+def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: "Namespace"):
     # This is where we make additional checks if the arguments provided are what we expect
     # For example, only one of -tbls and custom query options can be provided
     if hasattr(parsed_args, "tables_list") and hasattr(
@@ -223,7 +225,7 @@ def _check_custom_query_args(parser: argparse.ArgumentParser, parsed_args: Names
         return  # old format - only one of them is present
 
 
-def get_parsed_args() -> Namespace:
+def get_parsed_args() -> "Namespace":
     """Return ArgParser with configured CLI arguments."""
     parser = configure_arg_parser()
     args = ["--help"] if len(sys.argv) == 1 else None
@@ -393,6 +395,14 @@ def _configure_raw_query(subparsers):
     )
     query_parser.add_argument("--conn", "-c", help="Connection name to query")
     query_parser.add_argument("--query", "-q", help="Raw query to execute")
+    query_parser.add_argument(
+        "--format",
+        "-f",
+        dest="output_format",
+        choices=consts.RAW_QUERY_FORMAT_TYPES,
+        default=consts.FORMAT_TYPE_PYTHON,
+        help=f"Format for query output (default: {consts.FORMAT_TYPE_PYTHON})",
+    )
 
 
 def _configure_validation_config_parser(subparsers):
@@ -1027,7 +1037,7 @@ def _add_common_arguments(
     optional_arguments.add_argument(
         "--format",
         "-fmt",
-        default="table",
+        default=consts.FORMAT_TYPE_TABLE,
         help="Set the format for printing command output, Supported formats are (text, csv, json, table). Defaults "
         "to table",
     )
@@ -1443,7 +1453,7 @@ def _concat_column_count_configs(
     return return_list
 
 
-def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
+def get_pre_build_configs(args: "Namespace", validate_cmd: str) -> List[Dict]:
     """Return a dict of configurations to build ConfigManager object"""
 
     def cols_from_arg(concat_arg: str, client, table_obj: dict, query_str: str) -> list:
@@ -1503,7 +1513,7 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
     target_client = clients.get_data_client(mgr.get_connection_config(args.target_conn))
 
     # Get format: text, csv, json, table. Default is table
-    format = args.format if args.format else "table"
+    format = args.format if args.format else consts.FORMAT_TYPE_TABLE
 
     # Get random row arguments. Only in row validations these attributes can be present.
     # Bad coding here, but keeping it so as not to introduce a breaking change. See

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/data_validation/config_manager.py

@@ -459,7 +459,7 @@ class ConfigManager(object):
                 cols_filter_list = consts.COLUMN_FILTER_LIST
             # handler that display results either to output or in a file
             return TextResultHandler(
-                self._config.get(consts.CONFIG_FORMAT, "table"),
+                self._config.get(consts.CONFIG_FORMAT, consts.FORMAT_TYPE_TABLE),
                 self.filter_status,
                 cols_filter_list,
             )
@@ -486,7 +486,9 @@ class ConfigManager(object):
                 table_id=table_id,
                 credentials=credentials,
                 api_endpoint=api_endpoint,
-                text_format=self._config.get(consts.CONFIG_FORMAT, "table"),
+                text_format=self._config.get(
+                    consts.CONFIG_FORMAT, consts.FORMAT_TYPE_TABLE
+                ),
             )
         else:
             raise ValueError(f"Unknown ResultHandler Class: {result_type}")
@@ -874,7 +876,7 @@ class ConfigManager(object):
         target_column_ibis_type: dt.DataType,
         margin: int = 0,
     ) -> bool:
-        """Identifies Decimal columns that will cause problems in a Pandas Dataframe.
+        """Identifies numeric columns that will cause problems in a Pandas Dataframe.
 
         i.e. are of greater precision than a 64bit int/real can hold.
 
@@ -883,17 +885,23 @@ class ConfigManager(object):
         """
         return bool(
             (
-                isinstance(source_column_ibis_type, dt.Decimal)
-                and (
-                    source_column_ibis_type.precision is None
-                    or source_column_ibis_type.precision > (18 - margin)
+                (isinstance(source_column_ibis_type, dt.Int64) and margin > 0)
+                or (
+                    isinstance(source_column_ibis_type, dt.Decimal)
+                    and (
+                        source_column_ibis_type.precision is None
+                        or source_column_ibis_type.precision > (18 - margin)
+                    )
                 )
             )
             and (
-                isinstance(target_column_ibis_type, dt.Decimal)
-                and (
-                    target_column_ibis_type.precision is None
-                    or target_column_ibis_type.precision > (18 - margin)
+                (isinstance(target_column_ibis_type, dt.Int64) and margin > 0)
+                or (
+                    isinstance(target_column_ibis_type, dt.Decimal)
+                    and (
+                        target_column_ibis_type.precision is None
+                        or target_column_ibis_type.precision > (18 - margin)
+                    )
                 )
             )
         )

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/data_validation/consts.py

@@ -167,7 +167,14 @@ COUNT_STAR = "{count_star}"
 RESULT_TYPE_SOURCE = "source"
 RESULT_TYPE_TARGET = "target"
 
-FORMAT_TYPES = ["csv", "json", "table", "text"]
+FORMAT_TYPE_CSV = "csv"
+FORMAT_TYPE_JSON = "json"
+FORMAT_TYPE_MINIMAL = "minimal"
+FORMAT_TYPE_PYTHON = "python"
+FORMAT_TYPE_TABLE = "table"
+FORMAT_TYPE_TEXT = "text"
+FORMAT_TYPES = [FORMAT_TYPE_CSV, FORMAT_TYPE_JSON, FORMAT_TYPE_TABLE, FORMAT_TYPE_TEXT]
+RAW_QUERY_FORMAT_TYPES = [FORMAT_TYPE_MINIMAL, FORMAT_TYPE_PYTHON]
 
 # Text Result Handler column filter list
 COLUMN_FILTER_LIST = [

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/data_validation/partition_builder.py

@@ -17,8 +17,7 @@ import ibis
 import pandas
 import logging
 import re
-from typing import List, Dict
-from argparse import Namespace
+from typing import List, Dict, TYPE_CHECKING
 
 from data_validation import cli_tools, consts
 from data_validation.config_manager import ConfigManager
@@ -26,9 +25,12 @@ from data_validation.query_builder.partition_row_builder import PartitionRowBuil
 from data_validation.validation_builder import ValidationBuilder
 from data_validation.validation_builder import list_to_sublists
 
+if TYPE_CHECKING:
+    from argparse import Namespace
+
 
 class PartitionBuilder:
-    def __init__(self, config_managers: List[ConfigManager], args: Namespace) -> None:
+    def __init__(self, config_managers: List[ConfigManager], args: "Namespace") -> None:
         self.config_managers = config_managers
         self.table_count = len(config_managers)
         self.args = args
@@ -62,7 +64,7 @@ class PartitionBuilder:
         """
         # Create multiple yaml validation blocks corresponding to the filters provided
         yaml_validations = []
-        for (source_filter, target_filter) in zip(source_filters, target_filters):
+        for source_filter, target_filter in zip(source_filters, target_filters):
             filter_dict = {
                 "type": "custom",
                 "source": source_filter,

google_pso_data_validator-7.4.0/data_validation/raw_query.py

@@ -0,0 +1,74 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from sqlalchemy.engine.row import Row
+
+from data_validation import clients, consts, state_manager
+
+if TYPE_CHECKING:
+    from argparse import Namespace
+
+
+def run_raw_query_against_connection(args: "Namespace") -> list:
+    """Return results of raw query for ad hoc usage."""
+    mgr = state_manager.StateManager()
+    with clients.get_data_client_ctx(mgr.get_connection_config(args.conn)) as client:
+        cursor = client.raw_sql(args.query)
+        res = cursor.fetchall()
+        try:
+            cursor.close()
+        except Exception:
+            pass
+        return res
+
+
+def _minimize_raw_query_output(query_output: list) -> list:
+    """Reduce single column output from list(tuple(_)) to list(_)"""
+    if not query_output:
+        return query_output
+    if isinstance(query_output[0], (list, tuple)) and len(query_output[0]) == 1:
+        return [_[0] for _ in query_output]
+    else:
+        return query_output
+
+
+def print_raw_query_output(query_output: list, format: str = consts.FORMAT_TYPE_PYTHON):
+    """Print a query resultset avoiding SQLAlchemy "... (nn characters truncated) ..." behaviour.
+
+    Args:
+        query_output (list): A set of rows from a SQLAlchemy query.
+        format (str): Output format:
+            FORMAT_TYPE_PYTHON: Legacy format matching previous behaviour, simple print the Python object.
+            FORMAT_TYPE_MINIMAL: Attempt to minimize additional noise for bare minimum output.
+                                 This is useful for a specific customer who are using DVT raw queries to
+                                 generate other DVT commands and don't want any escaping or extra parentheses
+                                 in the output.
+    """
+
+    def row_to_tuple(row) -> tuple:
+        """This prevents SQLAlchemy string truncation inside Row() objects by first converting them to a tuple."""
+        if isinstance(row, (Row, list)):
+            return tuple(row)
+        else:
+            return row
+
+    sanitized_output = [row_to_tuple(_) for _ in query_output or []]
+
+    if format == consts.FORMAT_TYPE_MINIMAL:
+        for row in _minimize_raw_query_output(sanitized_output):
+            print(row)
+    else:
+        print(sanitized_output)

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/data_validation/result_handlers/bigquery.py

@@ -16,7 +16,7 @@
 
 import logging
 
-from data_validation import clients
+from data_validation import clients, consts
 from data_validation.result_handlers import text as text_handler
 
 
@@ -39,7 +39,7 @@ class BigQueryResultHandler(object):
         bigquery_client,
         status_list: list = None,
         table_id: str = "pso_data_validator.results",
-        text_format: str = "table",
+        text_format: str = consts.FORMAT_TYPE_TABLE,
     ):
         self._bigquery_client = bigquery_client
         self._table_id = table_id
@@ -53,7 +53,7 @@ class BigQueryResultHandler(object):
         table_id: str = "pso_data_validator.results",
         credentials=None,
         api_endpoint: str = None,
-        text_format: str = "table",
+        text_format: str = consts.FORMAT_TYPE_TABLE,
     ):
         """Return BigQueryResultHandler instance for given project.
 

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/data_validation/result_handlers/text.py

@@ -35,15 +35,17 @@ def filter_validation_status(status_list, result_df: "DataFrame"):
 
 
 def get_formatted(
-    result_df: "DataFrame", format: str = "table", cols_filter_list: list = None
+    result_df: "DataFrame",
+    format: str = consts.FORMAT_TYPE_TABLE,
+    cols_filter_list: list = None,
 ) -> str:
     """Expose formatting logic so it can be used in BigQuery handler."""
     cols_filter_list = cols_filter_list or consts.COLUMN_FILTER_LIST
-    if format == "text":
+    if format == consts.FORMAT_TYPE_TEXT:
         return result_df.drop(cols_filter_list, axis=1).to_string(index=False)
-    elif format == "csv":
+    elif format == consts.FORMAT_TYPE_CSV:
         return result_df.to_csv(index=False, lineterminator="\n")
-    elif format == "json":
+    elif format == consts.FORMAT_TYPE_JSON:
         return result_df.to_json(orient="index")
     else:
         return result_df.drop(cols_filter_list, axis=1).to_markdown(

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/google_pso_data_validator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.3.0
+Version: 7.4.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -651,6 +651,8 @@ data-validation query
           The connection name to be queried
   --query or -q QUERY
           The raw query to run against the supplied connection
+  [--format or -f {minimal,python}]
+          Format for query output (default: python)
 ```
 
 ### Building Matched Table Lists

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "7.3.0"
+version = "7.4.0"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:

{google_pso_data_validator-7.3.0 → google_pso_data_validator-7.4.0}/third_party/ibis/ibis_postgres/registry.py

@@ -23,6 +23,12 @@ def sa_format_hashbytes(translator, op):
 
 
 def sa_epoch_seconds(translator, op):
-    """Override for standard ExtractEpochSeconds but catering for larger second values."""
+    """Override for standard ExtractEpochSeconds but catering for larger second values.
+
+    This expression also truncates fractional seconds from the incoming datetime.
+    This matches behaviour on other SQL engines' epoch seconds expressions."""
     arg = translator.translate(op.arg)
-    return sa.cast(sa.extract("epoch", arg), sa.BIGINT)
+    return sa.cast(
+        sa.extract("epoch", sa.func.date_trunc(sa.sql.literal_column("'second'"), arg)),
+        sa.BIGINT,
+    )

google_pso_data_validator-7.3.0/data_validation/raw_query.py

@@ -1,47 +0,0 @@
-# Copyright 2024 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from sqlalchemy.engine.row import Row
-
-from data_validation import clients, state_manager
-
-
-def run_raw_query_against_connection(args) -> list:
-    """Return results of raw query for ad hoc usage."""
-    mgr = state_manager.StateManager()
-    with clients.get_data_client_ctx(mgr.get_connection_config(args.conn)) as client:
-        cursor = client.raw_sql(args.query)
-        res = cursor.fetchall()
-        try:
-            cursor.close()
-        except Exception:
-            pass
-        return res
-
-
-def print_raw_query_output(query_output: list):
-    """Print a query resultset avoiding SQLAlchemy "... (nn characters truncated) ..." behaviour.
-
-    Args:
-        query_output (list): A set of rows from a SQLAlchemy query.
-    """
-
-    def row_to_str(row) -> str:
-        """This prevents SQLAlchemy string truncation inside Row() objects by first converting them to a tuple."""
-        if isinstance(row, Row):
-            return str(tuple(row))
-        else:
-            return str(row)
-
-    print([row_to_str(_) for _ in query_output or []])