google-pso-data-validator 7.1.0__tar.gz → 7.2.0__tar.gz

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.1.0
+Version: 7.2.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -38,6 +38,14 @@ Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
 Requires-Dist: 3.0.0; extra == "pyspark"
+Provides-Extra: develop
+Requires-Dist: black==22.3.0; extra == "develop"
+Requires-Dist: flake8; extra == "develop"
+Requires-Dist: freezegun; extra == "develop"
+Requires-Dist: pyfakefs==4.6.2; extra == "develop"
+Requires-Dist: pytest; extra == "develop"
+Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/data_validation/__main__.py

@@ -640,6 +640,10 @@ def run_connections(args):
     """Run commands related to connection management."""
     if args.connect_cmd == "list":
         cli_tools.list_connections()
+    elif args.connect_cmd == "delete":
+        cli_tools.delete_connection(args.connection_name)
+    elif args.connect_cmd == "describe":
+        cli_tools.describe_connection(args.connection_name, args.output_format)
     elif args.connect_cmd == "add":
         conn = cli_tools.get_connection_config_from_args(args)
         # Test getting a client to validate connection details

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/data_validation/cli_tools.py

@@ -52,12 +52,18 @@ import sys
 import uuid
 import os
 import math
-import re
 from argparse import Namespace
 from typing import Dict, List, Optional
 from yaml import Dumper, Loader, dump, load
 
-from data_validation import clients, consts, find_tables, state_manager, gcs_helper
+from data_validation import (
+    clients,
+    consts,
+    find_tables,
+    state_manager,
+    gcs_helper,
+    util,
+)
 from data_validation.validation_builder import list_to_sublists
 
 
@@ -468,6 +474,27 @@ def _configure_connection_parser(subparsers):
     )
     _configure_database_specific_parsers(add_parser)
 
+    delete_parser = connect_subparsers.add_parser(
+        "delete", help="Delete an existing connection"
+    )
+    delete_parser.add_argument(
+        "--connection-name", "-c", required=True, help="Name of connection to delete"
+    )
+    describe_parser = connect_subparsers.add_parser(
+        "describe", help="Describe an existing connection"
+    )
+    describe_parser.add_argument(
+        "--connection-name", "-c", required=True, help="Name of connection to describe"
+    )
+    describe_parser.add_argument(
+        "--format",
+        "-f",
+        dest="output_format",
+        choices=["json", "yaml"],
+        default="yaml",
+        help="Output format for the configuration (default: yaml)",
+    )
+
 
 def _configure_database_specific_parsers(parser):
     """Configure a separate subparser for each supported DB."""
@@ -1085,13 +1112,29 @@ def store_connection(connection_name, conn):
     mgr.create_connection(connection_name, conn)
 
 
+def delete_connection(connection_name):
+    """Delete the connection config under the given name."""
+    mgr = state_manager.StateManager()
+    mgr.delete_connection(connection_name)
+
+
 def list_connections():
     """List all saved connections."""
     mgr = state_manager.StateManager()
     connections = mgr.list_connections()
     for conn_name in connections:
         source_type = mgr.get_connection_config(conn_name).get("source_type")
-        logging.info(f"Connection Name: {conn_name} : {source_type}")
+        print(f"Connection Name: {conn_name}")
+        print(f"Source Type:     {source_type}\n")
+    return connections
+
+
+def describe_connection(connection_name, output_format):
+    """Return yaml connection details for a specific connection"""
+    mgr = state_manager.StateManager()
+    connection_details = mgr.describe_connection(connection_name, output_format)
+    print(connection_details)
+    return connection_details
 
 
 def get_connection(connection_name):
@@ -1165,31 +1208,29 @@ def get_filters(filter_value: str) -> List[Dict]:
     If only one filter is specified, it applies to both source and target
     For a doc on regular expression for filters see docs/internal/filters_regex.md
     """
-
-    single_filter = r"([^':]*('[^']*')*)*"
-    double_filter = (
-        r"(?P<source>" + single_filter + r"):(?P<target>" + single_filter + r")"
-    )
-    filter_config = []
-    if result := re.fullmatch(single_filter, filter_value):
-        if result.group(0) == "":
+    filters = util.split_not_in_quotes(filter_value, ":")
+    if len(filters) not in (1, 2):
+        raise argparse.ArgumentTypeError("Unable to parse filter arguments.")
+    filters = [_.strip() for _ in filters]
+    if len(filters) == 1:
+        if not filters[0]:
             raise argparse.ArgumentTypeError("Empty string not allowed in filter")
         filter_dict = {
             "type": "custom",
-            "source": result.group(0),
-            "target": result.group(0),
+            "source": filters[0],
+            "target": filters[0],
         }
-    elif result := re.fullmatch(double_filter, filter_value):
-        if result.group("source") == "" or result.group("target") == "":
+    elif len(filters) == 2:
+        if not filters[0] or not filters[1]:
             raise argparse.ArgumentTypeError("Empty string not allowed in filter")
         filter_dict = {
             "type": "custom",
-            "source": result.group("source"),
-            "target": result.group("target"),
+            "source": filters[0],
+            "target": filters[1],
         }
-    else:
-        raise argparse.ArgumentTypeError("Unable to parse filter arguments.")
-    filter_config.append(filter_dict)
+    filter_config = [
+        filter_dict,
+    ]
     return filter_config
 
 

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/data_validation/combiner.py

@@ -124,8 +124,12 @@ def _calculate_difference(
 ):
     pct_threshold = ibis.literal(validation.threshold)
     if datatype.is_timestamp() or datatype.is_date():
-        source_value = field_differences["differences_source_value"].epoch_seconds()
-        target_value = field_differences["differences_target_value"].epoch_seconds()
+        source_value = (
+            field_differences["differences_source_value"].epoch_seconds().cast("int64")
+        )
+        target_value = (
+            field_differences["differences_target_value"].epoch_seconds().cast("int64")
+        )
     elif datatype.is_boolean() or (target_type and target_type.is_boolean()):
         source_value = field_differences["differences_source_value"].cast("boolean")
         target_value = field_differences["differences_target_value"].cast("boolean")

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/data_validation/config_manager.py

@@ -784,7 +784,9 @@ class ConfigManager(object):
         column_position: int,
     ) -> dict:
         """Append calculated field for length() or epoch_seconds(timestamp) for preprocessing before column validation aggregation."""
-        depth, cast_type = 0, None
+        depth = 0
+        cast_type = None
+        final_cast_type = None
         if any(_ in ["json", "!json"] for _ in [column_type, target_column_type]):
             # JSON data which needs casting to string before we apply a length function.
             pre_calculated_config = self.build_and_append_pre_agg_calc_config(
@@ -829,6 +831,10 @@ class ConfigManager(object):
                 depth = 1
 
             calc_func = "epoch_seconds"
+            if agg_type == consts.CONFIG_TYPE_SUM:
+                # It is possible to exceed int64 when summing epoch_seconds therefore cast to string.
+                # See issue 1391 for details.
+                final_cast_type = "string"
 
         elif column_type == "int32" or column_type == "!int32":
             calc_func = consts.CONFIG_CAST
@@ -856,6 +862,9 @@ class ConfigManager(object):
             ),
             consts.CONFIG_TYPE: agg_type,
         }
+        if final_cast_type:
+            # Adding to dict this way to avoid adding a lot of empty cast attributes.
+            aggregate_config[consts.CONFIG_CAST] = final_cast_type
 
         return aggregate_config
 
@@ -863,22 +872,28 @@ class ConfigManager(object):
         self,
         source_column_ibis_type: dt.DataType,
         target_column_ibis_type: dt.DataType,
+        margin: int = 0,
     ) -> bool:
-        """Identifies Decimal columns that will cause problems in a Pandas Dataframe, i.e.
-        is of greater precision than a 64bit int/real can hold."""
+        """Identifies Decimal columns that will cause problems in a Pandas Dataframe.
+
+        i.e. are of greater precision than a 64bit int/real can hold.
+
+        margin: Allows us to lower the precision threshold. This is helpful when summing column
+                values that are okay by themselves but cumulativaly could overflow a 64bit value.
+        """
         return bool(
             (
                 isinstance(source_column_ibis_type, dt.Decimal)
                 and (
                     source_column_ibis_type.precision is None
-                    or source_column_ibis_type.precision > 18
+                    or source_column_ibis_type.precision > (18 - margin)
                 )
             )
             and (
                 isinstance(target_column_ibis_type, dt.Decimal)
                 and (
                     target_column_ibis_type.precision is None
-                    or target_column_ibis_type.precision > 18
+                    or target_column_ibis_type.precision > (18 - margin)
                 )
             )
         )
@@ -1042,7 +1057,9 @@ class ConfigManager(object):
                     consts.CONFIG_TYPE: agg_type,
                 }
                 if self._decimal_column_too_big_for_pandas(
-                    source_column_ibis_type, target_column_ibis_type
+                    source_column_ibis_type,
+                    target_column_ibis_type,
+                    margin=(2 if agg_type == consts.CONFIG_TYPE_SUM else 0),
                 ):
                     aggregate_config[consts.CONFIG_CAST] = "string"
 

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/data_validation/gcs_helper.py

@@ -21,6 +21,7 @@ from data_validation import client_info
 
 
 WRITE_SUCCESS_STRING = "Success! Config output written to"
+DELETE_SUCCESS_STRING = "Successfully deleted"
 
 
 def _is_gcs_path(file_path: str) -> bool:
@@ -67,6 +68,13 @@ def _write_gcs_file(file_path: str, data: str):
     blob.upload_from_string(data)
 
 
+def _delete_gcs_file(file_path: str):
+    """Delete a file stored in GCS."""
+    gcs_bucket = get_gcs_bucket(file_path)
+    blob = gcs_bucket.blob(_get_gcs_file_path(file_path))
+    blob.delete()
+
+
 def read_file(file_path: str, download_as_text: bool = False):
     if _is_gcs_path(file_path):
         return _read_gcs_file(file_path, download_as_text)
@@ -87,6 +95,20 @@ def write_file(file_path: str, data: str, include_log: bool = True):
         logging.info(f"{WRITE_SUCCESS_STRING} {file_path}")
 
 
+def delete_file(file_path: str, include_log: bool = True):
+    """Delete a file from GCS or local filesystem, depending on the path."""
+    if _is_gcs_path(file_path):
+        _delete_gcs_file(file_path)
+    else:
+        if os.path.exists(file_path):
+            os.remove(file_path)
+        else:
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+    if include_log:
+        logging.info(f"{DELETE_SUCCESS_STRING}: {file_path}")
+
+
 def list_gcs_directory(directory_path: str) -> List[str]:
     gcs_prefix = _get_gcs_file_path(directory_path)
     gcs_bucket = get_gcs_bucket(directory_path)

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/data_validation/state_manager.py

@@ -19,8 +19,9 @@ and validation files.
 
 import enum
 import json
+import yaml
 import os
-from typing import Dict, List
+from typing import Dict, List, Union
 
 from data_validation import consts, gcs_helper
 
@@ -59,6 +60,15 @@ class StateManager(object):
         connection_path = self._get_connection_path(name)
         gcs_helper.write_file(connection_path, json.dumps(config))
 
+    def delete_connection(self, name: str):
+        """Delete an existing connection file by its connection name.
+
+        Args:
+            name (String): The name of the connection.
+        """
+        connection_path = self._get_connection_path(name)
+        gcs_helper.delete_file(connection_path)
+
     def get_connection_config(self, name: str) -> Dict[str, str]:
         """Get a connection configuration from the expected file.
 
@@ -72,6 +82,20 @@ class StateManager(object):
 
         return json.loads(conn_str)
 
+    def describe_connection(
+        self, connection_name: str, output_format: str
+    ) -> Union[dict, str]:
+        """Describe an existing connection by returning connection details in yaml format.
+
+        Args:
+            args (): The name of the connection.
+        """
+        conn_data = self.get_connection_config(connection_name)
+        if output_format == "json":
+            return conn_data
+        elif output_format == "yaml":
+            return yaml.dump(conn_data, default_flow_style=False, sort_keys=False)
+
     def list_connections(self) -> List[str]:
         """Returns a list of the connection names that exist."""
         file_names = self._list_directory(self._get_connections_directory())

google_pso_data_validator-7.2.0/data_validation/util.py

@@ -0,0 +1,46 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+import time
+
+
+def timed_call(log_txt, fn, *args, **kwargs):
+    t0 = time.time()
+    result = fn(*args, **kwargs)
+    elapsed = time.time() - t0
+    logging.debug(f"{log_txt} elapsed: {round(elapsed, 2)}s")
+    return result
+
+
+def split_not_in_quotes(
+    to_split: str, sep: str = " ", exclude_empty_tokens: bool = False
+) -> list:
+    """Split a string by a separator but only when the separator is not inside quotes.
+    re pattern taken from this comment:
+        https://stackoverflow.com/a/2787979/10979853
+    The commenter's words should the link ever go stale:
+        Each time it finds a semicolon, the lookahead scans the entire remaining string,
+        making sure there's an even number of single-quotes and an even number of double-quotes.
+        (Single-quotes inside double-quoted fields, or vice-versa, are ignored.) If the
+        lookahead succeeds, the semicolon is a delimiter.
+    The pattern doesn't cope with whitespace as sep, back to back spaces are multiple seps, therefore
+    we have exclude_empty_tokens parameter.
+    """
+    pattern = r"""%(sep)s(?=(?:[^'"]|'[^']*'|"[^"]*")*$)""" % {"sep": sep}
+    if exclude_empty_tokens:
+        return [t for t in re.split(pattern, to_split) if t]
+    else:
+        return re.split(pattern, to_split)

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/google_pso_data_validator.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: google-pso-data-validator
-Version: 7.1.0
+Version: 7.2.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -38,6 +38,14 @@ Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
 Requires-Dist: 3.0.0; extra == "pyspark"
+Provides-Extra: develop
+Requires-Dist: black==22.3.0; extra == "develop"
+Requires-Dist: flake8; extra == "develop"
+Requires-Dist: freezegun; extra == "develop"
+Requires-Dist: pyfakefs==4.6.2; extra == "develop"
+Requires-Dist: pytest; extra == "develop"
+Requires-Dist: pytest-cov; extra == "develop"
+Requires-Dist: pytest-timeout; extra == "develop"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/google_pso_data_validator.egg-info/SOURCES.txt

@@ -74,6 +74,7 @@ third_party/ibis/ibis_oracle/registry.py
 third_party/ibis/ibis_postgres/__init__.py
 third_party/ibis/ibis_postgres/client.py
 third_party/ibis/ibis_postgres/datatypes.py
+third_party/ibis/ibis_postgres/registry.py
 third_party/ibis/ibis_redshift/__init__.py
 third_party/ibis/ibis_redshift/api.py
 third_party/ibis/ibis_redshift/compiler.py

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/google_pso_data_validator.egg-info/requires.txt

@@ -22,5 +22,14 @@ tabulate>=0.9.0
 [apache-airflow]
 1.10.11
 
+[develop]
+black==22.3.0
+flake8
+freezegun
+pyfakefs==4.6.2
+pytest
+pytest-cov
+pytest-timeout
+
 [pyspark]
 3.0.0

{google_pso_data_validator-7.1.0 → google_pso_data_validator-7.2.0}/setup.py

@@ -18,7 +18,7 @@ import setuptools
 
 name = "google-pso-data-validator"
 description = "A package to enable easy data validation"
-version = "7.1.0"
+version = "7.2.0"
 release_status = "Development Status :: 4 - Beta"
 
 with open("README.md", "r") as fh:
@@ -50,6 +50,15 @@ dependencies = [
 extras_require = {
     "apache-airflow": "1.10.11",
     "pyspark": "3.0.0",
+    "develop": [
+        "black==22.3.0",
+        "flake8",
+        "freezegun",
+        "pyfakefs==4.6.2",
+        "pytest",
+        "pytest-cov",
+        "pytest-timeout",
+    ],
 }
 
 packages = [