google-pso-data-validator 6.1.0__tar.gz → 6.2.0__tar.gz

{google_pso_data_validator-6.1.0 → google_pso_data_validator-6.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-pso-data-validator
-Version: 6.1.0
+Version: 6.2.0
 Summary: A package to enable easy data validation
 Author: PSO DVT Engineering team
 Author-email: data-validator-eng@google.com
@@ -14,31 +14,26 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: attrs==23.1.0
-Requires-Dist: grpcio==1.53.2
-Requires-Dist: lazy-object-proxy==1.9.0
-Requires-Dist: marshmallow==3.19.0
-Requires-Dist: fsspec>=2022.8.2
-Requires-Dist: google-api-python-client==2.91.0
+Requires-Dist: Flask>=2.2
+Requires-Dist: fsspec>=2024.9.0
+Requires-Dist: google-api-python-client>=2.144.0
+Requires-Dist: google-cloud-bigquery>=3.25.0
+Requires-Dist: google-cloud-bigquery-storage>=2.26.0
+Requires-Dist: google-cloud-secret-manager>=2.20.2
+Requires-Dist: google-cloud-spanner>=3.49.1
+Requires-Dist: google-cloud-storage>=2.18.2
 Requires-Dist: ibis-framework==5.1.0
-Requires-Dist: impyla==0.18.0
-Requires-Dist: SQLAlchemy==1.4.49
-Requires-Dist: PyMySQL==1.1.1
-Requires-Dist: psycopg2-binary==2.9.6
-Requires-Dist: PyYAML==6.0
-Requires-Dist: pandas==1.5.3
-Requires-Dist: proto-plus==1.22.3
+Requires-Dist: impyla>=0.19.0
+Requires-Dist: jellyfish>=1.1.0
+Requires-Dist: pandas==2.0.3
+Requires-Dist: parsy>=2.1
+Requires-Dist: psycopg2-binary>=2.9.9
 Requires-Dist: pyarrow==14.0.1
-Requires-Dist: pydata-google-auth==1.8.0
-Requires-Dist: google-cloud-bigquery==3.11.3
-Requires-Dist: google-cloud-bigquery-storage==2.20.0
-Requires-Dist: google-cloud-spanner==3.36.0
-Requires-Dist: google-cloud-storage==2.10.0
-Requires-Dist: jellyfish==1.0.0
-Requires-Dist: tabulate==0.9.0
-Requires-Dist: Flask==2.3.2
-Requires-Dist: parsy==2.1
-Requires-Dist: google-cloud-secret-manager>=2.16.0
+Requires-Dist: pydata-google-auth>=1.8.2
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: PyYAML>=6.0.2
+Requires-Dist: SQLAlchemy==1.4.49
+Requires-Dist: tabulate>=0.9.0
 Provides-Extra: apache-airflow
 Requires-Dist: 1.10.11; extra == "apache-airflow"
 Provides-Extra: pyspark
@@ -133,7 +128,12 @@ over all columns ('*') will only run over numeric columns, unless the
 `--wildcard-include-string-len` or `--wildcard-include-timestamp` flags are present.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate column
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate column
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -141,7 +141,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         Target connection details
                         See: *Connections* section for each data source
   --tables-list or -tbls SOURCE_SCHEMA.SOURCE_TABLE=TARGET_SCHEMA.TARGET_TABLE
-                        Comma separated list of tables in the form schema.table=target_schema.target_table
+                        Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
                         Target schema name and table name are optional.
                         i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
   [--grouped-columns or -gc GROUPED_COLUMNS]
@@ -182,6 +182,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
+
 ```
 
 The default aggregation type is a 'COUNT *', which will run in addition to the validations you specify. To remove this default,
@@ -212,7 +213,12 @@ Under the hood, row validation uses
 apply functions such as IFNULL() or RTRIM(). These can be edited in the YAML or JSON config file to customize your row validation.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate row
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate row
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -270,8 +276,12 @@ The command generates and stores multiple YAML validations each representing a c
 The command takes the same parameters as required for `Row Validation` *plus* a few parameters to support partitioning. Single and multiple primary keys are supported and keys can be of any indexable type, except for date and timestamp type. A parameter used in earlier versions, ```partition-key``` is no longer supported.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) generate-table-partitions
-
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  generate-table-partitions
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -293,7 +303,7 @@ data-validation (--verbose or -v) (--log-level or -ll) generate-table-partitions
                         Directory Path to store YAML Config Files
                         GCS: Provide a full gs:// path of the target directory. Eg: `gs://<BUCKET>/partitions_dir`
                         Local: Provide a relative path of the target directory. Eg: `partitions_dir`
-  --partition-num INT, -pn INT 
+  --partition-num INT, -pn INT
                         Number of partitions into which the table should be split, e.g. 1000 or 10000
                         In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
@@ -301,7 +311,7 @@ data-validation (--verbose or -v) (--log-level or -ll) generate-table-partitions
                         See: *Validation Reports* section
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
-  [--parts-per-file INT], [-ppf INT] 
+  [--parts-per-file INT], [-ppf INT]
                         Number of partitions in a yaml file, default value 1.
   [--filters SOURCE_FILTER:TARGET_FILTER]
                         Colon separated string values of source and target filters.
@@ -326,7 +336,12 @@ types between source and target.
 Note: An exclamation point before a data type (`!string`) signifies the column is non-nullable or required.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate schema
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate schema
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -334,7 +349,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
                         Target connection details
                         See: *Connections* section for each data source
   --tables-list or -tbls SOURCE_SCHEMA.SOURCE_TABLE=TARGET_SCHEMA.TARGET_TABLE
-                        Comma separated list of tables in the form schema.table=target_schema.target_table
+                        Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
                         Target schema name and table name are optional.
                         e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
@@ -367,7 +382,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
 Below is the command syntax for custom query column validations.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate custom-query column
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate custom-query column
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -429,7 +449,12 @@ in the SELECT statement of both source_query.sql and target_query.sql.  See *Pri
 Below is the command syntax for custom query row validations.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate custom-query row
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -487,7 +512,12 @@ The `validate` command takes a `--dry-run` command line flag that prints source
 and target SQL to stdout as JSON in lieu of performing a validation:
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate
   [--dry-run or -dr]    Prints source and target SQL to stdout in lieu of performing a validation.
 ```
 
@@ -518,7 +548,12 @@ GCS and local paths.
 You can use the `data-validation configs` command to run and view YAMLs.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) configs run
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  configs run
   [--config-file or -c CONFIG_FILE]
                         Path to YAML config file to run. Supports local and GCS paths.
   [--config-dir or -cdir CONFIG_DIR]

{google_pso_data_validator-6.1.0 → google_pso_data_validator-6.2.0}/README.md

@@ -87,7 +87,12 @@ over all columns ('*') will only run over numeric columns, unless the
 `--wildcard-include-string-len` or `--wildcard-include-timestamp` flags are present.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate column
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate column
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -95,7 +100,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         Target connection details
                         See: *Connections* section for each data source
   --tables-list or -tbls SOURCE_SCHEMA.SOURCE_TABLE=TARGET_SCHEMA.TARGET_TABLE
-                        Comma separated list of tables in the form schema.table=target_schema.target_table
+                        Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
                         Target schema name and table name are optional.
                         i.e 'bigquery-public-data.new_york_citibike.citibike_trips'
   [--grouped-columns or -gc GROUPED_COLUMNS]
@@ -136,6 +141,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate column
                         Format for stdout output. Supported formats are (text, csv, json, table). Defaults to table.
   [--filter-status or -fs STATUSES_LIST]
                         Comma separated list of statuses to filter the validation results. Supported statuses are (success, fail). If no list is provided, all statuses are returned.
+
 ```
 
 The default aggregation type is a 'COUNT *', which will run in addition to the validations you specify. To remove this default,
@@ -166,7 +172,12 @@ Under the hood, row validation uses
 apply functions such as IFNULL() or RTRIM(). These can be edited in the YAML or JSON config file to customize your row validation.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate row
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate row
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -224,8 +235,12 @@ The command generates and stores multiple YAML validations each representing a c
 The command takes the same parameters as required for `Row Validation` *plus* a few parameters to support partitioning. Single and multiple primary keys are supported and keys can be of any indexable type, except for date and timestamp type. A parameter used in earlier versions, ```partition-key``` is no longer supported.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) generate-table-partitions
-
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  generate-table-partitions
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -247,7 +262,7 @@ data-validation (--verbose or -v) (--log-level or -ll) generate-table-partitions
                         Directory Path to store YAML Config Files
                         GCS: Provide a full gs:// path of the target directory. Eg: `gs://<BUCKET>/partitions_dir`
                         Local: Provide a relative path of the target directory. Eg: `partitions_dir`
-  --partition-num INT, -pn INT 
+  --partition-num INT, -pn INT
                         Number of partitions into which the table should be split, e.g. 1000 or 10000
                         In case this value exceeds the row count of the source/target table, it will be decreased to max(source_row_count, target_row_count)
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
@@ -255,7 +270,7 @@ data-validation (--verbose or -v) (--log-level or -ll) generate-table-partitions
                         See: *Validation Reports* section
   [--service-account or -sa PATH_TO_SA_KEY]
                         Service account to use for BigQuery result handler output.
-  [--parts-per-file INT], [-ppf INT] 
+  [--parts-per-file INT], [-ppf INT]
                         Number of partitions in a yaml file, default value 1.
   [--filters SOURCE_FILTER:TARGET_FILTER]
                         Colon separated string values of source and target filters.
@@ -280,7 +295,12 @@ types between source and target.
 Note: An exclamation point before a data type (`!string`) signifies the column is non-nullable or required.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate schema
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate schema
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -288,7 +308,7 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
                         Target connection details
                         See: *Connections* section for each data source
   --tables-list or -tbls SOURCE_SCHEMA.SOURCE_TABLE=TARGET_SCHEMA.TARGET_TABLE
-                        Comma separated list of tables in the form schema.table=target_schema.target_table
+                        Comma separated list of tables in the form schema.table=target_schema.target_table. Or shorthand schema.* for all tables.
                         Target schema name and table name are optional.
                         e.g.: 'bigquery-public-data.new_york_citibike.citibike_trips'
   [--bq-result-handler or -bqrh PROJECT_ID.DATASET.TABLE]
@@ -321,7 +341,12 @@ data-validation (--verbose or -v) (--log-level or -ll) validate schema
 Below is the command syntax for custom query column validations.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate custom-query column
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate custom-query column
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -383,7 +408,12 @@ in the SELECT statement of both source_query.sql and target_query.sql.  See *Pri
 Below is the command syntax for custom query row validations.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate custom-query row
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate custom-query row
   --source-conn or -sc SOURCE_CONN
                         Source connection details
                         See: *Data Source Configurations* section for each data source
@@ -441,7 +471,12 @@ The `validate` command takes a `--dry-run` command line flag that prints source
 and target SQL to stdout as JSON in lieu of performing a validation:
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) validate
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  validate
   [--dry-run or -dr]    Prints source and target SQL to stdout in lieu of performing a validation.
 ```
 
@@ -472,7 +507,12 @@ GCS and local paths.
 You can use the `data-validation configs` command to run and view YAMLs.
 
 ```
-data-validation (--verbose or -v) (--log-level or -ll) configs run
+data-validation
+  [--verbose or -v ]
+                        Verbose logging
+  [--log-level or -ll]
+                        Log Level to be assigned. Supported levels are (DEBUG,INFO,WARNING,ERROR,CRITICAL). Defaults to INFO.
+  configs run
   [--config-file or -c CONFIG_FILE]
                         Path to YAML config file to run. Supports local and GCS paths.
   [--config-dir or -cdir CONFIG_DIR]

{google_pso_data_validator-6.1.0 → google_pso_data_validator-6.2.0}/data_validation/__main__.py

@@ -24,11 +24,11 @@ from data_validation import (
     cli_tools,
     clients,
     consts,
-    jellyfish_distance,
     state_manager,
 )
 from data_validation.config_manager import ConfigManager
 from data_validation.data_validation import DataValidation
+from data_validation.find_tables import find_tables_using_string_matching
 from data_validation.partition_builder import PartitionBuilder
 
 # by default yaml dumps lists as pointers. This disables that feature
@@ -434,71 +434,6 @@ def build_config_managers_from_yaml(args, config_file_path):
     return config_managers
 
 
-def _compare_match_tables(source_table_map, target_table_map, score_cutoff=0.8):
-    """Return dict config object from matching tables."""
-    # TODO(dhercher): evaluate if improved comparison and score cutoffs should be used.
-    table_configs = []
-
-    target_keys = target_table_map.keys()
-    for source_key in source_table_map:
-        target_key = jellyfish_distance.extract_closest_match(
-            source_key, target_keys, score_cutoff=score_cutoff
-        )
-        if target_key is None:
-            continue
-
-        table_config = {
-            consts.CONFIG_SCHEMA_NAME: source_table_map[source_key][
-                consts.CONFIG_SCHEMA_NAME
-            ],
-            consts.CONFIG_TABLE_NAME: source_table_map[source_key][
-                consts.CONFIG_TABLE_NAME
-            ],
-            consts.CONFIG_TARGET_SCHEMA_NAME: target_table_map[target_key][
-                consts.CONFIG_SCHEMA_NAME
-            ],
-            consts.CONFIG_TARGET_TABLE_NAME: target_table_map[target_key][
-                consts.CONFIG_TABLE_NAME
-            ],
-        }
-        table_configs.append(table_config)
-
-    return table_configs
-
-
-def get_table_map(client, allowed_schemas=None):
-    """Return dict with searchable keys for table matching."""
-    table_map = {}
-    table_objs = clients.get_all_tables(client, allowed_schemas=allowed_schemas)
-
-    for table_obj in table_objs:
-        table_key = ".".join([t for t in table_obj if t])
-        table_map[table_key] = {
-            consts.CONFIG_SCHEMA_NAME: table_obj[0],
-            consts.CONFIG_TABLE_NAME: table_obj[1],
-        }
-
-    return table_map
-
-
-def find_tables_using_string_matching(args):
-    """Return JSON String with matched tables for use in validations."""
-    score_cutoff = args.score_cutoff or 1
-
-    mgr = state_manager.StateManager()
-    source_client = clients.get_data_client(mgr.get_connection_config(args.source_conn))
-    target_client = clients.get_data_client(mgr.get_connection_config(args.target_conn))
-
-    allowed_schemas = cli_tools.get_arg_list(args.allowed_schemas)
-    source_table_map = get_table_map(source_client, allowed_schemas=allowed_schemas)
-    target_table_map = get_table_map(target_client)
-
-    table_configs = _compare_match_tables(
-        source_table_map, target_table_map, score_cutoff=score_cutoff
-    )
-    return json.dumps(table_configs)
-
-
 def run_raw_query_against_connection(args):
     """Return results of raw query for ad hoc usage."""
     mgr = state_manager.StateManager()

{google_pso_data_validator-6.1.0 → google_pso_data_validator-6.2.0}/data_validation/cli_tools.py

@@ -52,13 +52,15 @@ import sys
 import uuid
 import os
 import math
+import re
 from argparse import Namespace
 from typing import Dict, List, Optional
 from yaml import Dumper, Loader, dump, load
 
-from data_validation import clients, consts, state_manager, gcs_helper
+from data_validation import clients, consts, find_tables, state_manager, gcs_helper
 from data_validation.validation_builder import list_to_sublists
 
+
 CONNECTION_SOURCE_FIELDS = {
     "BigQuery": [
         ["project_id", "GCP Project to use for BigQuery"],
@@ -645,7 +647,7 @@ def _configure_column_parser(column_parser):
         "-tbls",
         default=None,
         required=True,
-        help="Comma separated tables list in the form 'schema.table=target_schema.target_table'",
+        help="Comma separated tables list in the form 'schema.table=target_schema.target_table'. Or shorthand schema.* for all tables.",
     )
     _add_common_arguments(optional_arguments, required_arguments)
 
@@ -1064,26 +1066,33 @@ def get_labels(arg_labels):
     return labels
 
 
-def get_filters(filter_value):
-    """Returns parsed JSON from filter file. Backwards compatible for JSON input.
-
-    filter_value (str): Filter argument specified.
+def get_filters(filter_value: str) -> List[Dict]:
+    """Returns filters for source and target from --filters argument.
+    A filter is the condition that is used in a SQL WHERE clause.
+    If only one filter is specified, it applies to both source and target
+    For a doc on regular expression for filters see docs/internal/filters_regex.md
     """
+
+    single_filter = r"([^':]*('[^']*')*)*"
+    double_filter = (
+        r"(?P<source>" + single_filter + r"):(?P<target>" + single_filter + r")"
+    )
     filter_config = []
-    filter_vals = filter_value.split(":")
-    if len(filter_vals) == 1:
+    if result := re.fullmatch(single_filter, filter_value):
+        if result.group(0) == "":
+            raise ValueError("Empty string not allowed in filter")
         filter_dict = {
             "type": "custom",
-            "source": filter_vals[0],
-            "target": filter_vals[0],
+            "source": result.group(0),
+            "target": result.group(0),
         }
-    elif len(filter_vals) == 2:
-        if not filter_vals[1]:
-            raise ValueError("Please provide valid target filter.")
+    elif result := re.fullmatch(double_filter, filter_value):
+        if result.group("source") == "" or result.group("target") == "":
+            raise ValueError("Empty string not allowed in filter")
         filter_dict = {
             "type": "custom",
-            "source": filter_vals[0],
-            "target": filter_vals[1],
+            "source": result.group("source"),
+            "target": result.group("target"),
         }
     else:
         raise ValueError("Unable to parse filter arguments.")
@@ -1151,7 +1160,7 @@ def get_tables_list(arg_tables, default_value=None, is_filesystem=False):
 
     tables_list = []
     tables_mapping = list(csv.reader([arg_tables]))[0]
-    source_schema_required = False if is_filesystem else True
+    source_schema_required = bool(not is_filesystem)
 
     for mapping in tables_mapping:
         tables_map = mapping.split("=")
@@ -1388,6 +1397,10 @@ def get_pre_build_configs(args: Namespace, validate_cmd: str) -> List[Dict]:
         filter_status = None
 
     pre_build_configs_list = []
+    if config_type != consts.CUSTOM_QUERY:
+        tables_list = find_tables.expand_tables_of_asterisk(
+            tables_list, source_client, target_client
+        )
     for table_obj in tables_list:
         pre_build_configs = {
             "config_type": config_type,