dataqe-framework 0.2.4__tar.gz → 0.2.6__tar.gz

{dataqe_framework-0.2.4/src/dataqe_framework.egg-info → dataqe_framework-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataqe-framework
-Version: 0.2.4
+Version: 0.2.6
 Summary: Reusable Data Validation Framework for data migration, ETL validation, and cross-database reconciliation
 Author-email: Khadar Shaik <khadarmohiddin.shaik@apree.health>
 Project-URL: Homepage, https://github.com/ShaikKhadarmohiddin/dataqe-framework

{dataqe_framework-0.2.4 → dataqe_framework-0.2.6}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "dataqe-framework"
 dynamic = []
-version = "0.2.4"
+version = "0.2.6"
 description = "Reusable Data Validation Framework for data migration, ETL validation, and cross-database reconciliation"
 readme = "README.md"
 requires-python = ">=3.9"

{dataqe_framework-0.2.4 → dataqe_framework-0.2.6}/src/dataqe_framework/__init__.py

@@ -1,7 +1,7 @@
 from .validator import DataValidator
 from .credentials_extractor import CredentialsExtractor
 
-__version__ = "0.2.4"
+__version__ = "0.2.6"
 
 __all__ = ["DataValidator", "CredentialsExtractor"]
 

{dataqe_framework-0.2.4 → dataqe_framework-0.2.6}/src/dataqe_framework/cli.py

@@ -113,6 +113,7 @@ def main():
 
     # Get preprocessor queries path if specified
     preprocessor_queries_path = config_block["other"].get("preprocessor_queries")
+
     if preprocessor_queries_path:
         # Resolve relative path if needed
         if not os.path.isabs(preprocessor_queries_path):

{dataqe_framework-0.2.4 → dataqe_framework-0.2.6}/src/dataqe_framework/executor.py

@@ -17,11 +17,51 @@ class ValidationExecutor:
 
         self.source_connector = None
         self.target_connector = None
-        self.preprocessor = None
+        self.source_preprocessor = None
+        self.target_preprocessor = None
 
-        # Initialize preprocessor if path is provided
+        # Initialize preprocessors if path is provided
         if preprocessor_queries_path:
-            self.preprocessor = QueryPreprocessor(preprocessor_queries_path)
+            # Extract config_query_key from source config (under gcp/mysql/etc)
+            src_config = self._extract_preprocessor_config(source_config)
+            self.source_preprocessor = QueryPreprocessor(preprocessor_queries_path, src_config)
+
+            # Extract config_query_key from target config (under gcp/mysql/etc)
+            tgt_config = self._extract_preprocessor_config(target_config)
+            self.target_preprocessor = QueryPreprocessor(preprocessor_queries_path, tgt_config)
+
+    def _extract_preprocessor_config(self, config: dict) -> dict:
+        """
+        Extract preprocessor config (config_query_key) from database-specific config.
+
+        Args:
+            config: Source or target config block
+
+        Returns:
+            Dictionary with config_query_key if found, empty dict otherwise
+        """
+        if not config:
+            return {}
+
+        # Get database type from config
+        db_type = config.get("database_type")
+        if not db_type:
+            return {}
+
+        # Map database type to config key (gcpbq uses "gcp" config key)
+        config_key = "gcp" if db_type == "gcpbq" else db_type
+
+        # Extract database-specific config (gcp, mysql, etc.)
+        db_config = config.get(config_key)
+        if not db_config or not isinstance(db_config, dict):
+            return {}
+
+        # Extract config_query_key if present
+        config_query_key = db_config.get("config_query_key")
+        if config_query_key:
+            return {"config_query_key": config_query_key}
+
+        return {}
 
     def setup_connectors(self):
         if self.source_config:
@@ -59,11 +99,9 @@ class ValidationExecutor:
             if "source" in test_config:
                 source_query = test_config["source"]["query"]
 
-                # Process query with preprocessor if config_query_key exists
+                # Process query with source preprocessor (automatic replacement of all release labels)
                 source_query = self._process_query_with_preprocessor(
-                    source_query,
-                    test_config["source"],
-                    self.source_connector
+                    source_query, self.source_connector, self.source_preprocessor
                 )
 
                 source_query_start = datetime.now()
@@ -75,11 +113,9 @@ class ValidationExecutor:
             if "target" in test_config:
                 target_query = test_config["target"]["query"]
 
-                # Process query with preprocessor if config_query_key exists
+                # Process query with target preprocessor (automatic replacement of all release labels)
                 target_query = self._process_query_with_preprocessor(
-                    target_query,
-                    test_config["target"],
-                    self.target_connector
+                    target_query, self.target_connector, self.target_preprocessor
                 )
 
                 target_query_start = datetime.now()
@@ -128,43 +164,28 @@ class ValidationExecutor:
         # assuming single value queries
         return list(result[0].values())[0]
 
-    def _process_query_with_preprocessor(self, query: str, config_block: dict, connector) -> str:
+    def _process_query_with_preprocessor(self, query: str, connector, preprocessor) -> str:
         """
-        Process query with preprocessor if config_query_key is present in config block.
+        Process query with preprocessor to replace all release label placeholders.
+
+        Automatically replaces all SOURCE_CURR_WEEK and SOURCE_PREV_WEEK placeholders
+        without needing per-test configuration.
 
         Args:
             query: Original query string
-            config_block: Configuration block for source or target (should contain query and optionally config_query_key)
             connector: Database connector to use
+            preprocessor: QueryPreprocessor instance (source or target)
 
         Returns:
             Processed query (with replacements if applicable) or original query
         """
-        # Check if config_query_key exists in config block
-        config_query_key = config_block.get("config_query_key")
-
-        if not config_query_key:
-            # No preprocessor query key, return original query
+        if not preprocessor or not connector:
+            # Preprocessor not initialized or no connector, return original query
             return query
 
-        if not self.preprocessor:
-            # Preprocessor not initialized, return original query
-            logger.warning(
-                f"config_query_key '{config_query_key}' specified but preprocessor not initialized"
-            )
-            return query
-
-        # Get source_name from config block if provided
-        source_name = config_block.get("source_name")
-
         try:
-            # Process query through preprocessor
-            processed_query = self.preprocessor.process_query(
-                query,
-                config_query_key,
-                source_name,
-                connector
-            )
+            # Process query through preprocessor with automatic replacement
+            processed_query = preprocessor.replace_release_labels(query, connector)
             return processed_query
         except Exception as e:
             logger.error(f"Error processing query with preprocessor: {str(e)}")

{dataqe_framework-0.2.4 → dataqe_framework-0.2.6}/src/dataqe_framework/preprocessor.py

@@ -1,7 +1,7 @@
 import yaml
 import os
 import logging
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 from dataqe_framework.connectors import get_connector
 
 logger = logging.getLogger(__name__)
@@ -15,17 +15,19 @@ class QueryPreprocessor:
     and replaces placeholders in test queries with actual dataset names.
     """
 
-    def __init__(self, preprocessor_queries_path: str = None):
+    def __init__(self, preprocessor_queries_path: str = None, preprocessor_config: Dict[str, Any] = None):
         """
         Initialize the QueryPreprocessor.
 
         Args:
             preprocessor_queries_path: Path to preprocessor_queries.yml file.
-                                      If not provided, attempts to load from default location.
+            preprocessor_config: Configuration dict with config_query_key and other settings.
         """
         self.preprocessor_queries_path = preprocessor_queries_path
+        self.preprocessor_config = preprocessor_config or {}
         self.preprocessor_queries = {}
         self.dataset_mappings = {}
+        self.release_labels_cache = None
 
         if self.preprocessor_queries_path:
             self._load_preprocessor_queries()
@@ -98,6 +100,14 @@ class QueryPreprocessor:
                     }
 
             logger.info(f"Generated dataset mappings: {mappings}")
+            if mappings:
+                logger.info("=" * 60)
+                logger.info("PREPROCESSOR QUERY RESULTS:")
+                for source, mapping in mappings.items():
+                    logger.info(f"  Source: {source}")
+                    logger.info(f"    Current Release: {mapping.get('current_release')}")
+                    logger.info(f"    Previous Release: {mapping.get('previous_release')}")
+                logger.info("=" * 60)
             return mappings
 
         except Exception as e:
@@ -186,3 +196,91 @@ class QueryPreprocessor:
 
         # Replace placeholders in query
         return self.replace_placeholders_in_query(query, source_name, mappings)
+
+    def replace_release_labels(self, query: str, connector: Any) -> str:
+        """
+        Automatically replace all release label placeholders in query without needing
+        source_name or config_query_key specified per query.
+
+        This method executes the preprocessor query defined in preprocessor_config,
+        gets all release label mappings, and replaces all SOURCE_CURR_WEEK and SOURCE_PREV_WEEK
+        placeholders in the query.
+
+        Args:
+            query: Original query string with placeholders like SOURCE_CURR_WEEK, SOURCE_PREV_WEEK
+            connector: Database connector for executing preprocessor query
+
+        Returns:
+            Processed query with all placeholders replaced by actual dataset names
+        """
+        # If no preprocessor config or config_query_key, return original query
+        if not self.preprocessor_config or not self.preprocessor_config.get("config_query_key"):
+            return query
+
+        # Get release labels (cache to avoid multiple queries)
+        if self.release_labels_cache is None:
+            config_query_key = self.preprocessor_config.get("config_query_key")
+            release_labels = self.get_dataset_mappings(config_query_key, connector)
+
+            if not release_labels:
+                return query
+
+            # Convert mappings to list format for easier iteration
+            self.release_labels_cache = [
+                {
+                    "source": source,
+                    "curr_release_label": mapping.get("current_release"),
+                    "prev_release_label": mapping.get("previous_release")
+                }
+                for source, mapping in release_labels.items()
+            ]
+
+        # Replace all placeholders in query
+        return self._replace_all_release_labels(query, self.release_labels_cache)
+
+    def _replace_all_release_labels(self, query: str, release_labels: List[Dict[str, str]]) -> str:
+        """
+        Replace all SOURCE_CURR_WEEK and SOURCE_PREV_WEEK placeholders in query.
+
+        Args:
+            query: Original query string
+            release_labels: List of release label mappings
+
+        Returns:
+            Query with all placeholders replaced
+        """
+        modified_query = query
+
+        if not release_labels:
+            logger.debug("No release labels to replace")
+            return modified_query
+
+        replacements_made = False
+        for label in release_labels:
+            source = label.get("source", "").upper()
+            curr_label = label.get("curr_release_label")
+            prev_label = label.get("prev_release_label")
+
+            if not source or not curr_label or not prev_label:
+                logger.debug(f"Skipping incomplete label for source '{source}'")
+                continue
+
+            # Check if placeholders exist in query before replacing
+            curr_placeholder = f"{source}_CURR_WEEK"
+            prev_placeholder = f"{source}_PREV_WEEK"
+
+            if curr_placeholder in modified_query or prev_placeholder in modified_query:
+                replacements_made = True
+                modified_query = modified_query.replace(curr_placeholder, curr_label).replace(prev_placeholder, prev_label)
+                logger.info(
+                    f"Replaced placeholders for '{source}': "
+                    f"{curr_placeholder} → {curr_label}, "
+                    f"{prev_placeholder} → {prev_label}"
+                )
+            else:
+                logger.debug(f"No placeholders found for '{source}'")
+
+        if not replacements_made:
+            logger.info("No placeholder replacements made - query returned unchanged")
+
+        return modified_query

{dataqe_framework-0.2.4 → dataqe_framework-0.2.6/src/dataqe_framework.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataqe-framework
-Version: 0.2.4
+Version: 0.2.6
 Summary: Reusable Data Validation Framework for data migration, ETL validation, and cross-database reconciliation
 Author-email: Khadar Shaik <khadarmohiddin.shaik@apree.health>
 Project-URL: Homepage, https://github.com/ShaikKhadarmohiddin/dataqe-framework