dataqe-framework 0.2.0__tar.gz → 0.2.2__tar.gz

{dataqe_framework-0.2.0/src/dataqe_framework.egg-info → dataqe_framework-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataqe-framework
-Version: 0.2.0
+Version: 0.2.2
 Summary: Reusable Data Validation Framework for data migration, ETL validation, and cross-database reconciliation
 Author-email: Khadar Shaik <khadarmohiddin.shaik@apree.health>
 Project-URL: Homepage, https://github.com/ShaikKhadarmohiddin/dataqe-framework
@@ -8,7 +8,7 @@ Project-URL: Documentation, https://github.com/ShaikKhadarmohiddin/dataqe-framew
 Project-URL: Repository, https://github.com/ShaikKhadarmohiddin/dataqe-framework.git
 Project-URL: Issues, https://github.com/ShaikKhadarmohiddin/dataqe-framework/issues
 Keywords: data-validation,data-quality,testing,ETL,migration,mysql,bigquery
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Classifier: License :: OSI Approved :: MIT License
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Database
 Classifier: Topic :: Software Development :: Testing

{dataqe_framework-0.2.0 → dataqe_framework-0.2.2}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "dataqe-framework"
 dynamic = []
-version = "0.2.0"
+version = "0.2.2"
 description = "Reusable Data Validation Framework for data migration, ETL validation, and cross-database reconciliation"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -22,7 +22,7 @@ keywords = [
   "bigquery"
 ]
 classifiers = [
-  "Development Status :: 4 - Beta",
+  "Development Status :: 5 - Production/Stable",
   "Intended Audience :: Developers",
   "Intended Audience :: System Administrators",
   "License :: OSI Approved :: MIT License",
@@ -31,6 +31,7 @@ classifiers = [
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "Topic :: Software Development :: Libraries :: Python Modules",
   "Topic :: Database",
   "Topic :: Software Development :: Testing",

{dataqe_framework-0.2.0 → dataqe_framework-0.2.2}/src/dataqe_framework/__init__.py

@@ -1,7 +1,7 @@
 from .validator import DataValidator
 from .credentials_extractor import CredentialsExtractor
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 
 __all__ = ["DataValidator", "CredentialsExtractor"]
 

{dataqe_framework-0.2.0 → dataqe_framework-0.2.2}/src/dataqe_framework/connectors/bigquery_connector.py

@@ -3,7 +3,12 @@ import logging
 from google.cloud import bigquery
 from google.oauth2 import service_account
 from .base_connector import BaseConnector
-
+execution_env = os.environ['SPRING_PROFILES_ACTIVE']
+if (execution_env.upper() != "MYLOCAL"):
+    import castlight_common_lib.configfunctions as cfg
+    config_details = cfg.Config('dataqeteam', [os.environ.get('SPRING_PROFILES_ACTIVE')])
+else:
+    config_details = None
 logger = logging.getLogger(__name__)
 
 
@@ -21,35 +26,16 @@ class BigQueryConnector(BaseConnector):
         Args:
             config: Dictionary containing BigQuery configuration with keys:
                 - project_id: GCP project ID
-                - dataset_id: BigQuery dataset ID
                 - service_account (optional): Service account name for credential lookup
                 - credentials_path (optional): Path to service account JSON file
                 - location (optional): BigQuery location (default: us-central1)
-                - location_map (optional): Dict mapping environment to location
                 - infra_core (optional): Infrastructure core name
-                - infra_core_map (optional): Dict mapping environment to infra-core
+                - k8_db_details (optional): Details of GCP project_id dataset_id from K8 env
         """
         self.project_id = config.get("project_id")
-        self.dataset_id = config.get("dataset_id")
-        self.credentials_path = config.get("credentials_path")
-        self.service_account_name = config.get("service_account")
-
-        # Get execution environment
-        self.execution_env = os.environ.get("SPRING_PROFILES_ACTIVE", "mylocal").upper()
-
-        # Get location with fallback to default
-        location_map = config.get("location_map", {})
-        self.location = location_map.get(
-            os.environ.get("SPRING_PROFILES_ACTIVE", "mylocal"),
-            config.get("location", "us-central1")
-        )
-
-        # Get infra-core with fallback to default
-        infra_core_map = config.get("infra_core_map", {})
-        self.infra_core = infra_core_map.get(
-            os.environ.get("SPRING_PROFILES_ACTIVE", "mylocal"),
-            config.get("infra_core", "infra-core-us-central1")
-        )
+        self.service_account_name = config.get("service_account", "qe_kube_sa_key")
+        self.location = config.get("location", "us-central1")
+        self.infra_core = config.get("infra_core", "infra-core-us-central1")
 
         # KMS encryption settings for PHI data
         self.use_encryption = config.get("use_encryption", False)
@@ -58,6 +44,24 @@ class BigQueryConnector(BaseConnector):
         self._encryption_config = None
         self._query_job_config = None
 
+        if "k8_db_details" in config and config_details is not None:
+            try:
+                project, db_name = config.get("k8_db_details").split('_')
+                self.project_id = config_details.data['bigquery'][project]['datasets'][db_name]['project_id']
+                self.credentials_path = os.path.join(os.getcwd(), self.project_id.replace('-', '_') + '_sftp_client_secrets.json')
+                self.extract_service_account(config_details, self.credentials_path, self.service_account_name)
+            except (ValueError, KeyError) as e:
+                logger.error(f"Failed to extract Kubernetes configuration: {str(e)}")
+                raise ValueError(f"Invalid k8_db_details format or missing configuration: {str(e)}")
+        else:
+            self.credentials_path = config.get("credentials_path")
+
+
+    def extract_service_account(self, config_details, service_config_file, serv_acct_name):
+        with open(service_config_file, "w") as file:
+                file.write(config_details.data['gcp'][serv_acct_name])
+                # print(config_details.data['gcp'][serv_acct_name])
+
     def _setup_encryption(self):
         """
         Setup KMS encryption configuration for PHI data.

{dataqe_framework-0.2.0 → dataqe_framework-0.2.2}/src/dataqe_framework/connectors/mysql_connector.py

@@ -2,20 +2,37 @@ import os
 import pymysql
 import logging
 from .base_connector import BaseConnector
-
+execution_env = os.environ['SPRING_PROFILES_ACTIVE']
+if (execution_env.upper() != "MYLOCAL"):
+    import castlight_common_lib.configfunctions as cfg
+    config_details = cfg.Config('dataqeteam', [os.environ.get('SPRING_PROFILES_ACTIVE')])
+else:
+    config_details = None
 logger = logging.getLogger(__name__)
 
 
 class MySQLConnector(BaseConnector):
 
-    def __init__(self, host, port, user, password, database):
+    def __init__(self, host=None, port=None, user=None, password=None, database=None, k8_db_details=None):
         self.host = host
         self.port = port
         self.user = user
         self.password = password
         self.database = database
         self.connection = None
-        logger.info(f"MySQLConnector initialized for host={host}, database={database}")
+        if k8_db_details is None or config_details is None:
+            logger.info(f"Locally MySQLConnector initialized for host={host}, database={database}")
+        else:
+            try:
+                project, db_name = k8_db_details.split('_')
+                self.host = config_details.data['mysql'][project][db_name]['db_host']
+                self.port = config_details.data['mysql'][project][db_name]['db_port']
+                self.user = config_details.data['mysql'][project][db_name]['db_user']
+                self.password = config_details.data['mysql'][project][db_name]['db_password']
+                self.database = config_details.data['mysql'][project][db_name]['db_name']
+            except (ValueError, KeyError) as e:
+                logger.error(f"Failed to extract Kubernetes configuration: {str(e)}")
+                raise ValueError(f"Invalid k8_db_details format or missing configuration: {str(e)}")
 
     def connect(self):
         logger.info(f"Establishing MySQL connection to {self.host}:{self.port}/{self.database}")

{dataqe_framework-0.2.0 → dataqe_framework-0.2.2/src/dataqe_framework.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dataqe-framework
-Version: 0.2.0
+Version: 0.2.2
 Summary: Reusable Data Validation Framework for data migration, ETL validation, and cross-database reconciliation
 Author-email: Khadar Shaik <khadarmohiddin.shaik@apree.health>
 Project-URL: Homepage, https://github.com/ShaikKhadarmohiddin/dataqe-framework
@@ -8,7 +8,7 @@ Project-URL: Documentation, https://github.com/ShaikKhadarmohiddin/dataqe-framew
 Project-URL: Repository, https://github.com/ShaikKhadarmohiddin/dataqe-framework.git
 Project-URL: Issues, https://github.com/ShaikKhadarmohiddin/dataqe-framework/issues
 Keywords: data-validation,data-quality,testing,ETL,migration,mysql,bigquery
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Classifier: License :: OSI Approved :: MIT License
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Database
 Classifier: Topic :: Software Development :: Testing