xoverrr 1.1.4__tar.gz → 1.1.6__tar.gz

{xoverrr-1.1.4/src/xoverrr.egg-info → xoverrr-1.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xoverrr
-Version: 1.1.4
+Version: 1.1.6
 Summary: A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 Author-email: Dmitry Ischenko <hotmori@gmail.com>
 License: MIT
@@ -21,7 +21,7 @@ Requires-Dist: clickhouse-sqlalchemy>=0.2.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
-Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.15.0; extra == "dev"
 Requires-Dist: isort>=5.12.0; extra == "dev"
 Requires-Dist: mypy>=1.0.0; extra == "dev"
 Requires-Dist: pre-commit>=3.0.0; extra == "dev"
@@ -31,7 +31,7 @@ Requires-Dist: pytest>=7.0.0; extra == "test"
 Requires-Dist: pytest-cov>=4.0.0; extra == "test"
 Requires-Dist: tenacity>=8.2.0; extra == "test"
 Provides-Extra: lint
-Requires-Dist: black>=23.0.0; extra == "lint"
+Requires-Dist: ruff>=0.15.0; extra == "lint"
 Requires-Dist: isort>=5.12.0; extra == "lint"
 Requires-Dist: flake8>=6.0.0; extra == "lint"
 Dynamic: license-file
@@ -40,6 +40,70 @@ Dynamic: license-file
 
 A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 
+## Usage Example
+**Sample comparison** (Greenplum vs Oracle):
+
+```python
+from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
+import os
+from datetime import date, timedelta
+
+USER_ORA = os.getenv('USER_ORA', '')
+PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
+
+USER_GP = os.getenv('USER_GP', '')
+PASSWORD_GP = os.getenv('PASSWORD_GP', '')
+
+HOST_ORA = os.getenv('HOST_ORA', '')
+HOST_GP = os.getenv('HOST_GP', '')
+
+def create_src_engine(user, password, host):
+    """Source engine (Oracle)"""
+    os.environ['NLS_LANG'] = '.AL32UTF8'
+    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
+
+def create_trg_engine(user, password, host):
+    """Target engine (Postgres/Greenplum)"""
+    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
+    engine = create_engine(connection_string)
+    return engine
+
+
+src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST_ORA)
+trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST_GP)
+
+comparator = DataQualityComparator(
+    source_engine=src_engine,
+    target_engine=trg_engine,
+    timezone='Europe/Athens'
+)
+
+source = DataReference("users", "schema1")
+target = DataReference("users", "schema2")
+
+FORMAT = '%Y-%m-%d'
+recent_range_end = date.today()
+recent_range_begin = recent_range_end - timedelta(days=1)
+
+status, report, stats, details = comparator.compare_sample(
+    source,
+    target,
+    date_column="created_at",
+    update_column="modified_date",
+    exclude_columns=["audit_timestamp", "internal_id"],
+    exclude_recent_hours=3,
+    date_range=(
+        recent_range_begin.strftime(FORMAT),
+        recent_range_end.strftime(FORMAT)
+    ),
+    tolerance_percentage=0
+)
+
+print(report)
+if status == COMPARISON_FAILED:
+    raise Exception("Sample check failed")
+```
+
 ## Key Features
 - **Multi‑DBMS support**: Oracle, PostgreSQL (+ Greenplum), ClickHouse (extensible via adapter layer) — tables and views.
 - **Universal connections**: Provide SQLAlchemy Engine objects for source and target databases.
@@ -273,71 +337,3 @@ Logs include timing information and structured context:
 - If `final_diff_score ≤ tolerance`: status = `COMPARISON_SUCCESS`
 - Enables configuration of acceptable discrepancy levels.
 
----
-
-## Usage Example
-**Sample comparison** (Greenplum vs Oracle):
-
-```python
-from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
-import os
-from datetime import date, timedelta
-
-USER_ORA = os.getenv('USER_ORA', '')
-PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
-
-USER_GP = os.getenv('USER_GP', '')
-PASSWORD_GP = os.getenv('PASSWORD_GP', '')
-
-HOST = os.getenv('HOST', '')
-
-def create_src_engine(user, password, host):
-    """Source engine (Oracle)"""
-    os.environ['NLS_LANG'] = '.AL32UTF8'
-    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
-
-def create_trg_engine(user, password, host):
-    """Target engine (Postgres/Greenplum)"""
-    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
-    engine = create_engine(connection_string)
-    return engine
-
-
-
-src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST)
-trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST)
-
-comparator = DataQualityComparator(
-    source_engine=src_engine,
-    target_engine=trg_engine,
-    timezone='Asia/Yekaterinburg'
-)
-
-source = DataReference("users", "schema1")
-target = DataReference("users", "schema2")
-
-FORMAT = '%Y-%m-%d'
-recent_range_end = date.today()
-recent_range_begin = recent_range_end - timedelta(days=1)
-
-status, report, stats, details = comparator.compare_sample(
-    source,
-    target,
-    date_column="created_at",
-    update_column="modified_date",
-    exclude_columns=["audit_timestamp", "internal_id"],
-    exclude_recent_hours=24,
-    date_range=(
-        recent_range_begin.strftime(FORMAT),
-        recent_range_end.strftime(FORMAT)
-    ),
-    tolerance_percentage=0
-)
-
-print(report)
-if status == COMPARISON_FAILED:
-    raise Exception("Sample check failed")
-```
-
----
-

{xoverrr-1.1.4 → xoverrr-1.1.6}/README.md

@@ -2,6 +2,70 @@
 
 A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 
+## Usage Example
+**Sample comparison** (Greenplum vs Oracle):
+
+```python
+from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
+import os
+from datetime import date, timedelta
+
+USER_ORA = os.getenv('USER_ORA', '')
+PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
+
+USER_GP = os.getenv('USER_GP', '')
+PASSWORD_GP = os.getenv('PASSWORD_GP', '')
+
+HOST_ORA = os.getenv('HOST_ORA', '')
+HOST_GP = os.getenv('HOST_GP', '')
+
+def create_src_engine(user, password, host):
+    """Source engine (Oracle)"""
+    os.environ['NLS_LANG'] = '.AL32UTF8'
+    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
+
+def create_trg_engine(user, password, host):
+    """Target engine (Postgres/Greenplum)"""
+    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
+    engine = create_engine(connection_string)
+    return engine
+
+
+src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST_ORA)
+trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST_GP)
+
+comparator = DataQualityComparator(
+    source_engine=src_engine,
+    target_engine=trg_engine,
+    timezone='Europe/Athens'
+)
+
+source = DataReference("users", "schema1")
+target = DataReference("users", "schema2")
+
+FORMAT = '%Y-%m-%d'
+recent_range_end = date.today()
+recent_range_begin = recent_range_end - timedelta(days=1)
+
+status, report, stats, details = comparator.compare_sample(
+    source,
+    target,
+    date_column="created_at",
+    update_column="modified_date",
+    exclude_columns=["audit_timestamp", "internal_id"],
+    exclude_recent_hours=3,
+    date_range=(
+        recent_range_begin.strftime(FORMAT),
+        recent_range_end.strftime(FORMAT)
+    ),
+    tolerance_percentage=0
+)
+
+print(report)
+if status == COMPARISON_FAILED:
+    raise Exception("Sample check failed")
+```
+
 ## Key Features
 - **Multi‑DBMS support**: Oracle, PostgreSQL (+ Greenplum), ClickHouse (extensible via adapter layer) — tables and views.
 - **Universal connections**: Provide SQLAlchemy Engine objects for source and target databases.
@@ -235,71 +299,3 @@ Logs include timing information and structured context:
 - If `final_diff_score ≤ tolerance`: status = `COMPARISON_SUCCESS`
 - Enables configuration of acceptable discrepancy levels.
 
----
-
-## Usage Example
-**Sample comparison** (Greenplum vs Oracle):
-
-```python
-from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
-import os
-from datetime import date, timedelta
-
-USER_ORA = os.getenv('USER_ORA', '')
-PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
-
-USER_GP = os.getenv('USER_GP', '')
-PASSWORD_GP = os.getenv('PASSWORD_GP', '')
-
-HOST = os.getenv('HOST', '')
-
-def create_src_engine(user, password, host):
-    """Source engine (Oracle)"""
-    os.environ['NLS_LANG'] = '.AL32UTF8'
-    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
-
-def create_trg_engine(user, password, host):
-    """Target engine (Postgres/Greenplum)"""
-    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
-    engine = create_engine(connection_string)
-    return engine
-
-
-
-src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST)
-trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST)
-
-comparator = DataQualityComparator(
-    source_engine=src_engine,
-    target_engine=trg_engine,
-    timezone='Asia/Yekaterinburg'
-)
-
-source = DataReference("users", "schema1")
-target = DataReference("users", "schema2")
-
-FORMAT = '%Y-%m-%d'
-recent_range_end = date.today()
-recent_range_begin = recent_range_end - timedelta(days=1)
-
-status, report, stats, details = comparator.compare_sample(
-    source,
-    target,
-    date_column="created_at",
-    update_column="modified_date",
-    exclude_columns=["audit_timestamp", "internal_id"],
-    exclude_recent_hours=24,
-    date_range=(
-        recent_range_begin.strftime(FORMAT),
-        recent_range_end.strftime(FORMAT)
-    ),
-    tolerance_percentage=0
-)
-
-print(report)
-if status == COMPARISON_FAILED:
-    raise Exception("Sample check failed")
-```
-
----
-

{xoverrr-1.1.4 → xoverrr-1.1.6}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xoverrr"
-version = "1.1.4"
+version = "1.1.6"
 description = "A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -36,7 +36,7 @@ Homepage = "https://github.com/dima-ischenko/xoverrr"
 dev = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0",
-    "black>=23.0.0",
+    "ruff>=0.15.0",
     "isort>=5.12.0",
     "mypy>=1.0.0",
     "pre-commit>=3.0.0",
@@ -48,7 +48,7 @@ test = [
     "tenacity>=8.2.0"
 ]
 lint = [
-    "black>=23.0.0",
+    "ruff>=0.15.0",
     "isort>=5.12.0",
     "flake8>=6.0.0",
 ]
@@ -59,4 +59,10 @@ where = ["src"]
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 testpaths = ["tests"]
-addopts = "-v"
+addopts = "-v"
+
+[tool.ruff]
+target-version = "py39"
+
+[tool.ruff.format]
+quote-style = "single"

xoverrr-1.1.6/src/xoverrr/__init__.py

@@ -0,0 +1,13 @@
+from .constants import (COMPARISON_FAILED, COMPARISON_SKIPPED,
+                        COMPARISON_SUCCESS)
+from .core import DataQualityComparator, DataReference
+
+__all__ = [
+    'DataQualityComparator',
+    'DataReference',
+    'COMPARISON_SUCCESS',
+    'COMPARISON_FAILED',
+    'COMPARISON_SKIPPED',
+]
+
+__version__ = '1.1.6'

{xoverrr-1.1.4 → xoverrr-1.1.6}/src/xoverrr/adapters/__init__.py

@@ -1,6 +1,11 @@
 from .base import BaseDatabaseAdapter
+from .clickhouse import ClickHouseAdapter
 from .oracle import OracleAdapter
 from .postgres import PostgresAdapter
-from .clickhouse import ClickHouseAdapter
 
-__all__ = ['BaseDatabaseAdapter', 'OracleAdapter', 'PostgresAdapter', 'ClickHouseAdapter']
+__all__ = [
+    'BaseDatabaseAdapter',
+    'OracleAdapter',
+    'PostgresAdapter',
+    'ClickHouseAdapter',
+]

{xoverrr-1.1.4 → xoverrr-1.1.6}/src/xoverrr/adapters/base.py

@@ -1,18 +1,23 @@
-from abc import ABC, abstractmethod
-import pandas as pd
-from typing import Dict, Callable, List, Tuple, Optional, Union
 import re
+from abc import ABC, abstractmethod
 from datetime import datetime, timedelta
-from ..models import DataReference, ObjectType
-from ..constants import RESERVED_WORDS
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+import pandas as pd
 from sqlalchemy.engine import Engine
+
+from ..constants import RESERVED_WORDS
 from ..logger import app_logger
-from ..logger import app_logger
+from ..models import DataReference, ObjectType
+
 
 class BaseDatabaseAdapter(ABC):
     """Abstract base class with updated method signatures for parameterized queries"""
+
     @abstractmethod
-    def _execute_query(self, query: Union[str, Tuple[str, Dict]], engine: Engine, timezone:str) -> pd.DataFrame:
+    def _execute_query(
+        self, query: Union[str, Tuple[str, Dict]], engine: Engine, timezone: str
+    ) -> pd.DataFrame:
         """Execute query with DBMS-specific optimizations"""
         pass
 
@@ -30,42 +35,66 @@ class BaseDatabaseAdapter(ABC):
         pass
 
     @abstractmethod
-    def build_count_query(self, data_ref: DataReference, date_column: str,
-                         start_date: Optional[str], end_date: Optional[str]
-                         ) -> Tuple[str, Dict]:
+    def build_count_query(
+        self,
+        data_ref: DataReference,
+        date_column: str,
+        start_date: Optional[str],
+        end_date: Optional[str],
+    ) -> Tuple[str, Dict]:
         """Returns tuple of (query, params) with recent data exclusion"""
         pass
 
-    def build_data_query_common(self, data_ref: DataReference, columns: List[str],
-                        date_column: Optional[str], update_column: Optional[str],
-                        start_date: Optional[str], end_date: Optional[str],
-                        exclude_recent_hours: Optional[int] = None) -> Tuple[str, Dict]:
+    def build_data_query_common(
+        self,
+        data_ref: DataReference,
+        columns: List[str],
+        date_column: Optional[str],
+        update_column: Optional[str],
+        start_date: Optional[str],
+        end_date: Optional[str],
+        exclude_recent_hours: Optional[int] = None,
+    ) -> Tuple[str, Dict]:
         """Build data query for the DBMS with recent data exclusion"""
         # Handle reserved words
         cols_select = [
-            f'"{col}"' if col.lower() in RESERVED_WORDS
-            else col
-            for col in columns
+            f'"{col}"' if col.lower() in RESERVED_WORDS else col for col in columns
         ]
 
-        result = self.build_data_query(data_ref, cols_select, date_column, update_column,
-                                     start_date, end_date, exclude_recent_hours)
+        result = self.build_data_query(
+            data_ref,
+            cols_select,
+            date_column,
+            update_column,
+            start_date,
+            end_date,
+            exclude_recent_hours,
+        )
         return result
 
     @abstractmethod
-    def build_data_query(self, data_ref: DataReference, columns: List[str],
-                        date_column: Optional[str], update_column: Optional[str],
-                        start_date: Optional[str], end_date: Optional[str],
-                        exclude_recent_hours: Optional[int] = None) -> Tuple[str, Dict]:
+    def build_data_query(
+        self,
+        data_ref: DataReference,
+        columns: List[str],
+        date_column: Optional[str],
+        update_column: Optional[str],
+        start_date: Optional[str],
+        end_date: Optional[str],
+        exclude_recent_hours: Optional[int] = None,
+    ) -> Tuple[str, Dict]:
         pass
 
     @abstractmethod
-    def _build_exclusion_condition(self, update_column: str,
-                                 exclude_recent_hours: int) -> Tuple[str, Dict]:
+    def _build_exclusion_condition(
+        self, update_column: str, exclude_recent_hours: int
+    ) -> Tuple[str, Dict]:
         """DBMS-specific implementation for recent data exclusion"""
         pass
 
-    def convert_types(self, df: pd.DataFrame, metadata: pd.DataFrame, timezone: str) -> pd.DataFrame:
+    def convert_types(
+        self, df: pd.DataFrame, metadata: pd.DataFrame, timezone: str
+    ) -> pd.DataFrame:
         """Convert DBMS-specific types to standardized formats"""
         # there is need to specify timezone for covnersion as
         #   pandas implicitly converts to UTC tz aware cols
@@ -78,8 +107,9 @@ class BaseDatabaseAdapter(ABC):
         """Get type conversion rules for specific DBMS"""
         pass
 
-    def _apply_type_conversion(self, df: pd.DataFrame, metadata: pd.DataFrame,
-                             type_rules: Dict[str, Callable]) -> pd.DataFrame:
+    def _apply_type_conversion(
+        self, df: pd.DataFrame, metadata: pd.DataFrame, type_rules: Dict[str, Callable]
+    ) -> pd.DataFrame:
         """Apply type conversion rules to DataFrame"""
         if df.empty:
             return df
@@ -94,7 +124,6 @@ class BaseDatabaseAdapter(ABC):
             if col_name not in df.columns:
                 continue
 
-
             col_type = col_info['data_type'].lower()
             # Find matching conversion rule
             converter = None
@@ -105,15 +134,15 @@ class BaseDatabaseAdapter(ABC):
                     break
 
             if converter is None:
-                continue # Skip columns without converters
+                continue  # Skip columns without converters
 
             try:
                 df[col_name] = converter(df[col_name])
             except Exception as e:
-                app_logger.warning(f"Type conversion failed for {col_name}: {str(e)}")
+                app_logger.warning(f'Type conversion failed for {col_name}: {str(e)}')
                 df[col_name] = df[col_name].astype(str)
 
             new_type = df[col_name].dtype
             app_logger.debug(f'old: {col_type}, new: {new_type}')
 
-        return df
+        return df