xoverrr 1.1.4__tar.gz → 1.1.5__tar.gz

{xoverrr-1.1.4/src/xoverrr.egg-info → xoverrr-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xoverrr
-Version: 1.1.4
+Version: 1.1.5
 Summary: A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 Author-email: Dmitry Ischenko <hotmori@gmail.com>
 License: MIT
@@ -40,6 +40,70 @@ Dynamic: license-file
 
 A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 
+## Usage Example
+**Sample comparison** (Greenplum vs Oracle):
+
+```python
+from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
+import os
+from datetime import date, timedelta
+
+USER_ORA = os.getenv('USER_ORA', '')
+PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
+
+USER_GP = os.getenv('USER_GP', '')
+PASSWORD_GP = os.getenv('PASSWORD_GP', '')
+
+HOST_ORA = os.getenv('HOST_ORA', '')
+HOST_GP = os.getenv('HOST_GP', '')
+
+def create_src_engine(user, password, host):
+    """Source engine (Oracle)"""
+    os.environ['NLS_LANG'] = '.AL32UTF8'
+    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
+
+def create_trg_engine(user, password, host):
+    """Target engine (Postgres/Greenplum)"""
+    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
+    engine = create_engine(connection_string)
+    return engine
+
+
+src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST_ORA)
+trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST_GP)
+
+comparator = DataQualityComparator(
+    source_engine=src_engine,
+    target_engine=trg_engine,
+    timezone='Europe/Athens'
+)
+
+source = DataReference("users", "schema1")
+target = DataReference("users", "schema2")
+
+FORMAT = '%Y-%m-%d'
+recent_range_end = date.today()
+recent_range_begin = recent_range_end - timedelta(days=1)
+
+status, report, stats, details = comparator.compare_sample(
+    source,
+    target,
+    date_column="created_at",
+    update_column="modified_date",
+    exclude_columns=["audit_timestamp", "internal_id"],
+    exclude_recent_hours=3,
+    date_range=(
+        recent_range_begin.strftime(FORMAT),
+        recent_range_end.strftime(FORMAT)
+    ),
+    tolerance_percentage=0
+)
+
+print(report)
+if status == COMPARISON_FAILED:
+    raise Exception("Sample check failed")
+```
+
 ## Key Features
 - **Multi‑DBMS support**: Oracle, PostgreSQL (+ Greenplum), ClickHouse (extensible via adapter layer) — tables and views.
 - **Universal connections**: Provide SQLAlchemy Engine objects for source and target databases.
@@ -273,71 +337,3 @@ Logs include timing information and structured context:
 - If `final_diff_score ≤ tolerance`: status = `COMPARISON_SUCCESS`
 - Enables configuration of acceptable discrepancy levels.
 
----
-
-## Usage Example
-**Sample comparison** (Greenplum vs Oracle):
-
-```python
-from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
-import os
-from datetime import date, timedelta
-
-USER_ORA = os.getenv('USER_ORA', '')
-PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
-
-USER_GP = os.getenv('USER_GP', '')
-PASSWORD_GP = os.getenv('PASSWORD_GP', '')
-
-HOST = os.getenv('HOST', '')
-
-def create_src_engine(user, password, host):
-    """Source engine (Oracle)"""
-    os.environ['NLS_LANG'] = '.AL32UTF8'
-    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
-
-def create_trg_engine(user, password, host):
-    """Target engine (Postgres/Greenplum)"""
-    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
-    engine = create_engine(connection_string)
-    return engine
-
-
-
-src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST)
-trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST)
-
-comparator = DataQualityComparator(
-    source_engine=src_engine,
-    target_engine=trg_engine,
-    timezone='Asia/Yekaterinburg'
-)
-
-source = DataReference("users", "schema1")
-target = DataReference("users", "schema2")
-
-FORMAT = '%Y-%m-%d'
-recent_range_end = date.today()
-recent_range_begin = recent_range_end - timedelta(days=1)
-
-status, report, stats, details = comparator.compare_sample(
-    source,
-    target,
-    date_column="created_at",
-    update_column="modified_date",
-    exclude_columns=["audit_timestamp", "internal_id"],
-    exclude_recent_hours=24,
-    date_range=(
-        recent_range_begin.strftime(FORMAT),
-        recent_range_end.strftime(FORMAT)
-    ),
-    tolerance_percentage=0
-)
-
-print(report)
-if status == COMPARISON_FAILED:
-    raise Exception("Sample check failed")
-```
-
----
-

{xoverrr-1.1.4 → xoverrr-1.1.5}/README.md

@@ -2,6 +2,70 @@
 
 A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 
+## Usage Example
+**Sample comparison** (Greenplum vs Oracle):
+
+```python
+from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
+import os
+from datetime import date, timedelta
+
+USER_ORA = os.getenv('USER_ORA', '')
+PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
+
+USER_GP = os.getenv('USER_GP', '')
+PASSWORD_GP = os.getenv('PASSWORD_GP', '')
+
+HOST_ORA = os.getenv('HOST_ORA', '')
+HOST_GP = os.getenv('HOST_GP', '')
+
+def create_src_engine(user, password, host):
+    """Source engine (Oracle)"""
+    os.environ['NLS_LANG'] = '.AL32UTF8'
+    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
+
+def create_trg_engine(user, password, host):
+    """Target engine (Postgres/Greenplum)"""
+    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
+    engine = create_engine(connection_string)
+    return engine
+
+
+src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST_ORA)
+trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST_GP)
+
+comparator = DataQualityComparator(
+    source_engine=src_engine,
+    target_engine=trg_engine,
+    timezone='Europe/Athens'
+)
+
+source = DataReference("users", "schema1")
+target = DataReference("users", "schema2")
+
+FORMAT = '%Y-%m-%d'
+recent_range_end = date.today()
+recent_range_begin = recent_range_end - timedelta(days=1)
+
+status, report, stats, details = comparator.compare_sample(
+    source,
+    target,
+    date_column="created_at",
+    update_column="modified_date",
+    exclude_columns=["audit_timestamp", "internal_id"],
+    exclude_recent_hours=3,
+    date_range=(
+        recent_range_begin.strftime(FORMAT),
+        recent_range_end.strftime(FORMAT)
+    ),
+    tolerance_percentage=0
+)
+
+print(report)
+if status == COMPARISON_FAILED:
+    raise Exception("Sample check failed")
+```
+
 ## Key Features
 - **Multi‑DBMS support**: Oracle, PostgreSQL (+ Greenplum), ClickHouse (extensible via adapter layer) — tables and views.
 - **Universal connections**: Provide SQLAlchemy Engine objects for source and target databases.
@@ -235,71 +299,3 @@ Logs include timing information and structured context:
 - If `final_diff_score ≤ tolerance`: status = `COMPARISON_SUCCESS`
 - Enables configuration of acceptable discrepancy levels.
 
----
-
-## Usage Example
-**Sample comparison** (Greenplum vs Oracle):
-
-```python
-from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
-import os
-from datetime import date, timedelta
-
-USER_ORA = os.getenv('USER_ORA', '')
-PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
-
-USER_GP = os.getenv('USER_GP', '')
-PASSWORD_GP = os.getenv('PASSWORD_GP', '')
-
-HOST = os.getenv('HOST', '')
-
-def create_src_engine(user, password, host):
-    """Source engine (Oracle)"""
-    os.environ['NLS_LANG'] = '.AL32UTF8'
-    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
-
-def create_trg_engine(user, password, host):
-    """Target engine (Postgres/Greenplum)"""
-    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
-    engine = create_engine(connection_string)
-    return engine
-
-
-
-src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST)
-trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST)
-
-comparator = DataQualityComparator(
-    source_engine=src_engine,
-    target_engine=trg_engine,
-    timezone='Asia/Yekaterinburg'
-)
-
-source = DataReference("users", "schema1")
-target = DataReference("users", "schema2")
-
-FORMAT = '%Y-%m-%d'
-recent_range_end = date.today()
-recent_range_begin = recent_range_end - timedelta(days=1)
-
-status, report, stats, details = comparator.compare_sample(
-    source,
-    target,
-    date_column="created_at",
-    update_column="modified_date",
-    exclude_columns=["audit_timestamp", "internal_id"],
-    exclude_recent_hours=24,
-    date_range=(
-        recent_range_begin.strftime(FORMAT),
-        recent_range_end.strftime(FORMAT)
-    ),
-    tolerance_percentage=0
-)
-
-print(report)
-if status == COMPARISON_FAILED:
-    raise Exception("Sample check failed")
-```
-
----
-

{xoverrr-1.1.4 → xoverrr-1.1.5}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xoverrr"
-version = "1.1.4"
+version = "1.1.5"
 description = "A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting."
 readme = "README.md"
 requires-python = ">=3.9"

{xoverrr-1.1.4 → xoverrr-1.1.5}/src/xoverrr/__init__.py

@@ -14,4 +14,4 @@ __all__ = [
     "COMPARISON_SKIPPED",
 ]
 
-__version__ = "1.1.4"
+__version__ = "1.1.5"

{xoverrr-1.1.4 → xoverrr-1.1.5}/src/xoverrr/adapters/clickhouse.py

@@ -101,7 +101,7 @@ class ClickHouseAdapter(BaseDatabaseAdapter):
                          start_date: Optional[str], end_date: Optional[str]) -> Tuple[str, Dict]:
         query = f"""
             SELECT
-                toDate({date_column}) as dt,
+                formatDateTime(toDate({date_column}), '%%Y-%%m-%%d') as dt,
                 count(*) as cnt
             FROM {data_ref.full_name}
             WHERE 1=1
@@ -161,10 +161,14 @@ class ClickHouseAdapter(BaseDatabaseAdapter):
 
         return None, None
 
-    def _get_type_conversion_rules(self, timezone:str ) -> Dict[str, Callable]:
+    def _get_type_conversion_rules(self, timezone: str) -> Dict[str, Callable]:
         return {
-            r'datetime\(': lambda x: pd.to_datetime(x, utc=True, errors='coerce').dt.tz_convert(timezone).dt.tz_localize(None).strftime(DATETIME_FORMAT).str.replace(r'\s00:00:00$', '', regex=True),
-            r'datetime64': lambda x: pd.to_datetime(x, utc=True, errors='coerce').dt.tz_convert(timezone).dt.tz_localize(None).strftime(DATETIME_FORMAT).str.replace(r'\s00:00:00$', '', regex=True),
-            r'date': lambda x: pd.to_datetime(x, errors='coerce').dt.strftime(DATE_FORMAT).str.replace(r'\s00:00:00$', '', regex=True),
-            r'uint64|uint8|float|decimal|int32': lambda x: x.astype(str).str.replace(r'\.0+$', '', regex=True),
-        }
+                r'datetime64|datetime': lambda x: pd.to_datetime(x, utc=True, errors='coerce')
+                                                .dt.tz_convert(timezone)
+                                                .dt.strftime(DATETIME_FORMAT)
+                                                .str.replace(r'\s00:00:00$', '', regex=True),
+                r'date': lambda x: pd.to_datetime(x, errors='coerce')
+                                    .dt.strftime(DATE_FORMAT)
+                                    .str.replace(r'\s00:00:00$', '', regex=True),
+                r'uint64|uint8|float|decimal|int32': lambda x: x.astype(str).str.replace(r'\.0+$', '', regex=True),
+        } 

{xoverrr-1.1.4 → xoverrr-1.1.5}/src/xoverrr/adapters/oracle.py

@@ -1,7 +1,7 @@
 import pandas as pd
 from typing import Optional, Dict, Callable, List, Tuple, Union
-from datetime import datetime, timedelta
-from ..constants import DATE_FORMAT,DATETIME_FORMAT
+
+from ..constants import DATETIME_FORMAT
 from .base import BaseDatabaseAdapter, Engine
 from ..models import DataReference, ObjectType
 from ..exceptions import QueryExecutionError

{xoverrr-1.1.4 → xoverrr-1.1.5}/src/xoverrr/core.py

@@ -63,7 +63,12 @@ class DataQualityComparator:
             DBMSType.CLICKHOUSE: ClickHouseAdapter(),
         }
         self._reset_stats()
+        from . import __version__
         app_logger.info('start')
+        app_logger.info(f'Version: v{__version__}')
+        app_logger.info(f'Source DB: {self.source_db_type.name}')
+        app_logger.info(f'Target DB: {self.target_db_type.name}')        
+
 
     def reset_stats(self):
         self._reset_stats()
@@ -208,9 +213,8 @@ class DataQualityComparator:
             )
             target_counts = self._execute_query((target_query, target_params), self.target_engine, self.timezone)
 
+
             source_counts_filled, target_counts_filled = cross_fill_missing_dates(source_counts, target_counts)
-            source_counts_filled['dt'] = pd.to_datetime(source_counts_filled['dt'], format='%Y-%m-%d')
-            target_counts_filled['dt'] = pd.to_datetime(target_counts_filled['dt'], format='%Y-%m-%d')
 
             merged = source_counts_filled.merge(target_counts_filled, on='dt')
             total_count_source = source_counts_filled['cnt'].sum()

{xoverrr-1.1.4 → xoverrr-1.1.5}/src/xoverrr/utils.py

@@ -3,13 +3,9 @@ import numpy as np
 from typing import Dict, Any, List, Optional, Tuple, defaultdict
 from datetime import datetime
 
-try:
-    from .constants import NULL_REPLACEMENT, DEFAULT_MAX_EXAMPLES, DATETIME_FORMAT
-    from .logger import app_logger
-except ImportError:
-    # for cases when used as standalone script
-    from constants import NULL_REPLACEMENT, DEFAULT_MAX_EXAMPLES, DATETIME_FORMAT
-    from logger import app_logger
+
+from .constants import NULL_REPLACEMENT, DEFAULT_MAX_EXAMPLES, DATETIME_FORMAT
+from .logger import app_logger
 
 from dataclasses import dataclass, field
 
@@ -528,7 +524,7 @@ def prepare_dataframe(df: pd.DataFrame) -> pd.DataFrame:
 
 
     df = df.fillna(NULL_REPLACEMENT)
-    df = df.replace(r'(?i)^(None|nan|NaN|\s*)$', NULL_REPLACEMENT, regex=True)
+    df = df.replace(r'(?i)^(None|nan|NaN|NaT|\s*)$', NULL_REPLACEMENT, regex=True)
 
     df = df.astype(str)
 

{xoverrr-1.1.4 → xoverrr-1.1.5/src/xoverrr.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xoverrr
-Version: 1.1.4
+Version: 1.1.5
 Summary: A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 Author-email: Dmitry Ischenko <hotmori@gmail.com>
 License: MIT
@@ -40,6 +40,70 @@ Dynamic: license-file
 
 A tool for cross-database and intra-source data comparison with detailed discrepancy analysis and reporting.
 
+## Usage Example
+**Sample comparison** (Greenplum vs Oracle):
+
+```python
+from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
+import os
+from datetime import date, timedelta
+
+USER_ORA = os.getenv('USER_ORA', '')
+PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
+
+USER_GP = os.getenv('USER_GP', '')
+PASSWORD_GP = os.getenv('PASSWORD_GP', '')
+
+HOST_ORA = os.getenv('HOST_ORA', '')
+HOST_GP = os.getenv('HOST_GP', '')
+
+def create_src_engine(user, password, host):
+    """Source engine (Oracle)"""
+    os.environ['NLS_LANG'] = '.AL32UTF8'
+    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
+
+def create_trg_engine(user, password, host):
+    """Target engine (Postgres/Greenplum)"""
+    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
+    engine = create_engine(connection_string)
+    return engine
+
+
+src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST_ORA)
+trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST_GP)
+
+comparator = DataQualityComparator(
+    source_engine=src_engine,
+    target_engine=trg_engine,
+    timezone='Europe/Athens'
+)
+
+source = DataReference("users", "schema1")
+target = DataReference("users", "schema2")
+
+FORMAT = '%Y-%m-%d'
+recent_range_end = date.today()
+recent_range_begin = recent_range_end - timedelta(days=1)
+
+status, report, stats, details = comparator.compare_sample(
+    source,
+    target,
+    date_column="created_at",
+    update_column="modified_date",
+    exclude_columns=["audit_timestamp", "internal_id"],
+    exclude_recent_hours=3,
+    date_range=(
+        recent_range_begin.strftime(FORMAT),
+        recent_range_end.strftime(FORMAT)
+    ),
+    tolerance_percentage=0
+)
+
+print(report)
+if status == COMPARISON_FAILED:
+    raise Exception("Sample check failed")
+```
+
 ## Key Features
 - **Multi‑DBMS support**: Oracle, PostgreSQL (+ Greenplum), ClickHouse (extensible via adapter layer) — tables and views.
 - **Universal connections**: Provide SQLAlchemy Engine objects for source and target databases.
@@ -273,71 +337,3 @@ Logs include timing information and structured context:
 - If `final_diff_score ≤ tolerance`: status = `COMPARISON_SUCCESS`
 - Enables configuration of acceptable discrepancy levels.
 
----
-
-## Usage Example
-**Sample comparison** (Greenplum vs Oracle):
-
-```python
-from xoverrr import DataQualityComparator, DataReference, COMPARISON_SUCCESS, COMPARISON_FAILED, COMPARISON_SKIPPED
-import os
-from datetime import date, timedelta
-
-USER_ORA = os.getenv('USER_ORA', '')
-PASSWORD_ORA = os.getenv('PASSWORD_ORA', '')
-
-USER_GP = os.getenv('USER_GP', '')
-PASSWORD_GP = os.getenv('PASSWORD_GP', '')
-
-HOST = os.getenv('HOST', '')
-
-def create_src_engine(user, password, host):
-    """Source engine (Oracle)"""
-    os.environ['NLS_LANG'] = '.AL32UTF8'
-    return create_engine(f'oracle+oracledb://{user}:{password}@{host}:1521/?service_name=dwh')
-
-def create_trg_engine(user, password, host):
-    """Target engine (Postgres/Greenplum)"""
-    connection_string = f'postgresql+psycopg2://{user}:{password}@{host}:5432/adb'
-    engine = create_engine(connection_string)
-    return engine
-
-
-
-src_engine = create_src_engine(USER_ORA, PASSWORD_ORA, HOST)
-trg_engine = create_trg_engine(USER_GP, PASSWORD_GP, HOST)
-
-comparator = DataQualityComparator(
-    source_engine=src_engine,
-    target_engine=trg_engine,
-    timezone='Asia/Yekaterinburg'
-)
-
-source = DataReference("users", "schema1")
-target = DataReference("users", "schema2")
-
-FORMAT = '%Y-%m-%d'
-recent_range_end = date.today()
-recent_range_begin = recent_range_end - timedelta(days=1)
-
-status, report, stats, details = comparator.compare_sample(
-    source,
-    target,
-    date_column="created_at",
-    update_column="modified_date",
-    exclude_columns=["audit_timestamp", "internal_id"],
-    exclude_recent_hours=24,
-    date_range=(
-        recent_range_begin.strftime(FORMAT),
-        recent_range_end.strftime(FORMAT)
-    ),
-    tolerance_percentage=0
-)
-
-print(report)
-if status == COMPARISON_FAILED:
-    raise Exception("Sample check failed")
-```
-
----
-