ibor-audit-tool 0.1.0__tar.gz

ibor_audit_tool-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+/AGENTS.md

ibor_audit_tool-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

ibor_audit_tool-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: ibor-audit-tool
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: >=3.12
+Requires-Dist: psycopg2-binary>=2.9
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: typer>=0.12

ibor_audit_tool-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "ibor-audit-tool"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "psycopg2-binary>=2.9",
+    "pyyaml>=6.0",
+    "sqlalchemy>=2.0",
+    "typer>=0.12",
+]
+
+[project.scripts]
+xaudit= "audit_tool.cli:main"
+
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple"
+publish-url = "https://upload.pypi.org/legacy/"
+explicit = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/audit_tool"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

ibor_audit_tool-0.1.0/sample/config.yml

@@ -0,0 +1,32 @@
+connections:
+  - name: FA
+    url: postgresql://ibor_user:ibor_user@localhost:5432/ibor-demo
+  - name: IBOR
+    url: postgresql://ibor_user:ibor_user@localhost:5432/ibor-demo
+
+compares:
+  - name: total_pnl
+    mode: full
+    left:
+      sql: |
+        select
+          cast(replace(enddate,'-','') as int) as trade_date,
+          concat(t1.stkid,'.','HK')  as secu_code,
+          'XIR_'||book as portfolio_id,
+          reportpl as  ytd_total_pnl
+        from ods_faas.bd_ods_faas_glhs_bond_rpt_nxt_his t1
+        left join irmp.dim_dict_code_mapping t5
+          on t5.src_sys_name = 'PG'
+          and t5.src_cls_code = 'mkt_cd'
+          and t5.src_cls_cdval=t1.exch_id
+          and t5.valid_flag = '1'
+        where  enddate='2026-06-04'
+    right:
+      sql: |
+        select
+          trade_date,
+          portfolio_id,
+          secu_code,
+          ytd_total_pnl
+        from irmp.ibor_bond_position
+        where secu_code like '%HK' and trade_date =20260604 and calc_basis ='1' and invest_type ='1'

ibor_audit_tool-0.1.0/src/audit_tool/__init__.py

@@ -0,0 +1,19 @@
+"""Audit tool command-line application."""
+
+from audit_tool.compare import Compare, CompareResult
+from audit_tool.factor import Connection, Factor, SQLFactor, TableData
+from audit_tool.report import CompareReportItem, ReportResult
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Compare",
+    "CompareResult",
+    "Connection",
+    "Factor",
+    "CompareReportItem",
+    "ReportResult",
+    "SQLFactor",
+    "TableData",
+    "__version__",
+]

ibor_audit_tool-0.1.0/src/audit_tool/__main__.py

@@ -0,0 +1,5 @@
+from audit_tool.cli import main
+
+
+if __name__ == "__main__":
+    main()

ibor_audit_tool-0.1.0/src/audit_tool/cli.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+import typer
+
+from audit_tool.errors import ConfigError, FactorLoadError
+from audit_tool.report import run_report, write_html_report
+
+app = typer.Typer()
+
+
+@app.command()
+def run(
+    config_path: Path,
+    output: Path = typer.Option(Path("audit-report.html"), "--output", "-o"),
+) -> None:
+    try:
+        report = run_report(config_path)
+        write_html_report(report, output)
+    except ConfigError as exc:
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(2) from exc
+    except FactorLoadError as exc:
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(2) from exc
+
+    typer.echo(
+        f"wrote report to {output} "
+        f"({report.total_count} total, {report.passed_count} passed, {report.failed_count} failed)"
+    )
+
+
+def main() -> None:
+    app()

ibor_audit_tool-0.1.0/src/audit_tool/compare.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from typing import Any
+
+from audit_tool.errors import ConfigError
+from audit_tool.factor import Factor, TableData
+
+
+@dataclass(frozen=True)
+class CompareResult:
+    passed: bool
+    message: str
+    columns: tuple[str, ...] = field(default_factory=tuple)
+    diff_summary: dict[str, Any] = field(default_factory=dict)
+
+
+class Compare:
+    def __init__(
+        self,
+        name: str,
+        left: Factor,
+        right: Factor,
+        *,
+        mode: str = "full",
+    ) -> None:
+        self.name = name
+        self.left = left
+        self.right = right
+        self.mode = mode
+
+    def run(self) -> CompareResult:
+        if self.mode != "full":
+            raise ConfigError(f"unsupported compare mode {self.mode!r}")
+
+        left_data = self.left.load()
+        right_data = self.right.load()
+        return compare_full(
+            left_data,
+            right_data,
+            left_name=_factor_result_name(self.left),
+            right_name=_factor_result_name(self.right),
+        )
+
+
+def compare_full(
+    left: TableData,
+    right: TableData,
+    *,
+    left_name: str = "left",
+    right_name: str = "right",
+) -> CompareResult:
+    if Counter(left.columns) != Counter(right.columns):
+        return CompareResult(
+            passed=False,
+            message="columns differ",
+            columns=left.columns,
+            diff_summary={
+                f"{left_name}_columns": left.columns,
+                f"{right_name}_columns": right.columns,
+            },
+        )
+
+    right_column_indexes = _column_indexes(right.columns, left.columns)
+    left_rows = Counter(_normalize_row(row) for row in left.rows)
+    right_rows = Counter(
+        _normalize_row(_reorder_row(row, right_column_indexes))
+        for row in right.rows
+    )
+
+    if left_rows == right_rows:
+        return CompareResult(
+            passed=True,
+            message="tables match",
+            columns=left.columns,
+            diff_summary={
+                "columns": left.columns,
+                "row_count": len(left.rows),
+            },
+        )
+
+    left_only = left_rows - right_rows
+    right_only = right_rows - left_rows
+    return CompareResult(
+        passed=False,
+        message="rows differ",
+        columns=left.columns,
+        diff_summary={
+            f"{left_name}_row_count": len(left.rows),
+            f"{right_name}_row_count": len(right.rows),
+            f"{left_name}_only_count": left_only.total(),
+            f"{right_name}_only_count": right_only.total(),
+            f"{left_name}_only_sample": _sample_rows(left_only),
+            f"{right_name}_only_sample": _sample_rows(right_only),
+        },
+    )
+
+
+def _factor_result_name(factor: Factor) -> str:
+    connection = getattr(factor, "connection", None)
+    if connection is not None:
+        return str(connection.name)
+    return factor.name
+
+
+def _normalize_row(row: tuple[Any, ...]) -> tuple[Any, ...]:
+    return tuple(_normalize_value(value) for value in row)
+
+
+def _column_indexes(
+    source_columns: tuple[str, ...],
+    target_columns: tuple[str, ...],
+) -> tuple[int, ...]:
+    source_indexes: dict[tuple[str, int], int] = {}
+    occurrences: defaultdict[str, int] = defaultdict(int)
+    for index, column in enumerate(source_columns):
+        occurrence = occurrences[column]
+        source_indexes[(column, occurrence)] = index
+        occurrences[column] += 1
+
+    target_indexes: list[int] = []
+    occurrences.clear()
+    for column in target_columns:
+        occurrence = occurrences[column]
+        target_indexes.append(source_indexes[(column, occurrence)])
+        occurrences[column] += 1
+    return tuple(target_indexes)
+
+
+def _reorder_row(row: tuple[Any, ...], indexes: tuple[int, ...]) -> tuple[Any, ...]:
+    return tuple(row[index] for index in indexes)
+
+
+def _normalize_value(value: Any) -> Any:
+    if isinstance(value, dict):
+        return tuple(
+            sorted(
+                ((key, _normalize_value(item)) for key, item in value.items()),
+                key=repr,
+            )
+        )
+    if isinstance(value, list | tuple):
+        return tuple(_normalize_value(item) for item in value)
+    if isinstance(value, set):
+        return tuple(sorted((_normalize_value(item) for item in value), key=repr))
+    return value
+
+
+def _sample_rows(rows: Counter[tuple[Any, ...]], limit: int = 5) -> list[tuple[Any, ...]]:
+    sample: list[tuple[Any, ...]] = []
+    for row, count in rows.items():
+        sample.extend([row] * min(count, limit - len(sample)))
+        if len(sample) >= limit:
+            break
+    return sample

ibor_audit_tool-0.1.0/src/audit_tool/config.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from audit_tool.compare import Compare
+from audit_tool.errors import ConfigError
+from audit_tool.factor import Connection, SQLFactor
+
+DEFAULT_SIDE_CONNECTIONS = {
+    "left": "FA",
+    "right": "IBOR",
+}
+ENV_VAR_REFERENCE = re.compile(
+    r"(\$\{[A-Za-z_][A-Za-z0-9_]*\}|\$[A-Z_][A-Z0-9_]*|^[A-Z_][A-Z0-9_]*$)"
+)
+
+
+def load_config(path: Path) -> dict[str, Compare]:
+    raw_config = _read_yaml(path)
+    connections = _build_connections(raw_config.get("connections"))
+    compare_configs = _index_named_items(raw_config.get("compares"), "compares")
+
+    return {
+        name: _build_compare(
+            name,
+            _require_mapping(config, f"compares.{name}"),
+            connections,
+        )
+        for name, config in compare_configs.items()
+    }
+
+
+def _read_yaml(path: Path) -> dict[str, Any]:
+    try:
+        with path.open("r", encoding="utf-8") as file:
+            data = yaml.safe_load(file)
+    except OSError as exc:
+        raise ConfigError(f"failed to read config {str(path)!r}: {exc}") from exc
+
+    return _require_mapping(data, "config")
+
+
+def _build_compare(
+    name: str,
+    config: dict[str, Any],
+    connections: dict[str, Connection],
+) -> Compare:
+    mode = config.get("mode", "full")
+    if mode != "full":
+        raise ConfigError(f"compares.{name}.mode must be 'full'")
+
+    left = _build_compare_side(name, "left", config.get("left"), connections)
+    right = _build_compare_side(name, "right", config.get("right"), connections)
+
+    return Compare(name=name, left=left, right=right, mode=mode)
+
+
+def _build_compare_side(
+    compare_name: str,
+    side_name: str,
+    side_value: Any,
+    connections: dict[str, Connection],
+) -> SQLFactor:
+    side_config = _require_mapping(side_value, f"compares.{compare_name}.{side_name}")
+    connection = _resolve_connection(
+        side_name,
+        side_config,
+        connections,
+    )
+    sql = _require_string(side_config.get("sql"), f"compares.{compare_name}.{side_name}.sql")
+
+    return SQLFactor(
+        name=f"{compare_name}.{side_name}",
+        sql=sql,
+        connection=connection,
+    )
+
+
+def _resolve_connection(
+    side_name: str,
+    side_config: dict[str, Any],
+    connections: dict[str, Connection],
+) -> Connection:
+    connection_name = _resolve_connection_name(side_name, side_config)
+    try:
+        return connections[connection_name]
+    except KeyError as exc:
+        raise ConfigError(f"connections.{connection_name} must be configured") from exc
+
+
+def _resolve_connection_name(side_name: str, side_config: dict[str, Any]) -> str:
+    connection_name = side_config.get("connection")
+    if connection_name is None:
+        return DEFAULT_SIDE_CONNECTIONS[side_name]
+    return _require_string(connection_name, f"{side_name}.connection")
+
+
+def _build_connections(value: Any) -> dict[str, Connection]:
+    connection_configs = _index_named_items(value, "connections")
+    return {
+        name: _build_connection(name, config)
+        for name, config in connection_configs.items()
+    }
+
+
+def _build_connection(name: str, config: dict[str, Any]) -> Connection:
+    url = _require_string(config.get("url"), f"connections.{name}.url")
+    _validate_connection_url(name, url)
+    return Connection(name=name, url=url)
+
+
+def _validate_connection_url(name: str, url: str) -> None:
+    if ENV_VAR_REFERENCE.search(url):
+        raise ConfigError(
+            f"connections.{name}.url must be a literal database URL, not an environment variable"
+        )
+
+    if "://" not in url:
+        raise ConfigError(f"connections.{name}.url must be a literal database URL")
+
+
+def _require_mapping(value: Any, name: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise ConfigError(f"{name} must be a mapping")
+    return value
+
+
+def _index_named_items(value: Any, name: str) -> dict[str, dict[str, Any]]:
+    if not isinstance(value, list):
+        raise ConfigError(f"{name} must be a list")
+
+    items: dict[str, dict[str, Any]] = {}
+    for index, item in enumerate(value):
+        item_config = _require_mapping(item, f"{name}[{index}]")
+        item_name = _require_string(item_config.get("name"), f"{name}[{index}].name")
+        if item_name in items:
+            raise ConfigError(f"{name} contains duplicate name {item_name!r}")
+        items[item_name] = item_config
+    return items
+
+
+def _require_string(value: Any, name: str) -> str:
+    if not isinstance(value, str) or not value:
+        raise ConfigError(f"{name} must be a non-empty string")
+    return value

ibor_audit_tool-0.1.0/src/audit_tool/errors.py

@@ -0,0 +1,10 @@
+class AuditToolError(Exception):
+    """Base exception for audit-tool errors."""
+
+
+class ConfigError(AuditToolError):
+    """Raised when the audit configuration is invalid."""
+
+
+class FactorLoadError(AuditToolError):
+    """Raised when a factor cannot load its data."""

ibor_audit_tool-0.1.0/src/audit_tool/factor.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.exc import SQLAlchemyError
+
+from audit_tool.errors import FactorLoadError
+
+
+@dataclass(frozen=True)
+class TableData:
+    columns: tuple[str, ...]
+    rows: tuple[tuple[Any, ...], ...]
+
+
+@dataclass(frozen=True)
+class Connection:
+    name: str
+    url: str
+
+
+class Factor(ABC):
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    @abstractmethod
+    def load(self) -> TableData:
+        """Load this factor as table-shaped data."""
+
+
+class SQLFactor(Factor):
+    def __init__(
+        self,
+        name: str,
+        sql: str,
+        connection: Connection,
+    ) -> None:
+        super().__init__(name)
+        self.sql = sql
+        self.connection = connection
+
+    def load(self) -> TableData:
+        try:
+            engine = create_engine(self.connection.url)
+            with engine.connect() as connection:
+                result = connection.execute(text(self.sql))
+                columns = tuple(str(column) for column in result.keys())
+                rows = tuple(tuple(row) for row in result.fetchall())
+        except SQLAlchemyError as exc:
+            raise FactorLoadError(f"failed to load SQL factor {self.name!r}: {exc}") from exc
+
+        return TableData(columns=columns, rows=rows)