execsql2 2.15.0__py3-none-any.whl → 2.15.2__py3-none-any.whl

execsql/db/duckdb.py

@@ -81,11 +81,10 @@ class DuckDBDatabase(Database):
     def schema_exists(self, schema_name: str) -> bool:
         """Return True if the named schema exists in the current DuckDB catalog."""
         # In DuckDB, the 'schemata' view is not limited to the current database.
-        curs = self.cursor()
-        curs.execute(
-            "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ? and catalog_name = ?;",
-            (schema_name, self.catalog_name),
-        )
-        rows = curs.fetchall()
-        curs.close()
+        with self._cursor() as curs:
+            curs.execute(
+                "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ? and catalog_name = ?;",
+                (schema_name, self.catalog_name),
+            )
+            rows = curs.fetchall()
         return len(rows) > 0

execsql/db/sqlite.py

@@ -82,21 +82,21 @@ class SQLiteDatabase(Database):
 
     def table_exists(self, table_name: str, schema_name: str | None = None) -> bool:
         """Return True if the named table exists in the SQLite database."""
-        curs = self.cursor()
-        sql = "select name from sqlite_master where type='table' and name=?;"
-        try:
-            curs.execute(sql, (table_name,))
-        except ErrInfo:
-            raise
-        except Exception as e:
-            self.rollback()
-            raise ErrInfo(
-                type="db",
-                command_text=sql,
-                exception_msg=exception_desc(),
-                other_msg=f'Failed test for existence of SQLite table "{table_name}";',
-            ) from e
-        rows = curs.fetchall()
+        with self._cursor() as curs:
+            sql = "select name from sqlite_master where type='table' and name=?;"
+            try:
+                curs.execute(sql, (table_name,))
+            except ErrInfo:
+                raise
+            except Exception as e:
+                self.rollback()
+                raise ErrInfo(
+                    type="db",
+                    command_text=sql,
+                    exception_msg=exception_desc(),
+                    other_msg=f'Failed test for existence of SQLite table "{table_name}";',
+                ) from e
+            rows = curs.fetchall()
         return len(rows) > 0
 
     def column_exists(
@@ -111,40 +111,40 @@ class SQLiteDatabase(Database):
 
     def table_columns(self, table_name: str, schema_name: str | None = None) -> list[str]:
         """Return a list of column names for the given SQLite table."""
-        curs = self.cursor()
-        quoted_tbl = self.quote_identifier(table_name)
-        sql = f"select * from {quoted_tbl} where 1=0;"
-        try:
-            curs.execute(sql)
-        except ErrInfo:
-            raise
-        except Exception as e:
-            self.rollback()
-            raise ErrInfo(
-                type="db",
-                command_text=sql,
-                exception_msg=exception_desc(),
-                other_msg=f"Failed to get column names for table {table_name} of {self.name()}",
-            ) from e
-        return [d[0] for d in curs.description]
+        with self._cursor() as curs:
+            quoted_tbl = self.quote_identifier(table_name)
+            sql = f"select * from {quoted_tbl} where 1=0;"
+            try:
+                curs.execute(sql)
+            except ErrInfo:
+                raise
+            except Exception as e:
+                self.rollback()
+                raise ErrInfo(
+                    type="db",
+                    command_text=sql,
+                    exception_msg=exception_desc(),
+                    other_msg=f"Failed to get column names for table {table_name} of {self.name()}",
+                ) from e
+            return [d[0] for d in curs.description]
 
     def view_exists(self, view_name: str) -> bool:
         """Return True if the named view exists in the SQLite database."""
-        curs = self.cursor()
-        sql = "select name from sqlite_master where type='view' and name=?;"
-        try:
-            curs.execute(sql, (view_name,))
-        except ErrInfo:
-            raise
-        except Exception as e:
-            self.rollback()
-            raise ErrInfo(
-                type="db",
-                command_text=sql,
-                exception_msg=exception_desc(),
-                other_msg=f'Failed test for existence of SQLite view "{view_name}";',
-            ) from e
-        rows = curs.fetchall()
+        with self._cursor() as curs:
+            sql = "select name from sqlite_master where type='view' and name=?;"
+            try:
+                curs.execute(sql, (view_name,))
+            except ErrInfo:
+                raise
+            except Exception as e:
+                self.rollback()
+                raise ErrInfo(
+                    type="db",
+                    command_text=sql,
+                    exception_msg=exception_desc(),
+                    other_msg=f'Failed test for existence of SQLite view "{view_name}";',
+                ) from e
+            rows = curs.fetchall()
         return len(rows) > 0
 
     def schema_exists(self, schema_name: str) -> bool:
@@ -194,7 +194,6 @@ class SQLiteDatabase(Database):
                         type="error",
                         other_msg=f"Too few values on data line {datalineno} of input.",
                     )
-                linedata = [line[ix] for ix in data_indexes]
                 if _state.conf.trim_strings or _state.conf.replace_newlines or not _state.conf.empty_strings:
                     for i in range(len(line)):
                         if line[i] is not None and isinstance(line[i], _state.stringtypes):
@@ -204,6 +203,7 @@ class SQLiteDatabase(Database):
                                 line[i] = re.sub(r"[\s\t]*[\r\n]+[\s\t]*", " ", line[i])
                             if not _state.conf.empty_strings and line[i].strip() == "":
                                 line[i] = None
+                linedata = [line[ix] for ix in data_indexes]
                 # Convert datetime, time, and Decimal values to strings.
                 for i in range(len(linedata)):
                     if type(linedata[i]) in (datetime.datetime, datetime.time, Decimal):

execsql/gui/base.py

@@ -3,53 +3,166 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     pass
 
-__all__ = ["GuiBackend", "compare_stats"]
+__all__ = ["DiffResult", "GuiBackend", "compare_stats", "compute_row_diffs"]
 
+DIFF_MARKER = "● "
 
-def compare_stats(
+
+# ---------------------------------------------------------------------------
+# Value comparison helpers
+# ---------------------------------------------------------------------------
+
+
+def _values_equal(a: Any, b: Any) -> bool:
+    """Compare two cell values using native equality.
+
+    Rules:
+    - ``None == None`` → True
+    - ``None`` vs any non-None → False  (NULL is distinct from empty string)
+    - Numeric types are compared numerically (``int(1) == float(1.0)``,
+      ``Decimal("10.00") == Decimal("10.0")``)
+    - All other types use ``==``
+    - Falls back to ``repr()`` comparison for exotic types that raise on ``==``
+    """
+    if a is None and b is None:
+        return True
+    if a is None or b is None:
+        return False
+    try:
+        return bool(a == b)
+    except (TypeError, ValueError):
+        return repr(a) == repr(b)
+
+
+def _pk_tuple(row: list | tuple, pk_indices: list[int]) -> tuple:
+    """Extract a PK value tuple from *row*.
+
+    PK values are stringified so that type differences in key columns
+    (e.g. ``int(1)`` vs ``str("1")``) still match rows correctly.
+    ``None`` is preserved as-is so distinct NULL-keyed rows are not
+    collapsed (in practice PK columns are NOT NULL).
+    """
+    return tuple(str(row[i]) if row[i] is not None else None for i in pk_indices)
+
+
+# ---------------------------------------------------------------------------
+# DiffResult and compute_row_diffs
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class DiffResult:
+    """Per-row diff state for compare dialogs.
+
+    Attributes:
+        table1_row_states: ``"match"`` / ``"changed"`` / ``"only_t1"`` per row.
+        table2_row_states: Same for table 2.
+        table1_changed_cols: For each table-1 row, set of column names that differ.
+        table2_changed_cols: Same for table 2.
+        summary: Human-readable summary string.
+    """
+
+    table1_row_states: list[str] = field(default_factory=list)
+    table2_row_states: list[str] = field(default_factory=list)
+    table1_changed_cols: list[set[str]] = field(default_factory=list)
+    table2_changed_cols: list[set[str]] = field(default_factory=list)
+    summary: str = ""
+
+
+def compute_row_diffs(
     headers1: list,
     rows1: list,
     headers2: list,
     rows2: list,
     keylist: list,
-) -> str:
-    """Return a one-line diff summary for compare dialogs.
+) -> DiffResult | None:
+    """Compare two tables row-by-row and return per-cell diff information.
+
+    Rows are matched by the key columns in *keylist*, not by position.
+    Columns are matched by header name, not index — headers may differ in
+    order or membership.  Only columns present in **both** headers (minus
+    key columns) are compared for cell-level diffs.
 
-    Computes matching rows, differing rows, and rows only in one table
-    based on the given key columns.  Returns an empty string when *keylist*
-    is empty (stats cannot be computed without keys).
+    Values are compared with native Python equality via :func:`_values_equal`:
+    numeric types compare numerically, ``None`` is distinct from ``""``,
+    and exotic types fall back to ``repr()``.
+
+    When duplicate PK values exist in a table, the **first** row with that
+    key is kept and later duplicates are ignored.
+
+    Returns ``None`` when *keylist* is empty or a key column is missing
+    from either header.
     """
     if not keylist:
-        return ""
-    key_idx1 = [i for i, h in enumerate(headers1) if str(h) in keylist]
-    key_idx2 = [i for i, h in enumerate(headers2) if str(h) in keylist]
+        return None
+    headers1_str = [str(h) for h in headers1]
+    headers2_str = [str(h) for h in headers2]
+    key_idx1 = [i for i, h in enumerate(headers1_str) if h in keylist]
+    key_idx2 = [i for i, h in enumerate(headers2_str) if h in keylist]
     if not key_idx1 or not key_idx2:
-        return ""
-
-    def _kv(row: list | tuple, idxs: list) -> tuple:
-        return tuple(str(row[i]) if row[i] is not None else "" for i in idxs)
+        return None
+
+    # Build PK -> row-index maps (first occurrence wins for duplicates).
+    pk_map1: dict[tuple, int] = {}
+    for i, r in enumerate(rows1):
+        k = _pk_tuple(r, key_idx1)
+        if k not in pk_map1:
+            pk_map1[k] = i
+    pk_map2: dict[tuple, int] = {}
+    for i, r in enumerate(rows2):
+        k = _pk_tuple(r, key_idx2)
+        if k not in pk_map2:
+            pk_map2[k] = i
+
+    keys1 = set(pk_map1)
+    keys2 = set(pk_map2)
+    common = keys1 & keys2
+
+    # Shared non-key columns eligible for cell comparison.
+    key_set = set(keylist)
+    h1_idx = {h: i for i, h in enumerate(headers1_str)}
+    h2_idx = {h: i for i, h in enumerate(headers2_str)}
+    shared_cols = [h for h in headers1_str if h in h2_idx and h not in key_set]
+
+    # Initialise result lists.
+    t1_states: list[str] = [""] * len(rows1)
+    t2_states: list[str] = [""] * len(rows2)
+    t1_changed: list[set[str]] = [set() for _ in rows1]
+    t2_changed: list[set[str]] = [set() for _ in rows2]
+
+    for k in keys1 - keys2:
+        t1_states[pk_map1[k]] = "only_t1"
+    for k in keys2 - keys1:
+        t2_states[pk_map2[k]] = "only_t2"
 
-    keys1 = {_kv(r, key_idx1) for r in rows1}
-    keys2 = {_kv(r, key_idx2) for r in rows2}
-    only1 = len(keys1 - keys2)
-    only2 = len(keys2 - keys1)
-    common_keys = keys1 & keys2
-    row_map1 = {_kv(r, key_idx1): r for r in rows1}
-    row_map2 = {_kv(r, key_idx2): r for r in rows2}
     matching = 0
     differing = 0
-    for k in common_keys:
-        r1 = [str(v) if v is not None else "" for v in row_map1[k]]
-        r2 = [str(v) if v is not None else "" for v in row_map2[k]]
-        if r1 == r2:
-            matching += 1
-        else:
+    for k in common:
+        i1 = pk_map1[k]
+        i2 = pk_map2[k]
+        changed: set[str] = set()
+        for col in shared_cols:
+            if not _values_equal(rows1[i1][h1_idx[col]], rows2[i2][h2_idx[col]]):
+                changed.add(col)
+        if changed:
+            t1_states[i1] = "changed"
+            t2_states[i2] = "changed"
+            t1_changed[i1] = changed
+            t2_changed[i2] = set(changed)
             differing += 1
+        else:
+            t1_states[i1] = "match"
+            t2_states[i2] = "match"
+            matching += 1
+
+    only1 = len(keys1 - keys2)
+    only2 = len(keys2 - keys1)
     parts: list[str] = []
     if matching:
         parts.append(f"{matching:,} matching")
@@ -59,7 +172,39 @@ def compare_stats(
         parts.append(f"{only1:,} only in Table 1")
     if only2:
         parts.append(f"{only2:,} only in Table 2")
-    return " | ".join(parts) if parts else "Tables are identical"
+    summary = " | ".join(parts) if parts else "Tables are identical"
+
+    return DiffResult(
+        table1_row_states=t1_states,
+        table2_row_states=t2_states,
+        table1_changed_cols=t1_changed,
+        table2_changed_cols=t2_changed,
+        summary=summary,
+    )
+
+
+# ---------------------------------------------------------------------------
+# compare_stats — delegates to compute_row_diffs for consistency
+# ---------------------------------------------------------------------------
+
+
+def compare_stats(
+    headers1: list,
+    rows1: list,
+    headers2: list,
+    rows2: list,
+    keylist: list,
+) -> str:
+    """Return a one-line diff summary for compare dialogs.
+
+    Delegates to :func:`compute_row_diffs` so that the summary and the
+    cell-level diff always agree.  Returns an empty string when *keylist*
+    is empty or key columns are missing.
+    """
+    result = compute_row_diffs(headers1, rows1, headers2, rows2, keylist)
+    if result is None:
+        return ""
+    return result.summary
 
 
 class GuiBackend(ABC):

execsql/gui/console.py

@@ -11,7 +11,7 @@ import sys
 import time
 from typing import Any
 
-from execsql.gui.base import GuiBackend, compare_stats as _compare_stats
+from execsql.gui.base import DIFF_MARKER, GuiBackend, compare_stats as _compare_stats, compute_row_diffs
 
 __all__ = ["ConsoleBackend"]
 
@@ -28,22 +28,47 @@ def _row_count_text(n: int) -> str:
     return f"{n:,} row{'s' if n != 1 else ''}"
 
 
-def _print_table(headers: list, rows: list, file: Any = None) -> None:
-    """Print a simple ASCII table to the given file (default stderr)."""
+def _print_table(
+    headers: list,
+    rows: list,
+    file: Any = None,
+    row_states: list[str] | None = None,
+    changed_cols: list[set[str]] | None = None,
+) -> None:
+    """Print a simple ASCII table to the given file (default stderr).
+
+    When *row_states* and *changed_cols* are provided, cells that differ in
+    ``"changed"`` rows are prefixed with the diff marker (``"● "``).
+    """
     if file is None:
         file = sys.stderr
     if not headers:
         return
-    col_widths = [len(str(h)) for h in headers]
-    for row in rows:
-        for i, cell in enumerate(row):
-            col_widths[i] = max(col_widths[i], len(str(cell) if cell is not None else ""))
+    headers_str = [str(h) for h in headers]
+
+    # Pre-compute display values so column widths account for markers.
+    display_rows: list[list[str]] = []
+    for ridx, row in enumerate(rows):
+        cells: list[str] = []
+        state = row_states[ridx] if row_states and ridx < len(row_states) else ""
+        diff_set = changed_cols[ridx] if changed_cols and ridx < len(changed_cols) else set()
+        for ci, cell in enumerate(row):
+            val = str(cell) if cell is not None else ""
+            if state == "changed" and ci < len(headers_str) and headers_str[ci] in diff_set:
+                val = f"{DIFF_MARKER}{val}"
+            cells.append(val)
+        display_rows.append(cells)
+
+    col_widths = [len(h) for h in headers_str]
+    for cells in display_rows:
+        for i, cell in enumerate(cells):
+            col_widths[i] = max(col_widths[i], len(cell))
     fmt = "  " + "  ".join(f"{{:<{w}}}" for w in col_widths)
     sep = "  " + "  ".join("-" * w for w in col_widths)
-    print(fmt.format(*[str(h) for h in headers]), file=file)
+    print(fmt.format(*headers_str), file=file)
     print(sep, file=file)
-    for row in rows:
-        print(fmt.format(*[str(c) if c is not None else "" for c in row]), file=file)
+    for cells in display_rows:
+        print(fmt.format(*cells), file=file)
 
 
 def _prompt_buttons(button_list: list) -> int | None:
@@ -256,19 +281,31 @@ class ConsoleBackend(GuiBackend):
         headers2 = args.get("headers2", [])
         rows2 = args.get("rows2", [])
         button_list = args.get("button_list", [("Continue", 1, "<Return>")])
+        keylist = [str(k) for k in args.get("keylist", [])]
+
+        diff = compute_row_diffs(headers1, rows1, headers2, rows2, keylist) if keylist else None
 
         print(f"\n=== {title} ===", file=sys.stderr)
         if message:
             print(message, file=sys.stderr)
         _print_help_url(args)
         print("\n--- Table 1 ---", file=sys.stderr)
-        _print_table(headers1, rows1)
+        _print_table(
+            headers1,
+            rows1,
+            row_states=diff.table1_row_states if diff else None,
+            changed_cols=diff.table1_changed_cols if diff else None,
+        )
         print(f"  {_row_count_text(len(rows1))}", file=sys.stderr)
         print("\n--- Table 2 ---", file=sys.stderr)
-        _print_table(headers2, rows2)
+        _print_table(
+            headers2,
+            rows2,
+            row_states=diff.table2_row_states if diff else None,
+            changed_cols=diff.table2_changed_cols if diff else None,
+        )
         print(f"  {_row_count_text(len(rows2))}", file=sys.stderr)
 
-        keylist = [str(k) for k in args.get("keylist", [])]
         summary = _compare_stats(headers1, rows1, headers2, rows2, keylist)
         if summary:
             print(f"\n  {summary}", file=sys.stderr)

execsql/gui/desktop.py

@@ -50,7 +50,7 @@ def _center_window(win: tk.Tk | tk.Toplevel, width: int = 600, height: int = 400
     win.geometry(f"{width}x{height}+{x}+{y}")
 
 
-from execsql.gui.base import compare_stats as _compare_stats
+from execsql.gui.base import DIFF_MARKER, compare_stats as _compare_stats, compute_row_diffs
 
 
 def _add_help_button(frame: tk.Frame, url: str | None) -> None:
@@ -516,6 +516,10 @@ class CompareDialog:
             key_idx1 = [i for i, h in enumerate(headers1_str) if h in keylist]
             key_idx2 = [i for i, h in enumerate(headers2_str) if h in keylist]
 
+            # Syncing uses display-consistent string normalization (treeview
+            # values are strings).  The diff engine in base.py has its own
+            # _pk_tuple with native-equality semantics — it does not share
+            # these maps.
             def _kv(row, idxs):
                 return tuple(str(row[i]) if row[i] is not None else "" for i in idxs)
 
@@ -523,8 +527,15 @@ class CompareDialog:
             iids2 = tree2.get_children()
             iid_to_kv1 = {iid: _kv(rows1[i], key_idx1) for i, iid in enumerate(iids1)}
             iid_to_kv2 = {iid: _kv(rows2[i], key_idx2) for i, iid in enumerate(iids2)}
-            kv_to_iid1 = {v: k for k, v in iid_to_kv1.items()}
-            kv_to_iid2 = {v: k for k, v in iid_to_kv2.items()}
+            # First occurrence wins for duplicate PKs (consistent with compute_row_diffs).
+            kv_to_iid1: dict[tuple, str] = {}
+            for iid, kv in iid_to_kv1.items():
+                if kv not in kv_to_iid1:
+                    kv_to_iid1[kv] = iid
+            kv_to_iid2: dict[tuple, str] = {}
+            for iid, kv in iid_to_kv2.items():
+                if kv not in kv_to_iid2:
+                    kv_to_iid2[kv] = iid
 
             def _on_click1(event):
                 sel_item = tree1.focus()
@@ -556,35 +567,66 @@ class CompareDialog:
             tree2.tag_configure("diff_changed", background="#f5d98e", foreground="#3a2e00")
             tree2.tag_configure("diff_only", background="#f5a3a3", foreground="#3a0a0a")
             _diff_on = [False]
+            _diff_result = compute_row_diffs(headers1, rows1, headers2, rows2, keylist)
+            _original_values1: dict[str, tuple] = {}
+            _original_values2: dict[str, tuple] = {}
+
+            def _apply_diffs(
+                tree: ttk.Treeview,
+                iids: tuple,
+                row_states: list[str],
+                changed_cols: list[set[str]],
+                headers_str: list[str],
+                originals: dict[str, tuple],
+                turn_on: bool,
+            ) -> None:
+                iids_list = list(iids)
+                if not turn_on:
+                    for iid in iids:
+                        tree.item(iid, tags=())
+                        if iid in originals:
+                            tree.item(iid, values=originals[iid])
+                    originals.clear()
+                    return
+                for iid in iids:
+                    ridx = iids_list.index(iid)
+                    state = row_states[ridx]
+                    if state == "only_t1" or state == "only_t2":
+                        tree.item(iid, tags=("diff_only",))
+                    elif state == "match":
+                        tree.item(iid, tags=("diff_match",))
+                    elif state == "changed":
+                        tree.item(iid, tags=("diff_changed",))
+                        originals[iid] = tree.item(iid, "values")
+                        vals = list(tree.item(iid, "values"))
+                        diff_set = changed_cols[ridx]
+                        for ci, col_name in enumerate(headers_str):
+                            if col_name in diff_set and ci < len(vals):
+                                vals[ci] = f"{DIFF_MARKER}{vals[ci]}"
+                        tree.item(iid, values=vals)
 
             def _toggle_diffs():
                 _diff_on[0] = not _diff_on[0]
-                if not _diff_on[0]:
-                    for iid in tree1.get_children():
-                        tree1.item(iid, tags=())
-                    for iid in tree2.get_children():
-                        tree2.item(iid, tags=())
+                if _diff_result is None:
                     return
-                keys1_set = set(iid_to_kv1.values())
-                keys2_set = set(iid_to_kv2.values())
-                for iid, kv in iid_to_kv1.items():
-                    if kv not in keys2_set:
-                        tree1.item(iid, tags=("diff_only",))
-                    else:
-                        r1 = [str(v) if v is not None else "" for v in rows1[list(iids1).index(iid)]]
-                        match_iid = kv_to_iid2.get(kv)
-                        if match_iid:
-                            r2 = [str(v) if v is not None else "" for v in rows2[list(iids2).index(match_iid)]]
-                            tree1.item(iid, tags=("diff_match",) if r1 == r2 else ("diff_changed",))
-                for iid, kv in iid_to_kv2.items():
-                    if kv not in keys1_set:
-                        tree2.item(iid, tags=("diff_only",))
-                    else:
-                        r2 = [str(v) if v is not None else "" for v in rows2[list(iids2).index(iid)]]
-                        match_iid = kv_to_iid1.get(kv)
-                        if match_iid:
-                            r1 = [str(v) if v is not None else "" for v in rows1[list(iids1).index(match_iid)]]
-                            tree2.item(iid, tags=("diff_match",) if r1 == r2 else ("diff_changed",))
+                _apply_diffs(
+                    tree1,
+                    iids1,
+                    _diff_result.table1_row_states,
+                    _diff_result.table1_changed_cols,
+                    [str(h) for h in headers1],
+                    _original_values1,
+                    _diff_on[0],
+                )
+                _apply_diffs(
+                    tree2,
+                    iids2,
+                    _diff_result.table2_row_states,
+                    _diff_result.table2_changed_cols,
+                    [str(h) for h in headers2],
+                    _original_values2,
+                    _diff_on[0],
+                )
 
             ttk.Button(diff_frame, text="Highlight Diffs", command=_toggle_diffs).pack(side=tk.LEFT)
             ttk.Label(diff_frame, text="  ").pack(side=tk.LEFT)