shortcutxl 0.2.13 → 0.2.14

package/xll/modules/shortcut_xl/_diff_highlight.py

@@ -1,37 +1,47 @@
-"""Diff highlighting — CF overlay + MessageBox approval for cell changes.
+"""Diff highlighting — CF overlay for cell changes + revert.
 
 After user code executes and dirty cells are collected, this module:
 1. Applies Conditional Formatting to highlight changed cells (yellow)
-2. Shows a Win32 MessageBox asking the user to keep or reject changes
-3. If rejected, reverts cells using oldValue/oldFormula from the diff
-4. Always cleans up CF rules before returning
+   — skipped for >10K changes (too many COM calls)
+2. Stores CF state and revert data keyed by transaction ID
 
-Performance: CF rules are applied via batched Union ranges (not per-cell),
-so highlighting cost is O(n / batch_size) COM calls, not O(n).
-Revert writes are per-cell but in-process COM (~microseconds each).
+TS only receives the cfTxId. Revert data stays server-side — no
+round-trip through HTTP/JSON. Cleanup and revert are triggered by
+the TS layer via /exec control requests referencing the cfTxId.
+
+Thread safety: all public functions are called via _run_on_main
+(serialized on the main thread) AND s_exec_lock in C serializes
+all /exec requests. No concurrent access to module-level state.
 """
 
-import ctypes
 from shortcut_xl._log import xl_log
 
-# Win32 MessageBox constants
-_MB_YESNO = 0x04
-_MB_ICONQUESTION = 0x20
-_IDYES = 6
-
 # Excel constants
 _XL_EXPRESSION = 2          # xlExpression CF type
 _XL_CALC_MANUAL = -4135     # xlCalculationManual
 _HIGHLIGHT_COLOR = 0x00FFFF  # yellow (BGR for #FFFF00) — Track Changes color
 
-# Max length for comma-separated address string (Excel limit)
-_ADDR_BATCH_LIMIT = 255
+# Thresholds
+_MAX_HIGHLIGHT_CELLS = 10_000  # skip CF above this — too many COM calls
+_ADDR_BATCH_LIMIT = 255        # max chars per comma-separated address string
+
+# ---------------------------------------------------------------------------
+# Transaction state keyed by ID
+# ---------------------------------------------------------------------------
+_next_tx_id = 0
+_cf_state = {}      # tx_id -> list of CF rule COM objects
+_revert_data = {}   # tx_id -> list of dirty cell dicts (for server-side revert)
+_MAX_TX_ENTRIES = 10  # evict oldest if exceeded (guards against leaked entries)
 
 
 def _group_by_sheet(dirty_cells):
     by_sheet = {}
     for cell in dirty_cells:
-        by_sheet.setdefault(cell['sheet'], []).append(cell)
+        sheet = cell.get('sheet', '')
+        if not sheet:
+            xl_log(f"_group_by_sheet: skipping cell with no sheet key: {cell}")
+            continue
+        by_sheet.setdefault(sheet, []).append(cell)
     return by_sheet
 
 
@@ -58,40 +68,116 @@ def _build_union(app, ws, addresses):
 
 
 def _apply_cf_highlights(app, by_sheet):
-    """Apply CF rules to highlight changed cells. Returns cleanup state."""
-    cf_state = []  # (ws, count_before) for cleanup
+    """Apply CF rules to highlight changed cells. Returns list of rule COM objects."""
+    rules = []
     for sheet_name, cells in by_sheet.items():
         try:
             ws = app.Worksheets(sheet_name)
-            count_before = ws.Cells.FormatConditions.Count
             rng = _build_union(app, ws, [c['address'] for c in cells])
             if rng is None:
                 continue
-            rng.FormatConditions.Add(Type=_XL_EXPRESSION, Formula1="=TRUE")
-            rule = rng.FormatConditions(rng.FormatConditions.Count)
+            rule = rng.FormatConditions.Add(_XL_EXPRESSION, None, "=TRUE")
             rule.Interior.Color = _HIGHLIGHT_COLOR
-            cf_state.append((ws, count_before))
+            rules.append(rule)
         except Exception as e:
             xl_log(f"_apply_cf_highlights({sheet_name}): {e}")
-    return cf_state
+    return rules
+
+
+def highlight_changes(app, dirty_cells):
+    """Apply CF highlights and store revert data for changed cells.
+
+    Returns a transaction ID (string) for later cleanup/revert via
+    cleanup_by_tx or revert_by_tx. Always returns an ID, even if no
+    highlights were applied (for consistent API).
+    """
+    global _next_tx_id
+
+    tx_id = str(_next_tx_id)
+    _next_tx_id += 1
+
+    # Evict oldest entries if we're at the cap (guards against leaks
+    # when TS crashes before sending cleanup).
+    while len(_revert_data) >= _MAX_TX_ENTRIES:
+        oldest = next(iter(_revert_data))
+        _evict_tx(oldest)
+
+    # Always store revert data (even above highlight threshold).
+    if dirty_cells:
+        _revert_data[tx_id] = dirty_cells
+
+    if not dirty_cells or len(dirty_cells) > _MAX_HIGHLIGHT_CELLS:
+        return tx_id
+
+    by_sheet = _group_by_sheet(dirty_cells)
+    try:
+        app.ScreenUpdating = False
+    except Exception:
+        pass
 
+    rules = _apply_cf_highlights(app, by_sheet)
+    if rules:
+        _cf_state[tx_id] = rules
 
-def _cleanup_cf(cf_state):
-    """Remove CF rules added during highlighting."""
-    for ws, count_before in cf_state:
+    try:
+        app.ScreenUpdating = True
+    except Exception:
+        pass
+
+    return tx_id
+
+
+def _evict_tx(tx_id):
+    """Remove all state for a transaction (CF rules + revert data)."""
+    _revert_data.pop(tx_id, None)
+    for rule in _cf_state.pop(tx_id, []):
+        try:
+            rule.Delete()
+        except Exception:
+            pass
+
+
+def cleanup_by_tx(tx_id):
+    """Remove CF highlights and revert data for a transaction.
+
+    Called after user accepts changes — clears both CF rules and stored
+    revert data so the transaction is fully released.
+    """
+    tx_id = str(tx_id)
+    _revert_data.pop(tx_id, None)
+    rules = _cf_state.pop(tx_id, [])
+    for rule in rules:
         try:
-            while ws.Cells.FormatConditions.Count > count_before:
-                ws.Cells.FormatConditions(
-                    ws.Cells.FormatConditions.Count).Delete()
+            rule.Delete()
         except Exception as e:
-            xl_log(f"_cleanup_cf: {e}")
+            xl_log(f"cleanup_by_tx: {e}")
+
 
+def revert_by_tx(app, tx_id):
+    """Look up revert data by transaction ID and restore old values.
 
-def _revert_cells(app, dirty_cells):
+    Returns the number of cells reverted (0 if tx_id not found).
+    Also cleans up CF highlights for the transaction.
+    """
+    tx_id = str(tx_id)
+    dirty_cells = _revert_data.pop(tx_id, None)
+    # Clean up CF highlights too.
+    for rule in _cf_state.pop(tx_id, []):
+        try:
+            rule.Delete()
+        except Exception as e:
+            xl_log(f"revert_by_tx cleanup: {e}")
+    if not dirty_cells:
+        return 0
+    revert_cells(app, dirty_cells)
+    return len(dirty_cells)
+
+
+def revert_cells(app, dirty_cells):
     """Restore old values/formulas for rejected changes.
 
-    Sets calc to manual during revert to avoid expensive recalcs
-    on each write, then restores original calc mode.
+    Takes an explicit list of dirty cell dicts. All data stays in-process
+    (no JSON round-trip) — datetime and other Python types are preserved.
     """
     orig_calc = None
     try:
@@ -99,78 +185,34 @@ def _revert_cells(app, dirty_cells):
         app.Calculation = _XL_CALC_MANUAL
     except Exception:
         pass
+    try:
+        app.ScreenUpdating = False
+    except Exception:
+        pass
 
     for sheet_name, cells in _group_by_sheet(dirty_cells).items():
         try:
             ws = app.Worksheets(sheet_name)
             for cell in cells:
-                addr = cell['address']
+                addr = cell.get('address', '')
+                if not addr:
+                    continue
                 try:
                     if cell.get('oldFormula'):
                         ws.Range(addr).Formula = cell['oldFormula']
                     else:
-                        ws.Range(addr).Value = cell['oldValue']
+                        ws.Range(addr).Value = cell.get('oldValue')
                 except Exception as e:
-                    xl_log(f"_revert_cells({sheet_name}!{addr}): {e}")
+                    xl_log(f"revert_cells({sheet_name}!{addr}): {e}")
         except Exception as e:
-            xl_log(f"_revert_cells({sheet_name}): {e}")
+            xl_log(f"revert_cells({sheet_name}): {e}")
 
+    try:
+        app.ScreenUpdating = True
+    except Exception:
+        pass
     if orig_calc is not None:
         try:
             app.Calculation = orig_calc
         except Exception:
             pass
-
-
-def highlight_and_approve(app, dirty_cells):
-    """Highlight changed cells with CF, show MessageBox for user approval.
-
-    Returns True if approved, False if rejected.
-    On rejection, all changes are reverted from dirty_cells' oldValue/oldFormula.
-    CF highlighting is always cleaned up before returning.
-    """
-    if not dirty_cells:
-        return True
-
-    by_sheet = _group_by_sheet(dirty_cells)
-    cf_state = []
-
-    try:
-        # Phase 1: Apply CF highlights
-        try:
-            app.ScreenUpdating = False
-        except Exception:
-            pass
-        cf_state = _apply_cf_highlights(app, by_sheet)
-        try:
-            app.ScreenUpdating = True
-        except Exception:
-            pass
-
-        # Phase 2: MessageBox — modal to Excel window
-        n = len(dirty_cells)
-        sheet_summary = ", ".join(
-            f"{name} ({len(cells)})" for name, cells in by_sheet.items()
-        )
-        msg = (
-            f"ShortcutXL modified {n} cell{'s' if n != 1 else ''} "
-            f"in: {sheet_summary}\n\n"
-            "Keep these changes?"
-        )
-        try:
-            hwnd = app.Hwnd
-        except Exception:
-            hwnd = 0
-        result = ctypes.windll.user32.MessageBoxW(
-            hwnd, msg, "ShortcutXL", _MB_YESNO | _MB_ICONQUESTION
-        )
-        approved = (result == _IDYES)
-
-        # Phase 3: Revert if rejected
-        if not approved:
-            _revert_cells(app, dirty_cells)
-
-        return approved
-
-    finally:
-        _cleanup_cf(cf_state)

package/xll/modules/shortcut_xl/_exec_entry.py

@@ -0,0 +1,150 @@
+"""HTTP entry point — bridges run_managed to the C HTTP server.
+
+C calls dispatch_request(body_json) for every POST /exec.
+This function always returns valid JSON. C never builds JSON.
+
+Two request shapes:
+  {"code": "..."}                              → exec_and_respond
+  {"control": "revert"|"cleanup", "cfTxId": "..."} → handle_control
+"""
+
+import io
+import json
+import sys
+import traceback
+
+from shortcut_xl._com import xl_app
+from shortcut_xl._diff_highlight import (
+    cleanup_by_tx,
+    highlight_changes,
+    revert_by_tx,
+)
+from shortcut_xl._managed import run_managed
+from shortcut_xl._threading import _run_on_main
+
+_MAX_DIFF_JSON_CELLS = 50
+
+
+def _format_cell_value(v):
+    """Format a cell value for JSON display."""
+    if v is None:
+        return 'empty'
+    if isinstance(v, bool):
+        return 'TRUE' if v else 'FALSE'
+    if isinstance(v, int):
+        return v
+    if isinstance(v, str):
+        return 'empty' if len(v) == 0 else v
+    try:
+        f = float(v)
+        # Only convert to int if it fits exactly (avoids precision loss for large floats)
+        i = int(f)
+        return i if f == i and abs(i) <= 2**53 else round(f, 10)
+    except (TypeError, ValueError):
+        return str(v)
+
+
+def _build_diff(dirty):
+    """Build structured diff payload for the TUI approval component."""
+    cells = []
+    for c in dirty[:_MAX_DIFF_JSON_CELLS]:
+        entry = {
+            'cell': f"{c.get('sheet', '')}!{c.get('address', '')}",
+            'before': _format_cell_value(c.get('oldValue')),
+            'after': _format_cell_value(c.get('value')),
+        }
+        if c.get('oldFormula'):
+            entry['oldFormula'] = c['oldFormula']
+        if c.get('formula'):
+            entry['formula'] = c['formula']
+        cells.append(entry)
+    return {'cells': cells, 'total': len(dirty)}
+
+
+def _json_default(obj):
+    """Fallback serializer for non-standard types (e.g. datetime)."""
+    try:
+        return str(obj)
+    except Exception:
+        return None
+
+
+def dispatch_request(body_json):
+    """Single entry point for all POST /exec requests.
+
+    Always returns a valid JSON string. C sends this verbatim.
+    """
+    try:
+        request = json.loads(body_json)
+    except (json.JSONDecodeError, TypeError) as e:
+        return json.dumps({'ok': False, 'error': f'Invalid JSON: {e}'})
+
+    if not isinstance(request, dict):
+        return json.dumps({'ok': False, 'error': "request must be a JSON object"})
+
+    if 'code' in request:
+        return exec_and_respond(request['code'], auto_approve=request.get('autoApprove', False))
+    elif 'control' in request:
+        return handle_control(request)
+    else:
+        return json.dumps({'ok': False, 'error': "request must have 'code' or 'control' field"})
+
+
+def exec_and_respond(code_str, *, auto_approve=False):
+    """Execute user code and return a JSON response string.
+
+    Owns stdout/stderr capture, calls run_managed (pure), applies
+    CF highlights, and builds the response. Always returns valid JSON.
+
+    When *auto_approve* is True the caller already knows it will accept
+    any changes, so we skip CF highlighting entirely — no highlights to
+    apply, no revert data to store, no cleanup round-trip needed.
+    """
+    sio = io.StringIO()
+    old_out, old_err = sys.stdout, sys.stderr
+    sys.stdout = sys.stderr = sio
+    try:
+        dirty = run_managed(code_str)
+        output = sio.getvalue()
+
+        response = {'ok': True, 'output': output}
+        if dirty:
+            response['diff'] = _build_diff(dirty)
+            # Only apply CF highlights when the user may review changes.
+            if not auto_approve:
+                cf_tx_id = _run_on_main(lambda: highlight_changes(xl_app(), dirty))
+                response['cfTxId'] = cf_tx_id
+        return json.dumps(response, default=_json_default)
+    except Exception:
+        output = sio.getvalue()
+        tb = traceback.format_exc()
+        error = f"{output}{tb}" if output else tb
+        return json.dumps({'ok': False, 'error': error})
+    finally:
+        sys.stdout, sys.stderr = old_out, old_err
+
+
+def handle_control(request):
+    """Handle control commands (revert, cleanup).
+
+    Revert data is stored server-side in _diff_highlight, keyed by cfTxId.
+    TS only sends the ID — no dirty cell data round-trips through HTTP.
+    """
+    action = request.get('control')
+    cf_tx_id = request.get('cfTxId')
+
+    if action == 'revert':
+        if cf_tx_id is None:
+            return json.dumps({'ok': False, 'error': "revert requires cfTxId"})
+        n = _run_on_main(lambda: revert_by_tx(xl_app(), cf_tx_id))
+        if n == 0:
+            return json.dumps({'ok': True, 'output': 'Nothing to revert (transaction not found or already cleaned up).'})
+        return json.dumps({'ok': True, 'output': f"Reverted {n} cell{'s' if n != 1 else ''}."})
+
+    elif action == 'cleanup':
+        if cf_tx_id is not None:
+            _run_on_main(lambda: cleanup_by_tx(cf_tx_id))
+        return json.dumps({'ok': True, 'output': ''})
+
+    else:
+        return json.dumps({'ok': False, 'error': f"Unknown control action: {action}"})

package/xll/modules/shortcut_xl/_log.py

@@ -1,7 +1,7 @@
 """Logging utility for ShortcutXL."""
 
-import os
 import datetime
+import os
 
 
 def xl_log(msg):

package/xll/modules/shortcut_xl/_managed.py

@@ -1,17 +1,21 @@
 """run_managed — execute user code against Excel on the main thread."""
 
 import threading
-from shortcut_xl._log import xl_log
+
 from shortcut_xl._com import xl_app
+from shortcut_xl._log import xl_log
 from shortcut_xl._threading import _run_on_main
 from shortcut_xl._tracking import DirtyTracker, TrackedApp
 from shortcut_xl.api.format import format_cell_diff
-from shortcut_xl.api.workbook import Workbook
-from shortcut_xl.api.worksheet import Worksheet
 from shortcut_xl.api.utils.helpers import (
+    address_to_index,
+    col_index,
+    col_letter,
     hex_to_bgr,
-    index_to_address, address_to_index, col_letter, col_index,
+    index_to_address,
 )
+from shortcut_xl.api.workbook import Workbook
+from shortcut_xl.api.worksheet import Worksheet
 
 # Excel xlCalculation enum values
 _XL_CALCULATION_MANUAL = -4135
@@ -34,10 +38,11 @@ def run_managed(code_str):
 
     Suspends Calculation/Events/Alerts, runs code with a tracking
     proxy, collects dirty cells, then restores Excel state.
-    Prints a human-readable diff summary of any cell changes to stdout.
 
-    All COM calls happen directly on the main thread — no cross-apartment
-    marshaling overhead.
+    Returns the list of dirty cells. Caller (_exec_entry) is responsible
+    for CF highlighting, JSON response building, and stdout capture.
+    The text summary is still printed to stdout (captured by _exec_entry)
+    so the agent sees what changed in the tool result output.
     """
     code = compile(code_str, '<excel_exec>', 'exec')
 
@@ -92,13 +97,14 @@ def run_managed(code_str):
                     app.Calculation = _XL_CALCULATION_AUTOMATIC
             except Exception:
                 pass
+
         return dirty
 
     try:
         dirty = _run_on_main(_work)
     except Exception as e:
-        # Strip infrastructure traceback so PyRun_SimpleString only prints
-        # the exception type + message, not the full frame chain.
+        # Strip infrastructure traceback — _exec_entry formats it via
+        # traceback.format_exc(), so the extra frames are noise.
         e.__traceback__ = None
         raise
     summary = format_cell_diff(dirty)

package/xll/modules/shortcut_xl/_threading.py

@@ -6,12 +6,12 @@ This matches the pattern used by PyXLL, Excel-DNA, and xlwings:
 background threads enqueue work, a timer on the main thread drains it.
 """
 
-import threading
-import queue
 import ctypes
+import queue
+import threading
 from ctypes import wintypes
-from shortcut_xl._log import xl_log
 
+from shortcut_xl._log import xl_log
 
 # COM error code: Excel is busy (VBA_E_IGNORE = 0x800AC472)
 _VBA_E_IGNORE = -2146777998
@@ -44,6 +44,7 @@ def _is_excel_busy(e):
 
 # Persist main thread ID in builtins so it survives hot-reload.
 import builtins
+
 _main_thread_id = getattr(builtins, '_shortcutxl_main_thread_id', None)
 if _main_thread_id is None:
     _main_thread_id = threading.current_thread().ident

package/xll/modules/shortcut_xl/_tracking.py

@@ -7,9 +7,9 @@ sheets) not O(workbook), and the agent gets raw COM objects for everything
 below the Worksheet level — zero overhead on Range reads/writes.
 """
 
-from shortcut_xl.api.utils.helpers import col_letter
-from shortcut_xl.api.utils.com_utils import normalize_2d
 from shortcut_xl._log import xl_log
+from shortcut_xl.api.utils.com_utils import normalize_2d
+from shortcut_xl.api.utils.helpers import col_letter
 
 
 class _SheetSnapshot:
@@ -69,6 +69,9 @@ def _diff_snapshots(sheet_name, before, after):
                     formula = _get_formula(after, ri, ci)
                     if formula:
                         entry['formula'] = formula
+                    old_formula = _get_formula(before, ri, ci)
+                    if old_formula:
+                        entry['oldFormula'] = old_formula
                     changes.append(entry)
         return changes
 
@@ -105,6 +108,9 @@ def _diff_snapshots(sheet_name, before, after):
                 formula = _get_formula_abs(after, row, col)
                 if formula:
                     entry['formula'] = formula
+                old_formula = _get_formula_abs(before, row, col)
+                if old_formula:
+                    entry['oldFormula'] = old_formula
                 changes.append(entry)
     return changes
 

package/xll/modules/shortcut_xl/api/__init__.py

@@ -1,4 +1,4 @@
 """API layer — Workbook/Worksheet wrappers and supporting modules."""
 
-from shortcut_xl.api.workbook import Workbook
-from shortcut_xl.api.worksheet import Worksheet
+from shortcut_xl.api.workbook import Workbook as Workbook
+from shortcut_xl.api.worksheet import Worksheet as Worksheet

package/xll/modules/shortcut_xl/api/format.py

@@ -5,14 +5,19 @@ with Good / Issues breakdown and row sampling.
 """
 
 import re
-from shortcut_xl.api.utils.numerical import NUMERIC_TYPES
+
 from shortcut_xl.api.categorize import (
+    GOOD,
+    HARDCODED_NUMBER,
+    HARDCODED_NUMBER_IN_FORMULA,
+    INVALID_FORMULA,
+    LARGE_PERCENTAGE,
     categorize_cells,
-    GOOD, LARGE_PERCENTAGE, HARDCODED_NUMBER,
-    HARDCODED_NUMBER_IN_FORMULA, INVALID_FORMULA,
-    is_com_error, com_error_to_str,
+    com_error_to_str,
+    is_com_error,
 )
 from shortcut_xl.api.utils.helpers import col_letter
+from shortcut_xl.api.utils.numerical import NUMERIC_TYPES
 
 # Display constants
 MAX_ROWS_TO_SHOW = 10
@@ -165,7 +170,7 @@ def _format_row(sheet, row, entries, indent=2, show_issues=True, show_count=True
     lines.append(f'{ind}{sheet}!Row {row + 1} ({col_range}){count_suffix}')
 
     samples = _sample_first_last(entries, MAX_SAMPLES_PER_ROW)
-    for cell, issues, _col in samples:
+    for cell, _issues, _col in samples:
         fmt = cell.get('numberFormat')
         old_str = format_value(cell.get('oldValue'), fmt)
         new_str = _display_value(cell)

package/xll/modules/shortcut_xl/api/range_formatter.py

@@ -9,11 +9,11 @@ Replicates the TypeScript processCellRange / RangeFormatter logic:
 
 from __future__ import annotations
 
-from shortcut_xl.api.utils.helpers import index_to_address
 from shortcut_xl.api.style import read_style
-from shortcut_xl.api.utils.numerical import is_numerical, NUMERIC_TYPES
+from shortcut_xl.api.utils.helpers import index_to_address
+from shortcut_xl.api.utils.numerical import NUMERIC_TYPES, is_numerical
 from shortcut_xl.api.utils.ranges import consolidate_ranges
-from shortcut_xl.api.utils.style_utils import get_style_key, get_style_description
+from shortcut_xl.api.utils.style_utils import get_style_description, get_style_key
 
 # ---------------------------------------------------------------------------
 # Constants (match TypeScript range-processing/types.ts)
@@ -349,8 +349,8 @@ def format_cell_range(
     5. Context from cells to the left
     6. Context from cells above
     """
+    from shortcut_xl.api.utils.com_utils import make_range, normalize_2d
     from shortcut_xl.api.utils.helpers import parse_range
-    from shortcut_xl.api.utils.com_utils import normalize_2d, make_range
 
     sr, sc, er, ec = parse_range(range_addr)
     rows = er - sr + 1

package/xll/modules/shortcut_xl/api/workbook.py

@@ -5,17 +5,14 @@ Output formats match the SpreadJS equivalents.
 
 from __future__ import annotations
 
-from typing import Any
-
+from shortcut_xl.api.categorize import com_error_to_str, is_com_error
+from shortcut_xl.api.named_ranges import get_named_range_info
 from shortcut_xl.api.utils.helpers import (
-    col_letter,
+    extract_sheet_and_range,
     index_to_address,
     is_error_value,
     parse_range,
-    extract_sheet_and_range,
 )
-from shortcut_xl.api.categorize import is_com_error, com_error_to_str
-from shortcut_xl.api.named_ranges import get_named_range_info
 from shortcut_xl.api.worksheet import Worksheet
 
 # Error check constants (match SpreadJS error-checker.ts)
@@ -263,8 +260,6 @@ class Workbook:
                 rng.Value = com_error_to_str(data)
             return
         # Multi-cell range
-        rows = rng.Rows.Count
-        cols = rng.Columns.Count
         for r_idx, row in enumerate(data):
             if not isinstance(row, tuple):
                 row = (row,)