shortcutxl 0.2.13 → 0.2.15

package/xll/modules/shortcut_xl/__init__.py

@@ -21,19 +21,34 @@ hex_to_bgr              — convert "#RRGGBB" to BGR int for COM
 index_to_address, address_to_index, parse_range, range_to_excel, col_letter, col_index
 """
 
-from shortcut_xl._log import xl_log
-from shortcut_xl._registry import xl_func, _registry
-from shortcut_xl._com import xl_app
-from shortcut_xl._managed import run_managed, xl_batch, schedule_call
-from shortcut_xl.api.format import format_cell_diff
-from shortcut_xl.api.workbook import Workbook
-from shortcut_xl.api.worksheet import Worksheet
+from shortcut_xl._com import xl_app as xl_app
+from shortcut_xl._log import xl_log as xl_log
+from shortcut_xl._managed import run_managed as run_managed
+from shortcut_xl._managed import schedule_call as schedule_call
+from shortcut_xl._managed import xl_batch as xl_batch
+from shortcut_xl._registry import _registry as _registry
+from shortcut_xl._registry import xl_func as xl_func
+from shortcut_xl.api.format import format_cell_diff as format_cell_diff
+from shortcut_xl.api.utils.helpers import (  # noqa: I001
+    address_to_index as address_to_index,
+)
+from shortcut_xl.api.utils.helpers import (
+    col_index as col_index,
+)
+from shortcut_xl.api.utils.helpers import (
+    col_letter as col_letter,
+)
+from shortcut_xl.api.utils.helpers import (
+    hex_to_bgr as hex_to_bgr,
+)
+from shortcut_xl.api.utils.helpers import (
+    index_to_address as index_to_address,
+)
+from shortcut_xl.api.utils.helpers import (
+    parse_range as parse_range,
+)
 from shortcut_xl.api.utils.helpers import (
-    hex_to_bgr,
-    index_to_address,
-    address_to_index,
-    parse_range,
-    range_to_excel,
-    col_letter,
-    col_index,
+    range_to_excel as range_to_excel,
 )
+from shortcut_xl.api.workbook import Workbook as Workbook
+from shortcut_xl.api.worksheet import Worksheet as Worksheet

package/xll/modules/shortcut_xl/_com.py

@@ -2,6 +2,7 @@
 
 import ctypes
 from ctypes import wintypes
+
 from shortcut_xl._log import xl_log
 
 # AccessibleObjectFromWindow native object ID

package/xll/modules/shortcut_xl/_diff_highlight.py

@@ -1,37 +1,47 @@
-"""Diff highlighting — CF overlay + MessageBox approval for cell changes.
+"""Diff highlighting — CF overlay for cell changes + revert.
 
 After user code executes and dirty cells are collected, this module:
 1. Applies Conditional Formatting to highlight changed cells (yellow)
-2. Shows a Win32 MessageBox asking the user to keep or reject changes
-3. If rejected, reverts cells using oldValue/oldFormula from the diff
-4. Always cleans up CF rules before returning
+   — skipped for >10K changes (too many COM calls)
+2. Stores CF state and revert data keyed by transaction ID
 
-Performance: CF rules are applied via batched Union ranges (not per-cell),
-so highlighting cost is O(n / batch_size) COM calls, not O(n).
-Revert writes are per-cell but in-process COM (~microseconds each).
+TS only receives the cfTxId. Revert data stays server-side — no
+round-trip through HTTP/JSON. Cleanup and revert are triggered by
+the TS layer via /exec control requests referencing the cfTxId.
+
+Thread safety: all public functions are called via _run_on_main
+(serialized on the main thread) AND s_exec_lock in C serializes
+all /exec requests. No concurrent access to module-level state.
 """
 
-import ctypes
 from shortcut_xl._log import xl_log
 
-# Win32 MessageBox constants
-_MB_YESNO = 0x04
-_MB_ICONQUESTION = 0x20
-_IDYES = 6
-
 # Excel constants
 _XL_EXPRESSION = 2          # xlExpression CF type
 _XL_CALC_MANUAL = -4135     # xlCalculationManual
 _HIGHLIGHT_COLOR = 0x00FFFF  # yellow (BGR for #FFFF00) — Track Changes color
 
-# Max length for comma-separated address string (Excel limit)
-_ADDR_BATCH_LIMIT = 255
+# Thresholds
+_MAX_HIGHLIGHT_CELLS = 10_000  # skip CF above this — too many COM calls
+_ADDR_BATCH_LIMIT = 255        # max chars per comma-separated address string
+
+# ---------------------------------------------------------------------------
+# Transaction state keyed by ID
+# ---------------------------------------------------------------------------
+_next_tx_id = 0
+_cf_state = {}      # tx_id -> list of CF rule COM objects
+_revert_data = {}   # tx_id -> list of dirty cell dicts (for server-side revert)
+_MAX_TX_ENTRIES = 10  # evict oldest if exceeded (guards against leaked entries)
 
 
 def _group_by_sheet(dirty_cells):
     by_sheet = {}
     for cell in dirty_cells:
-        by_sheet.setdefault(cell['sheet'], []).append(cell)
+        sheet = cell.get('sheet', '')
+        if not sheet:
+            xl_log(f"_group_by_sheet: skipping cell with no sheet key: {cell}")
+            continue
+        by_sheet.setdefault(sheet, []).append(cell)
     return by_sheet
 
 
@@ -58,40 +68,116 @@ def _build_union(app, ws, addresses):
 
 
 def _apply_cf_highlights(app, by_sheet):
-    """Apply CF rules to highlight changed cells. Returns cleanup state."""
-    cf_state = []  # (ws, count_before) for cleanup
+    """Apply CF rules to highlight changed cells. Returns list of rule COM objects."""
+    rules = []
     for sheet_name, cells in by_sheet.items():
         try:
             ws = app.Worksheets(sheet_name)
-            count_before = ws.Cells.FormatConditions.Count
             rng = _build_union(app, ws, [c['address'] for c in cells])
             if rng is None:
                 continue
-            rng.FormatConditions.Add(Type=_XL_EXPRESSION, Formula1="=TRUE")
-            rule = rng.FormatConditions(rng.FormatConditions.Count)
+            rule = rng.FormatConditions.Add(_XL_EXPRESSION, None, "=TRUE")
             rule.Interior.Color = _HIGHLIGHT_COLOR
-            cf_state.append((ws, count_before))
+            rules.append(rule)
         except Exception as e:
             xl_log(f"_apply_cf_highlights({sheet_name}): {e}")
-    return cf_state
+    return rules
+
+
+def highlight_changes(app, dirty_cells):
+    """Apply CF highlights and store revert data for changed cells.
+
+    Returns a transaction ID (string) for later cleanup/revert via
+    cleanup_by_tx or revert_by_tx. Always returns an ID, even if no
+    highlights were applied (for consistent API).
+    """
+    global _next_tx_id
+
+    tx_id = str(_next_tx_id)
+    _next_tx_id += 1
+
+    # Evict oldest entries if we're at the cap (guards against leaks
+    # when TS crashes before sending cleanup).
+    while len(_revert_data) >= _MAX_TX_ENTRIES:
+        oldest = next(iter(_revert_data))
+        _evict_tx(oldest)
+
+    # Always store revert data (even above highlight threshold).
+    if dirty_cells:
+        _revert_data[tx_id] = dirty_cells
+
+    if not dirty_cells or len(dirty_cells) > _MAX_HIGHLIGHT_CELLS:
+        return tx_id
+
+    by_sheet = _group_by_sheet(dirty_cells)
+    try:
+        app.ScreenUpdating = False
+    except Exception:
+        pass
 
+    rules = _apply_cf_highlights(app, by_sheet)
+    if rules:
+        _cf_state[tx_id] = rules
 
-def _cleanup_cf(cf_state):
-    """Remove CF rules added during highlighting."""
-    for ws, count_before in cf_state:
+    try:
+        app.ScreenUpdating = True
+    except Exception:
+        pass
+
+    return tx_id
+
+
+def _evict_tx(tx_id):
+    """Remove all state for a transaction (CF rules + revert data)."""
+    _revert_data.pop(tx_id, None)
+    for rule in _cf_state.pop(tx_id, []):
+        try:
+            rule.Delete()
+        except Exception:
+            pass
+
+
+def cleanup_by_tx(tx_id):
+    """Remove CF highlights and revert data for a transaction.
+
+    Called after user accepts changes — clears both CF rules and stored
+    revert data so the transaction is fully released.
+    """
+    tx_id = str(tx_id)
+    _revert_data.pop(tx_id, None)
+    rules = _cf_state.pop(tx_id, [])
+    for rule in rules:
         try:
-            while ws.Cells.FormatConditions.Count > count_before:
-                ws.Cells.FormatConditions(
-                    ws.Cells.FormatConditions.Count).Delete()
+            rule.Delete()
         except Exception as e:
-            xl_log(f"_cleanup_cf: {e}")
+            xl_log(f"cleanup_by_tx: {e}")
+
 
+def revert_by_tx(app, tx_id):
+    """Look up revert data by transaction ID and restore old values.
 
-def _revert_cells(app, dirty_cells):
+    Returns the number of cells reverted (0 if tx_id not found).
+    Also cleans up CF highlights for the transaction.
+    """
+    tx_id = str(tx_id)
+    dirty_cells = _revert_data.pop(tx_id, None)
+    # Clean up CF highlights too.
+    for rule in _cf_state.pop(tx_id, []):
+        try:
+            rule.Delete()
+        except Exception as e:
+            xl_log(f"revert_by_tx cleanup: {e}")
+    if not dirty_cells:
+        return 0
+    revert_cells(app, dirty_cells)
+    return len(dirty_cells)
+
+
+def revert_cells(app, dirty_cells):
     """Restore old values/formulas for rejected changes.
 
-    Sets calc to manual during revert to avoid expensive recalcs
-    on each write, then restores original calc mode.
+    Takes an explicit list of dirty cell dicts. All data stays in-process
+    (no JSON round-trip) — datetime and other Python types are preserved.
     """
     orig_calc = None
     try:
@@ -99,78 +185,34 @@ def _revert_cells(app, dirty_cells):
         app.Calculation = _XL_CALC_MANUAL
     except Exception:
         pass
+    try:
+        app.ScreenUpdating = False
+    except Exception:
+        pass
 
     for sheet_name, cells in _group_by_sheet(dirty_cells).items():
         try:
             ws = app.Worksheets(sheet_name)
             for cell in cells:
-                addr = cell['address']
+                addr = cell.get('address', '')
+                if not addr:
+                    continue
                 try:
                     if cell.get('oldFormula'):
                         ws.Range(addr).Formula = cell['oldFormula']
                     else:
-                        ws.Range(addr).Value = cell['oldValue']
+                        ws.Range(addr).Value = cell.get('oldValue')
                 except Exception as e:
-                    xl_log(f"_revert_cells({sheet_name}!{addr}): {e}")
+                    xl_log(f"revert_cells({sheet_name}!{addr}): {e}")
         except Exception as e:
-            xl_log(f"_revert_cells({sheet_name}): {e}")
+            xl_log(f"revert_cells({sheet_name}): {e}")
 
+    try:
+        app.ScreenUpdating = True
+    except Exception:
+        pass
     if orig_calc is not None:
         try:
             app.Calculation = orig_calc
         except Exception:
             pass
-
-
-def highlight_and_approve(app, dirty_cells):
-    """Highlight changed cells with CF, show MessageBox for user approval.
-
-    Returns True if approved, False if rejected.
-    On rejection, all changes are reverted from dirty_cells' oldValue/oldFormula.
-    CF highlighting is always cleaned up before returning.
-    """
-    if not dirty_cells:
-        return True
-
-    by_sheet = _group_by_sheet(dirty_cells)
-    cf_state = []
-
-    try:
-        # Phase 1: Apply CF highlights
-        try:
-            app.ScreenUpdating = False
-        except Exception:
-            pass
-        cf_state = _apply_cf_highlights(app, by_sheet)
-        try:
-            app.ScreenUpdating = True
-        except Exception:
-            pass
-
-        # Phase 2: MessageBox — modal to Excel window
-        n = len(dirty_cells)
-        sheet_summary = ", ".join(
-            f"{name} ({len(cells)})" for name, cells in by_sheet.items()
-        )
-        msg = (
-            f"ShortcutXL modified {n} cell{'s' if n != 1 else ''} "
-            f"in: {sheet_summary}\n\n"
-            "Keep these changes?"
-        )
-        try:
-            hwnd = app.Hwnd
-        except Exception:
-            hwnd = 0
-        result = ctypes.windll.user32.MessageBoxW(
-            hwnd, msg, "ShortcutXL", _MB_YESNO | _MB_ICONQUESTION
-        )
-        approved = (result == _IDYES)
-
-        # Phase 3: Revert if rejected
-        if not approved:
-            _revert_cells(app, dirty_cells)
-
-        return approved
-
-    finally:
-        _cleanup_cf(cf_state)

package/xll/modules/shortcut_xl/_exec_entry.py

@@ -0,0 +1,191 @@
+"""HTTP entry point — bridges run_managed to the C HTTP server.
+
+C calls dispatch_request(body_json) for every POST /exec.
+This function always returns valid JSON. C never builds JSON.
+
+Two request shapes:
+  {"code": "..."}                                        → exec_and_respond
+  {"control": "revert"|"cleanup"|"navigate", ...}        → handle_control
+"""
+
+import io
+import json
+import sys
+import traceback
+
+from shortcut_xl._com import xl_app
+from shortcut_xl._diff_highlight import (
+    cleanup_by_tx,
+    highlight_changes,
+    revert_by_tx,
+)
+from shortcut_xl._managed import run_managed
+from shortcut_xl._navigate import clear_navigate_cursor, navigate_to_cell
+from shortcut_xl._threading import _run_on_main
+from shortcut_xl.api.utils.helpers import address_to_index
+
+_MAX_DIFF_JSON_CELLS = 50
+
+
+def _format_cell_value(v):
+    """Format a cell value for JSON display."""
+    if v is None:
+        return 'empty'
+    if isinstance(v, bool):
+        return 'TRUE' if v else 'FALSE'
+    if isinstance(v, int):
+        return v
+    if isinstance(v, str):
+        return 'empty' if len(v) == 0 else v
+    try:
+        f = float(v)
+        # Only convert to int if it fits exactly (avoids precision loss for large floats)
+        i = int(f)
+        return i if f == i and abs(i) <= 2**53 else round(f, 10)
+    except (TypeError, ValueError):
+        return str(v)
+
+
+def _cell_sort_key(c):
+    """Sort key: (sheet, row, col) using address_to_index."""
+    addr = c.get('address', '')
+    try:
+        row, col = address_to_index(addr)
+    except (ValueError, AttributeError):
+        row, col = 0, 0
+    return (c.get('sheet', ''), row, col)
+
+
+def _sort_dirty(dirty):
+    """Sort dirty cells by (sheet, row, column) for row-wise display.
+
+    Applied once before both _build_diff and highlight_changes so the
+    entire pipeline sees a consistent order.
+    """
+    return sorted(dirty, key=_cell_sort_key)
+
+
+def _cell_ref(c):
+    """Build a full cell reference string from a dirty cell dict."""
+    wb = c.get('workbook', '')
+    prefix = f"[{wb}]" if wb else ''
+    return f"{prefix}{c.get('sheet', '')}!{c.get('address', '')}"
+
+
+def _build_diff(dirty):
+    """Build structured diff payload for the TUI approval component.
+
+    Expects *dirty* to be pre-sorted by _sort_dirty. One row per cell.
+    """
+    cells = []
+    for c in dirty[:_MAX_DIFF_JSON_CELLS]:
+        entry = {
+            'address': _cell_ref(c),
+            'before': _format_cell_value(c.get('oldValue')),
+            'after': _format_cell_value(c.get('value')),
+        }
+        if c.get('oldFormula'):
+            entry['oldFormula'] = c['oldFormula']
+        if c.get('formula'):
+            entry['formula'] = c['formula']
+        cells.append(entry)
+    return {'cells': cells, 'total': len(dirty)}
+
+
+def _json_default(obj):
+    """Fallback serializer for non-standard types (e.g. datetime)."""
+    try:
+        return str(obj)
+    except Exception:
+        return None
+
+
+def dispatch_request(body_json):
+    """Single entry point for all POST /exec requests.
+
+    Always returns a valid JSON string. C sends this verbatim.
+    """
+    try:
+        request = json.loads(body_json)
+    except (json.JSONDecodeError, TypeError) as e:
+        return json.dumps({'ok': False, 'error': f'Invalid JSON: {e}'})
+
+    if not isinstance(request, dict):
+        return json.dumps({'ok': False, 'error': "request must be a JSON object"})
+
+    if 'code' in request:
+        return exec_and_respond(request['code'], auto_approve=request.get('autoApprove', False))
+    elif 'control' in request:
+        return handle_control(request)
+    else:
+        return json.dumps({'ok': False, 'error': "request must have 'code' or 'control' field"})
+
+
+def exec_and_respond(code_str, *, auto_approve=False):
+    """Execute user code and return a JSON response string.
+
+    Owns stdout/stderr capture, calls run_managed (pure), applies
+    CF highlights, and builds the response. Always returns valid JSON.
+
+    When *auto_approve* is True the caller already knows it will accept
+    any changes, so we skip CF highlighting entirely — no highlights to
+    apply, no revert data to store, no cleanup round-trip needed.
+    """
+    sio = io.StringIO()
+    old_out, old_err = sys.stdout, sys.stderr
+    sys.stdout = sys.stderr = sio
+    try:
+        dirty = run_managed(code_str)
+        output = sio.getvalue()
+
+        response = {'ok': True, 'output': output}
+        if dirty:
+            dirty = _sort_dirty(dirty)
+            response['diff'] = _build_diff(dirty)
+            # Only apply CF highlights when the user may review changes.
+            if not auto_approve:
+                cf_tx_id = _run_on_main(lambda: highlight_changes(xl_app(), dirty))
+                response['cfTxId'] = cf_tx_id
+        return json.dumps(response, default=_json_default)
+    except Exception:
+        output = sio.getvalue()
+        tb = traceback.format_exc()
+        error = f"{output}{tb}" if output else tb
+        return json.dumps({'ok': False, 'error': error})
+    finally:
+        sys.stdout, sys.stderr = old_out, old_err
+
+
+def handle_control(request):
+    """Handle control commands (revert, cleanup).
+
+    Revert data is stored server-side in _diff_highlight, keyed by cfTxId.
+    TS only sends the ID — no dirty cell data round-trips through HTTP.
+    """
+    action = request.get('control')
+    cf_tx_id = request.get('cfTxId')
+
+    if action == 'revert':
+        if cf_tx_id is None:
+            return json.dumps({'ok': False, 'error': "revert requires cfTxId"})
+        _run_on_main(clear_navigate_cursor)
+        n = _run_on_main(lambda: revert_by_tx(xl_app(), cf_tx_id))
+        if n == 0:
+            return json.dumps({'ok': True, 'output': 'Nothing to revert (transaction not found or already cleaned up).'})
+        return json.dumps({'ok': True, 'output': f"Reverted {n} cell{'s' if n != 1 else ''}."})
+
+    elif action == 'navigate':
+        cell_ref = request.get('cellRef', '')
+        if not cell_ref:
+            return json.dumps({'ok': False, 'error': "navigate requires cellRef"})
+        ok = _run_on_main(lambda: navigate_to_cell(xl_app(), cell_ref))
+        return json.dumps({'ok': ok, 'output': '' if ok else f"Could not navigate to {cell_ref}"})
+
+    elif action == 'cleanup':
+        _run_on_main(clear_navigate_cursor)
+        if cf_tx_id is not None:
+            _run_on_main(lambda: cleanup_by_tx(cf_tx_id))
+        return json.dumps({'ok': True, 'output': ''})
+
+    else:
+        return json.dumps({'ok': False, 'error': f"Unknown control action: {action}"})

package/xll/modules/shortcut_xl/_log.py

@@ -1,7 +1,7 @@
 """Logging utility for ShortcutXL."""
 
-import os
 import datetime
+import os
 
 
 def xl_log(msg):

package/xll/modules/shortcut_xl/_managed.py

@@ -1,17 +1,21 @@
 """run_managed — execute user code against Excel on the main thread."""
 
 import threading
-from shortcut_xl._log import xl_log
+
 from shortcut_xl._com import xl_app
+from shortcut_xl._log import xl_log
 from shortcut_xl._threading import _run_on_main
 from shortcut_xl._tracking import DirtyTracker, TrackedApp
 from shortcut_xl.api.format import format_cell_diff
-from shortcut_xl.api.workbook import Workbook
-from shortcut_xl.api.worksheet import Worksheet
 from shortcut_xl.api.utils.helpers import (
+    address_to_index,
+    col_index,
+    col_letter,
     hex_to_bgr,
-    index_to_address, address_to_index, col_letter, col_index,
+    index_to_address,
 )
+from shortcut_xl.api.workbook import Workbook
+from shortcut_xl.api.worksheet import Worksheet
 
 # Excel xlCalculation enum values
 _XL_CALCULATION_MANUAL = -4135
@@ -34,10 +38,11 @@ def run_managed(code_str):
 
     Suspends Calculation/Events/Alerts, runs code with a tracking
     proxy, collects dirty cells, then restores Excel state.
-    Prints a human-readable diff summary of any cell changes to stdout.
 
-    All COM calls happen directly on the main thread — no cross-apartment
-    marshaling overhead.
+    Returns the list of dirty cells. Caller (_exec_entry) is responsible
+    for CF highlighting, JSON response building, and stdout capture.
+    The text summary is still printed to stdout (captured by _exec_entry)
+    so the agent sees what changed in the tool result output.
     """
     code = compile(code_str, '<excel_exec>', 'exec')
 
@@ -92,13 +97,14 @@ def run_managed(code_str):
                     app.Calculation = _XL_CALCULATION_AUTOMATIC
             except Exception:
                 pass
+
         return dirty
 
     try:
         dirty = _run_on_main(_work)
     except Exception as e:
-        # Strip infrastructure traceback so PyRun_SimpleString only prints
-        # the exception type + message, not the full frame chain.
+        # Strip infrastructure traceback — _exec_entry formats it via
+        # traceback.format_exc(), so the extra frames are noise.
         e.__traceback__ = None
         raise
     summary = format_cell_diff(dirty)