shortcutxl 0.2.0

package/xll/modules/shortcut_xl/_format.py

@@ -0,0 +1,252 @@
+"""Cell diff formatter — port of cell-diff-formatter.ts.
+
+Produces a human-readable summary of dirty cells for the LLM,
+with Good / Issues breakdown and row sampling.
+"""
+
+import re
+from ._categorize import (
+    categorize_cells,
+    GOOD, LARGE_PERCENTAGE, HARDCODED_NUMBER,
+    HARDCODED_NUMBER_IN_FORMULA, INVALID_FORMULA,
+    is_com_error, com_error_to_str,
+)
+from ._tracking import _col_to_letter
+
+# Display constants
+MAX_ROWS_TO_SHOW = 10
+MAX_SAMPLES_PER_ROW = 5
+DECIMAL_PLACES = 3
+MAX_FORMAT_DECIMALS = 2
+UNDEFINED_SYMBOL = '\u2205'  # ∅
+NO_CHANGES_MSG = 'No changes made.'
+
+# (category_key, must_fix)
+_PROBLEM_SECTIONS = (
+    (LARGE_PERCENTAGE, False),
+    (HARDCODED_NUMBER, False),
+    (HARDCODED_NUMBER_IN_FORMULA, False),
+    (INVALID_FORMULA, True),
+)
+
+# ---------- address parsing ----------
+
+_ADDR_RE = re.compile(r'^(\$?)([A-Z]+)(\$?)(\d+)$', re.IGNORECASE)
+
+
+def _parse_address(addr):
+    """Parse 'B12' -> (col_index_0based, row_0based)."""
+    m = _ADDR_RE.match(addr)
+    if not m:
+        return (0, 0)
+    letters = m.group(2).upper()
+    col = 0
+    for ch in letters:
+        col = col * 26 + (ord(ch) - 64)
+    return (col - 1, int(m.group(4)) - 1)
+
+
+def _col_letter(col_0):
+    """0-based column index to letter(s). 0->A, 25->Z, 26->AA."""
+    return _col_to_letter(col_0 + 1)
+
+
+# ---------- value formatting ----------
+
+def format_value(value, number_format=None):
+    """Format a raw cell value for display."""
+    if value is None:
+        return UNDEFINED_SYMBOL
+    if isinstance(value, bool):
+        return 'TRUE' if value else 'FALSE'
+    if is_com_error(value):
+        return com_error_to_str(value)
+    if isinstance(value, str):
+        return 'EMPTY' if len(value) == 0 else value
+    # COM may return Decimal or other numeric types — coerce to float
+    if not isinstance(value, (int, float)):
+        try:
+            value = float(value)
+        except (TypeError, ValueError):
+            return str(value)
+
+    if not number_format:
+        return '0' if value == 0 else f'{value:.{DECIMAL_PLACES}f}'
+
+    fmt = number_format
+    is_pct = '%' in fmt
+    currency_m = re.search(r'[$\xa3\u20ac\xa5\u20b9\u20a9\u20aa\u20b1\u0e3f]|kr', fmt)
+    currency = currency_m.group(0) if currency_m else None
+    dec_m = re.search(r'\.([0#]+)', fmt)
+    decimals = min(len(dec_m.group(1)), MAX_FORMAT_DECIMALS) if dec_m else MAX_FORMAT_DECIMALS
+    use_parens = '(' in fmt
+
+    num = value * 100 if is_pct else value
+    abs_str = f'{abs(num):,.{decimals}f}'
+    neg = num < 0
+
+    if is_pct:
+        return f'({abs_str}%)' if neg and use_parens else f'{"-" if neg else ""}{abs_str}%'
+    if currency:
+        return (f'({currency}{abs_str})' if neg and use_parens
+                else f'{"-" if neg else ""}{currency}{abs_str}')
+    return f'({abs_str})' if neg and use_parens else f'{"-" if neg else ""}{abs_str}'
+
+
+def _display_value(cell):
+    """Get display string for a cell — prefer displayValue, then format with numberFormat."""
+    dv = cell.get('displayValue')
+    if dv is not None:
+        return 'EMPTY' if dv == '' else dv
+    return format_value(cell.get('value'), cell.get('numberFormat'))
+
+
+# ---------- sampling ----------
+
+def _sample_first_last(items, max_n):
+    """Sample up to max_n items, always including first and last.
+
+    Uses evenly-spaced indices for deterministic output."""
+    n = len(items)
+    if n <= max_n:
+        return list(items)
+    if max_n <= 0:
+        return []
+    if max_n == 1:
+        return [items[0]]
+    # Evenly spaced indices including 0 and n-1
+    indices = [round(i * (n - 1) / (max_n - 1)) for i in range(max_n)]
+    return [items[i] for i in indices]
+
+
+# ---------- filtering ----------
+
+def _should_include(cell, issues):
+    """Include all changes except trivial None↔None (no actual change)."""
+    value = cell.get('value')
+    old = cell.get('oldValue')
+    if value is None and old is None:
+        return False
+    return True
+
+
+# ---------- grouping ----------
+
+def _group_by_sheet_row(entries):
+    """Group (cell, issues) entries by sheet -> row.
+    Returns {sheet: {row: [(cell, issues, col), ...]}}
+    """
+    grouped = {}
+    for cell, issues in entries:
+        if not _should_include(cell, issues):
+            continue
+        col, row = _parse_address(cell.get('address', 'A1'))
+        sheet = cell.get('sheet', 'Sheet1')
+        grouped.setdefault(sheet, {}).setdefault(row, []).append((cell, issues, col))
+    # Sort within each row by column
+    for sheet_rows in grouped.values():
+        for row_entries in sheet_rows.values():
+            row_entries.sort(key=lambda t: t[2])
+    return grouped
+
+
+# ---------- row formatting ----------
+
+def _format_row(sheet, row, entries, indent=2, show_issues=True, show_count=True):
+    """Format a single row with sampled cells."""
+    ind = ' ' * indent
+    sind = ' ' * (indent + 2)
+    lines = []
+
+    cols = [e[2] for e in entries]
+    min_c, max_c = min(cols), max(cols)
+    col_range = (_col_letter(min_c) if min_c == max_c
+                 else f'{_col_letter(min_c)}-{_col_letter(max_c)}')
+
+    count_suffix = f': {len(entries)} cells' if show_count else ':'
+    lines.append(f'{ind}{sheet}!Row {row + 1} ({col_range}){count_suffix}')
+
+    samples = _sample_first_last(entries, MAX_SAMPLES_PER_ROW)
+    for cell, issues, _col in samples:
+        fmt = cell.get('numberFormat')
+        old_str = format_value(cell.get('oldValue'), fmt)
+        new_str = _display_value(cell)
+        # Show formula text as annotation when relevant (not category names —
+        # the section header already tells you the category)
+        ann = f' [{cell["formula"]}]' if show_issues and cell.get('formula') else ''
+        lines.append(f'{sind}\u2192 {cell.get("address", "?")}: {old_str} -> {new_str}{ann}')
+
+    if len(entries) > MAX_SAMPLES_PER_ROW:
+        lines.append(f'{sind}... and {len(entries) - MAX_SAMPLES_PER_ROW} more cells')
+
+    return lines
+
+
+# ---------- section formatting ----------
+
+def _format_section(entries, title, must_fix=False):
+    """Format a titled section of (cell, issues) entries."""
+    filtered = [(c, iss) for c, iss in entries if _should_include(c, iss)]
+    if not filtered:
+        return ''
+
+    prefix = 'MUST FIX: ' if must_fix else ''
+    lines = [f'{prefix}{title}: {len(filtered)} total cells']
+
+    grouped = _group_by_sheet_row(filtered)
+    # Flatten to sorted list of (sheet, row, entries)
+    all_rows = []
+    for sheet in sorted(grouped):
+        for row in sorted(grouped[sheet]):
+            all_rows.append((sheet, row, grouped[sheet][row]))
+
+    selected = _sample_first_last(all_rows, MAX_ROWS_TO_SHOW)
+    for sheet, row, row_entries in selected:
+        lines.extend(_format_row(sheet, row, row_entries))
+
+    if len(all_rows) > MAX_ROWS_TO_SHOW:
+        lines.append(f'  ... and {len(all_rows) - MAX_ROWS_TO_SHOW} more rows')
+
+    return '\n'.join(lines)
+
+
+# ---------- public API ----------
+
+def format_cell_diff(dirty_cells):
+    """Format a list of dirty cell dicts into a human-readable summary.
+
+    This is the main entry point — call it with the raw dirty_cells list
+    from DirtyTracker.collect_changes().  It categorizes issues and produces
+    the same style of output as the Office.js client's cell-diff-formatter.
+
+    Returns a string ready to be printed to stdout for the LLM to see.
+    """
+    if not dirty_cells:
+        return NO_CHANGES_MSG
+
+    cats = categorize_cells(dirty_cells)
+
+    sections = [
+        '--- CELL DIFF SUMMARY ---',
+        f'({UNDEFINED_SYMBOL} = undefined/empty value)\n',
+    ]
+
+    # Good cells
+    if cats[GOOD]:
+        good_entries = [(c, []) for c in cats[GOOD]]
+        s = _format_section(good_entries, 'Changed without issues')
+        if s:
+            sections.append(s)
+
+    # Problem cells — driven by _PROBLEM_SECTIONS config
+    problem_parts = []
+    for cat_key, must_fix in _PROBLEM_SECTIONS:
+        if cats[cat_key]:
+            s = _format_section(cats[cat_key], cat_key, must_fix=must_fix)
+            if s:
+                problem_parts.append(s)
+
+    if problem_parts:
+        sections.append('Cells that need review:\n' + '\n\n'.join(problem_parts))
+
+    return '\n\n'.join(sections) if len(sections) > 2 else NO_CHANGES_MSG

package/xll/modules/shortcut_xl/_log.py

@@ -0,0 +1,12 @@
+"""Logging utility for ShortcutXL."""
+
+import os
+import datetime
+
+
+def xl_log(msg):
+    """Append a timestamped message to %TEMP%\\shortcutxl.log."""
+    log_path = os.path.join(os.environ.get("TEMP", "."), "shortcutxl.log")
+    timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+    with open(log_path, "a", encoding="utf-8") as f:
+        f.write(f"[{timestamp}] {msg}\n")

package/xll/modules/shortcut_xl/_managed.py

@@ -0,0 +1,116 @@
+"""run_managed — execute a function against Excel on the main thread."""
+
+import threading
+from ._log import xl_log
+from ._com import xl_app
+from ._threading import _run_on_main
+from ._tracking import DirtyTracker, TrackedApp
+
+# Excel xlCalculation enum values
+_XL_CALCULATION_MANUAL = -4135
+
+
+def run_managed(fn):
+    """Run fn(app) on the main thread with Excel state managed.
+
+    Suspends Calculation/Events/Alerts, runs fn(app) with a tracking
+    proxy, collects dirty cells, then restores Excel state.
+
+    All COM calls happen directly on the main thread — no cross-apartment
+    marshaling overhead.
+
+    Returns the list of dirty cell dicts.
+
+    Usage::
+
+        def do_work(app):
+            app.ActiveSheet.Range("A1").Value = 42
+
+        dirty = run_managed(do_work)
+    """
+
+    def _work():
+        app = xl_app()
+        has_workbooks = app.Workbooks.Count > 0
+        orig_ev = app.EnableEvents
+        orig_da = app.DisplayAlerts
+        orig_calc = app.Calculation if has_workbooks else None
+        if has_workbooks:
+            app.Calculation = _XL_CALCULATION_MANUAL
+        app.EnableEvents = False
+        app.DisplayAlerts = False
+        tracker = DirtyTracker(app)
+        tracked = TrackedApp(app, tracker)
+        dirty = []
+        try:
+            fn(tracked)
+        finally:
+            # Collect changes BEFORE restoring calc so we only see
+            # direct writes, not formula recalculations.
+            try:
+                dirty.extend(tracker.collect_changes())
+            except Exception as e:
+                xl_log(f"run_managed: collect_changes failed: {e}")
+            # Each restoration is independent — don't let one failure
+            # prevent the others from being restored.
+            try:
+                app.DisplayAlerts = orig_da
+            except Exception:
+                pass
+            try:
+                app.EnableEvents = orig_ev
+            except Exception:
+                pass
+            try:
+                # If user's calc mode was manual, restoring it won't
+                # trigger a recalc — force one so dependents are fresh.
+                if orig_calc == _XL_CALCULATION_MANUAL and app.Workbooks.Count > 0:
+                    app.Calculate()
+            except Exception:
+                pass
+            try:
+                # Restore original calc mode if we had workbooks,
+                # or reset to automatic if fn created the first workbook.
+                if orig_calc is not None:
+                    app.Calculation = orig_calc
+                elif app.Workbooks.Count > 0:
+                    # fn created workbooks — calc was never saved, reset
+                    # to automatic so we don't leave it stuck on manual.
+                    app.Calculation = -4105  # xlCalculationAutomatic
+            except Exception:
+                pass
+        return dirty
+
+    return _run_on_main(_work)
+
+
+def xl_batch(fn):
+    """Run fn(app) on the main thread with ScreenUpdating disabled."""
+    def _batched():
+        import pythoncom
+        app = xl_app()
+        app.ScreenUpdating = False
+        try:
+            fn(app)
+        finally:
+            app.ScreenUpdating = True
+            pythoncom.PumpWaitingMessages()
+    _run_on_main(_batched)
+
+
+def schedule_call(fn, delay_seconds=0.0):
+    """Schedule fn() to be called after delay_seconds on a background thread."""
+    import time
+
+    def _run():
+        if delay_seconds > 0:
+            time.sleep(delay_seconds)
+        try:
+            fn()
+        except Exception as e:
+            import traceback
+            xl_log(f"schedule_call error: {e}\n{traceback.format_exc()}")
+
+    t = threading.Thread(target=_run, daemon=True)
+    t.start()
+    return t

package/xll/modules/shortcut_xl/_registry.py

@@ -0,0 +1,44 @@
+"""@xl_func decorator — marks functions for registration as Excel UDFs.
+
+The C runtime scans _registry at xlAutoOpen time to register functions with Excel.
+"""
+
+import inspect
+
+# Internal registry: list of (name, callable, n_args)
+_registry = []
+
+
+def xl_func(fn=None, *, name=None):
+    """Decorator — mark a function as an Excel UDF.
+
+    Usage:
+        @xl_func
+        def my_func(a, b):
+            return a + b
+
+        @xl_func(name="myCustomName")
+        def my_func(a, b):
+            return a + b
+    """
+    def _register(f):
+        excel_name = name if name else f.__name__
+        sig = inspect.signature(f)
+        n_args = len([
+            p for p in sig.parameters.values()
+            if p.kind in (
+                inspect.Parameter.POSITIONAL_OR_KEYWORD,
+                inspect.Parameter.POSITIONAL_ONLY,
+            )
+        ])
+        # Check if already registered (e.g. after a hot-reload)
+        for i, (n, _, _) in enumerate(_registry):
+            if n == excel_name:
+                _registry[i] = (excel_name, f, n_args)
+                return f
+        _registry.append((excel_name, f, n_args))
+        return f
+
+    if fn is not None:
+        return _register(fn)
+    return _register

package/xll/modules/shortcut_xl/_threading.py

@@ -0,0 +1,161 @@
+"""Main-thread work queue for ShortcutXL.
+
+Python win32com cannot call Excel COM from background threads reliably
+(GetActiveObject fails from MTA, Dispatch creates a new instance).
+This matches the pattern used by PyXLL, Excel-DNA, and xlwings:
+background threads enqueue work, a timer on the main thread drains it.
+"""
+
+import threading
+import queue
+import ctypes
+from ctypes import wintypes
+from ._log import xl_log
+
+
+# COM error code: Excel is busy (VBA_E_IGNORE = 0x800AC472)
+_VBA_E_IGNORE = -2146777998
+
+_MAX_ITEMS_PER_TICK = 64      # cap per timer tick to avoid starving Excel
+_MAX_BUSY_RETRIES = 2400      # ~120s at 50ms intervals before giving up
+_DRAIN_INTERVAL_MS = 50       # SetTimer interval for queue drain
+_DEFAULT_TIMEOUT_S = 30.0     # max wait for main-thread dispatch + execution
+
+
+class ExecutionTimeout(BaseException):
+    """Raised on the main thread when a caller's timeout expires mid-execution.
+
+    Inherits from BaseException (not Exception) so that user code's
+    ``except Exception:`` blocks cannot swallow it."""
+    pass
+
+
+def _is_excel_busy(e):
+    """Check if a COM exception is VBA_E_IGNORE (Excel is busy)."""
+    if not hasattr(e, 'hresult'):
+        return False
+    if e.hresult == _VBA_E_IGNORE:
+        return True
+    ei = getattr(e, 'excepinfo', None)
+    if ei and len(ei) > 5 and ei[5] == _VBA_E_IGNORE:
+        return True
+    return False
+
+
+# Persist main thread ID in builtins so it survives hot-reload.
+import builtins
+_main_thread_id = getattr(builtins, '_shortcutxl_main_thread_id', None)
+if _main_thread_id is None:
+    _main_thread_id = threading.current_thread().ident
+    builtins._shortcutxl_main_thread_id = _main_thread_id
+
+_work_queue = queue.Queue()
+_draining = False
+
+
+def _drain_queue():
+    """Drain pending work items.  Runs on the main thread from the timer.
+
+    The reentrancy guard is essential: work items may call
+    PumpWaitingMessages(), which can dispatch another WM_TIMER and
+    re-enter this function.  Without the guard the queue can be
+    drained out-of-order and done_events can be missed."""
+    global _draining
+    if _draining:
+        return
+    _draining = True
+    try:
+        for _ in range(_MAX_ITEMS_PER_TICK):
+            try:
+                fn, result_box, done_event = _work_queue.get_nowait()
+            except queue.Empty:
+                break
+            if result_box.get('_cancelled'):
+                continue
+            result_box['_running'] = True
+            try:
+                result_box['result'] = fn()
+                result_box['error'] = None
+                done_event.set()
+            except ExecutionTimeout:
+                result_box['error'] = RuntimeError(
+                    "Execution interrupted: caller timed out")
+                done_event.set()
+            except Exception as e:
+                if _is_excel_busy(e):
+                    result_box['_retries'] = result_box.get('_retries', 0) + 1
+                    if result_box['_retries'] < _MAX_BUSY_RETRIES:
+                        _work_queue.put((fn, result_box, done_event))
+                        break
+                    xl_log(f"_drain_queue: Excel busy exceeded {_MAX_BUSY_RETRIES} retries")
+                result_box['error'] = e
+                done_event.set()
+    finally:
+        _draining = False
+
+
+# SetTimer callback — called by Windows on the main thread during message pump
+_TIMERPROC = ctypes.WINFUNCTYPE(
+    None, wintypes.HWND, ctypes.c_uint, ctypes.c_size_t, wintypes.DWORD)
+
+
+def _on_timer(hwnd, msg, id_event, dw_time):
+    _drain_queue()
+
+
+# prevent GC of the callback (prevents crash when timer fires)
+_timer_proc_ref = _TIMERPROC(_on_timer)
+
+
+def _start_drain_timer():
+    """Create a 50ms timer on the main thread.  Safe to call on hot-reload."""
+    try:
+        old_id = getattr(builtins, '_shortcutxl_timer_id', None)
+        if old_id:
+            ctypes.windll.user32.KillTimer(None, ctypes.c_size_t(old_id))
+        timer_id = ctypes.windll.user32.SetTimer(
+            None, ctypes.c_size_t(0), _DRAIN_INTERVAL_MS, _timer_proc_ref)
+        builtins._shortcutxl_timer_id = timer_id
+        xl_log(f"_start_drain_timer: timer_id={timer_id}")
+    except Exception as e:
+        xl_log(f"_start_drain_timer: FAILED — {e}")
+        import traceback
+        xl_log(traceback.format_exc())
+
+
+# Only start the drain timer on the main thread (xlAutoOpen).
+if threading.current_thread().ident == _main_thread_id:
+    _start_drain_timer()
+
+
+def _run_on_main(fn, timeout=_DEFAULT_TIMEOUT_S):
+    """Run fn() on the main thread and return its result.  Blocks the caller.
+    If already on the main thread, calls fn() directly."""
+    if threading.current_thread().ident == _main_thread_id:
+        return fn()
+    result_box = {'_retries': 0}
+    done_event = threading.Event()
+    _work_queue.put((fn, result_box, done_event))
+    if not done_event.wait(timeout=timeout):
+        if result_box.get('_running'):
+            # fn is mid-execution on the main thread — inject an async
+            # exception to interrupt it.  This fires at the next Python
+            # bytecode boundary (won't interrupt a blocking C call, but
+            # will fire between COM calls / sleep increments).
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(
+                ctypes.c_ulong(_main_thread_id),
+                ctypes.py_object(ExecutionTimeout))
+            # Race guard: fn may have completed between the timeout and
+            # the injection.  If done_event is already set, the exception
+            # would land on the NEXT work item — clear it immediately.
+            if done_event.is_set():
+                ctypes.pythonapi.PyThreadState_SetAsyncExc(
+                    ctypes.c_ulong(_main_thread_id), None)
+            elif not done_event.wait(timeout=5.0):
+                xl_log("_run_on_main: fn did not stop after async exception")
+        else:
+            result_box['_cancelled'] = True
+        raise RuntimeError("_run_on_main: timed out waiting for main thread")
+    if result_box.get('error') is not None:
+        raise result_box['error']
+    return result_box.get('result')