visidata 3.1.1__py3-none-any.whl → 3.3__py3-none-any.whl

visidata/deprecated.py

@@ -3,12 +3,16 @@ import functools
 from visidata import VisiData, vd
 import visidata
 
-alias = visidata.BaseSheet.bindkey
+def deprecated_alias(depver, *args, **kwargs):
+    # expand this to create cmd
+    # with cmd.deprecated=depver
+    return visidata.BaseSheet.bindkey(*args, **kwargs)
 
-def deprecated_warn(func, ver, instead):
+@VisiData.api
+def deprecated_warn(vd, funcname, ver, instead):
     import traceback
 
-    msg = f'{func.__name__} deprecated since v{ver}'
+    msg = f'{funcname} deprecated since v{ver}'
     if instead:
         msg += f'; use {instead}'
 
@@ -20,18 +24,27 @@ def deprecated_warn(func, ver, instead):
         vd.warning(f'Deprecated call traceback (most recent last):')
 
 
-def deprecated(ver, instead=''):
+def deprecated(ver, instead='', check=True):
     def decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
-            deprecated_warn(func, ver, instead)
+            vd.deprecated_warn(wrapper.__name__, ver, instead)
             return func(*args, **kwargs)
+
+        if check and hasattr(func, '_extensible_api'):
+            vd.error(f"{func.__name__}: @deprecated applied in wrong order")  #2623
+
         return wrapper
     return decorator
 
 
-@deprecated('1.6', 'vd instead of vd()')
+def _deprecated_api(ver, instead=''):
+    'Decorator to deliberately wrap non-deprecated .api functions as a deprecated global function.  Use @deprecated instead, except in deprecated.py.'
+    return deprecated(ver, instead, check=False)
+
+
 @VisiData.api
+@deprecated('1.6', 'vd instead of vd()')
 def __call__(vd):
     'Deprecated; use plain "vd"'
     return vd
@@ -39,7 +52,7 @@ def __call__(vd):
 
 @deprecated('1.6')
 def copyToClipboard(value):
-    vd.error("copyToClipboard longer implemented")
+    vd.error("copyToClipboard no longer implemented")
     return visidata.clipboard_copy(value)
 
 
@@ -62,32 +75,32 @@ def bindkey_override(keystrokes, longname):
 bindkey = visidata.BaseSheet.bindkey
 unbindkey = visidata.BaseSheet.unbindkey
 
-@deprecated('2.0')
 @visidata.Sheet.api
+@deprecated('2.0')
 def exec_keystrokes(self, keystrokes, vdglobals=None):
     return self.execCommand(self.getCommand(keystrokes), vdglobals, keystrokes=keystrokes)
 
 visidata.Sheet.exec_command = deprecated('2.0')(visidata.Sheet.execCommand)
 
-@deprecated('2.0', 'def open_<filetype> instead')
 @VisiData.api
+@deprecated('2.0', 'def open_<filetype> instead')
 def filetype(vd, ext, constructor):
     'Add constructor to handle the given file type/extension.'
     globals().setdefault('open_'+ext, lambda p,ext=ext: constructor(p.base_stem, source=p, filetype=ext))
 
-@deprecated('2.0', 'Sheet(namepart1, namepart2, ...)')
 @VisiData.global_api
+@deprecated('2.0', 'Sheet(namepart1, namepart2, ...)')
 def joinSheetnames(vd, *sheetnames):
     'Concatenate sheet names in a standard way'
     return visidata.options.name_joiner.join(str(x) for x in sheetnames)
 
-@deprecated('2.0', 'PyobjSheet')
 @VisiData.global_api
+@deprecated('2.0', 'PyobjSheet')
 def load_pyobj(*names, **kwargs):
     return visidata.PyobjSheet(*names, **kwargs)
 
-@deprecated('2.0', 'PyobjSheet')
 @VisiData.global_api
+@deprecated('2.0', 'PyobjSheet')
 def push_pyobj(name, pyobj):
     vs = visidata.PyobjSheet(name, source=pyobj)
     if vs:
@@ -103,33 +116,33 @@ visidata.addGlobals({'load_pyobj': load_pyobj, 'isNumeric': isNumeric})
 
 # The longnames on the left are deprecated for 2.0
 
-alias('edit-cells', 'setcol-input')
-alias('fill-nulls', 'setcol-fill')
-alias('paste-cells', 'setcol-clipboard')
-alias('frequency-rows', 'frequency-summary')
-alias('dup-cell', 'dive-cell')
-alias('dup-row', 'dive-row')
-alias('next-search', 'search-next')
-alias('prev-search', 'search-prev')
-alias('search-prev', 'searchr-next')
-alias('prev-sheet', 'jump-prev')
-alias('prev-value', 'go-prev-value')
-alias('next-value', 'go-next-value')
-alias('prev-selected', 'go-prev-selected')
-alias('next-selected', 'go-next-selected')
-alias('prev-null', 'go-prev-null')
-alias('next-null', 'go-next-null')
-alias('page-right', 'go-right-page')
-alias('page-left', 'go-left-page')
-alias('dive-cell', 'open-cell')
-alias('dive-row', 'open-row')
-alias('add-sheet', 'open-new')
-alias('save-sheets-selected', 'save-selected')
-alias('join-sheets', 'join-selected')
-alias('dive-rows', 'dive-selected')
+deprecated_alias('2.0', 'edit-cells', 'setcol-input')
+deprecated_alias('2.0', 'fill-nulls', 'setcol-fill')
+deprecated_alias('2.0', 'paste-cells', 'setcol-clipboard')
+deprecated_alias('2.0', 'frequency-rows', 'frequency-summary')
+deprecated_alias('2.0', 'dup-cell', 'dive-cell')
+deprecated_alias('2.0', 'dup-row', 'dive-row')
+deprecated_alias('2.0', 'next-search', 'search-next')
+deprecated_alias('2.0', 'prev-search', 'search-prev')
+deprecated_alias('2.0', 'search-prev', 'searchr-next')
+deprecated_alias('2.0', 'prev-sheet', 'jump-prev')
+deprecated_alias('2.0', 'prev-value', 'go-prev-value')
+deprecated_alias('2.0', 'next-value', 'go-next-value')
+deprecated_alias('2.0', 'prev-selected', 'go-prev-selected')
+deprecated_alias('2.0', 'next-selected', 'go-next-selected')
+deprecated_alias('2.0', 'prev-null', 'go-prev-null')
+deprecated_alias('2.0', 'next-null', 'go-next-null')
+deprecated_alias('2.0', 'page-right', 'go-right-page')
+deprecated_alias('2.0', 'page-left', 'go-left-page')
+deprecated_alias('2.0', 'dive-cell', 'open-cell')
+deprecated_alias('2.0', 'dive-row', 'open-row')
+deprecated_alias('2.0', 'add-sheet', 'open-new')
+deprecated_alias('2.0', 'save-sheets-selected', 'save-selected')
+deprecated_alias('2.0', 'join-sheets', 'join-selected')
+deprecated_alias('2.0', 'dive-rows', 'dive-selected')
 
 # v2.3
-alias('show-aggregate', 'memo-aggregate')
+deprecated_alias('2.3', 'show-aggregate', 'memo-aggregate')
 #theme('use_default_colors', True, 'curses use default terminal colors')
 #option('expand_col_scanrows', 1000, 'number of rows to check when expanding columns (0 = all)')
 
@@ -149,25 +162,25 @@ def load_tsv(fn):
 
 # NOTE: you cannot use deprecated() with nonfuncs
 
-cancelThread = deprecated('2.6', 'vd.cancelThread')(vd.cancelThread)
-status = deprecated('2.6', 'vd.status')(vd.status)
-warning = deprecated('2.6', 'vd.warning')(vd.warning)
-error = deprecated('2.6', 'vd.error')(vd.error)
-debug = deprecated('2.6', 'vd.debug')(vd.debug)
-fail = deprecated('2.6', 'vd.fail')(vd.fail)
+cancelThread = _deprecated_api('2.6', 'vd.cancelThread')(vd.cancelThread)
+status = _deprecated_api('2.6', 'vd.status')(vd.status)
+warning = _deprecated_api('2.6', 'vd.warning')(vd.warning)
+error = _deprecated_api('2.6', 'vd.error')(vd.error)
+debug = _deprecated_api('2.6', 'vd.debug')(vd.debug)
+fail = _deprecated_api('2.6', 'vd.fail')(vd.fail)
 
 option = theme = vd.option # deprecated('2.6', 'vd.option')(vd.option)
 jointypes = vd.jointypes # deprecated('2.6', 'vd.jointypes')(vd.jointypes)
-confirm = deprecated('2.6', 'vd.confirm')(vd.confirm)
-launchExternalEditor = deprecated('2.6', 'vd.launchExternalEditor')(vd.launchExternalEditor)
-launchEditor = deprecated('2.6', 'vd.launchEditor')(vd.launchEditor)
-exceptionCaught = deprecated('2.6', 'vd.exceptionCaught')(vd.exceptionCaught)
-openSource = deprecated('2.6', 'vd.openSource')(vd.openSource)
+confirm = _deprecated_api('2.6', 'vd.confirm')(vd.confirm)
+launchExternalEditor = _deprecated_api('2.6', 'vd.launchExternalEditor')(vd.launchExternalEditor)
+launchEditor = _deprecated_api('2.6', 'vd.launchEditor')(vd.launchEditor)
+exceptionCaught = _deprecated_api('2.6', 'vd.exceptionCaught')(vd.exceptionCaught)
+openSource = _deprecated_api('2.6', 'vd.openSource')(vd.openSource)
 globalCommand = visidata.BaseSheet.addCommand
-visidata.Sheet.StaticColumn = deprecated('2.11', 'Sheet.freeze_col')(visidata.Sheet.freeze_col)
+visidata.Sheet.StaticColumn = _deprecated_api('2.11', 'Sheet.freeze_col')(visidata.Sheet.freeze_col)
 #visidata.Path.open_text = deprecated('3.0', 'visidata.Path.open')(visidata.Path.open)  # undeprecated in 3.1
 
-vd.sysclip_value = deprecated('3.0', 'vd.sysclipValue')(vd.sysclipValue)
+vd.sysclip_value = _deprecated_api('3.0', 'vd.sysclipValue')(vd.sysclipValue)
 
 def itemsetter(i):
     def g(obj, v):
@@ -181,8 +194,8 @@ vd.optalias('confirm_overwrite', 'overwrite', 'confirm')
 vd.optalias('show_graph_labels', 'disp_graph_labels')
 vd.optalias('zoom_incr', 'disp_zoom_incr')
 
-alias('visibility-sheet', 'toggle-multiline')
-alias('visibility-col', 'toggle-multiline')
+deprecated_alias('3.0', 'visibility-sheet', 'toggle-multiline')
+deprecated_alias('3.0', 'visibility-col', 'toggle-multiline')
 
 def clean_to_id(s):
     return visidata.vd.cleanName(s)
@@ -204,7 +217,7 @@ class OnExit:
         except Exception as e:
             vd.exceptionCaught(e)
 
-alias('open-inputs', 'open-input-history')
+deprecated_alias('3.0', 'open-inputs', 'open-input-history')
 
 #vd.option('plugins_url', 'https://visidata.org/plugins/plugins.jsonl', 'source of plugins sheet')
 
@@ -216,37 +229,57 @@ def inputRegexSubstOld(vd, prompt):
     return dict(before=before, after=after)
 
 
-visidata.Sheet.addCommand('', 'addcol-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, **inputRegexSubstOld("transform column by regex: "))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)', deprecated=True)
-visidata.Sheet.addCommand('', 'setcol-subst', 'setValuesFromRegex([cursorCol], someSelectedRows, **inputRegexSubstOld("transform column by regex: "))', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)', deprecated=True)
-visidata.Sheet.addCommand('', 'setcol-subst-all', 'setValuesFromRegex(visibleCols, someSelectedRows, **inputRegexSubstOld(f"transform {nVisibleCols} columns by regex: "))', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)', deprecated=True)
+visidata.Sheet.addCommand('', 'addcol-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, **inputRegexSubstOld("transform column by regex: "))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)', deprecated='3.0')
+visidata.Sheet.addCommand('', 'setcol-subst', 'setValuesFromRegex([cursorCol], someSelectedRows, **inputRegexSubstOld("transform column by regex: "))', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)', deprecated='3.0')
+visidata.Sheet.addCommand('', 'setcol-subst-all', 'setValuesFromRegex(visibleCols, someSelectedRows, **inputRegexSubstOld(f"transform {nVisibleCols} columns by regex: "))', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)', deprecated='3.0')
 
-visidata.Sheet.addCommand('', 'split-col', 'addRegexColumns(makeRegexSplitter, cursorCol, inputRegex("split regex: ", type="regex-split"))', 'Add new columns from regex split', deprecated=True)
-visidata.Sheet.addCommand('', 'capture-col', 'addRegexColumns(makeRegexMatcher, cursorCol, inputRegex("capture regex: ", type="regex-capture"))', 'add new column from capture groups of regex; requires example row', deprecated=True)
+visidata.Sheet.addCommand('', 'split-col', 'addRegexColumns(makeRegexSplitter, cursorCol, inputRegex("split regex: ", type="regex-split"))', 'Add new columns from regex split', deprecated='3.0')
+visidata.Sheet.addCommand('', 'capture-col', 'addRegexColumns(makeRegexMatcher, cursorCol, inputRegex("capture regex: ", type="regex-capture"))', 'add new column from capture groups of regex; requires example row', deprecated='3.0')
 
 #vd.option('cmdlog_histfile', '', 'file to autorecord each cmdlog action to', sheettype=None)
 #BaseSheet.bindkey('KEY_BACKSPACE', 'menu-help')
 
-@deprecated('3.0', 'vd.callNoExceptions(col.setValue, row, value)')
 @visidata.Column.api
+@deprecated('3.0', 'vd.callNoExceptions(col.setValue, row, value)')
 def setValueSafe(self, row, value):
     'setValue and ignore exceptions.'
     return vd.callNoExceptions(self.setValue, row, value)
 
-@deprecated('3.0', 'vd.callNoExceptions(sheet.checkCursor)')
 @visidata.BaseSheet.api
+@deprecated('3.0', 'vd.callNoExceptions(sheet.checkCursor)')
 def checkCursorNoExceptions(sheet):
     return vd.callNoExceptions(sheet.checkCursor)
 
-@deprecated('3.1', 'vd.memoValue(name, value, displayvalue)')
 @VisiData.api
+@deprecated('3.1', 'vd.memoValue(name, value, displayvalue)')
 def memo(vd, name, col, row):
     return vd.memoValue(name, col.getTypedValue(row), col.getDisplayValue(row))
 
-alias('view-cell', 'pyobj-cell')
+deprecated_alias('3.1', 'view-cell', 'pyobj-cell')
 
 vd.optalias('textwrap_cells', 'disp_wrap_max_lines', 3) # wordwrap text for multiline rows
 
-@deprecated('3.1', 'sheet.rowname(row)')
 @visidata.TableSheet.api
+@deprecated('3.1', 'sheet.rowname(row)')
 def keystr(sheet, row):
     return sheet.rowname(row)
+
+vd.optalias('color_refline', 'color_graph_refline') # color_refline was used in v3.1 by mistake
+
+@visidata.TableSheet.api
+@deprecated('3.2', '[self.unsetKeys([c]) if c.keycol else self.setKeys([c]) for c in cols]')
+def toggleKeys(self, cols):
+    for col in cols:
+        if col.keycol:
+            self.unsetKeys([col])
+        else:
+            self.setKeys([col])
+
+vd.optalias('disp_pixel_random', 'disp_graph_pixel_random')  #2661
+
+vd.addGlobals(deprecated_warn=deprecated_warn)
+
+# v3.3
+
+#vd.option('disp_expert', 'max level of options and columns to include')
+#vd.option('disp_help', '')

visidata/errors.py

@@ -1,4 +1,6 @@
 import traceback
+import sys
+import re
 
 from visidata import vd, VisiData
 
@@ -10,10 +12,43 @@ class ExpectedException(Exception):
     pass
 
 
-def stacktrace(e=None):
-    if not e:
-        return traceback.format_exc().strip().splitlines()
-    return traceback.format_exception_only(type(e), e)
+def stacktrace(e=None, exclude_caller=False):
+    '''Return a list of strings for the stack trace, without newlines
+    at the end. If an exception handler is executing, and *e* is none,
+    the stack trace includes extra levels of callers beyond the level
+    where the exception was caught. If *exclude_caller* is True, the
+    trace will exclude the function that called stacktrace(). The
+    trace will exclude several uninformative levels that are run
+    in interactive visidata.'''
+
+    if e:
+        return traceback.format_exception_only(type(e), e)
+    #in Python 3.11 we can replace sys.exc_info() with sys.exception()
+    handling = (sys.exc_info() != (None, None, None))
+
+    stack = ''.join(traceback.format_stack()).strip().splitlines()
+
+    if handling:
+        trim_levels = 2       # remove levels for stacktrace() -> format_stack()
+        if exclude_caller:
+            trim_levels += 1
+        trace_above = stack[:-2*trim_levels]
+    else:
+        trace_above = stack
+    if trace_above:
+        trace_above[0] = '  ' + trace_above[0]  #fix indent level of first line
+    try:
+        # remove several levels of uninformative stacktrace in typical interactive vd
+        idx = trace_above.index('    ret = vd.mainloop(scr)')
+        trace_above = trace_above[idx+1:]
+    except ValueError:
+        pass
+    if not handling:
+        return trace_above
+    # remove lines that mark error columns with carets and sometimes tildes
+    trace_below = [ line for line in traceback.format_exc().strip().splitlines() if not re.match('^ *~*\\^+$', line) ]
+    # move the "Traceback (most recent call last) header to the top of the output
+    return [trace_below[0]] + trace_above + trace_below[1:]
 
 
 @VisiData.api
@@ -21,7 +56,8 @@ def exceptionCaught(vd, exc=None, status=True, **kwargs):
     'Add *exc* to list of last errors and add to status history.  Show on left status bar if *status* is True.  Reraise exception if options.debug is True.'
     if isinstance(exc, ExpectedException):  # already reported, don't log
         return
-    vd.lastErrors.append(stacktrace())
+    # save a stack trace that does not include this function
+    vd.lastErrors.append(stacktrace(exclude_caller=True))
     if status:
         vd.status(f'{type(exc).__name__}: {exc}', priority=2)
     else:

visidata/{features → experimental}/helloworld.py

@@ -7,4 +7,4 @@ from visidata import vd, BaseSheet
 
 vd.option('hello_world', '¡Hola mundo!', 'shown by the hello-world command')
 
-BaseSheet.addCommand('KEY_F(2)', 'hello-world', 'status(options.hello_world)', 'print greeting to status')
+BaseSheet.addCommand('F2', 'hello-world', 'status(options.hello_world)', 'print greeting to status')

visidata/experimental/liveupdate.py

@@ -42,4 +42,4 @@ def addcol_expr(sheet):
 
 
 Sheet.addCommand(None, 'addcol-expr', 'sheet.addcol_expr()', "create new column from Python expression, updating the column's calculated values live")
-Sheet.addCommand(None, 'addcol-new', 'c=addColumnAtIndex(SettableColumn(width=options.default_width)); draw(sheet._scr); cursorVisibleColIndex=visibleCols.index(c); c.name=editCell(cursorVisibleColIndex, -1); c.width=None', 'append new column, updating the column name live')
+Sheet.addCommand(None, 'addcol-new', 'c=addColumnAtCursor(SettableColumn(name="", width=options.default_width)); draw(sheet._scr); cursorVisibleColIndex=visibleCols.index(c); c.name=editCell(cursorVisibleColIndex, -1); c.width=None', 'append new column, updating the column name live')

visidata/expr.py

@@ -100,6 +100,7 @@ Sheet.addCommand('=', 'addcol-expr', 'addColumnAtCursor(ExprColumn(inputExpr("ne
 Sheet.addCommand('g=', 'setcol-expr', 'cursorCol.setValuesFromExpr(someSelectedRows, inputExpr("set selected="), curcol=cursorCol)', 'set current column for selected rows to result of Python expression')
 Sheet.addCommand('z=', 'setcell-expr', 'cursorCol.setValues([cursorRow], evalExpr(inputExpr("set expr="), row=cursorRow, curcol=cursorCol))', 'evaluate Python expression on current row and set current cell with result of Python expression')
 Sheet.addCommand('gz=', 'setcol-iter', 'cursorCol.setValues(someSelectedRows, *list(itertools.islice(eval(input("set column= ", "expr", completer=CompleteExpr())), len(someSelectedRows))))', 'set current column for selected rows to the items in result of Python sequence expression')
+Sheet.addCommand('', 'addcol-iter', 'iter_expr=inputExpr("new column iterator expr: "); it = eval(iter_expr, getGlobals()); c=SettableColumn(); addColumnAtCursor(c); c.setValues(rows, *it)', 'add column with values from a Python sequence expression, repeating it if needed to fill')
 
 Sheet.addCommand(None, 'show-expr', 'status(evalExpr(inputExpr("show expr="), row=cursorRow, curcol=cursorCol))', 'evaluate Python expression on current row and show result on status line')
 

visidata/extensible.py

@@ -50,6 +50,7 @@ class Extensible:
         from visidata import vd
         func.importingModule = vd.importingModule
         setattr(cls, func.__name__, func)
+        func._extensible_api = True
         return func
 
     @classmethod
@@ -97,6 +98,7 @@ class Extensible:
         oldfunc = getattr(cls, name, None)
         if oldfunc:
             func = wraps(oldfunc)(func)
+        func._extensible_api = True
         setattr(cls, name, func)
         return func
 
@@ -107,6 +109,7 @@ class Extensible:
         def dofunc(self):
             return func(self)
         setattr(cls, func.__name__, dofunc)
+        func._extensible_api = True
         return dofunc
 
     @classmethod
@@ -121,6 +124,7 @@ class Extensible:
                 setattr(self, name, func(self))
             return getattr(self, name)
         setattr(cls, func.__name__, get_if_not)
+        func._extensible_api = True
         return get_if_not
 
     @classmethod

visidata/features/cmdpalette.py

@@ -1,6 +1,7 @@
 import collections
+import math
 from functools import partial
-from visidata import DrawablePane, BaseSheet, vd, VisiData, CompleteKey, clipdraw, HelpSheet, colors, AcceptInput, AttrDict, drawcache_property
+from visidata import DrawablePane, BaseSheet, vd, VisiData, CompleteKey, clipdraw, HelpSheet, colors, AcceptInput, AttrDict, drawcache_property, dispwidth
 
 
 vd.theme_option('color_cmdpalette', 'black on 72', 'base color of command palette')
@@ -58,14 +59,21 @@ def inputPalette(sheet, prompt, items,
                  formatter=lambda m, item, trigger_key: f'{trigger_key} {item}',
                  multiple=False,
                  **kwargs):
-    if sheet.options.disp_expert >= 5:
+    if not vd.wantsHelp('cmdpalette'):
         return vd.input(prompt,
                 completer=CompleteKey(sorted(item[value_key] for item in items)),
                 **kwargs)
 
     bindings = dict()
 
+    #state variables for navigating display of matches
+    prev_value = None
     tabitem = -1
+    offset = 0
+    def reset_display():
+        nonlocal tabitem, offset
+        tabitem = -1
+        offset = 0
 
     def tab(n, nitems):
         nonlocal tabitem
@@ -73,7 +81,11 @@ def inputPalette(sheet, prompt, items,
         tabitem = (tabitem + n) % nitems
 
     def _draw_palette(value):
-        words = value.lower().split()
+        nonlocal prev_value
+        words = value.split()
+        if value != prev_value:
+            reset_display()
+            prev_value = value
 
         if multiple and words:
             if value.endswith(' '):
@@ -83,8 +95,8 @@ def inputPalette(sheet, prompt, items,
                 finished_words = words[:-1]
                 unfinished_words = [words[-1]]
         else:
-            unfinished_words = words
             finished_words = []
+            unfinished_words = words
 
         unuseditems = [item for item in items if item[value_key] not in finished_words]
 
@@ -92,25 +104,50 @@ def inputPalette(sheet, prompt, items,
 
         h = sheet.windowHeight
         w = min(100, sheet.windowWidth)
-        nitems = min(h-1, sheet.options.disp_cmdpal_max)
+        nitems = min(h-2, sheet.options.disp_cmdpal_max)
+        if nitems <= 0:
+            return None
 
         useditems = []
         palrows = []
-
-        for m in matches[:nitems]:
-            useditems.append(m.match)
-            palrows.append((m, m.match))
-
-        favitems = sorted([item for item in unuseditems if item not in useditems],
-                          key=lambda item: -vd.usedInputs.get(item[value_key], 0))
-
-        for item in favitems[:nitems-len(palrows)]:
-            palrows.append((None, item))
+        n_results = 0
+        def read_matches(offset):
+            nonlocal useditems, palrows, value, n_results
+
+            useditems = []
+            palrows = []
+            for m in matches[offset:offset+nitems]:
+                useditems.append(m.match)
+                palrows.append((m, m.match))
+            n_results += len(matches)
+
+            #List matches only, usually. But list the available choices when there's no input,
+            #or (if multiple is True) they've just pressed space after a word.
+            if not unfinished_words:
+                favitems = sorted([item for item in unuseditems if item not in useditems],
+                                key=lambda item: -vd.usedInputs.get(item[value_key], 0))
+                for item in favitems[offset-len(palrows):offset+nitems-len(palrows)]:
+                    palrows.append((None, item))
+                n_results += len(favitems)
+        read_matches(offset)
+
+        def change_page(dir=+1):
+            nonlocal offset, n_results, nitems
+            new_offset = offset + dir*nitems
+            # constrain offset to be a multiple of nitems
+            new_offset = min(new_offset, ((n_results-1) // nitems)*nitems)
+            new_offset = max(new_offset, 0)
+            if new_offset == offset: return None
+            offset = new_offset
 
         navailitems = min(len(palrows), nitems)
 
         bindings['^I'] = lambda *args: tab(1, navailitems) or args
         bindings['KEY_BTAB'] = lambda *args: tab(-1, navailitems) or args
+        bindings['KEY_PPAGE'] = lambda *args: (change_page(-1) and read_matches(offset)) or args
+        bindings['KEY_NPAGE'] = lambda *args: (change_page(+1) and read_matches(offset)) or args
+        for numkey in '1234567890':
+            bindings.pop(numkey, None)
 
         for i in range(nitems-len(palrows)):
             palrows.append((None, None))
@@ -129,14 +166,13 @@ def inputPalette(sheet, prompt, items,
 
             if tabitem < 0 and palrows:
                 _ , topitem = palrows[0]
-                if not topitem: return
-                if multiple:
-                    bindings[' '] = partial(add_to_input, value=topitem[value_key])
-                    bindings['^J'] = partial(accept_input_if_subset, value=topitem[value_key])
-                else:
-                    bindings['^J'] = partial(accept_input, value=topitem[value_key])
+                if topitem:
+                    if multiple:
+                        bindings['^J'] = partial(accept_input_if_subset, value=topitem[value_key])
+                        bindings[' '] = partial(add_to_input, value=topitem[value_key])
+                    else:
+                        bindings['^J'] = partial(accept_input, value=topitem[value_key])
             elif item and i == tabitem:
-                if not item: return
                 if multiple:
                     bindings['^J'] = partial(accept_input_if_subset, value=item[value_key])
                     bindings[' '] = partial(add_to_input, value=item[value_key])
@@ -146,7 +182,10 @@ def inputPalette(sheet, prompt, items,
 
             match_summary = formatter(m, item, trigger_key) if item else ' '
 
-            clipdraw(sheet._scr, h-nitems-1+i, 0, match_summary, attr, w=w)
+            clipdraw(sheet._scr, h-nitems-2+i, 0, match_summary, attr, w=w)
+        attr = colors.color_cmdpalette
+        instr = 'Press [:keystrokes]PgUp/PgDn[/] to scroll items, [:keystrokes]Tab/Shift+Tab/Enter[/] to choose, [:keystrokes]Esc[/] to cancel.'
+        clipdraw(sheet._scr, h-2, 0, instr, attr, w=w)
 
         return None
 
@@ -180,7 +219,7 @@ def inputLongname(sheet):
         formatted_name = f'[:bold][:onclick {row.longname}]{formatted_longname}[/][/]'
         if vd.options.debug and match:
             keystrokes = f'[{match.score}]'
-        r = f' [:keystrokes]{keystrokes.rjust(len(prompt)-5)}[/]  '
+        r = f' [:keystrokes]{keystrokes.rjust(dispwidth(prompt)-5)}[/]  '
         if trigger_key:
             r += f'[:keystrokes]{trigger_key}[/]'
         else:
@@ -213,4 +252,4 @@ def exec_longname(sheet, longname):
 
 
 vd.addCommand('Space', 'exec-longname', 'exec_longname(inputLongname())', 'execute command by its longname')
-vd.addCommand('zSpace', 'exec-longname-simple', 'exec_longname(inputLongnameSimple())', 'execute command by its longname')
+vd.addCommand('zSpace', 'exec-longname-simple', 'exec_longname(inputLongnameSimple())', 'execute command by its longname (without command palette)')

visidata/features/describe.py

@@ -2,7 +2,7 @@ from copy import copy
 from statistics import mode, median, mean, stdev
 
 from visidata import vd, Column, ColumnAttr, vlen, RowColorizer, asyncthread, Progress, wrapply
-from visidata import BaseSheet, TableSheet, ColumnsSheet, SheetsSheet
+from visidata import BaseSheet, TableSheet, ColumnsSheet, IndexSheet
 
 
 vd.option('describe_aggrs', 'mean stdev', 'numeric aggregators to calculate on Describe sheet', help=vd.help_aggregators)
@@ -114,7 +114,7 @@ class DescribeSheet(ColumnsSheet):
 
 TableSheet.addCommand('I', 'describe-sheet', 'vd.push(DescribeSheet(sheet.name+"_describe", source=[sheet]))', 'open Describe Sheet with descriptive statistics for all visible columns')
 BaseSheet.addCommand('gI', 'describe-all', 'vd.push(DescribeSheet("describe_all", source=vd.stackedSheets))', 'open Describe Sheet with description statistics for all visible columns from all sheets')
-SheetsSheet.addCommand('gI', 'describe-selected', 'vd.push(DescribeSheet("describe_all", source=selectedRows))', 'open Describe Sheet with all visible columns from selected sheets')
+IndexSheet.addCommand('gI', 'describe-selected', 'vd.push(DescribeSheet("describe_all", source=selectedRows))', 'open Describe Sheet with all visible columns from selected sheets')
 
 DescribeSheet.addCommand('zs', 'select-cell', 'cursorRow.sheet.select(cursorValue)', 'select rows on source sheet which are being described in current cell')
 DescribeSheet.addCommand('zu', 'unselect-cell', 'cursorRow.sheet.unselect(cursorValue)', 'unselect rows on source sheet which are being described in current cell')

visidata/features/expand_cols.py

@@ -121,8 +121,10 @@ class ExpandedColumn(Column):
     def calcValue(self, row):
         return getitemdef(self.origCol.getValue(row), self.expr)
 
-    def setValue(self, row, value):
+    def setValue(self, row, value, setModified=True):
         self.origCol.getValue(row)[self.expr] = value
+        if setModified:
+            self.origCol.sheet.setModified()
 
 
 @Sheet.api
@@ -181,10 +183,10 @@ Sheet.addCommand('g(', 'expand-cols', 'expand_cols_deep(visibleCols, depth=1)',
 Sheet.addCommand('z(', 'expand-col-depth', 'expand_cols_deep([cursorCol], depth=int(input("expand depth=", value=0)))', 'expand current column of containers to given depth (0=fully)')
 Sheet.addCommand('gz(', 'expand-cols-depth', 'expand_cols_deep(visibleCols, depth=int(input("expand depth=", value=0)))', 'expand all visible columns of containers to given depth (0=fully)')
 
-Sheet.addCommand(')', 'contract-col', 'contract_cols([cursorCol])', 'remove current column and siblings from sheet columns and unhide parent')
-Sheet.addCommand('g)', 'contract-cols', 'contract_cols(visibleCols)', 'remove all child columns and unhide toplevel parents')
-Sheet.addCommand('z)', 'contract-col-depth', 'contract_cols([cursorCol], depth=int(input("contract depth=", value=0)))', 'remove current column and siblings from sheet columns and unhide parent')
-Sheet.addCommand('gz)', 'contract-cols-depth', 'contract_cols(visibleCols, depth=int(input("contract depth=", value=0)))', 'remove all child columns and unhide toplevel parents')
+Sheet.addCommand(')', 'contract-col', 'contract_cols([cursorCol])', 'remove current column and siblings from sheet columns and unhide parent (1 level)')
+Sheet.addCommand('g)', 'contract-cols', 'contract_cols(visibleCols)', 'remove all child columns and unhide toplevel parents (1 level)')
+Sheet.addCommand('z)', 'contract-col-depth', 'contract_cols([cursorCol], depth=int(input("contract depth=", value=0)))', 'remove current column and siblings from sheet columns and unhide parent, prompting for depth')
+Sheet.addCommand('gz)', 'contract-cols-depth', 'contract_cols(visibleCols, depth=int(input("contract depth=", value=0)))', 'remove all child columns and unhide toplevel parents, prompting for depth')
 
 ColumnsSheet.addCommand(')', 'contract-source-cols', 'source[0].addColumn(contract_source_cols(someSelectedRows), index=cursorRowIndex)', 'contract selected columns into column group')  #1702
 

visidata/features/freeze.py

@@ -60,10 +60,22 @@ class StaticSheet(Sheet):
                     row.append(val)
 
 
+@Sheet.api
+def setcol_freeze(sheet, unfrozen):  #2660  Contributed by @midichef
+    frozen = sheet.freeze_col(unfrozen)
+    frozen.name = unfrozen.name
+    unfrozen.hide()
+    vd.addUndoColNames([unfrozen])
+    unfrozen.name = frozen.name + '_unfrozen'
+    sheet.addColumnAtCursor(frozen)
+    vd.status(f'replaced {frozen.name} with frozen copy')
+
+
+Sheet.addCommand("z'", 'setcol-freeze', 'setcol_freeze(cursorCol)', 'replace current column with a frozen copy, with all cells evaluated')
 Sheet.addCommand("'", 'freeze-col', 'sheet.addColumnAtCursor(freeze_col(cursorCol))', 'add a frozen copy of current column with all cells evaluated')
 Sheet.addCommand("g'", 'freeze-sheet', 'vd.push(StaticSheet(sheet)); status("pushed frozen copy of "+name)', 'open a frozen copy of current sheet with all visible columns evaluated')
-Sheet.addCommand("z'", 'cache-col', 'cursorCol.resetCache()', 'add/reset cache for current column')
+Sheet.addCommand(None, 'cache-col', 'cursorCol.resetCache()', 'add/reset cache for current column')
 Sheet.addCommand("gz'", 'cache-cols', 'for c in visibleCols: c.resetCache()', 'add/reset cache for all visible columns')
 
-vd.addMenuItem('Column', 'Freeze', 'freeze-col')
+vd.addMenuItem('Column', 'Freeze', 'setcol-freeze')
 vd.addMenuItem('File', 'Freeze', 'freeze-sheet')