visidata 3.1__py3-none-any.whl → 3.2__py3-none-any.whl

visidata/features/describe.py

@@ -2,7 +2,7 @@ from copy import copy
 from statistics import mode, median, mean, stdev
 
 from visidata import vd, Column, ColumnAttr, vlen, RowColorizer, asyncthread, Progress, wrapply
-from visidata import BaseSheet, TableSheet, ColumnsSheet, SheetsSheet
+from visidata import BaseSheet, TableSheet, ColumnsSheet, IndexSheet
 
 
 vd.option('describe_aggrs', 'mean stdev', 'numeric aggregators to calculate on Describe sheet', help=vd.help_aggregators)
@@ -114,7 +114,7 @@ class DescribeSheet(ColumnsSheet):
 
 TableSheet.addCommand('I', 'describe-sheet', 'vd.push(DescribeSheet(sheet.name+"_describe", source=[sheet]))', 'open Describe Sheet with descriptive statistics for all visible columns')
 BaseSheet.addCommand('gI', 'describe-all', 'vd.push(DescribeSheet("describe_all", source=vd.stackedSheets))', 'open Describe Sheet with description statistics for all visible columns from all sheets')
-SheetsSheet.addCommand('gI', 'describe-selected', 'vd.push(DescribeSheet("describe_all", source=selectedRows))', 'open Describe Sheet with all visible columns from selected sheets')
+IndexSheet.addCommand('gI', 'describe-selected', 'vd.push(DescribeSheet("describe_all", source=selectedRows))', 'open Describe Sheet with all visible columns from selected sheets')
 
 DescribeSheet.addCommand('zs', 'select-cell', 'cursorRow.sheet.select(cursorValue)', 'select rows on source sheet which are being described in current cell')
 DescribeSheet.addCommand('zu', 'unselect-cell', 'cursorRow.sheet.unselect(cursorValue)', 'unselect rows on source sheet which are being described in current cell')

visidata/features/expand_cols.py

@@ -121,8 +121,10 @@ class ExpandedColumn(Column):
     def calcValue(self, row):
         return getitemdef(self.origCol.getValue(row), self.expr)
 
-    def setValue(self, row, value):
+    def setValue(self, row, value, setModified=True):
         self.origCol.getValue(row)[self.expr] = value
+        if setModified:
+            self.origCol.sheet.setModified()
 
 
 @Sheet.api
@@ -141,6 +143,7 @@ def contract_cols(sheet, cols, depth=1):  # depth == 0 means contract all the wa
             col.width = sheet.options.default_width
 
         sheet.columns = [col for col in sheet.columns if getattr(col, 'origCol', None) not in origCols]
+        sheet.calcColLayout()
 
 
 @Sheet.api
@@ -181,10 +184,10 @@ Sheet.addCommand('g(', 'expand-cols', 'expand_cols_deep(visibleCols, depth=1)',
 Sheet.addCommand('z(', 'expand-col-depth', 'expand_cols_deep([cursorCol], depth=int(input("expand depth=", value=0)))', 'expand current column of containers to given depth (0=fully)')
 Sheet.addCommand('gz(', 'expand-cols-depth', 'expand_cols_deep(visibleCols, depth=int(input("expand depth=", value=0)))', 'expand all visible columns of containers to given depth (0=fully)')
 
-Sheet.addCommand(')', 'contract-col', 'contract_cols([cursorCol])', 'remove current column and siblings from sheet columns and unhide parent')
-Sheet.addCommand('g)', 'contract-cols', 'contract_cols(visibleCols)', 'remove all child columns and unhide toplevel parents')
-Sheet.addCommand('z)', 'contract-col-depth', 'contract_cols([cursorCol], depth=int(input("contract depth=", value=0)))', 'remove current column and siblings from sheet columns and unhide parent')
-Sheet.addCommand('gz)', 'contract-cols-depth', 'contract_cols(visibleCols, depth=int(input("contract depth=", value=0)))', 'remove all child columns and unhide toplevel parents')
+Sheet.addCommand(')', 'contract-col', 'contract_cols([cursorCol])', 'remove current column and siblings from sheet columns and unhide parent (1 level)')
+Sheet.addCommand('g)', 'contract-cols', 'contract_cols(visibleCols)', 'remove all child columns and unhide toplevel parents (1 level)')
+Sheet.addCommand('z)', 'contract-col-depth', 'contract_cols([cursorCol], depth=int(input("contract depth=", value=0)))', 'remove current column and siblings from sheet columns and unhide parent, prompting for depth')
+Sheet.addCommand('gz)', 'contract-cols-depth', 'contract_cols(visibleCols, depth=int(input("contract depth=", value=0)))', 'remove all child columns and unhide toplevel parents, prompting for depth')
 
 ColumnsSheet.addCommand(')', 'contract-source-cols', 'source[0].addColumn(contract_source_cols(someSelectedRows), index=cursorRowIndex)', 'contract selected columns into column group')  #1702
 

visidata/features/freeze.py

@@ -60,10 +60,22 @@ class StaticSheet(Sheet):
                     row.append(val)
 
 
+@Sheet.api
+def setcol_freeze(sheet, unfrozen):  #2660  Contributed by @midichef
+    frozen = sheet.freeze_col(unfrozen)
+    frozen.name = unfrozen.name
+    unfrozen.hide()
+    vd.addUndoColNames([unfrozen])
+    unfrozen.name = frozen.name + '_unfrozen'
+    sheet.addColumnAtCursor(frozen)
+    vd.status(f'replaced {frozen.name} with frozen copy')
+
+
+Sheet.addCommand("z'", 'setcol-freeze', 'setcol_freeze(cursorCol)', 'replace current column with a frozen copy, with all cells evaluated')
 Sheet.addCommand("'", 'freeze-col', 'sheet.addColumnAtCursor(freeze_col(cursorCol))', 'add a frozen copy of current column with all cells evaluated')
 Sheet.addCommand("g'", 'freeze-sheet', 'vd.push(StaticSheet(sheet)); status("pushed frozen copy of "+name)', 'open a frozen copy of current sheet with all visible columns evaluated')
-Sheet.addCommand("z'", 'cache-col', 'cursorCol.resetCache()', 'add/reset cache for current column')
+Sheet.addCommand(None, 'cache-col', 'cursorCol.resetCache()', 'add/reset cache for current column')
 Sheet.addCommand("gz'", 'cache-cols', 'for c in visibleCols: c.resetCache()', 'add/reset cache for all visible columns')
 
-vd.addMenuItem('Column', 'Freeze', 'freeze-col')
+vd.addMenuItem('Column', 'Freeze', 'setcol-freeze')
 vd.addMenuItem('File', 'Freeze', 'freeze-sheet')

visidata/features/go_col.py

@@ -32,7 +32,7 @@ def nextColName(sheet, show_cells=True):
 
     def _fmt_colname(match, row, trigger_key):
         name = match.formatted.get('name', row.name) if match else row.name
-        r = ' '*(len(prompt)-3)
+        r = ' '*(dispwidth(prompt)-3)
         r += f'[:keystrokes]{trigger_key}[/]  '
         if show_cells and len(sheet.rows) > 0:
             # pad the right side with spaces
@@ -46,6 +46,7 @@ def nextColName(sheet, show_cells=True):
         else:
             r += name
         return r
+
     name = vd.activeSheet.inputPalette(prompt,
             colnames,
             value_key='name',

visidata/features/graph_zoom_y.py

@@ -0,0 +1,47 @@
+# contributed by Ram Rachum (@cool-RR) via ChatGPT  #2751
+
+from visidata import GraphSheet
+
+@GraphSheet.api
+def zoom_all_y(sheet):
+    """Find the lowest and highest y values in the current visible plot and set the y range to that."""
+    ymin, ymax = None, None
+    
+    # Find the min and max y values of all points within the current x range
+    xmin, xmax = sheet.visibleBox.xmin, sheet.visibleBox.xmax
+    
+    for vertexes, attr, row in sheet.polylines:
+        if attr in sheet.hiddenAttrs:
+            continue
+            
+        for x, y in vertexes:
+            # Check if the point is within the current x range
+            if xmin <= x <= xmax:
+                if ymin is None or y < ymin:
+                    ymin = y
+                if ymax is None or y > ymax:
+                    ymax = y
+    
+    if ymin is None or ymax is None:
+        # No visible points found
+        return
+    
+    # Add a 5% margin on both top and bottom
+    y_range = ymax - ymin
+    margin = sheet.ycols[0].type(y_range * 0.05)
+    
+    # Calculate adjusted min/max with margin
+    adj_ymin = ymin - margin
+    adj_ymax = ymax + margin
+    
+    # Create 5 equally spaced y ticks from real min to real max (not adjusted with margin)
+    step = (ymax - ymin) / 4  # 5 ticks means 4 intervals
+    y_ticks = tuple(ymin + step * i for i in range(5))
+    
+    # Set the y range with margin
+    sheet.set_y(f"{adj_ymin} {adj_ymax}")
+    
+    # Set custom y ticks (using the real min/max, not the adjusted ones)
+    sheet.forced_y_ticks = y_ticks
+
+GraphSheet.addCommand('g_', 'zoom-all-y', 'zoom_all_y()', 'Zoom y-axis to fit all visible data points')

visidata/features/incr.py

@@ -1,4 +1,4 @@
-from visidata import VisiData, Sheet, vd, options
+from visidata import VisiData, Sheet, vd
 
 
 vd.option('incr_base', 1.0, 'start value for column increments', replay=True)
@@ -7,10 +7,14 @@ vd.option('incr_base', 1.0, 'start value for column increments', replay=True)
 @VisiData.api
 def numrange(vd, n, step=1):
     'Generate n values, starting from options.incr_base and increasing by step for each number.'
-    base = type(step)(options.incr_base)
-    yield from (base+x*step for x in range(n))
+    base = type(step)(vd.options.incr_base)
+    yield from ((base+x)*step for x in range(n))
 
 
+def test_numrange(vd=None):
+    assert list(vd.numrange(5)) == [1,2,3,4,5]
+    assert list(vd.numrange(5, step=5)) == [5,10,15,20,25]  #2769
+
 @VisiData.api
 def num(vd, *args):
     'Return parsed string as number, preferring int to float.'

visidata/features/join.py

@@ -3,7 +3,7 @@ import itertools
 import functools
 from copy import copy
 
-from visidata import vd, VisiData, asyncthread, Sheet, Progress, IndexSheet, Column, CellColorizer, ColumnItem, SubColumnItem, TypedWrapper, ColumnsSheet, AttrDict
+from visidata import vd, VisiData, asyncthread, Sheet, Progress, IndexSheet, Column, CellColorizer, ColumnItem, SubColumnItem, TypedWrapper, ColumnsSheet, AttrDict, dispwidth
 
 vd.help_join = '# Join Help\nHELPTODO'
 
@@ -166,7 +166,8 @@ class MergeColumn(Column):
 
     def putValue(self, row, value):
         for vs, c in reversed(list(self.cols.items())):
-            c.setValue(row[vs], value)
+            if row[vs] is not None:
+                c.setValue(row[vs], value)
 
     def isDiff(self, row, value):
         col = list(self.cols.values())[0]
@@ -362,11 +363,11 @@ class ConcatSheet(Sheet):
 
 
 @VisiData.api
-def chooseJointype(vd):
+def inputJointype(vd):
     prompt = 'choose jointype: '
     def _fmt_aggr_summary(match, row, trigger_key):
         formatted_jointype = match.formatted.get('key', row.key) if match else row.key
-        r = ' '*(len(prompt)-3)
+        r = ' '*(dispwidth(prompt)-3)
         r += f'[:keystrokes]{trigger_key}[/]  '
         r += formatted_jointype
         if row.desc:
@@ -382,19 +383,29 @@ def chooseJointype(vd):
             type='jointype')
 
 
-IndexSheet.addCommand('&', 'join-selected', 'left, rights = someSelectedRows[0], someSelectedRows[1:]; vd.push(left.openJoin(rights, jointype=chooseJointype()))', 'merge selected sheets with visible columns from all, keeping rows according to jointype')
+IndexSheet.addCommand('&', 'join-selected', 'left, rights = someSelectedRows[0], someSelectedRows[1:]; vd.push(left.openJoin(rights, jointype=inputJointype()))', 'join selected sheets with visible columns from all, keeping rows according to jointype')
 IndexSheet.bindkey('g&', 'join-selected')
-Sheet.addCommand('&', 'join-sheets-top2', 'vd.push(openJoin(vd.sheets[1:2], jointype=chooseJointype()))', 'concatenate top two sheets in Sheets Stack')
-Sheet.addCommand('g&', 'join-sheets-all', 'vd.push(openJoin(vd.sheets[1:], jointype=chooseJointype()))', 'concatenate all sheets in Sheets Stack')
-
-ColumnsSheet.addCommand('&', 'join-sheets-cols', 'vd.push(join_sheets_cols(selectedRows, jointype=chooseJointype()))', '')
+Sheet.addCommand('&', 'join-sheets-top2', 'vd.push(openJoin(vd.sheets[1:2], jointype=inputJointype()))', 'join top two sheets on Sheets Stack')
+Sheet.addCommand('g&', 'join-sheets-all', 'vd.push(openJoin(vd.sheets[1:], jointype=inputJointype()))', 'join all sheets on Sheets Stack')
+ColumnsSheet.addCommand('&', 'join-sheets-cols', 'vd.push(join_sheets_cols(selectedRows, jointype=inputJointype()))', 'join sheets for selected columns')
 
 vd.addMenuItems('''
-    Data > Join > selected sheets > join-selected
-    Data > Join > top two sheets > join-sheets-top2
-    Data > Join > all sheets > join-sheets-all
+    Data > Join > Selected Sheets > choose jointype > join-selected
+    Data > Join > Top Two Sheets > choose jointype > join-sheets-top2
+    Data > Join > All Sheets > choose jointype > join-sheets-all
 ''')
 
+for d in vd.jointypes:
+    jointype, joinhelp = d.key, d.desc
+    IndexSheet.addCommand('', f'join-selected-{jointype}', 'left, rights = someSelectedRows[0], someSelectedRows[1:]; vd.push(left.openJoin(rights, jointype="{jointype}))', f'join selected sheets, keeping {joinhelp}')
+    Sheet.addCommand('', f'join-sheets-top2-{jointype}', f'vd.push(openJoin(vd.sheets[1:2], jointype="{jointype}"))', f'join top two sheets on Sheets Stack, keeping {joinhelp}')
+    Sheet.addCommand('', f'join-sheets-all-{jointype}', f'vd.push(openJoin(vd.sheets[1:], jointype="{jointype}"))', f'join all sheets on Sheets Stack, keeping {joinhelp}')
+    ColumnsSheet.addCommand('', 'join-cols-{jointype}', 'vd.push(join_sheets_cols(selectedRows, jointype=inputJointype()))', f'join sheets for selected columns, keeping {joinhelp}')
+
+    vd.addMenuItems(f'''Data > Join > Selected Sheets > {jointype} > join-selected-{jointype}''')
+    vd.addMenuItems(f'''Data > Join > Top Two Sheets > {jointype} > join-sheets-top2-{jointype}''')
+    vd.addMenuItems(f'''Data > Join > All Sheets > {jointype} > join-selected-{jointype}''')
+
 IndexSheet.guide += '''
     - `&` to join the selected sheets together
 '''

visidata/features/layout.py

@@ -40,20 +40,25 @@ def hide_col(vd, col):
 def hide_uniform_cols(sheet):
     if len(sheet.rows) < 2:
         return
-    for col in sheet.visibleCols:
+    idx = sheet.cursorVisibleColIndex
+    for i, col in enumerate(sheet.visibleCols):
         vals = (col.getTypedValue(r) for r in sheet.rows)
         first = next(vals)
         if all(v == first for v in vals):
+            vd.status(f'hide {col.name} with value: {col.getDisplayValue(sheet.rows[0])}')
+            if i <= idx:
+                sheet.cursorRight(-1)
             col.hide()
+            Sheet.clear_all_caches()  #2578
 
 Sheet.addCommand('_', 'resize-col-max', 'if cursorCol: cursorCol.toggleWidth(cursorCol.getMaxWidth(visibleRows))', 'toggle width of current column between full and default width')
 Sheet.addCommand('z_', 'resize-col-input', 'width = int(input("set width= ", value=cursorCol.width)); cursorCol.setWidth(width)', 'adjust width of current column to N')
-Sheet.addCommand('g_', 'resize-cols-max', 'for c in visibleCols: c.setWidth(c.getMaxWidth(visibleRows))', 'toggle widths of all visible columns between full and default width')
+Sheet.addCommand('g_', 'resize-cols-max', 'for c in visibleCols: c.toggleWidth(c.getMaxWidth(visibleRows))', 'toggle widths of all visible columns between full and default width')
 Sheet.addCommand('gz_', 'resize-cols-input', 'width = int(input("set width= ", value=cursorCol.width)); Fanout(visibleCols).setWidth(width)', 'adjust widths of all visible columns to N')
 
 Sheet.addCommand('-', 'hide-col', 'hide_col(cursorCol)', 'hide the current column')
 Sheet.addCommand('z-', 'resize-col-half', 'cursorCol.setWidth(cursorCol.width//2)', 'reduce width of current column by half')
-Sheet.addCommand(None, 'hide-uniform-cols', 'sheet.hide_uniform_cols()', 'hide any column that has multiple rows but only one distinct value')
+Sheet.addCommand('g-', 'hide-uniform-cols', 'sheet.hide_uniform_cols()', 'hide any column that has multiple rows but only one distinct value')
 
 Sheet.addCommand('gv', 'unhide-cols', 'unhide_cols(columns, visibleRows)', 'unhide all hidden columns on current sheet')
 Sheet.addCommand('v', 'toggle-multiline', 'for c in visibleCols: c.toggleMultiline()', 'toggle multiline display')

visidata/features/melt.py

@@ -104,3 +104,4 @@ vd.addMenuItems('''
     Data > Melt > nonkey columns > melt
     Data > Melt > nonkey columns by regex > melt-regex
 ''')
+vd.addGlobals(MeltedSheet=MeltedSheet)

visidata/features/rank.py

@@ -0,0 +1,103 @@
+import itertools
+
+from visidata import Sheet, ListAggregator, SettableColumn
+from visidata import vd, anytype, asyncthread, Progress
+
+class RankAggregator(ListAggregator):
+    '''
+    Ranks start at 1, and each group's rank is 1 higher than the previous group.
+    When elements are tied in ranking, each of them gets the same rank.
+    '''
+    def aggregate(self, col, rows) -> [int]:
+        return self.aggregate_list(col, rows)
+
+    def aggregate_list(self, col, rows) -> [int]:
+        if not col.sheet.keyCols:
+            vd.error('ranking requires one or more key columns')
+        return self.rank(col, rows)
+
+    def rank(self, col, rows):
+        if col.keycol:
+            vd.warning('rank aggregator is uninformative for key columns')
+        def _key_progress(prog):
+            def identity(val):
+                prog.addProgress(1)
+                return val
+            return identity
+        with Progress(gerund='ranking', total=4*col.sheet.nRows) as prog:
+            p = _key_progress(prog) # increment progress every time p() is called
+            # compile row data, for each row a list of tuples: (group_key, rank_key, rownum)
+            rowdata = [(col.sheet.rowkey(r), col.getTypedValue(r), p(rownum)) for rownum, r in enumerate(rows)]
+            # sort by row key and column value to prepare for grouping
+            # If the column is in descending order, use descending order for within-group ranking.
+            reverse = next((r for (c, r) in col.sheet.ordering if c == col or c == col.name), False)
+            try:
+                rowdata.sort(reverse=reverse, key=p)
+                if reverse:
+                    vd.status('ranking {col.name} in descending order')
+            except TypeError as e:
+                vd.fail(f'elements in a ranking column must be comparable: {e.args[0]}')
+            rowvals = []
+            #group by row key
+            for _, group in itertools.groupby(rowdata, key=lambda v: v[0]):
+                # within a group, the rows have already been sorted by col_val
+                group = list(group)
+                # rank each group individually
+                group_ranks = rank_sorted_iterable([p(col_val) for _, col_val, rownum in group])
+                rowvals += [(rownum, rank) for (_, _, rownum), rank in zip(group, group_ranks)]
+            # sort by unique rownum, to make rank results match the original row order
+            rowvals.sort(key=p)
+            rowvals = [ rank for rownum, rank in rowvals ]
+        return rowvals
+
+vd.aggregators['rank'] = RankAggregator('rank', anytype, helpstr='list of ranks, when grouping by key columns', listtype=int)
+
+def rank_sorted_iterable(vals_sorted) -> [int]:
+    '''*vals_sorted* is an iterable whose elements form one or more groups.
+    The iterable must already be sorted.'''
+
+    ranks = []
+    val_groups = itertools.groupby(vals_sorted)
+    for rank, (_, val_group) in enumerate(val_groups, 1):
+        for _ in val_group:
+            ranks.append(rank)
+    return ranks
+
+@Sheet.api
+@asyncthread
+def addcol_sheetrank(sheet, rows):
+    '''
+    Each row is ranked within its sheet. Rows are ordered by the
+    value of their key columns.
+    '''
+    if not sheet.keyCols:
+        vd.error('ranking requires one or more key columns')
+    colname = f'{sheet.name}_sheetrank'
+    c = SettableColumn(name=colname, type=int)
+    sheet.addColumnAtCursor(c)
+    def _key_progress(prog):
+        def identity(val):
+            prog.addProgress(1)
+            return val
+        return identity
+    with Progress(gerund='ranking', total=5*sheet.nRows) as prog:
+        p = _key_progress(prog) # increment progress every time p() is called
+        ordering = [(col, reverse) for (col, reverse) in sheet.ordering if col.keycol]
+        rowkeys = [(sheet.rowkey(r), p(rownum), r) for rownum, r in enumerate(rows)]
+        if ordering:
+            vd.status('using custom ordering for keycol sort')
+            keycols_ordered = [col for (col, reverse) in ordering]
+            keycols_unordered = [keycol for keycol in sheet.keyCols if not keycol in keycols_ordered]
+            ordering += [(keycol, False) for keycol in keycols_unordered]
+            def _sortkey(e): # sort the rows by using the column
+                p(None)
+                return sheet.sortkey(e[2], ordering=ordering)
+            rowkeys.sort(key=_sortkey)
+        else:
+            rowkeys.sort(key=p)
+        ranks = rank_sorted_iterable([p(rowkey) for rowkey, _, _ in rowkeys])
+        row_ranks = sorted(zip((rownum for _, rownum, _ in rowkeys), ranks), key=p)
+        row_ranks = [rank for rownum, rank in row_ranks]
+        c.setValues(sheet.rows, *[p(row_rank) for row_rank in row_ranks])
+
+Sheet.addCommand('', 'addcol-rank-sheet', 'sheet.addcol_sheetrank(rows)', 'add column with the rank of each row based on its key columns')

visidata/features/reload_every.py

@@ -1,14 +1,17 @@
 import os
 import time
 
-from visidata import vd, BaseSheet, Sheet, asyncthread, Path, ScopedSetattr
+from visidata import vd, BaseSheet, Sheet, asyncignore, asyncthread, Path, ScopedSetattr
 
 
 @BaseSheet.api
-@asyncthread
+@asyncignore
 def reload_every(sheet, seconds:int):
     while True:
-        sheet.reload()
+        # continue reloading till vd.remove() runs, like when the sheet is quit
+        if not sheet in vd.sheets:
+            break
+        vd.sync(sheet.reload())
         time.sleep(seconds)
 
 
@@ -20,13 +23,13 @@ def reload_modified(sheet):
     assert isinstance(p, Path)
     assert not p.is_url()
 
-    mtime = os.stat(p).st_mtime
+    mtime = 0
     while True:
-        time.sleep(1)
         t = os.stat(p).st_mtime
         if t != mtime:
             mtime = t
             sheet.reload_rows()
+        time.sleep(1)
 
 
 @Sheet.api
@@ -46,7 +49,7 @@ def reload_rows(self):
 
 BaseSheet.addCommand('', 'reload-every', 'sheet.reload_every(input("reload interval (sec): ", value=1))', 'schedule sheet reload every N seconds') #683
 BaseSheet.addCommand('', 'reload-modified', 'sheet.reload_modified()', 'reload sheet when source file modified (tail-like behavior)')  #1686
-BaseSheet.addCommand('z^R', 'reload-rows', 'preloadHook(); reload_rows(); status("reloaded")', 'Reload current sheet')
+Sheet.addCommand('z^R', 'reload-rows', 'preloadHook(); reload_rows(); status("reloaded")', 'Reload current sheet leaving current columns intact')
 
 vd.addMenuItems('''
     File > Reload > rows only > reload-rows

visidata/features/sysedit.py

@@ -1,5 +1,5 @@
 from copy import copy
-from visidata import vd, asyncthread, Path, Sheet, IndexSheet
+from visidata import vd, asyncthread, Path, Sheet, IndexSheet, TableSheet
 
 
 @Sheet.api
@@ -37,7 +37,17 @@ def syseditCells_async(sheet, cols, rows, filetype=None):
             tempcol = tempvs.colsByName.get(col.name)
             if not tempcol: # column not in edited version
                 continue
-            col.setValuesTyped(rows, *[tempcol.getTypedValue(r) for r in tempvs.rows])
-
-
+            # only assign values that were changed by the editor
+            edited_rows = []
+            edited_vals = []
+            for r, r_edited in zip(rows, tempvs.rows):
+                v = tempcol.getDisplayValue(r_edited)
+                if col.getDisplayValue(r) != v:
+                    edited_rows.append(r)
+                    edited_vals.append(v)
+            if edited_rows:
+                col.setValuesTyped(edited_rows, *edited_vals)
+
+
+TableSheet.addCommand('^O', 'sysedit-cell', 'cd = cursorDisplay; e = vd.launchExternalEditor(cd); cursorCol.setValues([cursorRow], e) if e != cd else None', 'edit current cell in external $EDITOR')
 Sheet.addCommand('g^O', 'sysedit-selected', 'syseditCells(visibleCols, onlySelectedRows)', 'edit rows in $EDITOR')

visidata/features/transpose.py

@@ -25,3 +25,4 @@ class TransposeSheet(Sheet):
 Sheet.addCommand('T', 'transpose', 'vd.push(TransposeSheet(name+"_T", source=sheet))', 'open new sheet with rows and columns transposed')
 
 vd.addMenuItems('Data > Transpose > transpose')
+vd.addGlobals(TransposeSheet=TransposeSheet)

visidata/features/window.py

@@ -36,6 +36,16 @@ class WindowColumn(Column):
 
         return self._windowrows
 
+    def __getstate__(self):
+        r = super().__getstate__()
+        r['before'] = self.before
+        r['after'] = self.after
+        r['sourcecol'] = self.sourcecol.name
+        return r
+
+    def __setstate__(self, r):
+        self.sourcecol = self.sheet.column(r.pop('sourcecol', None))
+        return super().__setstate__(r)
 
 @Sheet.api
 def addcol_window(sheet, curcol):
@@ -54,3 +64,5 @@ Sheet.addCommand('w', 'addcol-window', 'addcol_window(cursorCol)', 'add column w
 Sheet.addCommand('', 'select-around-n', 'select_around(input("select rows around selected: ", value=1))', 'select additional N rows before/after each selected row')
 
 vd.addMenuItem('Row', 'Select', 'N rows around each selected row', 'select-around-n')
+
+vd.addGlobals(WindowColumn=WindowColumn)

visidata/form.py

@@ -40,7 +40,7 @@ class FormCanvas(BaseSheet):
                 continue
             x, y = r.x, r.y
             if isinstance(y, float) and (0 < y < 1) or (-1 < y < 0): y = h*y
-            if isinstance(x, float) and (0 < x < 1) or (-1 < x < 0): x = w*x-(len(r.text)/2)
+            if isinstance(x, float) and (0 < x < 1) or (-1 < x < 0): x = w*x-(dispwidth(r.text)/2)
             x = int(x)
             y = int(y)
             if y < 0: y += h
@@ -52,7 +52,7 @@ class FormCanvas(BaseSheet):
             # underline first occurrence of r.key in r.text
             if hasattr(r, 'key') and r.key:
                 index = r.text.find(r.key)
-                clipdraw(scr, y, x+index, r.text[index:len(r.key)+1], colors[color + " underline"])
+                clipdraw(scr, y, x+index, r.text[index:index+len(r.key)], colors[color + " underline"])
             vd.onMouse(scr, x, y, dispwidth(r.text), 1,
                     BUTTON1_PRESSED=lambda y,x,key,r=r,sheet=self: sheet.onPressed(r),
                     BUTTON1_RELEASED=lambda y,x,key,r=r,sheet=self: sheet.onReleased(r))
@@ -61,7 +61,7 @@ class FormCanvas(BaseSheet):
         vd.setWindows(vd.scrFull)
         drawnrows = [r for r in self.source.rows if r.text]
         inputs = [r for r in self.source.rows if r.input]
-        maxw = max(int(r.x)+len(r.text) for r in drawnrows)
+        maxw = max(int(r.x)+dispwidth(r.text) for r in drawnrows)
         maxh = max(int(r.y) for r in drawnrows)
         h, w = vd.scrFull.getmaxyx()
         y, x = max(0, (h-maxh)//2-1), max(0, (w-maxw)//2-1)
@@ -103,7 +103,7 @@ class FormCanvas(BaseSheet):
 @VisiData.api
 def confirm(vd, prompt, exc=EscapeException):
     'Display *prompt* on status line and demand input that starts with "Y" or "y" to proceed.  Raise *exc* otherwise.  Return True.'
-    if vd.options.batch and not vd.options.interactive:
+    if vd.options.batch:
         return vd.fail('cannot confirm in batch mode: ' + prompt)
 
     form = FormSheet('confirm', rows=[

visidata/freqtbl.py

@@ -1,7 +1,7 @@
 from copy import copy
 import itertools
 
-from visidata import vd, asyncthread, vlen, VisiData, Column, AttrColumn, Sheet, ColumnsSheet, ENTER, Fanout
+from visidata import vd, vlen, VisiData, Column, AttrColumn, Sheet, ColumnsSheet, ENTER, Fanout
 from visidata.pivot import PivotSheet, PivotGroupRow
 
 
@@ -60,13 +60,57 @@ Each row on this sheet corresponds to a *bin* of rows on the source sheet that h
         return '+'.join(c.name for c in self.groupByCols)
 
     def selectRow(self, row):
-        self.source.select(row.sourcerows)     # select all entries in the bin on the source sheet
+        # Does not create an undo-operation for the select on the source rows. The caller should create undo-information itself.
+        self.source.select(row.sourcerows, add_undo=False)     # select all entries in the bin on the source sheet
         return super().selectRow(row)  # then select the bin itself on this sheet
 
     def unselectRow(self, row):
-        self.source.unselect(row.sourcerows)
+        self.source.unselect(row.sourcerows, add_undo=False)
         return super().unselectRow(row)
 
+    def addUndoSelection(self):
+        self.source.addUndoSelection()
+        super().addUndoSelection()
+
+    # override Sheet operations that handle multiple rows:
+    #     select(), unselect(), and toggle()
+    # to make undo more efficient. Without this optimization, the memory
+    # use for the undo-tracking on the source sheet is O(n^2) in the number
+    # of bins selected, which can easily exceed all available memory.
+    def select(self, rows, status=True, progress=True, add_undo=True):
+        if add_undo:
+            self.addUndoSelection()
+        super().select(rows, status, progress, add_undo=False)
+
+    def unselect(self, rows, status=True, progress=True, add_undo=True):
+        if add_undo:
+            self.addUndoSelection()
+        super().unselect(rows, status, progress, add_undo=False)
+
+    def toggle(self, rows, add_undo=True):
+        'Toggle selection of given *rows* and corresponding rows in source sheet.'
+        if add_undo:
+            self.addUndoSelection()
+        super().toggle(rows, add_undo=False)
+
+    def select_row(self, row, add_undo=True):
+        'Add single *row* to set of selected rows, and corresponding rows in source sheet.'
+        if add_undo:
+            self.addUndoSelection()
+        super().select_row(row, add_undo=False)
+
+    def unselect_row(self, row, add_undo=True):
+        'Remove single *row* from set of selected rows, and remove corresponding rows in source sheet.'
+        if add_undo:
+            self.addUndoSelection()
+        super().unselect_row(row, add_undo=False)
+
+    def toggle_row(self, row, add_undo=True):
+        'Toggle selection of given *row* and of corresponding rows in source sheet.'
+        if add_undo:
+            self.addUndoSelection()
+        super().toggle_row(row, add_undo=False)
+
     def resetCols(self):
         super().resetCols()
 

visidata/fuzzymatch.py

@@ -367,23 +367,26 @@ CombinedMatch = collections.namedtuple('CombinedMatch', 'score formatted match')
 
 @VisiData.api
 def fuzzymatch(vd, haystack:"list[dict[str, str]]", needles:"list[str]) -> list[CombinedMatch]"):
-    'Return sorted list of matching dict values in haystack, augmenting the input dicts with _score:int and _positions:dict[k,set[int]] where k is each non-_ key in the haystack dict.'
-
+    '''Perform case-insensitive matching. Return sorted list of matching dict values in haystack, augmenting the input dicts with _score:int and _positions:dict[k,set[int]] where k is each non-_ key in the haystack dict.'''
+    needles = [ p.lower() for p in needles]
     matches = []
     for h in haystack:
         match = {}
         formatted_hay = {}
         for k, v in h.items():
             if k[0] == '_': continue
+            positions = set()
+            v = v.lower()
             for p in needles:
                 mr = _fuzzymatch(v, p)
                 if mr.score > 0:
-                    match[k] = mr
-                    formatted_hay[k] = _format_match(v, mr.positions)
+                    match.setdefault(k, []).append(mr)
+                    positions |= set(mr.positions)
+            formatted_hay[k] = _format_match(v, positions)
 
         if match:
             # square to prefer larger scores in a single haystack
-            score = int(sum(mr.score**2 for mr in match.values()))
+            score = int(sum([mr.score**2 for mrs in match.values() for mr in mrs]))
             matches.append(CombinedMatch(score=score, formatted=formatted_hay, match=h))
 
     return sorted(matches, key=lambda m: -m.score)