visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/sheets.py

@@ -3,87 +3,35 @@ import itertools
 from copy import copy, deepcopy
 import textwrap
 
-from visidata import VisiData, Extensible, globalCommand, ColumnAttr, ColumnItem, vd, ENTER, EscapeException, drawcache, drawcache_property, LazyChainMap, asyncthread, ExpectedException, setitem
+from visidata import VisiData, Extensible, globalCommand, ColumnAttr, ColumnItem, vd, ENTER, EscapeException, drawcache, drawcache_property, LazyChainMap, asyncthread, ExpectedException
 from visidata import (options, Column, namedlist, SettableColumn,
 TypedExceptionWrapper, BaseSheet, UNLOADED,
-clipdraw, ColorAttr, update_attr, colors, undoAttrFunc, vlen)
+clipdraw, clipdraw_chunks, ColorAttr, update_attr, colors, undoAttrFunc, vlen, dispwidth)
 import visidata
 
 
 vd.activePane = 1   # pane numbering starts at 1; pane 0 means active pane
 
 
-__all__ = ['RowColorizer', 'CellColorizer', 'ColumnColorizer', 'Sheet', 'TableSheet', 'IndexSheet', 'SheetsSheet', 'LazyComputeRow', 'SequenceSheet']
-
-
-vd.option('default_width', 20, 'default column width', replay=True)   # TODO: make not replay and remove from markdown saver
-vd.option('default_height', 4, 'default column height')
-vd.option('textwrap_cells', True, 'wordwrap text for multiline rows')
-
-vd.option('quitguard', False, 'confirm before quitting modified sheet')
-vd.option('debug', False, 'exit on error and display stacktrace')
-vd.option('skip', 0, 'skip N rows before header', replay=True)
-vd.option('header', 1, 'parse first N rows as column names', replay=True)
-vd.option('load_lazy', False, 'load subsheets always (False) or lazily (True)')
-
-vd.option('force_256_colors', False, 'use 256 colors even if curses reports fewer')
-
-vd.option('disp_note_none', '⌀',  'visible contents of a cell whose value is None')
-vd.option('disp_truncator', '…', 'indicator that the contents are only partially visible')
-vd.option('disp_oddspace', '\u00b7', 'displayable character for odd whitespace')
-vd.option('disp_more_left', '<', 'header note indicating more columns to the left')
-vd.option('disp_more_right', '>', 'header note indicating more columns to the right')
-vd.option('disp_error_val', '', 'displayed contents for computation exception')
-vd.option('disp_ambig_width', 1, 'width to use for unicode chars marked ambiguous')
-
-vd.option('disp_pending', '', 'string to display in pending cells')
-vd.option('note_pending', '⌛', 'note to display for pending cells')
-vd.option('note_format_exc', '?', 'cell note for an exception during formatting')
-vd.option('note_getter_exc', '!', 'cell note for an exception during computation')
-vd.option('note_type_exc', '!', 'cell note for an exception during type conversion')
-
-vd.option('color_note_pending', 'bold magenta', 'color of note in pending cells')
-vd.option('color_note_type', '226 yellow', 'color of cell note for non-str types in anytype columns')
-vd.option('color_note_row', '220 yellow', 'color of row note on left edge')
-vd.option('scroll_incr', -3, 'amount to scroll with scrollwheel')
-vd.option('disp_column_sep', '│', 'separator between columns')
-vd.option('disp_keycol_sep', '║', 'separator between key columns and rest of columns')
-vd.option('disp_rowtop_sep', '│', '') # ╷│┬╽⌜⌐▇
-vd.option('disp_rowmid_sep', '⁝', '') # ┃┊│█
-vd.option('disp_rowbot_sep', '⁝', '') # ┊┴╿⌞█⍿╵⎢┴⌊  ⋮⁝
-vd.option('disp_rowend_sep', '║', '') # ┊┴╿⌞█⍿╵⎢┴⌊
-vd.option('disp_keytop_sep', '║', '') # ╽╿┃╖╟
-vd.option('disp_keymid_sep', '║', '') # ╽╿┃
-vd.option('disp_keybot_sep', '║', '') # ╽╿┃╜‖
-vd.option('disp_endtop_sep', '║', '') # ╽╿┃╖╢
-vd.option('disp_endmid_sep', '║', '') # ╽╿┃
-vd.option('disp_endbot_sep', '║', '') # ╽╿┃╜‖
-vd.option('disp_selected_note', '•', '') #
-vd.option('disp_sort_asc', '↑↟⇞⇡⇧⇑', 'characters for ascending sort') # ↑▲↟↥↾↿⇞⇡⇧⇈⤉⤒⥔⥘⥜⥠⍏˄ˆ
-vd.option('disp_sort_desc', '↓↡⇟⇣⇩⇓', 'characters for descending sort') # ↓▼↡↧⇂⇃⇟⇣⇩⇊⤈⤓⥕⥙⥝⥡⍖˅ˇ
-vd.option('color_default', 'white on black', 'the default fg and bg colors')
-vd.option('color_default_hdr', 'bold', 'color of the column headers')
-vd.option('color_bottom_hdr', 'underline', 'color of the bottom header row')
-vd.option('color_current_row', 'reverse', 'color of the cursor row')
-vd.option('color_current_col', 'bold', 'color of the cursor column')
-vd.option('color_current_hdr', 'bold reverse', 'color of the header for the cursor column')
-vd.option('color_column_sep', '246 blue', 'color of column separators')
-vd.option('color_key_col', '81 cyan', 'color of key columns')
-vd.option('color_hidden_col', '8', 'color of hidden columns on metasheets')
-vd.option('color_selected_row', '215 yellow', 'color of selected rows')
-vd.option('name_joiner', '_', 'string to join sheet or column names')
-vd.option('value_joiner', ' ', 'string to join display values')
+vd.option('name_joiner', '_', 'string to join sheet or column names', max_help=0)
+vd.option('value_joiner', ' ', 'string to join display values', max_help=0)
 
 
-@VisiData.api
 @drawcache
-def splitcell(sheet, s, width=0):
+def _splitcell(sheet, s, width=0, maxheight=1):
     if width <= 0 or not sheet.options.textwrap_cells:
         return [s]
 
     ret = []
-    for L in s.splitlines():
-        ret.extend(textwrap.wrap(L, width=width, break_long_words=False, replace_whitespace=False))
+    for attr, text in s:
+        for line in textwrap.wrap(
+            text, width=width, break_long_words=False, replace_whitespace=False
+            ):
+            if len(ret) >= maxheight:
+                ret[-1][0][1] += ' ' + line
+                break
+            else:
+                ret.append([[attr, line]])
     return ret
 
 disp_column_fill = ' ' # pad chars after column value
@@ -92,6 +40,7 @@ disp_column_fill = ' ' # pad chars after column value
 # coloropt is the color option name (like 'color_error')
 # func(sheet,col,row,value) should return a true value if coloropt should be applied
 # if coloropt is None, func() should return a coloropt (or None) instead
+Colorizer = collections.namedtuple('Colorizer', 'precedence coloropt func')
 RowColorizer = collections.namedtuple('RowColorizer', 'precedence coloropt func')
 CellColorizer = collections.namedtuple('CellColorizer', 'precedence coloropt func')
 ColumnColorizer = collections.namedtuple('ColumnColorizer', 'precedence coloropt func')
@@ -169,6 +118,21 @@ class TableSheet(BaseSheet):
     _coltype = SettableColumn
 
     rowtype = 'rows'
+    guide = '# {sheet.help_title}\n{sheet.help_columns}\n'
+
+    @property
+    def help_title(self):
+        if isinstance(self.source, visidata.Path):
+            return 'Source Table'
+        else:
+            return 'Table Sheet'
+
+    @property
+    def help_columns(self):
+        hiddenCols = [c for c in self.columns if c.hidden]
+        if hiddenCols:
+            return f'- `gv` to unhide {len(hiddenCols)} hidden columns'
+        return ''
 
     columns = []  # list of Column
     colorizers = [ # list of Colorizer
@@ -176,14 +140,13 @@ class TableSheet(BaseSheet):
         ColumnColorizer(2, 'color_current_col', lambda s,c,r,v: c is s.cursorCol),
         ColumnColorizer(1, 'color_key_col', lambda s,c,r,v: c and c.keycol),
         CellColorizer(0, 'color_default', lambda s,c,r,v: True),
-        RowColorizer(2, 'color_selected_row', lambda s,c,r,v: r is not None and s.isSelected(r)),
         RowColorizer(1, 'color_error', lambda s,c,r,v: isinstance(r, (Exception, TypedExceptionWrapper))),
+        CellColorizer(3, 'color_current_cell', lambda s,c,r,v: c is s.cursorCol and r is s.cursorRow)
     ]
     nKeys = 0  # columns[:nKeys] are key columns
 
-    def __init__(self, *names, **kwargs):
-        super().__init__(*names, **kwargs)
-        self.rows = UNLOADED      # list of opaque row objects (UNLOADED before first reload)
+    def __init__(self, *names, rows=UNLOADED, **kwargs):
+        super().__init__(*names, rows=rows, **kwargs)
         self.cursorRowIndex = 0  # absolute index of cursor into self.rows
         self.cursorVisibleColIndex = 0  # index of cursor into self.visibleCols
 
@@ -196,11 +159,11 @@ class TableSheet(BaseSheet):
         self._visibleColLayout = {}      # [vcolidx] -> (x, w)
 
         # list of all columns in display order
-        self.columns = kwargs.get('columns') or [copy(c) for c in self.columns] or [Column('_')]
-        self._colorizers = []
-        self.recalc()  # set .sheet on columns and start caches
+        self.initialCols = kwargs.pop('columns', None) or type(self).columns
+        self.resetCols()
 
-        self.setKeys(self.columns[:self.nKeys])  # initial list of key columns
+        self._colorizers = self.classColorizers
+        self.recalc()  # set .sheet on columns and start caches
 
         self.__dict__.update(kwargs)  # also done earlier in BaseSheet.__init__
 
@@ -216,33 +179,30 @@ class TableSheet(BaseSheet):
     def addColorizer(self, c):
         'Add Colorizer *c* to the list of colorizers for this sheet.'
         self._colorizers.append(c)
+        self._colorizers = sorted(self._colorizers, key=lambda x: x.precedence, reverse=True)
 
     def removeColorizer(self, c):
         'Remove Colorizer *c* from the list of colorizers for this sheet.'
         self._colorizers.remove(c)
 
-    @drawcache_property
-    def allColorizers(self):
+    @property
+    def classColorizers(self) -> list:
+        'List of all colorizers from sheet class hierarchy in precedence order (highest precedence first)'
         # all colorizers must be in the same bucket
         # otherwise, precedence does not get applied properly
         _colorizers = set()
-        def allParents(cls):
-            yield from cls.__bases__
-            for b in cls.__bases__:
-                yield from allParents(b)
 
-        for b in [self] + list(allParents(self.__class__)):
+        for b in [self] + list(type(self).superclasses()):
             for c in getattr(b, 'colorizers', []):
                 _colorizers.add(c)
 
-        _colorizers |= set(self._colorizers)
         return sorted(_colorizers, key=lambda x: x.precedence, reverse=True)
 
     def _colorize(self, col, row, value=None) -> ColorAttr:
-        'Returns ColorAttr for the given colorizers/col/row/value'
+        'Return ColorAttr for the given colorizers/col/row/value'
 
         colorstack = []
-        for colorizer in self.allColorizers:
+        for colorizer in self._colorizers:
             try:
                 r = colorizer.func(self, col, row, value)
                 if r:
@@ -283,7 +243,33 @@ class TableSheet(BaseSheet):
 
     @asyncthread
     def reload(self):
-        'Load rows and/or columns from ``self.source``.  Async.  Override in subclass.'
+        'Load or reload rows and columns from ``self.source``.  Async.  Override resetCols() or loader() in subclass.'
+        with visidata.ScopedSetattr(self, 'loading', True):
+            self.resetCols()
+            self.beforeLoad()
+            try:
+                self.loader()
+                vd.debug(f'finished loading {self}')
+            finally:
+                self.afterLoad()
+
+        self.recalc()
+
+    def beforeLoad(self):
+        pass
+
+    def resetCols(self):
+        'Reset columns to class settings'
+        self.columns = []
+        for c in self.initialCols:
+            self.addColumn(deepcopy(c))
+            if self.options.disp_help > c.max_help:
+                c.hide()
+
+        self.setKeys(self.columns[:self.nKeys])
+
+    def loader(self):
+        'Reset rows and sync load ``source`` via iterload.  Overrideable.'
         self.rows = []
         try:
             with vd.Progress(gerund='loading', total=0):
@@ -292,15 +278,17 @@ class TableSheet(BaseSheet):
         except FileNotFoundError:
             return  # let it be a blank sheet without error
 
-        # if an ordering has been specified, sort the sheet
-        if self._ordering:
-            vd.sync(self.sort())
-
     def iterload(self):
         'Generate rows from ``self.source``.  Override in subclass.'
         if False:
             yield vd.fail('no iterload for this loader yet')
 
+    def afterLoad(self):
+        'hook for after loading has finished.  Overrideable (be sure to call super).'
+        # if an ordering has been specified, sort the sheet
+        if self._ordering:
+            vd.sync(self.sort())
+
     def iterrows(self):
         if self.rows is UNLOADED:
             try:
@@ -324,10 +312,24 @@ class TableSheet(BaseSheet):
         'Copy sheet design but remain unloaded. Deepcopy columns so their attributes (width, type, name) may be adjusted independently of the original.'
         ret = super().__copy__()
         ret.rows = UNLOADED
-        ret.columns = [copy(c) for c in self.keyCols]
-        ret.setKeys(ret.columns)
-        ret.columns.extend(copy(c) for c in self.columns if c not in self.keyCols)
-        ret.recalc()  # set .sheet on columns
+
+        ret.columns = []
+
+        col_mapping = {}
+        for c in self.columns:
+            new_col = copy(c)
+            col_mapping[c] = new_col
+            ret.addColumn(new_col)
+
+        ret.setKeys([col_mapping[c] for c in self.columns if c.keycol])
+
+        ret._ordering = []
+        for sortcol,reverse in self._ordering:
+            if isinstance(sortcol, str):
+                ret._ordering.append((sortcol,reverse))
+            else:
+                ret._ordering.append((col_mapping[sortcol],reverse))
+
         ret.topRowIndex = ret.cursorRowIndex = 0
         return ret
 
@@ -350,9 +352,12 @@ class TableSheet(BaseSheet):
         memo[id(self)] = ret
         return ret
 
-    def __repr__(self):
+    def __str__(self):
         return self.name
 
+    def __repr__(self):
+        return f'<{type(self).__name__}: {self.name}>'
+
     def evalExpr(self, expr, row=None, col=None):
         if row is not None:
             # contexts are cached by sheet/rowid for duration of drawcycle
@@ -390,7 +395,8 @@ class TableSheet(BaseSheet):
     @property
     def cursorRow(self):
         'The row object at the row cursor.'
-        return self.rows[self.cursorRowIndex] if self.nRows > 0 else None
+        idx = self.cursorRowIndex
+        return self.rows[idx] if self.nRows > idx else None
 
     @property
     def visibleRows(self):  # onscreen rows
@@ -402,16 +408,6 @@ class TableSheet(BaseSheet):
         'List of non-hidden columns in display order.'
         return self.keyCols + [c for c in self.columns if not c.hidden and not c.keycol]
 
-    def visibleColAtX(self, x):
-        for vcolidx, (colx, w) in self._visibleColLayout.items():
-            if colx <= x <= colx+w:
-                return vcolidx
-
-    def visibleRowAtY(self, y):
-        for rowidx, (rowy, h) in self._rowLayout.items():
-            if rowy <= y <= rowy+h-1:
-                return rowidx
-
     @drawcache_property
     def keyCols(self):
         'List of visible key columns.'
@@ -430,7 +426,10 @@ class TableSheet(BaseSheet):
     @property
     def cursorColIndex(self):
         'Index of current column into `Sheet.columns`. Linear search; prefer `cursorCol` or `cursorVisibleColIndex`.'
-        return self.columns.index(self.cursorCol)
+        try:
+            return self.columns.index(self.cursorCol)
+        except ValueError:
+            return None
 
     @property
     def nonKeyVisibleCols(self):
@@ -442,6 +441,13 @@ class TableSheet(BaseSheet):
         'String of key column names, for SheetsSheet convenience.'
         return ' '.join(c.name for c in self.keyCols)
 
+    @keyColNames.setter
+    def keyColNames(self, v):  #2122
+        'Set key columns on this sheet to the space-separated list of column names.'
+        newkeys = [self.column(colname) for colname in v.split()]
+        self.unsetKeys(self.keyCols)
+        self.setKeys(newkeys)
+
     @property
     def cursorCell(self):
         'Displayed value (DisplayWrapper) at current row and column.'
@@ -449,7 +455,7 @@ class TableSheet(BaseSheet):
 
     @property
     def cursorDisplay(self):
-        'Displayed value (DisplayWrapper.display) at current row and column.'
+        'Displayed value (DisplayWrapper.text) at current row and column.'
         return self.cursorCol.getDisplayValue(self.cursorRow)
 
     @property
@@ -506,12 +512,10 @@ class TableSheet(BaseSheet):
 
         if index is not None:
             self.setModified()
-        else:
-            for col in cols:
-                col.defer = self.defer
 
         for i, col in enumerate(cols):
             col.name = self.maybeClean(col.name)
+            col.defer = self.defer
 
             vd.addUndo(self.columns.remove, col)
             idx = len(self.columns) if index is None else index
@@ -607,7 +611,7 @@ class TableSheet(BaseSheet):
                     break
                 mincolidx, maxcolidx = min(self._visibleColLayout.keys()), max(self._visibleColLayout.keys())
                 if self.cursorVisibleColIndex < mincolidx:
-                    self.leftVisibleColIndex -= max((self.cursorVisibleColIndex - mincolid)//2, 1)
+                    self.leftVisibleColIndex -= max((self.cursorVisibleColIndex - mincolidx)//2, 1)
                     continue
                 elif self.cursorVisibleColIndex > maxcolidx:
                     self.leftVisibleColIndex += max((maxcolidx - self.cursorVisibleColIndex)//2, 1)
@@ -620,16 +624,25 @@ class TableSheet(BaseSheet):
 
     def calcColLayout(self):
         'Set right-most visible column, based on calculation.'
-        minColWidth = len(self.options.disp_more_left)+len(self.options.disp_more_right)+2
-        sepColWidth = len(self.options.disp_column_sep)
+        minColWidth = dispwidth(self.options.disp_more_left)+dispwidth(self.options.disp_more_right)+2
+        sepColWidth = dispwidth(self.options.disp_column_sep)
         winWidth = self.windowWidth
         self._visibleColLayout = {}
         x = 0
         vcolidx = 0
         for vcolidx in range(0, self.nVisibleCols):
+            width = self.calcSingleColLayout(vcolidx, x, minColWidth)
+            if width:
+                x += width+sepColWidth
+            if x > winWidth-1:
+                break
+
+        self.rightVisibleColIndex = vcolidx
+
+    def calcSingleColLayout(self, vcolidx:int, x:int=0, minColWidth:int=4):
             col = self.visibleCols[vcolidx]
             if col.width is None and len(self.visibleRows) > 0:
-                vrows = self.visibleRows if self.nRows > 1000 else self.rows
+                vrows = self.visibleRows if self.nRows > 1000 else self.rows[:1000]  #1964
                 # handle delayed column width-finding
                 col.width = max(col.getMaxWidth(vrows), minColWidth)
                 if vcolidx != self.nVisibleCols-1:  # let last column fill up the max width
@@ -638,12 +651,9 @@ class TableSheet(BaseSheet):
             if col in self.keyCols:
                 width = max(width, 1)  # keycols must all be visible
             if col in self.keyCols or vcolidx >= self.leftVisibleColIndex:  # visible columns
-                self._visibleColLayout[vcolidx] = [x, min(width, winWidth-x)]
-                x += width+sepColWidth
-            if x > winWidth-1:
-                break
+                self._visibleColLayout[vcolidx] = [x, min(width, self.windowWidth-x)]
+                return width
 
-        self.rightVisibleColIndex = vcolidx
 
     def drawColHeader(self, scr, y, h, vcolidx):
         'Compose and draw column header for given vcolidx.'
@@ -670,24 +680,23 @@ class TableSheet(BaseSheet):
 
         hdrs = col.name.split('\n')
         for i in range(h):
-            name = ' '  # save room at front for LeftMore or sorted arrow
+            name = ''
+            if colwidth > 2:
+                name = ' '  # save room at front for LeftMore or sorted arrow
 
             if h-i-1 < len(hdrs):
                 name += hdrs[::-1][h-i-1]
 
-            if len(name) > colwidth-1:
-                name = name[:colwidth-len(self.options.disp_truncator)] + self.options.disp_truncator
-
             if i == h-1:
                 hdrcattr = update_attr(hdrcattr, colors.color_bottom_hdr, 5)
 
-            clipdraw(scr, y+i, x, name, hdrcattr.attr, colwidth)
-            vd.onMouse(scr, y+i, x, 1, colwidth, BUTTON3_RELEASED='rename-col')
+            clipdraw(scr, y+i, x, name, hdrcattr, w=colwidth)
+            vd.onMouse(scr, x, y+i, colwidth, 1, BUTTON3_RELEASED='rename-col')
 
             if C and x+colwidth+len(C) < self.windowWidth and y+i < self.windowWidth:
                 scr.addstr(y+i, x+colwidth, C, sepcattr.attr)
 
-        clipdraw(scr, y+h-1, x+colwidth-len(T), T, hdrcattr.attr)
+        clipdraw(scr, y+h-1, x+colwidth-len(T), T, hdrcattr)
 
         try:
             if vcolidx == self.leftVisibleColIndex and col not in self.keyCols and self.nonKeyVisibleCols.index(col) > 0:
@@ -699,6 +708,8 @@ class TableSheet(BaseSheet):
         try:
             A = ''
             for j, (sortcol, sortdir) in enumerate(self._ordering):
+                if isinstance(sortcol, str):
+                    sortcol = self.column(sortcol)
                 if col is sortcol:
                     A = self.options.disp_sort_desc[j] if sortdir else self.options.disp_sort_asc[j]
                     scr.addstr(y+h-1, x, A, hdrcattr.attr)
@@ -713,10 +724,7 @@ class TableSheet(BaseSheet):
     def draw(self, scr):
         'Draw entire screen onto the `scr` curses object.'
         if not self.columns:
-            if self.options.debug:
-                self.addColumn(Column())
-            else:
-                return
+            return
 
         drawparams = {
             'isNull': self.isNullFunc(),
@@ -751,7 +759,7 @@ class TableSheet(BaseSheet):
         y = headerRow + numHeaderRows
 
         rows = self.rows[self.topRowIndex:min(self.topRowIndex+self.nScreenRows+1, self.nRows)]
-        self.checkCursorNoExceptions()
+        vd.callNoExceptions(self.checkCursor)
 
         for rowidx, row in enumerate(rows):
             if y >= self.windowHeight-1:
@@ -762,9 +770,10 @@ class TableSheet(BaseSheet):
             y += self.drawRow(scr, row, self.topRowIndex+rowidx, y, rowcattr, maxheight=self.windowHeight-y-1, **drawparams)
 
         if vcolidx+1 < self.nVisibleCols:
-            scr.addstr(headerRow, self.windowWidth-2, self.options.disp_more_right, colors.color_column_sep)
+            scr.addstr(headerRow, self.windowWidth-2, self.options.disp_more_right, colors.color_column_sep.attr)
 
-    def calc_height(self, row, displines=None, isNull=None):
+    def calc_height(self, row, displines=None, isNull=None, maxheight=1):
+            'render cell contents ifor row into displines'
             if displines is None:
                 displines = {}  # [vcolidx] -> list of lines in that cell
 
@@ -775,8 +784,8 @@ class TableSheet(BaseSheet):
                         continue
                     col = vcols[vcolidx]
                     cellval = col.getCell(row)
-                    if colwidth > 1 and vd.isNumeric(col):
-                        cellval.display = cellval.display.rjust(colwidth-2)
+
+                    cellval.display = col.display(cellval, colwidth)
 
                     try:
                         if isNull and isNull(cellval.value):
@@ -785,13 +794,15 @@ class TableSheet(BaseSheet):
                     except (TypeError, ValueError):
                         pass
 
-                    if col.voffset or col.height > 1:
-                        lines = splitcell(self, cellval.display, width=colwidth-2)
+                    if maxheight > 1:
+                        lines = _splitcell(self, cellval.display, width=colwidth-2, maxheight=maxheight)
                     else:
                         lines = [cellval.display]
                     displines[vcolidx] = (col, cellval, lines)
 
-            return self.rowHeight
+            if len(displines) == 0:
+                return 0
+            return max(len(lines) for _, _, lines in displines.values())
 
     def drawRow(self, scr, row, rowidx, ybase, rowcattr: ColorAttr, maxheight,
             isNull='',
@@ -820,8 +831,11 @@ class TableSheet(BaseSheet):
             else:
                 basecellcattr = rowcattr
 
+            # calc_height renders cell contents into displines
             displines = {}  # [vcolidx] -> list of lines in that cell
-            height = min(self.calc_height(row, displines), maxheight) or 1  # display even empty rows
+            self.calc_height(row, displines, maxheight=self.rowHeight)
+
+            height = min(self.rowHeight, maxheight) or 1  # display even empty rows
             self._rowLayout[rowidx] = (ybase, height)
 
             for vcolidx, (col, cellval, lines) in displines.items():
@@ -838,23 +852,16 @@ class TableSheet(BaseSheet):
                     notewidth = 1 if note else 0
                     if note:
                         notecattr = update_attr(cattr, colors.get_color(cellval.notecolor), 10)
-                        clipdraw(scr, ybase, x+colwidth-notewidth, note, notecattr.attr)
-
-                    if voffset >= 0:
-                        if len(lines)-voffset > height:
-                            # last line should always include as much as possible
-                            firstn = sum(len(i)+1 for i in lines[:voffset+height-1])
-                            lines = lines[:voffset+height]
-                            lines[-1] = cellval.display[firstn:][:col.width]
+                        scr.addstr(ybase, x+colwidth-notewidth, note, notecattr.attr)
 
                     lines = lines[voffset:]
 
                     if len(lines) > height:
                         lines = lines[:height]
                     elif len(lines) < height:
-                        lines.extend(['']*(height-len(lines)))
+                        lines.extend([[('', '')]]*(height-len(lines)))
 
-                    for i, line in enumerate(lines):
+                    for i, chunks in enumerate(lines):
                         y = ybase+i
 
                         if vcolidx == self.rightVisibleColIndex:  # right edge of sheet
@@ -889,8 +896,15 @@ class TableSheet(BaseSheet):
                                     sepchars = midsep
 
                         pre = disp_truncator if hoffset != 0 else disp_column_fill
-                        clipdraw(scr, y, x, (pre if colwidth > 2 else '')+line[hoffset:], cattr.attr, w=colwidth-notewidth)
-                        vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='edit-cell')
+                        prechunks = []
+                        if colwidth > 2:
+                            prechunks.append(('', pre))
+
+                        for attr, text in chunks:
+                            prechunks.append((attr, text[hoffset:]))
+
+                        clipdraw_chunks(scr, y, x, prechunks, cattr, w=colwidth-notewidth)
+                        vd.onMouse(scr, x, y, colwidth, 1, BUTTON3_RELEASED='edit-cell')
 
                         if x+colwidth+len(sepchars) <= self.windowWidth:
                             scr.addstr(y, x+colwidth, sepchars, sepcattr.attr)
@@ -904,7 +918,7 @@ class TableSheet(BaseSheet):
             return height
 
 vd.rowNoters = [
-        lambda sheet, row: sheet.isSelected(row) and sheet.options.disp_selected_note,
+    # f(sheet, row) -> character to be displayed on the left side of row
 ]
 
 Sheet = TableSheet  # deprecated in 2.0 but still widely used internally
@@ -914,6 +928,7 @@ class SequenceSheet(Sheet):
     'Sheets with ``ColumnItem`` columns, and rows that are Python sequences (list, namedtuple, etc).'
     def setCols(self, headerrows):
         self.columns = []
+        vd.clearCaches()  #1997
         for i, colnamelines in enumerate(itertools.zip_longest(*headerrows, fillvalue='')):
             colnamelines = ['' if c is None else c for c in colnamelines]
             self.addColumn(ColumnItem(''.join(map(str, colnamelines)), i))
@@ -939,8 +954,7 @@ class SequenceSheet(Sheet):
             except StopIteration:
                 break
 
-    @asyncthread
-    def reload(self):
+    def loader(self):
         'Skip first options.skip rows; set columns from next options.header rows.'
 
         itsource = self.iterload()
@@ -956,72 +970,6 @@ class SequenceSheet(Sheet):
         for r in vd.Progress(itsource, gerund='loading', total=0):
             self.addRow(r)
 
-        # if an ordering has been specified, sort the sheet
-        if self._ordering:
-            vd.sync(self.sort())
-
-
-class IndexSheet(Sheet):
-    'Base class for tabular sheets with rows that are Sheets.'
-    rowtype = 'sheets'  # rowdef: Sheet
-
-    columns = [
-        Column('name', getter=lambda c,r: r.names[-1], setter=lambda c,r,v: setitem(r.names, -1, v)),
-        ColumnAttr('rows', 'nRows', type=int, width=9),
-        ColumnAttr('cols', 'nCols', type=int),
-        ColumnAttr('keys', 'keyColNames'),
-        ColumnAttr('source'),
-    ]
-    nKeys = 1
-
-    def newRow(self):
-        return Sheet('', columns=[ColumnItem('', 0)], rows=[])
-
-    def openRow(self, row):
-        return row  # rowdef is Sheet
-
-    def getSheet(self, k):
-        for vs in self.rows:
-            if vs.name == k:
-                return vs
-
-    def addRow(self, sheet, **kwargs):
-        super().addRow(sheet, **kwargs)
-        if not self.options.load_lazy and not sheet.options.load_lazy:
-            sheet.ensureLoaded()
-
-    @asyncthread
-    def reloadSheets(self, sheets):
-        for vs in vd.Progress(sheets):
-            vs.reload()
-
-
-class SheetsSheet(IndexSheet):
-    columns = [
-        ColumnAttr('name'),
-        ColumnAttr('type', '__class__.__name__'),
-        ColumnAttr('pane', type=int),
-        Column('shortcut', getter=lambda c,r: getattr(r, 'shortcut'), setter=lambda c,r,v: setattr(r, '_shortcut', v)),
-        ColumnAttr('nRows', type=int),
-        ColumnAttr('nCols', type=int),
-        ColumnAttr('nVisibleCols', type=int),
-        ColumnAttr('cursorDisplay'),
-        ColumnAttr('keyColNames'),
-        ColumnAttr('source'),
-        ColumnAttr('progressPct'),
-#        ColumnAttr('threads', 'currentThreads', type=vlen),
-    ]
-    precious = False
-    nKeys = 1
-    def reload(self):
-        self.rows = self.source
-
-    def sort(self):
-        self.rows[1:] = sorted(self.rows[1:], key=self.sortkey)
-
-class GlobalSheetsSheet(SheetsSheet):  #1620
-    def sort(self):
-        IndexSheet.sort(self)
 
 @VisiData.property
 @drawcache
@@ -1077,15 +1025,6 @@ def push(vd, vs, pane=0, load=True):
         vs.ensureLoaded()
 
 
-@VisiData.lazy_property
-def allSheetsSheet(vd):
-    return GlobalSheetsSheet("sheets_all", source=vd.allSheets)
-
-@VisiData.lazy_property
-def sheetsSheet(vd):
-    return SheetsSheet("sheets", source=vd.sheets)
-
-
 @VisiData.api
 def quit(vd, *sheets):
     'Remove *sheets* from sheets stack, asking for confirmation if needed.'
@@ -1098,10 +1037,10 @@ def quit(vd, *sheets):
 
 @BaseSheet.api
 def confirmQuit(vs, verb='quit'):
-    if vs.options.quitguard and vs.precious and vs.hasBeenModified:
+    if vs.options.quitguard and vs.precious and vs.hasBeenModified and not vd._nextCommands:
         vd.draw_all()
         vd.confirm(f'{verb} modified sheet "{vs.name}"? ')
-    elif vs.options.getonly('quitguard', vs, False):  # if this sheet is specifically guarded
+    elif vs.options.getonly('quitguard', vs, False) and not vd._nextCommands:  # if this sheet is specifically guarded
         vd.draw_all()
         vd.confirm(f'{verb} guarded sheet "{vs.name}"? ')
 
@@ -1126,7 +1065,9 @@ def quitAndReleaseMemory(vs):
         vs.source.lines.clear() # clear cache of read lines
 
     if vs.precious: # only precious sheets have meaningful data
+        vs.confirmQuit('quit')
         vs.rows.clear()
+        vs.rows = UNLOADED
         vd.remove(vs)
         vd.allSheets.remove(vs)
 
@@ -1161,18 +1102,14 @@ def async_deepcopy(sheet, rowlist):
     return ret
 
 
-IndexSheet.options.header = 0
-IndexSheet.options.skip = 0
 
 BaseSheet.init('pane', lambda: 1)
 
-Sheet.init('_ordering', list, copy=True)  # (col:Column, reverse:bool)
+Sheet.init('_ordering', list, copy=False)  # (col:Column, reverse:bool)
 
-globalCommand('S', 'sheets-stack', 'vd.push(vd.sheetsSheet)', 'open Sheets Stack: join or jump between the active sheets on the current stack')
-globalCommand('gS', 'sheets-all', 'vd.push(vd.allSheetsSheet)', 'open Sheets Sheet: join or jump between all sheets from current session')
 
-BaseSheet.addCommand('^R', 'reload-sheet', 'preloadHook(); reload(); recalc(); status("reloaded")', 'Reload current sheet'),
-Sheet.addCommand('^G', 'show-cursor', 'status(statusLine)', 'show cursor position and bounds of current sheet on status line'),
+BaseSheet.addCommand('^R', 'reload-sheet', 'preloadHook(); reload()', 'Reload current sheet')
+Sheet.addCommand('^G', 'show-cursor', 'status(statusLine)', 'show cursor position and bounds of current sheet on status line')
 
 Sheet.addCommand('!', 'key-col', 'toggleKeys([cursorCol])', 'toggle current column as a key column')
 Sheet.addCommand('z!', 'key-col-off', 'unsetKeys([cursorCol])', 'unset current column as a key column')
@@ -1180,10 +1117,10 @@ Sheet.addCommand('z!', 'key-col-off', 'unsetKeys([cursorCol])', 'unset current c
 Sheet.addCommand('e', 'edit-cell', 'cursorCol.setValues([cursorRow], editCell(cursorVisibleColIndex)) if not (cursorRow is None) else fail("no rows to edit")', 'edit contents of current cell')
 Sheet.addCommand('ge', 'setcol-input', 'cursorCol.setValuesTyped(selectedRows, input("set selected to: ", value=cursorDisplay))', 'set contents of current column for selected rows to same input')
 
-Sheet.addCommand('"', 'dup-selected', 'vs=copy(sheet); vs.name += "_selectedref"; vs.reload=lambda vs=vs,rows=selectedRows: setattr(vs, "rows", list(rows)); vd.push(vs)', 'open duplicate sheet with only selected rows'),
-Sheet.addCommand('g"', 'dup-rows', 'vs=copy(sheet); vs.name+="_copy"; vs.rows=list(rows); status("copied "+vs.name); vs.select(selectedRows); vd.push(vs)', 'open duplicate sheet with all rows'),
-Sheet.addCommand('z"', 'dup-selected-deep', 'vs = deepcopy(sheet); vs.name += "_selecteddeepcopy"; vs.rows = vs.async_deepcopy(selectedRows); vd.push(vs); status("pushed sheet with async deepcopy of selected rows")', 'open duplicate sheet with deepcopy of selected rows'),
-Sheet.addCommand('gz"', 'dup-rows-deep', 'vs = deepcopy(sheet); vs.name += "_deepcopy"; vs.rows = vs.async_deepcopy(rows); vd.push(vs); status("pushed sheet with async deepcopy of all rows")', 'open duplicate sheet with deepcopy of all rows'),
+Sheet.addCommand('"', 'dup-selected', 'vs=copy(sheet); vs.name += "_selectedref"; vs.reload=lambda vs=vs,rows=selectedRows: setattr(vs, "rows", list(rows)); vd.push(vs)', 'open a duplicate sheet with only the selected rows')
+Sheet.addCommand('g"', 'dup-rows', 'vs=copy(sheet); vs.name+="_copy"; vs.rows=list(rows); status("copied "+vs.name); vs.select(selectedRows); vd.push(vs)', 'open a duplicate sheet with all rows')
+Sheet.addCommand('z"', 'dup-selected-deep', 'vs = deepcopy(sheet); vs.name += "_selecteddeepcopy"; vs.rows = vs.async_deepcopy(selectedRows); vd.push(vs); status("pushed sheet with async deepcopy of selected rows")', 'open duplicate sheet with deepcopy of selected rows')
+Sheet.addCommand('gz"', 'dup-rows-deep', 'vs = deepcopy(sheet); vs.name += "_deepcopy"; vs.rows = vs.async_deepcopy(rows); vd.push(vs); status("pushed sheet with async deepcopy of all rows")', 'open duplicate sheet with deepcopy of all rows')
 
 Sheet.addCommand('z~', 'type-any', 'cursorCol.type = anytype', 'set type of current column to anytype')
 Sheet.addCommand('~', 'type-string', 'cursorCol.type = str', 'set type of current column to str')
@@ -1192,14 +1129,6 @@ Sheet.addCommand('z#', 'type-len', 'cursorCol.type = vlen', 'set type of current
 Sheet.addCommand('%', 'type-float', 'cursorCol.type = float', 'set type of current column to float')
 Sheet.addCommand('', 'type-floatlocale', 'cursorCol.type = floatlocale', 'set type of current column to float using system locale set in LC_NUMERIC')
 
-# when diving into a sheet, remove the index unless it is precious
-IndexSheet.addCommand('g^R', 'reload-selected', 'reloadSheets(selectedRows or rows)', 'reload all selected sheets')
-SheetsSheet.addCommand('gC', 'columns-selected', 'vd.push(ColumnsSheet("all_columns", source=selectedRows))', 'open Columns Sheet with all visible columns from selected sheets')
-SheetsSheet.addCommand('gI', 'describe-selected', 'vd.push(DescribeSheet("describe_all", source=selectedRows))', 'open Describe Sheet with all visible columns from selected sheets')
-SheetsSheet.addCommand('z^C', 'cancel-row', 'cancelThread(*cursorRow.currentThreads)', 'abort async thread for current sheet')
-SheetsSheet.addCommand('gz^C', 'cancel-rows', 'for vs in selectedRows: cancelThread(*vs.currentThreads)', 'abort async threads for selected sheets')
-SheetsSheet.addCommand(ENTER, 'open-row', 'dest=cursorRow; vd.sheets.remove(sheet) if not sheet.precious else None; vd.push(openRow(dest))', 'open sheet referenced in current row')
-
 BaseSheet.addCommand('q', 'quit-sheet',  'vd.quit(sheet)', 'quit current sheet')
 BaseSheet.addCommand('Q', 'quit-sheet-free',  'quitAndReleaseMemory()', 'discard current sheet and free memory')
 globalCommand('gq', 'quit-all', 'vd.quit(*vd.sheets)', 'quit all sheets (clean exit)')
@@ -1210,20 +1139,19 @@ BaseSheet.addCommand('^I', 'splitwin-swap', 'vd.activePane = 1 if sheet.pane ==
 BaseSheet.addCommand('g^I', 'splitwin-swap-pane', 'vd.options.disp_splitwin_pct=-vd.options.disp_splitwin_pct', 'swap panes onscreen')
 BaseSheet.addCommand('zZ', 'splitwin-input', 'vd.options.disp_splitwin_pct = input("% height for split window: ", value=vd.options.disp_splitwin_pct)', 'set split pane to specific size')
 
-BaseSheet.addCommand('^L', 'redraw', 'vd.redraw(); sheet.refresh()', 'Refresh screen')
+BaseSheet.addCommand('^L', 'redraw', 'sheet.refresh(); vd.redraw()', 'Refresh screen')
 BaseSheet.addCommand(None, 'guard-sheet', 'options.set("quitguard", True, sheet); status("guarded")', 'Set quitguard on current sheet to confirm before quit')
 BaseSheet.addCommand(None, 'guard-sheet-off', 'options.set("quitguard", False, sheet); status("unguarded")', 'Unset quitguard on current sheet to not confirm before quit')
-BaseSheet.addCommand(None, 'open-source', 'vd.push(source)', 'jump to the source of this sheet')
+BaseSheet.addCommand(None, 'open-source', 'vd.replace(source)', 'jump to the source of this sheet')
 
 BaseSheet.bindkey('KEY_RESIZE', 'redraw')
 
 BaseSheet.addCommand('A', 'open-new', 'vd.push(vd.newSheet("unnamed", 1))', 'Open new empty sheet')
 
-Sheet.addCommand('^', 'rename-col', 'vd.addUndoColNames([cursorCol]); cursorCol.name = editCell(cursorVisibleColIndex, -1)', 'rename current column')
-Sheet.addCommand('z^', 'rename-col-selected', 'updateColNames(selectedRows or [cursorRow], [sheet.cursorCol], overwrite=True)', 'rename current column to combined contents of current cell in selected rows (or current row)')
-Sheet.addCommand('g^', 'rename-cols-row', 'updateColNames(selectedRows or [cursorRow], sheet.visibleCols)', 'rename all unnamed visible columns to contents of selected rows (or current row)')
-Sheet.addCommand('gz^', 'rename-cols-selected', 'updateColNames(selectedRows or [cursorRow], sheet.visibleCols, overwrite=True)', 'rename all visible columns to combined contents of selected rows (or current row)')
-BaseSheet.addCommand(None, 'rename-sheet', 'sheet.name = input("rename sheet to: ", value=sheet.name)', 'Rename current sheet')
+BaseSheet.addCommand('`', 'open-source', 'vd.push(source)', 'open source sheet')
+BaseSheet.addCommand(None, 'rename-sheet', 'sheet.name = input("rename sheet to: ", value=cleanName(sheet.name))', 'Rename current sheet')
+
+Sheet.addCommand('', 'addcol-source', 'source .addColumn(copy(cursorCol)) if isinstance (source, BaseSheet) else error("source must be sheet")', 'add copy of current column to source sheet')  #988  frosencrantz
 
 
 @Column.api
@@ -1232,3 +1160,47 @@ def formatter_enum(col, fmtdict):
 
 Sheet.addCommand('', 'setcol-formatter', 'cursorCol.formatter=input("set formatter to: ", value=cursorCol.formatter or "generic")', 'set formatter for current column (generic, json, python)')
 Sheet.addCommand('', 'setcol-format-enum', 'cursorCol.fmtstr=input("format replacements (k=v): ", value=f"{cursorDisplay}=", i=len(cursorDisplay)+1); cursorCol.formatter="enum"', 'add secondary type translator to current column from input enum (space-separated)')
+
+
+vd.addGlobals(
+    RowColorizer=RowColorizer,
+    CellColorizer=CellColorizer,
+    ColumnColorizer=ColumnColorizer,
+    RecursiveExprException=RecursiveExprException,
+    LazyComputeRow=LazyComputeRow,
+    Sheet=Sheet,
+    TableSheet=TableSheet,
+    SequenceSheet=SequenceSheet)
+
+vd.addMenuItems('''
+    File > New > open-new
+    File > Rename > rename-sheet
+    File > Guard > on > guard-sheet
+    File > Guard > off > guard-sheet-off
+    File > Duplicate > selected rows by ref > dup-selected
+    File > Duplicate > all rows by ref > dup-rows
+    File > Duplicate > selected rows deep > dup-selected-deep
+    File > Duplicate > all rows deep > dup-rows-deep
+    File > Reload > rows and columns > reload-sheet
+    File > Quit > top sheet > quit-sheet
+    File > Quit > all sheets > quit-all
+    Edit > Modify > current cell > input > edit-cell
+    Edit > Modify > selected cells > from input > setcol-input
+    View > Sheets > stack > sheets-stack
+    View > Sheets > all > sheets-all
+    View > Other sheet > source sheet > open-source
+    View > Split pane > in half > splitwin-half
+    View > Split pane > in percent > splitwin-input
+    View > Split pane > unsplit > splitwin-close
+    View > Split pane > swap panes > splitwin-swap-pane
+    View > Split pane > goto other pane > splitwin-swap
+    View > Refresh screen > redraw
+    Column > Type as > anytype > type-any
+    Column > Type as > string > type-string
+    Column > Type as > integer > type-int
+    Column > Type as > float > type-float
+    Column > Type as > locale float > type-floatlocale
+    Column > Type as > length > type-len
+    Column > Key > toggle current column > key-col
+    Column > Key > unkey current column > key-col-off
+''')