visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/{join.py → features/join.py}

@@ -3,13 +3,16 @@ import itertools
 import functools
 from copy import copy
 
-from visidata import *
+from visidata import vd, VisiData, asyncthread, Sheet, Progress, IndexSheet, Column, CellColorizer, ColumnItem, SubColumnItem, TypedWrapper, ColumnsSheet, AttrDict
+
+vd.help_join = '# Join Help\nHELPTODO'
 
 @VisiData.api
 def ensureLoaded(vd, sheets):
     threads = [vs.ensureLoaded() for vs in sheets]
     threads = [t for t in threads if t]
-    vd.status('loading %d sheets' % len(threads))
+    if threads:
+        vd.status('loading %d source sheets' % len(threads))
     return threads
 
 
@@ -28,6 +31,20 @@ def _appendRowsAfterLoading(joinsheet, origsheets):
                 joinsheet.addColumn(newcol)
 
 
+@VisiData.api
+def join_sheets_cols(vd, cols, jointype:str=''):
+    'match joinkeys by cols in order per sheet.'
+    sheetkeys = collections.defaultdict(list)  # [sheet] -> list of keycols on that sheet
+    for c in cols:
+        sheetkeys[c.sheet].append(c)
+
+    sheets = list(sheetkeys.keys())
+    return JoinSheet('+'.join(vs.name for vs in sheets),
+                     sources=sheets,
+                     sheetKeyCols=sheetkeys,
+                     jointype=jointype)
+
+
 @Sheet.api
 def openJoin(sheet, others, jointype=''):
     sheets = [sheet] + others
@@ -53,19 +70,24 @@ def openJoin(sheet, others, jointype=''):
         name = '&'.join(vs.name for vs in sheets)
         return ConcatSheet(name, source=sheets)
 
-    for s in sheets:
-        s.keyCols or vd.fail(f'{s.name} has no key cols to join')
+    nkeys = set(len(s.keyCols) for s in sheets)
+    if 0 in nkeys or len(nkeys) != 1:
+        vd.fail(f'all sheets must have the same number of key columns')
 
     if jointype == 'extend':
         vs = copy(sheets[0])
         vs.name = '+'.join(vs.name for vs in sheets)
+        vs.sheetKeyCols = {vs:vs.keyCols for vs in sheets}
         vs.reload = functools.partial(ExtendedSheet_reload, vs, sheets)
         return vs
     else:
-        return JoinSheet('+'.join(vs.name for vs in sheets), sources=sheets, jointype=jointype)
+        return JoinSheet('+'.join(vs.name for vs in sheets),
+                         sources=sheets,
+                         jointype=jointype,
+                         sheetKeyCols={s:s.keyCols for s in sheets})
 
 
-vd.jointypes = [{'key': k, 'desc': v} for k, v in {
+vd.jointypes = [AttrDict(key=k, desc=v) for k, v in {
     'inner': 'only rows with matching keys on all sheets',
     'outer': 'only rows with matching keys on first selected sheet',
     'full': 'all rows from all sheets (union)',
@@ -73,27 +95,27 @@ vd.jointypes = [{'key': k, 'desc': v} for k, v in {
     'append': 'all rows from all sheets; columns from all sheets',
     'concat': 'all rows from all sheets; columns and type from first sheet',
     'extend': 'only rows from first sheet; type from first sheet; columns from all sheets',
-    'merge': 'merge differences from other sheets into first sheet',
+    'merge': 'merge differences from other sheets into first sheet (including new rows)',
 }.items()]
 
-def joinkey(sheet, row):
-    return tuple(c.getDisplayValue(row) for c in sheet.keyCols)
+def joinkey(sheetKeyCols, row):
+    return tuple(c.getDisplayValue(row) for c in sheetKeyCols)
 
 
-def groupRowsByKey(sheets, rowsBySheetKey, rowsByKey):
+def groupRowsByKey(sheets:dict, rowsBySheetKey, rowsByKey):
     with Progress(gerund='grouping', total=sum(len(vs.rows) for vs in sheets)*2) as prog:
         for vs in sheets:
             # tally rows by keys for each sheet
             rowsBySheetKey[vs] = collections.defaultdict(list)
             for r in vs.rows:
                 prog.addProgress(1)
-                key = joinkey(vs, r)
+                key = joinkey(sheets[vs], r)
                 rowsBySheetKey[vs][key].append(r)
 
         for vs in sheets:
             for r in vs.rows:
                 prog.addProgress(1)
-                key = joinkey(vs, r)
+                key = joinkey(sheets[vs], r)
                 if key not in rowsByKey: # gather for this key has not been done yet
                     # multiplicative for non-unique keys
                     rowsByKey[key] = [
@@ -114,11 +136,11 @@ class JoinKeyColumn(Column):
         vals = set()
         for i, c in enumerate(self.keycols):
             if row[c.sheet] is not None:
-                vals.add(c.getDisplayValue(row[c.sheet]))
-        if len(vals) == 1:
-            return vals.pop()
-        else:
-            raise Exception(f'inconsistent keys: ' + str(vals))
+                vals.add(c.getTypedValue(row[c.sheet]))
+        if len(vals) != 1:
+            keycolnames = ', '.join([f'{col.sheet.name}:{col.name}' for col in self.keycols])
+            vd.warning(f"source key columns ({keycolnames}) have different types")
+        return vals.pop()
 
     def putValue(self, row, value):
         for i, c in enumerate(self.keycols):
@@ -133,22 +155,23 @@ class JoinKeyColumn(Column):
 
 class MergeColumn(Column):
     def calcValue(self, row):
-        for vs, c in self.cols.items():
+        for vs, c in reversed(list(self.cols.items())):
             if c:
                 v = c.getTypedValue(row[vs])
                 if v and not isinstance(v, TypedWrapper):
                     return v
 
     def putValue(self, row, value):
-        for vs, c in reversed(self.cols.items()):
+        for vs, c in reversed(list(self.cols.items())):
             c.setValue(row[vs], value)
 
     def isDiff(self, row, value):
         col = list(self.cols.values())[0]
         return col and value != col.getValue(row[col.sheet])
 
+
 #### slicing and dicing
-# rowdef: [sheet1_row, sheet2_row, ...]
+# rowdef: {sheet1:sheet1_row, sheet2:sheet2_row, ...}
 #   if a sheet does not have this key, sheet#_row is None
 class JoinSheet(Sheet):
     'Column-wise join/merge. `jointype` constructor arg should be one of jointypes.'
@@ -156,8 +179,9 @@ class JoinSheet(Sheet):
         CellColorizer(0, 'color_diff', lambda s,c,r,v: c and r and isinstance(c, MergeColumn) and c.isDiff(r, v.value))
     ]
 
-    @asyncthread
-    def reload(self):
+    sheetKeyCols = {}  # [sheet] -> list of joinkeycols for that sheet
+
+    def loader(self):
         sheets = self.sources
 
         vd.ensureLoaded(sheets)
@@ -167,30 +191,32 @@ class JoinSheet(Sheet):
         # first columns are the key columns from the first sheet, using its row (0)
         self.columns = []
 
-        for i, cols in enumerate(itertools.zip_longest(*(s.keyCols for s in sheets))):
+        for i, cols in enumerate(itertools.zip_longest(*list(self.sheetKeyCols.values()))):
             self.addColumn(JoinKeyColumn(cols[0].name, keycols=cols)) # ColumnItem(c.name, i, sheet=sheets[0], type=c.type, width=c.width)))
         self.setKeys(self.columns)
 
         allcols = collections.defaultdict(dict) # colname: { sheet: origcol, ... }
         for sheetnum, vs in enumerate(sheets):
-            for c in vs.nonKeyVisibleCols:
-                allcols[c.name][vs] = c
+            for c in vs.visibleCols:
+                if c not in self.sheetKeyCols[vs]:
+                    allcols[c.name][vs] = c
 
         if self.jointype == 'merge':
             for colname, cols in allcols.items():
                 self.addColumn(MergeColumn(colname, cols=cols))
         else:
-          ctr = collections.Counter(c.name for vs in sheets for c in vs.nonKeyVisibleCols)
+          ctr = collections.Counter(c.name for vs in sheets for c in vs.visibleCols if c not in self.sheetKeyCols[vs])
           for sheetnum, vs in enumerate(sheets):
-            # subsequent elements are the rows from each source, in order of the source sheets
-            for c in vs.nonKeyVisibleCols:
-                newname = c.name if ctr[c.name] == 1 else '%s_%s' % (vs.name, c.name)
-                self.addColumn(SubColumnItem(vs, c, name=newname))
+              # subsequent elements are the rows from each source, in order of the source sheets
+              for c in vs.visibleCols:
+                  if c not in self.sheetKeyCols[vs]:
+                      newname = c.name if ctr[c.name] == 1 else '%s_%s' % (vs.name, c.name)
+                      self.addColumn(SubColumnItem(vs, c, name=newname))
 
         rowsBySheetKey = {}   # [sheet] -> { key:list(rows), ... }
         rowsByKey = {}  # [key] -> [{sheet1:row1, sheet2:row1, ... }, ...]
 
-        groupRowsByKey(sheets, rowsBySheetKey, rowsByKey)
+        groupRowsByKey(self.sheetKeyCols, rowsBySheetKey, rowsByKey)
 
         self.rows = []
 
@@ -221,13 +247,13 @@ class JoinSheet(Sheet):
 ## for ExtendedSheet_reload below
 class ExtendedColumn(Column):
     def calcValue(self, row):
-        key = joinkey(self.firstJoinSource, row)
+        key = joinkey(self.firstJoinSource.keyCols, row)
         srcrow = self.rowsBySheetKey[self.srcsheet][key]
         if srcrow:
             return self.sourceCol.calcValue(srcrow[0])
 
     def putValue(self, row, value):
-        key = joinkey(self.firstJoinSource, row)
+        key = joinkey(self.firstJoinSource.keyCols, row)
         srcrow = self.rowsBySheetKey[self.srcsheet][key]
         if len(srcrow) == 1:
             self.sourceCol.putValue(srcrow[0], value)
@@ -248,21 +274,23 @@ def ExtendedSheet_reload(self, sheets):
         self.addColumn(copy(c))
     self.setKeys(self.columns)
 
-    for i, c in enumerate(sheets[0].nonKeyVisibleCols):
-        self.addColumn(copy(c))
+    for i, c in enumerate(sheets[0].visibleCols):
+        if c not in self.sheetKeyCols[c.sheet]:
+            self.addColumn(copy(c))
 
     self.rowsBySheetKey = {}  # [srcSheet][key] -> list(rowobjs from sheets[0])
     rowsByKey = {}  # [key] -> [{sheet1:row1, sheet2:row1, ... }, ...]
 
     for sheetnum, vs in enumerate(sheets[1:]):
         # subsequent elements are the rows from each source, in order of the source sheets
-#        ctr = collections.Counter(c.name for c in vs.nonKeyVisibleCols)
-        for c in vs.nonKeyVisibleCols:
-            newname = '%s_%s' % (vs.name, c.name)
-            newcol = ExtendedColumn(newname, srcsheet=vs, rowsBySheetKey=self.rowsBySheetKey, firstJoinSource=sheets[0], sourceCol=c)
-            self.addColumn(newcol)
+#        ctr = collections.Counter(c.name for c in vs.visibleCols if c not in sheetkeys[vs])
+        for c in vs.visibleCols:
+            if c not in self.sheetKeyCols[c.sheet]:
+                newname = '%s_%s' % (vs.name, c.name)
+                newcol = ExtendedColumn(newname, srcsheet=vs, rowsBySheetKey=self.rowsBySheetKey, firstJoinSource=sheets[0], sourceCol=c)
+                self.addColumn(newcol)
 
-    groupRowsByKey(sheets, self.rowsBySheetKey, rowsByKey)
+    groupRowsByKey(self.sheetKeyCols, self.rowsBySheetKey, rowsByKey)
 
     self.rows = []
 
@@ -298,14 +326,10 @@ class ConcatColumn(Column):
 # rowdef: (srcSheet, srcRow)
 class ConcatSheet(Sheet):
     'combination of multiple sheets by row concatenation. source=list of sheets. '
-    @asyncthread
-    def reload(self):
-        self.columns = []
-        self.addColumn(ColumnItem('origin_sheet', 0, width=0))
-
+    columns = [ColumnItem('origin_sheet', 0, width=0)]
+    def iterload(self):
         # only one column with each name allowed per sheet
         keyedcols = collections.defaultdict(dict)  # name -> { sheet -> col }
-        self.rows = []
 
         with Progress(gerund='joining', sheet=self, total=sum(vs.nRows for vs in self.source)) as prog:
             for sheet in self.source:
@@ -313,7 +337,7 @@ class ConcatSheet(Sheet):
                     vd.sync()
 
                 for r in sheet.rows:
-                    self.addRow((sheet, r))
+                    yield (sheet, r)
                     prog.addProgress(1)
 
                 for idx, col in enumerate(sheet.visibleCols):
@@ -327,10 +351,40 @@ class ConcatSheet(Sheet):
                         keyedcols[col.name][sheet] = col
 
 
-
-IndexSheet.addCommand('&', 'join-selected', 'left, rights = someSelectedRows[0], someSelectedRows[1:]; vd.push(left.openJoin(rights, jointype=chooseOne(jointypes)))', 'merge selected sheets with visible columns from all, keeping rows according to jointype')
+@VisiData.api
+def chooseJointype(vd):
+    prompt = 'choose jointype: '
+    def _fmt_aggr_summary(match, row, trigger_key):
+        formatted_jointype = match.formatted.get('key', row.key) if match else row.key
+        r = ' '*(len(prompt)-3)
+        r += f'[:keystrokes]{trigger_key}[/]  '
+        r += formatted_jointype
+        if row.desc:
+            r += ' - '
+            r += match.formatted.get('desc', row.desc) if match else row.desc
+        return r
+
+    return vd.activeSheet.inputPalette(prompt,
+            vd.jointypes,
+            value_key='key',
+            formatter=_fmt_aggr_summary,
+            help=vd.help_join,
+            type='jointype')
+
+
+IndexSheet.addCommand('&', 'join-selected', 'left, rights = someSelectedRows[0], someSelectedRows[1:]; vd.push(left.openJoin(rights, jointype=chooseJointype()))', 'merge selected sheets with visible columns from all, keeping rows according to jointype')
 IndexSheet.bindkey('g&', 'join-selected')
-Sheet.addCommand('&', 'join-sheets-top2', 'vd.push(openJoin(vd.sheets[1:2], jointype=chooseOne(jointypes)))', 'concatenate top two sheets in Sheets Stack')
-Sheet.addCommand('g&', 'join-sheets-all', 'vd.push(openJoin(vd.sheets[1:], jointype=chooseOne(jointypes)))', 'concatenate all sheets in Sheets Stack')
+Sheet.addCommand('&', 'join-sheets-top2', 'vd.push(openJoin(vd.sheets[1:2], jointype=chooseJointype()))', 'concatenate top two sheets in Sheets Stack')
+Sheet.addCommand('g&', 'join-sheets-all', 'vd.push(openJoin(vd.sheets[1:], jointype=chooseJointype()))', 'concatenate all sheets in Sheets Stack')
+
+ColumnsSheet.addCommand('&', 'join-sheets-cols', 'vd.push(join_sheets_cols(selectedRows, jointype=chooseJointype()))', '')
+
+vd.addMenuItems('''
+    Data > Join > selected sheets > join-selected
+    Data > Join > top two sheets > join-sheets-top2
+    Data > Join > all sheets > join-sheets-all
+''')
 
-vd.addMenuItem('Data', 'Join', 'selected sheets', 'join-selected')
+IndexSheet.guide += '''
+    - `&` to join the selected sheets together
+'''

visidata/features/known_cols.py

@@ -0,0 +1,21 @@
+'''#1488
+Usage (in .visidatarc):
+
+
+    DirSheet.knownCols.directory.width = 0
+    Sheet.knownCols.date.type = date
+'''
+
+from visidata import Sheet, DefaultAttrDict
+
+
+Sheet.knownCols = DefaultAttrDict()
+
+
+@Sheet.before
+def afterLoad(sheet):
+    for colname, attrs in sheet.knownCols.items():
+        col = sheet.colsByName.get(colname)
+        if col:
+            for k, v in attrs.items():
+                setattr(col, k, v)

visidata/features/layout.py

@@ -0,0 +1,62 @@
+from visidata import VisiData, vd, Column, Sheet, Fanout
+
+@Column.api
+def setWidth(self, w):
+    if self.width != w:
+        if self.width == 0 or w == 0:  # hide/unhide
+            vd.addUndo(setattr, self, '_width', self.width)
+    self._width = w
+
+
+@Column.api
+def toggleWidth(self, width):
+    'Change column width to either given `width` or default value.'
+    if self.width != width:
+        self.width = width
+    else:
+        self.width = int(self.sheet.options.default_width)
+
+
+@Column.api
+def toggleMultiline(self):
+    if self.height == 1:
+        self.height = self.sheet.options.default_height
+    else:
+        self.height = 1
+
+@VisiData.api
+def unhide_cols(vd, cols, rows):
+    'sets appropriate width if column was either hidden (0) or unseen (None)'
+    for c in cols:
+        c.setWidth(abs(c.width or 0) or c.getMaxWidth(rows))
+
+@VisiData.api
+def hide_col(vd, col):
+    if not col: vd.fail("no columns to hide")
+    col.hide()
+
+Sheet.addCommand('_', 'resize-col-max', 'if cursorCol: cursorCol.toggleWidth(cursorCol.getMaxWidth(visibleRows))', 'toggle width of current column between full and default width')
+Sheet.addCommand('z_', 'resize-col-input', 'width = int(input("set width= ", value=cursorCol.width)); cursorCol.setWidth(width)', 'adjust width of current column to N')
+Sheet.addCommand('g_', 'resize-cols-max', 'for c in visibleCols: c.setWidth(c.getMaxWidth(visibleRows))', 'toggle widths of all visible columns between full and default width')
+Sheet.addCommand('gz_', 'resize-cols-input', 'width = int(input("set width= ", value=cursorCol.width)); Fanout(visibleCols).setWidth(width)', 'adjust widths of all visible columns to N')
+
+Sheet.addCommand('-', 'hide-col', 'hide_col(cursorCol)', 'Hide current column')
+Sheet.addCommand('z-', 'resize-col-half', 'cursorCol.setWidth(cursorCol.width//2)', 'reduce width of current column by half')
+
+Sheet.addCommand('gv', 'unhide-cols', 'unhide_cols(columns, visibleRows)', 'Unhide all hidden columns')
+Sheet.addCommand('v', 'toggle-multiline', 'for c in visibleCols: c.toggleMultiline()', 'toggle multiline display')
+Sheet.addCommand('zv', 'resize-height-input', 'Fanout(visibleCols).height=int(input("set height for all columns to: ", value=max(c.height for c in sheet.visibleCols)))', 'resize row height to N')
+Sheet.addCommand('gzv', 'resize-height-max', 'h=calc_height(cursorRow, {}, maxheight=windowHeight-1); vd.status(f"set height for all columns to {h}"); Fanout(visibleCols).height=h', 'resize row height to max height needed to see this row')
+
+vd.addMenuItems('''
+    Column > Hide > hide-col
+    Column > Unhide all > unhide-cols
+    Column > Resize > half width > resize-col-half
+    Column > Resize > current column width to max > resize-col-max
+    Column > Resize > current column width to N > resize-col-input
+    Column > Resize > all columns width to max > resize-cols-max
+    Column > Resize > all columns width to N > resize-cols-input
+    Row > Resize > height to N > resize-height-input
+    Row > Resize > height to max > resize-height-max
+    View > Toggle display > multiline > toggle-multiline
+''')

visidata/{melt.py → features/melt.py}

@@ -1,34 +1,27 @@
 import collections
 import re
+from copy import copy
 
-from visidata import *
-
+from visidata import Sheet, SubColumnItem, ColumnItem, Column, Progress
+from visidata import asyncthread, vd
 
 melt_var_colname = 'Variable' # column name to use for the melted variable name
 melt_value_colname = 'Value'  # column name to use for the melted value
-melt_null = False   # whether to melt null values
+melt_null = False   # whether to include rows for null values during melt
 
 
 # rowdef: {0:sourceRow, 1:Category1, ..., N:CategoryN, ColumnName:Column, ...}
 class MeltedSheet(Sheet):
-    "Perform 'melt', the inverse of 'pivot', on input sheet."
+    "Perform 'melt', the inverse of 'pivot', on input `source` sheet.  Pass `regex` to parse column names into additional columns"
 
     rowtype = 'melted values'
 
-    @asyncthread
-    def reload(self):
-        self.columns = []
-        isNull = self.isNullFunc()
-
-        sheet = self.source
-        for c in sheet.keyCols:
-            self.addColumn(SubColumnItem(0, c))
-        self.setKeys(self.columns)
-
-        colsToMelt = [copy(c) for c in sheet.nonKeyVisibleCols]
+    def getValueCols(self) -> dict:
+        '''Return dict of ('Category1', 'Category2') -> list of tuple('ColumnName', Column)'''
+        colsToMelt = [copy(c) for c in self.source.nonKeyVisibleCols]
 
         # break down Category1_Category2_ColumnName as per regex
-        valcols = collections.OrderedDict()  # ('Category1', 'Category2') -> list of tuple('ColumnName', Column)
+        valcols = collections.OrderedDict()
         for c in colsToMelt:
             c.aggregators = [vd.aggregators['max']]
             m = re.match(self.regex, c.name)
@@ -44,11 +37,23 @@ class MeltedSheet(Sheet):
                 valcols[cats].append((valcolname, c))
                 ncats = len(varvals)
             else:
-                vd.status('"%s" column does not match regex, skipping' % c.name)
+                vd.status(f'"{c.name}" column does not match regex, skipping')
                 ncats = 0
 
+        return valcols
+
+    def resetCols(self):
+        self.columns = []
+        sheet = self.source
+        for c in sheet.keyCols:
+            self.addColumn(SubColumnItem(0, c))
+        self.setKeys(self.columns)
+
+        valcols = self.getValueCols()
         othercols = set()
+        ncats = 0
         for colnames, cols in valcols.items():
+            ncats = max(ncats, len(colnames))
             for cname, _ in cols:
                 othercols.add(cname)
 
@@ -56,7 +61,7 @@ class MeltedSheet(Sheet):
             self.addColumn(ColumnItem(melt_var_colname, 1))
         else:
             for i in range(ncats):
-                self.addColumn(ColumnItem('%s%d' % (melt_var_colname, i+1), i+1))
+                self.addColumn(ColumnItem(f'{melt_var_colname}{i+1}', i+1))
 
         for cname in othercols:
             self.addColumn(Column(cname,
@@ -64,7 +69,10 @@ class MeltedSheet(Sheet):
                 setter=lambda col,row,val,cname=cname: row[cname].setValues([row[0]], val),
                 aggregators=[vd.aggregators['max']]))
 
-        self.rows = []
+    def iterload(self):
+        isNull = self.isNullFunc()
+
+        valcols = self.getValueCols()
         for r in Progress(self.source.rows, 'melting'):
             for colnames, cols in valcols.items():
                 meltedrow = {}
@@ -80,7 +88,7 @@ class MeltedSheet(Sheet):
                     for i, colname in enumerate(colnames):
                         meltedrow[i+1] = colname
 
-                    self.addRow(meltedrow)
+                    yield meltedrow
 
 
 @Sheet.api
@@ -90,5 +98,9 @@ def openMelt(sheet, regex='(.*)'):
 
 Sheet.addCommand('M', 'melt', 'vd.push(openMelt())', 'open Melted Sheet (unpivot), with key columns retained and all non-key columns reduced to Variable-Value rows')
 
+Sheet.addCommand('gM', 'melt-regex', 'vd.push(openMelt(vd.inputRegex("regex to split colname: ", value="(.*)_(.*)", type="regex-capture")))', 'open Melted Sheet (unpivot), with key columns retained and regex capture groups determining how the non-key columns will be reduced to Variable-Value rows')
 
-Sheet.addCommand('gM', 'melt-regex', 'vd.push(openMelt(vd.input("regex to split colname: ", value="(.*)_(.*)", type="regex-capture")))', 'open Melted Sheet (unpivot), with key columns retained and regex capture groups determining how the non-key columns will be reduced to Variable-Value rows')
+vd.addMenuItems('''
+    Data > Melt > nonkey columns > melt
+    Data > Melt > nonkey columns by regex > melt-regex
+''')

visidata/features/normcol.py

@@ -0,0 +1,118 @@
+"""
+# Usage
+
+This plugin normalizes column names in any given sheet, so that the names are:
+
+- Composed only of lowercase letters, numbers, and underscores.
+
+- Valid Python identifiers. This is mostly handled by the rule above, but also
+  prohibits names beginning with a digit; that is handled by prefixing those
+  names with an underscore.
+
+- Unique within the sheet. Non-unique names are suffixed with "__" and an
+  integer.
+
+Unnamed columns are left as such.
+
+For instance, a sheet with the following columns names:
+
+- "Genus, Species"
+- "Height"
+- "5-score"
+- "Height"
+- ""
+- ""
+
+... would be converted to have the following column names:
+
+- "genus_species"
+- "height__0"
+- "_5_score"
+- "height__1"
+- ""
+- ""
+
+## Commands
+
+- `normalize-col-names` normalizes the names of all *non-hidden* columns in the
+  active sheet, per the approach described above.
+
+"""
+
+__author__ = "Jeremy Singer-Vine <jsvine@gmail.com>"
+
+from visidata import vd, Sheet, asyncthread, Progress
+from collections import Counter
+import re
+import string
+
+nonalphanum_pat = re.compile(r"[^a-z0-9]+")
+
+
+def normalize_name(name):
+    """
+    Given a string, return a normalized string, per the first two rules
+    described above.
+    """
+    # Lowercase and replace all non-alphanumeric characters with _
+    subbed = re.sub(nonalphanum_pat, "_", name.lower())
+
+    # Remove leading and trailing _s
+    stripped = subbed.strip("_")
+
+    # To ensure it's a valid Python identifier
+    if (stripped or "_")[0] in string.digits:
+        stripped = "_" + stripped
+
+    return stripped
+
+
+def gen_normalize_names(names):
+    """
+    Given a list of strings, yield fully-normalized conversions of those
+    strings, ensuring that each is unique.
+    """
+    base = list(map(normalize_name, names))
+    counts = Counter(base)
+
+    # Append __{i} to non-unique names
+    seen = dict((key, 0) for key in counts.keys())
+    for name in base:
+        if counts[name] == 1 or name == "":
+            norm_name = name
+        else:
+            norm_name = name + "__" + str(seen[name])
+            seen[name] += 1
+        yield norm_name
+
+
+@Sheet.api
+@asyncthread
+def normalize_column_names(sheet):
+    """
+    Normalize the names of all non-hidden columns on the active sheet.
+    """
+
+    init_names = {}
+    gen = gen_normalize_names(c.name for c in sheet.visibleCols)
+    prog = Progress(gen, gerund="normalizing", total=sheet.nVisibleCols)
+
+    for i, norm_name in enumerate(prog):
+        col = sheet.visibleCols[i]
+        init_names[col] = col.name  # Store for undo
+        col.name = norm_name
+
+    @asyncthread
+    def undo():
+        for c, oldname in init_names.items():
+            c.name = oldname
+
+    vd.addUndo(undo)
+
+
+# Add longname-commands to VisiData to execute these methods
+Sheet.addCommand(None, "normalize-col-names", "vd.sheet.normalize_column_names()", "normalize the names of all non-hidden columns")
+
+vd.addMenuItems('''
+    Column > Rename > normalize all > normalize-col-names
+''')

visidata/features/open_config.py

@@ -0,0 +1,7 @@
+from visidata import vd, BaseSheet
+
+BaseSheet.addCommand('gO', 'open-config', 'vd.push(open_txt(Path(options.config)))', 'open options.config as text sheet')
+
+vd.addMenuItems('''
+    File > Options > edit config file > open-config
+''')

visidata/features/open_syspaste.py

@@ -0,0 +1,18 @@
+'''
+Load new table from system clipboard
+'''
+
+from visidata import vd, BaseSheet, Path
+
+
+@BaseSheet.api
+def open_syspaste(sheet, filetype='tsv'):
+    import io
+
+    v = vd.sysclipValue().strip() or vd.fail('nothing to open')
+
+    p = Path('syspaste'+'.'+filetype, fp=io.BytesIO(v.encode('utf-8')))
+    return vd.openSource(p, filetype=filetype)
+
+
+BaseSheet.addCommand('gShift+P', 'open-syspaste', 'vd.push(open_syspaste(filetype=vd.input("paste as filetype: ", value="tsv")))', 'open clipboard as filetype')