visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/features/ping.py

@@ -0,0 +1,157 @@
+# requires shell tools: ping traceroute
+#   these tools aren't inlined as they require root privs
+
+import re
+import time
+from statistics import mean
+
+from visidata import vd, VisiData, BaseSheet, Sheet, Column, AttrColumn, Progress
+
+vd.theme_option('color_shellcmd', '21 on 114 green', '')
+vd.theme_option('color_colname', 'underline', '')
+vd.theme_option('color_longname', 'bold 52 on 114 green', '')
+
+
+@VisiData.api
+def new_ping(vd, p):
+    'Open a sheet with the round-trip time for each hop along the path to the given host.'
+    pingsheet = PingSheet(p.given, source=p.given)
+    return PingStatsSheet("traceroute_"+pingsheet.name, source=pingsheet)
+
+
+class PingStatsSheet(Sheet):
+    help='''# ping/traceroute
+This sheet runs {shellcmd}traceroute{} to generate intermediate hops, then runs {shellcmd}ping{} against each hop N times to get the {colname}avg_ms{} and {colname}max_ms{} ping time.
+
+{longname}open-source{} to open the raw ping data.
+'''
+    rowtype='hosts' # rowdef: PingColumn
+    columns = [
+        Column('hop', type=int, width=5, getter=lambda col,r: col.sheet.source.sources.index(r.ip)+1),
+        Column('hostname', getter=lambda col,r: r.name, width=40),
+        Column('ip',  getter=lambda col,r: r.ip, width=17),
+        Column('avg_ms', type=float, fmtstr='%.1f', getter=lambda col,r: mean(r.getValues(col.sheet.source.rows))),
+        Column('max_ms', type=float, fmtstr='%.1f', getter=lambda col,r: max(r.getValues(col.sheet.source.rows))),
+        Column('count',  type=int, getter=lambda col,r: len(list(r.getValues(col.sheet.source.rows)))),
+    ]
+    def loader(self):
+        sheet = self.source
+        sheet.ensureLoaded()
+
+        self.rows = sheet.columns
+
+
+def PingColumn(name, ip):
+    return Column(name, ip=ip, getter=lambda c,r: r.get(c.ip), type=float, fmtstr='%0.1f')
+
+
+class PingSheet(Sheet):
+    rowtype = 'pings'  # rowdef: {'time':time.time(), 'hostname': pingtime}
+    def __init__(self, *names, source=None, **kwargs):
+        super().__init__(*names, source=source, **kwargs)
+        self.sources = [source]
+
+    def ping_response(self, row, ip, data):
+        for line in data.splitlines():
+            m = re.search(r'from ([0-9\.]+).* time=(.*) ms', line)
+            if m:
+                row[ip] = m.group(2)
+#                assert ip == m.group(1)
+
+    def traceroute_response(self, ip, data):
+        for line in data.splitlines():
+            m = re.search(r'(\d+)  (\S+) \((\S+)\)  (.*) ms', line)
+            if m:
+                ttl, hostname, inner_ip, latency_ms = m.groups()
+                self.routes[ip][int(ttl)-1] = inner_ip
+                if inner_ip == ip:
+                    return  # traceroute work is done, let ping do the rest
+
+                if inner_ip not in self.sources:
+                    self.sources.insert(-1, inner_ip)
+                    self.addColumn(PingColumn(hostname, inner_ip), index=-1)
+                    self.send_trace(ip, int(ttl)+1)  # get next hop
+                break
+
+    def ping_error(self, ip, data):
+        if ip in self.sources:
+            self.sources.remove(ip)
+            vd.warning("%s removed: %s" % (ip, data))
+
+    def update_traces(self, row, ip):
+        rtes = self.routes.get(ip)
+        if rtes is None:
+            rtes = []
+            self.routes[ip] = rtes
+
+        if ip in rtes:
+            return
+
+        for i, inner_ip in enumerate(rtes):
+            if inner_ip is None:
+                self.send_trace(ip, i+1)
+
+        self.send_trace(ip, len(rtes)+1)  # and one more for the road
+
+    def send_trace(self, ip, n):
+        sh = vd.importExternal('sh')
+
+        while n > len(self.routes[ip]):
+            self.routes[ip].append(None)
+
+        sh.traceroute('--sim-queries=1',
+                      '--queries=1',
+                      '--first=%s' % n,
+                      '--max-hops=%s' % n,
+                      ip,
+                      _bg=True, _bg_exc=False,
+                      _out=lambda data,self=self,ip=ip,a=1: self.traceroute_response(ip, data),
+                      _err=lambda data,self=self,ip=ip,a=1: self.ping_error(ip, data))
+
+    def iterload(self):
+        sh = vd.importExternal('sh')
+
+        self.stop = False
+        self.start_time = time.time()
+        self.columns.clear()
+
+        for ip in self.sources:
+            self.addColumn(PingColumn(ip, ip))
+
+        self.routes = {}
+        self.rows = []
+        pings_sent = {}
+        ping_count = self.options.ping_count
+        with Progress(total=ping_count*len(self.sources), gerund='pinging') as prog:
+          while not self.stop:
+            r = {'time':time.time()}
+            yield r
+
+            npings = 0  # this loop
+            for ip in self.sources:
+                self.update_traces(r, ip)
+                pings_sent[ip] = pings_sent.get(ip, 0)+1
+                if ping_count and pings_sent[ip] <= ping_count:
+                    sh.ping('-c', '1', ip, _bg=True, _bg_exc=False, _timeout=30,
+                            _out=lambda data,self=self,r=r,ip=ip: self.ping_response(r,ip,data),
+                            _err=lambda data,self=self,ip=ip,a=1: self.ping_error(ip, data))
+                    npings += 1
+                else:
+                    r[ip] = None
+
+            if npings == 0:
+                vd.status('no more pings to send')
+                break
+
+            time.sleep(self.options.ping_interval)
+
+
+vd.option('ping_count', 3, 'send this many pings to each host', sheettype=PingSheet)
+vd.option('ping_interval', 0.1, 'wait between ping rounds, in seconds', sheettype=PingSheet)
+
+PingSheet.options.null_value = False
+BaseSheet.addCommand('', 'open-ping', 'vd.push(openSource(input("ping: ", type="hostip"), filetype="ping"))', 'open sheet to ping input IP Address')
+
+vd.addMenuItems('''
+    System > Ping IP/hostname > open-ping
+''')

visidata/features/procmgr.py

@@ -0,0 +1,208 @@
+from visidata import *
+
+
+@VisiData.api
+def new_top(vd, p):
+    return vd.processes
+
+
+class CPUStatsSheet(Sheet):
+    rowtype='CPUs'  # rowdef = (count, perfect, times, freq) from psutil (see below)
+    columns = [
+        ColumnItem('cpu', 0, type=int, keycol=0),
+        ColumnItem('cpu_pct', 1, type=float)
+    ]
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+
+        for i, k in enumerate(psutil.cpu_times()._fields):
+            self.addColumn(SubColumnItem(2, ColumnItem(k+'_s', i, type=float, sheet=self)))
+
+        for i, k in enumerate(psutil.cpu_freq()._fields):
+            self.addColumn(SubColumnItem(3, ColumnItem(k+'_MHz', i, type=float, sheet=self)))
+
+        for r in zip(range(psutil.cpu_count()),
+                        psutil.cpu_percent(percpu=True),
+                        psutil.cpu_times(percpu=True),
+                        psutil.cpu_freq(percpu=True)):
+            yield r
+
+
+
+class MemStatsSheet(Sheet):
+    rowtype = ''  # rowdef: (name, value)
+    columns = [
+        AttrColumn('t', type=date, fmtstr='%H:%M:%S'),
+        AttrColumn('meminfo'),
+        AttrColumn('virtmem'),
+        AttrColumn('swapmem'),
+    ]
+    nKeys = 1
+
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+        import time
+
+        proc = psutil.Process()
+
+        while True:
+            yield AttrDict(t=time.time(),
+                         virtmem=psutil.virtual_memory(),
+                         swapmem=psutil.swap_memory(),
+                         meminfo=proc.memory_info(),
+                        )
+
+            if self.nRows == 1:  # first time through
+                for n in 'meminfo virtmem swapmem'.split():
+                    c = self.column(n)
+                    if not c.hidden:
+                        c.expand([self.rows[0]])
+
+            time.sleep(1)
+
+
+class UsefulProcessesSheet(Sheet):
+    columns = [
+        Column('pid', type=int, getter=lambda c,r: r[0].pid),
+        Column('name', getter=lambda c,r: r[0].name()),
+        Column('status', getter=lambda c,r: r[0].status()),
+        Column('cmdline', getter=lambda c,r: ' '.join(r[0].cmdline())),
+        Column('user', getter=lambda c,r: r[0].username()),
+        Column('real_uid', type=int, width=0, getter=lambda c,r: r[0].uids()[0]),
+        Column('effective_uid', type=int, width=0, getter=lambda c,r: r[0].uids()[1]),
+        Column('mem_uss', type=int, getter=lambda c,r: r[1].mem_uss),
+        Column('user_time_s', type=float, getter=lambda c,r: r[0].cpu_times()[0]),
+        Column('system_time_s', type=float, getter=lambda c,r: r[0].cpu_times()[1]),
+    ]
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+        for pr in psutil.process_iter():
+            yield (pr, pr.memory_full_info())
+
+
+class ProcessesSheet(Sheet):
+    columns = [
+        Column('pid', type=int, getter=lambda c,r: r.pid),
+        Column('name', getter=lambda c,r: r.name()),
+        Column('status', getter=lambda c,r: r.status()),
+        Column('parent_pid', type=int, getter=lambda c,r: r.ppid()),
+        Column('exe', getter=lambda c,r: r.exe()),
+        Column('cmdline', getter=lambda c,r: ' '.join(r.cmdline())),
+        Column('cwd', getter=lambda c,r: r.cwd()),
+        Column('user', getter=lambda c,r: r.username()),
+
+        Column('real_uid', type=int, width=0, getter=lambda c,r: r.uids()[0]),
+        Column('effective_uid', type=int, width=0, getter=lambda c,r: r.uids()[1]),
+        Column('saved_uid', type=int, width=0, getter=lambda c,r: r.uids()[2]),
+
+        Column('real_gid', type=int, width=0, getter=lambda c,r: r.gids()[0]),
+        Column('effective_gid', type=int, width=0, getter=lambda c,r: r.gids()[1]),
+        Column('saved_gid', type=int, width=0, getter=lambda c,r: r.gids()[2]),
+
+        Column('create_time', type=date, getter=lambda c,r: r.create_time()),
+        Column('cpu_num', type=int, getter=lambda c,r: r.cpu_num()),
+        Column('cpu_percent', type=float, getter=lambda c,r: r.cpu_percent()),
+
+        Column('tty', getter=lambda c,r: r.terminal()),
+        Column('nice', getter=lambda c,r: r.nice()),
+        Column('ioclass', getter=lambda c,r: r.ionice()[0]),
+        Column('ionice', getter=lambda c,r: r.ionice()[1]),
+
+        Column('user_time_s', type=float, getter=lambda c,r: r.cpu_times()[0]),
+        Column('system_time_s', type=float, getter=lambda c,r: r.cpu_times()[1]),
+        Column('children_user_time_s', type=float, width=0, getter=lambda c,r: r.cpu_times()[2]),
+        Column('children_system_time_s', type=float, width=0, getter=lambda c,r: r.cpu_times()[3]),
+
+        Column('read_ops', type=int, getter=lambda c,r: r.io_counters()[0]),
+        Column('write_ops', type=int, getter=lambda c,r: r.io_counters()[1]),
+        Column('read_bytes', type=int, getter=lambda c,r: r.io_counters()[2]),
+        Column('write_bytes', type=int, getter=lambda c,r: r.io_counters()[3]),
+
+        Column('voluntary_ctx_switches', type=int, getter=lambda c,r: r.num_ctx_switches()[0]),
+        Column('involuntary_ctx_switches', type=int, getter=lambda c,r: r.num_ctx_switches()[1]),
+        Column('num_fds', type=int, getter=lambda c,r: r.num_fds()),
+        Column('num_threads', type=int, getter=lambda c,r: r.num_threads()),
+    ]
+    nKeys = 2
+
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+#        self.columns = []
+#        for c in ProcessesSheet.columns:
+#            self.addColumn(c)
+
+        for i,k in enumerate(psutil.Process().memory_full_info()._fields):
+            self.addColumn(Column('mem_'+k, type=int, getter=lambda c,r,i=i: r.memory_full_info()[i], cache=True))
+        # mem_uss is probably the most representative metric for determining how much memory is actually being used by a process. It represents the amount of memory that would be freed if the process was terminated right now.
+
+        yield from psutil.process_iter()
+#            try:
+#                self.addRow((pr, pr.memory_full_info()))
+#            except:
+#                self.addRow((pr, None))
+
+class RlimitsSheet(Sheet):
+    columns = [
+        ColumnItem('rlimit', 0),
+        Column('soft', type=int, getter=lambda c,r: c.sheet.soft(r), setter=lambda c,r,v: c.sheet.set_soft(r, v)),
+        Column('hard', type=int, getter=lambda c,r: c.sheet.hard(r), setter=lambda c,r,v: c.sheet.set_hard(r, v))
+    ]
+
+    def soft(self, r):
+        return self.source.rlimit(r[1])[0]
+    def hard(self, r):
+        return self.source.rlimit(r[1])[1]
+    def set_soft(self, r, v):
+        self.source.rlimit(r[1], (v, self.hard(r)))
+    def set_hard(self, r, v):
+        self.source.rlimit(r[1], (self.soft(r), v))
+
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+        for r in dir(psutil):
+            if r.startswith('RLIMIT'):
+                yield (r[7:], getattr(psutil, r))
+
+
+@VisiData.lazy_property
+def cpuStats(vd):
+    return CPUStatsSheet('cpu_stats')
+
+@VisiData.lazy_property
+def memStats(vd):
+    return MemStatsSheet('memory_stats')
+
+@VisiData.lazy_property
+def processes(vd):
+    return ProcessesSheet('processes')
+
+
+BaseSheet.addCommand('', 'open-cpustats', 'vd.push(vd.cpuStats)', 'open CPU usage stats' )
+BaseSheet.addCommand('', 'open-memstats', 'vd.push(vd.memStats)', 'open memory usage stats')
+BaseSheet.addCommand('', 'open-processes', 'vd.push(vd.processes)', 'open system process stats')
+
+@VisiData.api
+def chooseSignal(vd):
+    import signal
+    d = [{'key': attr[3:]} for attr in dir(signal) if attr.startswith('SIG') and not attr.startswith('SIG_')]
+    return getattr(signal, 'SIG'+vd.chooseOne(d, type='signal'))
+
+ProcessesSheet.addCommand('d', 'term-process', 'os.kill(cursorRow.pid, signal.SIGTERM)', 'send SIGTERM to process')
+ProcessesSheet.addCommand('gd', 'term-selected', 'for r in someSelectedRows: os.kill(r.pid, signal.SIGTERM)', 'send SIGTERM to selected processes')
+ProcessesSheet.addCommand('zd', 'kill-process', 'os.kill(cursorRow.pid, signal.SIGKILL)', 'send SIGKILL to process')
+ProcessesSheet.addCommand('gzd', 'kill-selected', 'for r in someSelectedRows: os.kill(r.pid, signal.SIGKILL)', 'send SIGKILL to selected processes')
+
+ProcessesSheet.addCommand('^K', 'signal-process', 'os.kill(cursorRow.pid, chooseSignal())', 'send chosen signal to process')
+UsefulProcessesSheet.addCommand('^K', 'signal-selected', 'os.kill(cursorRow.pid, chooseSignal())', 'kill(2) send chosen signal to process')
+UsefulProcessesSheet.addCommand('', 'open-rlimits', 'vd.push(RlimitsSheet(cursorRow.name() + "_rlimits", cursorRow))', 'push rlimits for this process')
+
+vd.addMenuItem('System', 'Signal', 'current process', 'TERMinate', 'term-process')
+vd.addMenuItem('System', 'Signal', 'current process', 'KILL', 'kill-process')
+vd.addMenuItem('System', 'Signal', 'current process', 'choose signal', 'signal-process')
+vd.addMenuItem('System', 'Signal', 'selected processes', 'TERMinate', 'term-selected')
+vd.addMenuItem('System', 'Signal', 'selected processes', 'KILL', 'kill-selected')
+vd.addMenuItem('System', 'Signal', 'selected processes', 'choose signal', 'signal-selected')
+vd.addMenuItem('System', 'Stats', 'CPU', 'open-cpustats')
+vd.addMenuItem('System', 'Stats', 'memory', 'open-memstats')
+vd.addMenuItem('System', 'Stats', 'processes', 'open-processes')
+vd.addMenuItem('System', 'Stats', 'resource limits', 'open-rlimits')

visidata/features/random_sample.py

@@ -0,0 +1,6 @@
+from visidata import vd, Sheet
+
+Sheet.addCommand(None, 'random-rows', 'nrows=int(input("random number to filter: ", value=nRows)); vs=copy(sheet); vs.name=name+"_sample"; vs.rows=sample(rows, nrows or nRows); vd.push(vs)', 'open duplicate sheet with a random population subset of N rows')
+Sheet.addCommand(None, 'select-random', 'nrows=int(input("random number to select: ", value=nRows)); select(sample(rows, nrows or nRows))', 'select random sample of N rows')
+
+vd.addMenuItems('Row > Select > random sample > select-random')

visidata/{regex.py → features/regex.py}

@@ -5,17 +5,11 @@ from visidata import asyncthread, options, vd
 from visidata import VisiData, BaseSheet, Sheet, Column, Progress
 
 
-@Sheet.api
-def setSubst(sheet, cols, rows):
-    if not rows:
-        vd.warning('no %s selected' % sheet.rowtype)
-        return
-    modified = 'column' if len(cols) == 1 else 'columns'
-    rex = vd.input("transform %s by regex: " % modified, type="regex-subst")
-    setValuesFromRegex(cols, rows, rex)
+@VisiData.lazy_property
+def help_regex(vd):
+    return vd.getHelpPane('regex', module='visidata')
 
 
-vd.option('regex_flags', 'I', 'flags to pass to re.compile() [AILMSUX]', replay=True)
 vd.option('regex_maxsplit', 0, 'maxsplit to pass to regex.split', replay=True)
 
 @VisiData.api
@@ -24,6 +18,8 @@ def makeRegexSplitter(vd, regex, origcol):
 
 @VisiData.api
 def makeRegexMatcher(vd, regex, origcol):
+    if not regex.groups:
+        vd.fail('specify a capture group')  #1778
     def _regexMatcher(row):
         m = regex.search(origcol.getDisplayValue(row))
         if m:
@@ -52,12 +48,9 @@ def addRegexColumns(vs, regexMaker, origcol, regexstr):
     cols = {}
     ncols = 0  # number of new columns added already
     for r in Progress(vs.getSampleRows()):
-        try:
-            m = func(r)
-            if not m:
-                continue
-        except Exception as e:
-            vd.exceptionCaught(e)
+        m = vd.callNoExceptions(func, r)
+        if not m:
+            continue
 
         if isinstance(m, dict):
             for name in m:
@@ -74,12 +67,15 @@ def addRegexColumns(vs, regexMaker, origcol, regexstr):
         else:
             raise TypeError("addRegexColumns() expects a dict, list, or tuple from regexMaker, but got a "+type(m).__name__)
 
+    if not cols:
+        vd.warning("no regex matches found, didn't add column")
+        return
+
     vs.addColumnAtCursor(*cols.values())
 
 
 @VisiData.api
-def regexTransform(vd, origcol, instr):
-    before, after = vd.parse_sed_transform(instr)
+def regexTransform(vd, origcol, before='', after=''):
     return lambda col,row,origcol=origcol,before=before,after=after,flags=origcol.sheet.regex_flags(): re.sub(before, after, origcol.getDisplayValue(row), flags=flags)
 
 
@@ -104,27 +100,47 @@ def indexWithEscape(s, char, escape_char='\\'):
     return None
 
 
+@Sheet.api
 @asyncthread
-def setValuesFromRegex(cols, rows, rex):
-    transforms = [vd.regexTransform(col, rex) for col in cols]
+def setValuesFromRegex(sheet, cols, rows, before='', after=''):
+    transforms = [vd.regexTransform(col, before=before, after=after) for col in cols]
     vd.addUndoSetValues(cols, rows)
     for r in Progress(rows, 'replacing'):
         for col, transform in zip(cols, transforms):
-            col.setValueSafe(r, transform(col, r))
+            vd.callNoExceptions(col.setValue, r, transform(col, r))
+
     for col in cols:
         col.recalc()
 
 
-@BaseSheet.api
-def regex_flags(sheet):
-    'Return flags to pass to regex functions from options'
-    return sum(getattr(re, f.upper()) for f in options.regex_flags)
+@VisiData.api
+def inputRegex(vd, prompt, type='regex', **kwargs):
+    return vd.input(prompt, type=type, help=vd.help_regex, **kwargs)
+
 
 
-Sheet.addCommand(':', 'split-col', 'addRegexColumns(makeRegexSplitter, cursorCol, input("split regex: ", type="regex-split"))', 'Add new columns from regex split')
-Sheet.addCommand(';', 'capture-col', 'addRegexColumns(makeRegexMatcher, cursorCol, input("capture regex: ", type="regex-capture"))', 'add new column from capture groups of regex; requires example row')
-Sheet.addCommand('', 'addcol-split', 'addColumnAtCursor(RegexColumn(makeRegexSplitter, cursorCol, input("split regex: ", type="regex-split")))', 'Add column split by regex')
-Sheet.addCommand('', 'addcol-capture', 'addColumnAtCursor(RegexColumn(makeRegexMatcher, cursorCol, input("capture regex: ", type="regex-capture")))', 'Add column captured by regex')
-Sheet.addCommand('*', 'addcol-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, input("transform column by regex: ", type="regex-subst"))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)')
-Sheet.addCommand('g*', 'setcol-subst', 'setSubst([cursorCol], someSelectedRows)', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)')
-Sheet.addCommand('gz*', 'setcol-subst-all', 'setSubst(visibleCols, someSelectedRows)', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)')
+@VisiData.api
+def inputRegexSubst(vd, prompt):
+    'Input regex transform via oneliner (separated with `/`).  Return parsed transformer as dict(before=, after=).'
+    return vd.inputMultiple(before=dict(type='regex', prompt='search: ', help=prompt),
+                            after=dict(type='regex-replace', prompt='replace: ', help=prompt))
+
+
+Sheet.addCommand(':', 'addcol-split', 'addColumnAtCursor(RegexColumn(makeRegexSplitter, cursorCol, inputRegex("split regex: ", type="regex-split")))', 'Add column split by regex')
+Sheet.addCommand(';', 'addcol-capture', 'addColumnAtCursor(RegexColumn(makeRegexMatcher, cursorCol, inputRegex("capture regex: ", type="regex-capture")))', 'Add column captured by regex')
+
+Sheet.addCommand('*', 'addcol-regex-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, **inputRegexSubst("regex transform column"))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)')
+Sheet.addCommand('g*', 'setcol-regex-subst', 'setValuesFromRegex([cursorCol], someSelectedRows, **inputRegexSubst("regex transform column"))', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)')
+Sheet.addCommand('gz*', 'setcol-regex-subst-all', 'setValuesFromRegex(visibleCols, someSelectedRows, **inputRegexSubst(f"regex transform {nVisibleCols} columns"))', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)')
+
+
+vd.addMenuItems('''
+    Edit > Modify > selected cells > regex substitution > setcol-regex-subst
+    Column > Add column > capture by regex > addcol-capture
+    Column > Add column > split by regex > addcol-split
+    Column > Add column > subst by regex > addcol-regex-subst
+    Row > Select > by regex > current column > select-col-regex
+    Row > Select > by regex > all columns > select-cols-regex
+    Row > Unselect > by regex > current column > unselect-col-regex
+    Row > Unselect > by regex > all columns > unselect-cols-regex
+''')

visidata/features/reload_every.py

@@ -0,0 +1,55 @@
+import os
+import time
+
+from visidata import vd, BaseSheet, Sheet, asyncthread, Path, ScopedSetattr
+
+
+@BaseSheet.api
+@asyncthread
+def reload_every(sheet, seconds:int):
+    while True:
+        sheet.reload()
+        time.sleep(seconds)
+
+
+@BaseSheet.api
+@asyncthread
+def reload_modified(sheet):
+    'Spawn thread to call sheet.reload_rows when sheet.source mtime has changed.'
+    p = sheet.source
+    assert isinstance(p, Path)
+    assert not p.is_url()
+
+    mtime = os.stat(p).st_mtime
+    while True:
+        time.sleep(1)
+        t = os.stat(p).st_mtime
+        if t != mtime:
+            mtime = t
+            sheet.reload_rows()
+
+
+@Sheet.api
+@asyncthread
+def reload_rows(self):
+    'Reload rows from ``self.source``, keeping current columns intact.  Async.'
+    with (ScopedSetattr(self, 'loading', True),
+          ScopedSetattr(self, 'checkCursor', lambda: True),
+          ScopedSetattr(self, 'cursorRowIndex', self.cursorRowIndex)):
+            self.beforeLoad()
+            try:
+                self.loader()
+                vd.status("finished loading rows")
+            finally:
+                self.afterLoad()
+
+
+BaseSheet.addCommand('', 'reload-every', 'sheet.reload_every(input("reload interval (sec): ", value=1))', 'schedule sheet reload every N seconds') #683
+BaseSheet.addCommand('', 'reload-modified', 'sheet.reload_modified()', 'reload sheet when source file modified (tail-like behavior)')  #1686
+BaseSheet.addCommand('z^R', 'reload-rows', 'preloadHook(); reload_rows(); status("reloaded")', 'Reload current sheet')
+
+vd.addMenuItems('''
+    File > Reload > rows only > reload-rows
+    File > Reload > every N seconds > reload-every
+    File > Reload > when source modified > reload-modified
+''')

visidata/features/rename_col_cascade.py

@@ -0,0 +1,30 @@
+import ast
+
+from visidata import vd, Column, ExprColumn, Sheet
+
+vd.option('rename_cascade', False, 'cascade column renames into expressions')  #2088
+
+
+class Renamer(ast.NodeTransformer):
+    def __init__(self, find, replace):
+        self.find = find
+        self.replace = replace
+
+    def visit_Name(self, node):
+        if node.id == self.find:
+            node.id = self.replace
+
+        return node
+
+
+@Column.before
+def setName(col, newname):
+    if col.sheet and col.sheet.options.rename_cascade:
+        for c in col.sheet.columns:
+            if isinstance(c, ExprColumn):
+                parsed_expr = ast.parse(c.expr)
+                canon_expr = ast.unparse(parsed_expr)
+                new_expr = ast.unparse(Renamer(col.name, newname).visit(parsed_expr))
+                if new_expr != canon_expr:
+                    vd.addUndo(setattr, c, 'expr', c.expr)
+                    c.expr = new_expr

visidata/features/scroll_context.py

@@ -0,0 +1,60 @@
+'''
+Enables two functions:
+1. options.disp_scroll_context: the minimum number of lines to keep visible above and below the cursor when scrolling
+2. toggle-scrollfix: Lock the cursor to a particular row number in the view
+
+Allows the user to fix the position on the page where they would like the
+cursor to "stick". Helps to provide context about surrounding rows when
+near the top and bottom of the page.
+
+Usage: scroll a few lines down, `toggle-scrollfix' and scroll again!
+
+NOTE:
+    - disables scroll-middle command (zz)
+'''
+
+__author__ = 'Geekscrapy'
+__version__ = '1.2'
+
+from visidata import vd, Sheet
+
+vd.option('disp_scroll_context', 0, 'minimum number of lines to keep visible above/below cursor when scrolling')
+vd.optalias('disp_scrolloff', 'disp_scroll_context')
+
+@Sheet.after
+def checkCursor(sheet):
+    nctx = sheet.options.disp_scroll_context
+    if nctx:
+        if nctx >= int(sheet.nScreenRows/2):
+            if sheet.cursorRowIndex+1 > int(sheet.nScreenRows/2):
+                sheet.topRowIndex = sheet.cursorRowIndex - int(sheet.nScreenRows/2)
+        else:
+            if sheet.cursorRowIndex-sheet.topRowIndex < nctx:
+                sheet.topRowIndex = max(0, sheet.cursorRowIndex-nctx)
+
+            if sheet.bottomRowIndex-sheet.cursorRowIndex < nctx:
+                sheet.bottomRowIndex = sheet.cursorRowIndex+nctx
+
+    nfix = getattr(sheet, 'disp_scrollfix', 0)
+    if nfix > 0:
+        cursorViewIndex = sheet.cursorRowIndex - sheet.topRowIndex
+        if cursorViewIndex > nfix:
+            sheet.topRowIndex += abs(nfix - cursorViewIndex)
+        if cursorViewIndex < nfix and sheet.topRowIndex > 0:
+            sheet.topRowIndex -= abs(nfix - cursorViewIndex)
+
+
+@Sheet.api
+def toggle_scrollfix(sheet):
+    if getattr(sheet, 'disp_scrollfix', -1) >= 0:
+        sheet.disp_scrollfix = -1
+        vd.status("cursor unlocked")
+    else:
+        sheet.disp_scrollfix = sheet.cursorRowIndex - sheet.topRowIndex
+        vd.status(f"cursor locked to screen row {sheet.disp_scrollfix}")
+
+
+Sheet.addCommand('', 'toggle-scrollfix', 'toggle_scrollfix()', helpstr='toggle cursor lock to current screen row')
+
+
+vd.addMenuItems('''View > Toggle display > lock cursor to screen row > toggle-scrollfix''')

visidata/features/select_equal_selected.py

@@ -0,0 +1,11 @@
+from visidata import Sheet, asyncthread, Progress
+
+
+@Sheet.api
+@asyncthread
+def select_equal_selected(sheet, col):
+    selectedVals = set(col.getDisplayValue(row) for row in Progress(sheet.selectedRows))
+    sheet.select(sheet.gatherBy(lambda r,c=col,vals=selectedVals: c.getDisplayValue(r) in vals), progress=False)
+
+
+Sheet.addCommand('', 'select-equal-selected', 'select_equal_selected(cursorCol)', 'select rows with values in current column in already selected rows')