visidata 2.11.1__py3-none-any.whl → 3.0.1__py3-none-any.whl

visidata/features/normcol.py

@@ -0,0 +1,118 @@
+"""
+# Usage
+
+This plugin normalizes column names in any given sheet, so that the names are:
+
+- Composed only of lowercase letters, numbers, and underscores.
+
+- Valid Python identifiers. This is mostly handled by the rule above, but also
+  prohibits names beginning with a digit; that is handled by prefixing those
+  names with an underscore.
+
+- Unique within the sheet. Non-unique names are suffixed with "__" and an
+  integer.
+
+Unnamed columns are left as such.
+
+For instance, a sheet with the following columns names:
+
+- "Genus, Species"
+- "Height"
+- "5-score"
+- "Height"
+- ""
+- ""
+
+... would be converted to have the following column names:
+
+- "genus_species"
+- "height__0"
+- "_5_score"
+- "height__1"
+- ""
+- ""
+
+## Commands
+
+- `normalize-col-names` normalizes the names of all *non-hidden* columns in the
+  active sheet, per the approach described above.
+
+"""
+
+__author__ = "Jeremy Singer-Vine <jsvine@gmail.com>"
+
+from visidata import vd, Sheet, asyncthread, Progress
+from collections import Counter
+import re
+import string
+
+nonalphanum_pat = re.compile(r"[^a-z0-9]+")
+
+
+def normalize_name(name):
+    """
+    Given a string, return a normalized string, per the first two rules
+    described above.
+    """
+    # Lowercase and replace all non-alphanumeric characters with _
+    subbed = re.sub(nonalphanum_pat, "_", name.lower())
+
+    # Remove leading and trailing _s
+    stripped = subbed.strip("_")
+
+    # To ensure it's a valid Python identifier
+    if (stripped or "_")[0] in string.digits:
+        stripped = "_" + stripped
+
+    return stripped
+
+
+def gen_normalize_names(names):
+    """
+    Given a list of strings, yield fully-normalized conversions of those
+    strings, ensuring that each is unique.
+    """
+    base = list(map(normalize_name, names))
+    counts = Counter(base)
+
+    # Append __{i} to non-unique names
+    seen = dict((key, 0) for key in counts.keys())
+    for name in base:
+        if counts[name] == 1 or name == "":
+            norm_name = name
+        else:
+            norm_name = name + "__" + str(seen[name])
+            seen[name] += 1
+        yield norm_name
+
+
+@Sheet.api
+@asyncthread
+def normalize_column_names(sheet):
+    """
+    Normalize the names of all non-hidden columns on the active sheet.
+    """
+
+    init_names = {}
+    gen = gen_normalize_names(c.name for c in sheet.visibleCols)
+    prog = Progress(gen, gerund="normalizing", total=sheet.nVisibleCols)
+
+    for i, norm_name in enumerate(prog):
+        col = sheet.visibleCols[i]
+        init_names[col] = col.name  # Store for undo
+        col.name = norm_name
+
+    @asyncthread
+    def undo():
+        for c, oldname in init_names.items():
+            c.name = oldname
+
+    vd.addUndo(undo)
+
+
+# Add longname-commands to VisiData to execute these methods
+Sheet.addCommand(None, "normalize-col-names", "vd.sheet.normalize_column_names()", "normalize the names of all non-hidden columns")
+
+vd.addMenuItems('''
+    Column > Rename > normalize all > normalize-col-names
+''')

visidata/features/open_config.py

@@ -0,0 +1,7 @@
+from visidata import vd, BaseSheet
+
+BaseSheet.addCommand('gO', 'open-config', 'vd.push(open_txt(Path(options.config)))', 'open options.config as text sheet')
+
+vd.addMenuItems('''
+    File > Options > edit config file > open-config
+''')

visidata/features/open_syspaste.py

@@ -0,0 +1,18 @@
+'''
+Load new table from system clipboard
+'''
+
+from visidata import vd, BaseSheet, Path
+
+
+@BaseSheet.api
+def open_syspaste(sheet, filetype='tsv'):
+    import io
+
+    v = vd.sysclipValue().strip() or vd.fail('nothing to open')
+
+    p = Path('syspaste'+'.'+filetype, fp=io.BytesIO(v.encode('utf-8')))
+    return vd.openSource(p, filetype=filetype)
+
+
+BaseSheet.addCommand('gShift+P', 'open-syspaste', 'vd.push(open_syspaste(filetype=vd.input("paste as filetype: ", value="tsv")))', 'open clipboard as filetype')

visidata/features/ping.py

@@ -0,0 +1,157 @@
+# requires shell tools: ping traceroute
+#   these tools aren't inlined as they require root privs
+
+import re
+import time
+from statistics import mean
+
+from visidata import vd, VisiData, BaseSheet, Sheet, Column, AttrColumn, Progress
+
+vd.theme_option('color_shellcmd', '21 on 114 green', '')
+vd.theme_option('color_colname', 'underline', '')
+vd.theme_option('color_longname', 'bold 52 on 114 green', '')
+
+
+@VisiData.api
+def new_ping(vd, p):
+    'Open a sheet with the round-trip time for each hop along the path to the given host.'
+    pingsheet = PingSheet(p.given, source=p.given)
+    return PingStatsSheet("traceroute_"+pingsheet.name, source=pingsheet)
+
+
+class PingStatsSheet(Sheet):
+    help='''# ping/traceroute
+This sheet runs {shellcmd}traceroute{} to generate intermediate hops, then runs {shellcmd}ping{} against each hop N times to get the {colname}avg_ms{} and {colname}max_ms{} ping time.
+
+{longname}open-source{} to open the raw ping data.
+'''
+    rowtype='hosts' # rowdef: PingColumn
+    columns = [
+        Column('hop', type=int, width=5, getter=lambda col,r: col.sheet.source.sources.index(r.ip)+1),
+        Column('hostname', getter=lambda col,r: r.name, width=40),
+        Column('ip',  getter=lambda col,r: r.ip, width=17),
+        Column('avg_ms', type=float, fmtstr='%.1f', getter=lambda col,r: mean(r.getValues(col.sheet.source.rows))),
+        Column('max_ms', type=float, fmtstr='%.1f', getter=lambda col,r: max(r.getValues(col.sheet.source.rows))),
+        Column('count',  type=int, getter=lambda col,r: len(list(r.getValues(col.sheet.source.rows)))),
+    ]
+    def loader(self):
+        sheet = self.source
+        sheet.ensureLoaded()
+
+        self.rows = sheet.columns
+
+
+def PingColumn(name, ip):
+    return Column(name, ip=ip, getter=lambda c,r: r.get(c.ip), type=float, fmtstr='%0.1f')
+
+
+class PingSheet(Sheet):
+    rowtype = 'pings'  # rowdef: {'time':time.time(), 'hostname': pingtime}
+    def __init__(self, *names, source=None, **kwargs):
+        super().__init__(*names, source=source, **kwargs)
+        self.sources = [source]
+
+    def ping_response(self, row, ip, data):
+        for line in data.splitlines():
+            m = re.search(r'from ([0-9\.]+).* time=(.*) ms', line)
+            if m:
+                row[ip] = m.group(2)
+#                assert ip == m.group(1)
+
+    def traceroute_response(self, ip, data):
+        for line in data.splitlines():
+            m = re.search(r'(\d+)  (\S+) \((\S+)\)  (.*) ms', line)
+            if m:
+                ttl, hostname, inner_ip, latency_ms = m.groups()
+                self.routes[ip][int(ttl)-1] = inner_ip
+                if inner_ip == ip:
+                    return  # traceroute work is done, let ping do the rest
+
+                if inner_ip not in self.sources:
+                    self.sources.insert(-1, inner_ip)
+                    self.addColumn(PingColumn(hostname, inner_ip), index=-1)
+                    self.send_trace(ip, int(ttl)+1)  # get next hop
+                break
+
+    def ping_error(self, ip, data):
+        if ip in self.sources:
+            self.sources.remove(ip)
+            vd.warning("%s removed: %s" % (ip, data))
+
+    def update_traces(self, row, ip):
+        rtes = self.routes.get(ip)
+        if rtes is None:
+            rtes = []
+            self.routes[ip] = rtes
+
+        if ip in rtes:
+            return
+
+        for i, inner_ip in enumerate(rtes):
+            if inner_ip is None:
+                self.send_trace(ip, i+1)
+
+        self.send_trace(ip, len(rtes)+1)  # and one more for the road
+
+    def send_trace(self, ip, n):
+        sh = vd.importExternal('sh')
+
+        while n > len(self.routes[ip]):
+            self.routes[ip].append(None)
+
+        sh.traceroute('--sim-queries=1',
+                      '--queries=1',
+                      '--first=%s' % n,
+                      '--max-hops=%s' % n,
+                      ip,
+                      _bg=True, _bg_exc=False,
+                      _out=lambda data,self=self,ip=ip,a=1: self.traceroute_response(ip, data),
+                      _err=lambda data,self=self,ip=ip,a=1: self.ping_error(ip, data))
+
+    def iterload(self):
+        sh = vd.importExternal('sh')
+
+        self.stop = False
+        self.start_time = time.time()
+        self.columns.clear()
+
+        for ip in self.sources:
+            self.addColumn(PingColumn(ip, ip))
+
+        self.routes = {}
+        self.rows = []
+        pings_sent = {}
+        ping_count = self.options.ping_count
+        with Progress(total=ping_count*len(self.sources), gerund='pinging') as prog:
+          while not self.stop:
+            r = {'time':time.time()}
+            yield r
+
+            npings = 0  # this loop
+            for ip in self.sources:
+                self.update_traces(r, ip)
+                pings_sent[ip] = pings_sent.get(ip, 0)+1
+                if ping_count and pings_sent[ip] <= ping_count:
+                    sh.ping('-c', '1', ip, _bg=True, _bg_exc=False, _timeout=30,
+                            _out=lambda data,self=self,r=r,ip=ip: self.ping_response(r,ip,data),
+                            _err=lambda data,self=self,ip=ip,a=1: self.ping_error(ip, data))
+                    npings += 1
+                else:
+                    r[ip] = None
+
+            if npings == 0:
+                vd.status('no more pings to send')
+                break
+
+            time.sleep(self.options.ping_interval)
+
+
+vd.option('ping_count', 3, 'send this many pings to each host', sheettype=PingSheet)
+vd.option('ping_interval', 0.1, 'wait between ping rounds, in seconds', sheettype=PingSheet)
+
+PingSheet.options.null_value = False
+BaseSheet.addCommand('', 'open-ping', 'vd.push(openSource(input("ping: ", type="hostip"), filetype="ping"))', 'open sheet to ping input IP Address')
+
+vd.addMenuItems('''
+    System > Ping IP/hostname > open-ping
+''')

visidata/features/procmgr.py

@@ -0,0 +1,208 @@
+from visidata import *
+
+
+@VisiData.api
+def new_top(vd, p):
+    return vd.processes
+
+
+class CPUStatsSheet(Sheet):
+    rowtype='CPUs'  # rowdef = (count, perfect, times, freq) from psutil (see below)
+    columns = [
+        ColumnItem('cpu', 0, type=int, keycol=0),
+        ColumnItem('cpu_pct', 1, type=float)
+    ]
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+
+        for i, k in enumerate(psutil.cpu_times()._fields):
+            self.addColumn(SubColumnItem(2, ColumnItem(k+'_s', i, type=float, sheet=self)))
+
+        for i, k in enumerate(psutil.cpu_freq()._fields):
+            self.addColumn(SubColumnItem(3, ColumnItem(k+'_MHz', i, type=float, sheet=self)))
+
+        for r in zip(range(psutil.cpu_count()),
+                        psutil.cpu_percent(percpu=True),
+                        psutil.cpu_times(percpu=True),
+                        psutil.cpu_freq(percpu=True)):
+            yield r
+
+
+
+class MemStatsSheet(Sheet):
+    rowtype = ''  # rowdef: (name, value)
+    columns = [
+        AttrColumn('t', type=date, fmtstr='%H:%M:%S'),
+        AttrColumn('meminfo'),
+        AttrColumn('virtmem'),
+        AttrColumn('swapmem'),
+    ]
+    nKeys = 1
+
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+        import time
+
+        proc = psutil.Process()
+
+        while True:
+            yield AttrDict(t=time.time(),
+                         virtmem=psutil.virtual_memory(),
+                         swapmem=psutil.swap_memory(),
+                         meminfo=proc.memory_info(),
+                        )
+
+            if self.nRows == 1:  # first time through
+                for n in 'meminfo virtmem swapmem'.split():
+                    c = self.column(n)
+                    if not c.hidden:
+                        c.expand([self.rows[0]])
+
+            time.sleep(1)
+
+
+class UsefulProcessesSheet(Sheet):
+    columns = [
+        Column('pid', type=int, getter=lambda c,r: r[0].pid),
+        Column('name', getter=lambda c,r: r[0].name()),
+        Column('status', getter=lambda c,r: r[0].status()),
+        Column('cmdline', getter=lambda c,r: ' '.join(r[0].cmdline())),
+        Column('user', getter=lambda c,r: r[0].username()),
+        Column('real_uid', type=int, width=0, getter=lambda c,r: r[0].uids()[0]),
+        Column('effective_uid', type=int, width=0, getter=lambda c,r: r[0].uids()[1]),
+        Column('mem_uss', type=int, getter=lambda c,r: r[1].mem_uss),
+        Column('user_time_s', type=float, getter=lambda c,r: r[0].cpu_times()[0]),
+        Column('system_time_s', type=float, getter=lambda c,r: r[0].cpu_times()[1]),
+    ]
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+        for pr in psutil.process_iter():
+            yield (pr, pr.memory_full_info())
+
+
+class ProcessesSheet(Sheet):
+    columns = [
+        Column('pid', type=int, getter=lambda c,r: r.pid),
+        Column('name', getter=lambda c,r: r.name()),
+        Column('status', getter=lambda c,r: r.status()),
+        Column('parent_pid', type=int, getter=lambda c,r: r.ppid()),
+        Column('exe', getter=lambda c,r: r.exe()),
+        Column('cmdline', getter=lambda c,r: ' '.join(r.cmdline())),
+        Column('cwd', getter=lambda c,r: r.cwd()),
+        Column('user', getter=lambda c,r: r.username()),
+
+        Column('real_uid', type=int, width=0, getter=lambda c,r: r.uids()[0]),
+        Column('effective_uid', type=int, width=0, getter=lambda c,r: r.uids()[1]),
+        Column('saved_uid', type=int, width=0, getter=lambda c,r: r.uids()[2]),
+
+        Column('real_gid', type=int, width=0, getter=lambda c,r: r.gids()[0]),
+        Column('effective_gid', type=int, width=0, getter=lambda c,r: r.gids()[1]),
+        Column('saved_gid', type=int, width=0, getter=lambda c,r: r.gids()[2]),
+
+        Column('create_time', type=date, getter=lambda c,r: r.create_time()),
+        Column('cpu_num', type=int, getter=lambda c,r: r.cpu_num()),
+        Column('cpu_percent', type=float, getter=lambda c,r: r.cpu_percent()),
+
+        Column('tty', getter=lambda c,r: r.terminal()),
+        Column('nice', getter=lambda c,r: r.nice()),
+        Column('ioclass', getter=lambda c,r: r.ionice()[0]),
+        Column('ionice', getter=lambda c,r: r.ionice()[1]),
+
+        Column('user_time_s', type=float, getter=lambda c,r: r.cpu_times()[0]),
+        Column('system_time_s', type=float, getter=lambda c,r: r.cpu_times()[1]),
+        Column('children_user_time_s', type=float, width=0, getter=lambda c,r: r.cpu_times()[2]),
+        Column('children_system_time_s', type=float, width=0, getter=lambda c,r: r.cpu_times()[3]),
+
+        Column('read_ops', type=int, getter=lambda c,r: r.io_counters()[0]),
+        Column('write_ops', type=int, getter=lambda c,r: r.io_counters()[1]),
+        Column('read_bytes', type=int, getter=lambda c,r: r.io_counters()[2]),
+        Column('write_bytes', type=int, getter=lambda c,r: r.io_counters()[3]),
+
+        Column('voluntary_ctx_switches', type=int, getter=lambda c,r: r.num_ctx_switches()[0]),
+        Column('involuntary_ctx_switches', type=int, getter=lambda c,r: r.num_ctx_switches()[1]),
+        Column('num_fds', type=int, getter=lambda c,r: r.num_fds()),
+        Column('num_threads', type=int, getter=lambda c,r: r.num_threads()),
+    ]
+    nKeys = 2
+
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+#        self.columns = []
+#        for c in ProcessesSheet.columns:
+#            self.addColumn(c)
+
+        for i,k in enumerate(psutil.Process().memory_full_info()._fields):
+            self.addColumn(Column('mem_'+k, type=int, getter=lambda c,r,i=i: r.memory_full_info()[i], cache=True))
+        # mem_uss is probably the most representative metric for determining how much memory is actually being used by a process. It represents the amount of memory that would be freed if the process was terminated right now.
+
+        yield from psutil.process_iter()
+#            try:
+#                self.addRow((pr, pr.memory_full_info()))
+#            except:
+#                self.addRow((pr, None))
+
+class RlimitsSheet(Sheet):
+    columns = [
+        ColumnItem('rlimit', 0),
+        Column('soft', type=int, getter=lambda c,r: c.sheet.soft(r), setter=lambda c,r,v: c.sheet.set_soft(r, v)),
+        Column('hard', type=int, getter=lambda c,r: c.sheet.hard(r), setter=lambda c,r,v: c.sheet.set_hard(r, v))
+    ]
+
+    def soft(self, r):
+        return self.source.rlimit(r[1])[0]
+    def hard(self, r):
+        return self.source.rlimit(r[1])[1]
+    def set_soft(self, r, v):
+        self.source.rlimit(r[1], (v, self.hard(r)))
+    def set_hard(self, r, v):
+        self.source.rlimit(r[1], (self.soft(r), v))
+
+    def iterload(self):
+        psutil = vd.importExternal('psutil')
+        for r in dir(psutil):
+            if r.startswith('RLIMIT'):
+                yield (r[7:], getattr(psutil, r))
+
+
+@VisiData.lazy_property
+def cpuStats(vd):
+    return CPUStatsSheet('cpu_stats')
+
+@VisiData.lazy_property
+def memStats(vd):
+    return MemStatsSheet('memory_stats')
+
+@VisiData.lazy_property
+def processes(vd):
+    return ProcessesSheet('processes')
+
+
+BaseSheet.addCommand('', 'open-cpustats', 'vd.push(vd.cpuStats)', 'open CPU usage stats' )
+BaseSheet.addCommand('', 'open-memstats', 'vd.push(vd.memStats)', 'open memory usage stats')
+BaseSheet.addCommand('', 'open-processes', 'vd.push(vd.processes)', 'open system process stats')
+
+@VisiData.api
+def chooseSignal(vd):
+    import signal
+    d = [{'key': attr[3:]} for attr in dir(signal) if attr.startswith('SIG') and not attr.startswith('SIG_')]
+    return getattr(signal, 'SIG'+vd.chooseOne(d, type='signal'))
+
+ProcessesSheet.addCommand('d', 'term-process', 'os.kill(cursorRow.pid, signal.SIGTERM)', 'send SIGTERM to process')
+ProcessesSheet.addCommand('gd', 'term-selected', 'for r in someSelectedRows: os.kill(r.pid, signal.SIGTERM)', 'send SIGTERM to selected processes')
+ProcessesSheet.addCommand('zd', 'kill-process', 'os.kill(cursorRow.pid, signal.SIGKILL)', 'send SIGKILL to process')
+ProcessesSheet.addCommand('gzd', 'kill-selected', 'for r in someSelectedRows: os.kill(r.pid, signal.SIGKILL)', 'send SIGKILL to selected processes')
+
+ProcessesSheet.addCommand('^K', 'signal-process', 'os.kill(cursorRow.pid, chooseSignal())', 'send chosen signal to process')
+UsefulProcessesSheet.addCommand('^K', 'signal-selected', 'os.kill(cursorRow.pid, chooseSignal())', 'kill(2) send chosen signal to process')
+UsefulProcessesSheet.addCommand('', 'open-rlimits', 'vd.push(RlimitsSheet(cursorRow.name() + "_rlimits", cursorRow))', 'push rlimits for this process')
+
+vd.addMenuItem('System', 'Signal', 'current process', 'TERMinate', 'term-process')
+vd.addMenuItem('System', 'Signal', 'current process', 'KILL', 'kill-process')
+vd.addMenuItem('System', 'Signal', 'current process', 'choose signal', 'signal-process')
+vd.addMenuItem('System', 'Signal', 'selected processes', 'TERMinate', 'term-selected')
+vd.addMenuItem('System', 'Signal', 'selected processes', 'KILL', 'kill-selected')
+vd.addMenuItem('System', 'Signal', 'selected processes', 'choose signal', 'signal-selected')
+vd.addMenuItem('System', 'Stats', 'CPU', 'open-cpustats')
+vd.addMenuItem('System', 'Stats', 'memory', 'open-memstats')
+vd.addMenuItem('System', 'Stats', 'processes', 'open-processes')
+vd.addMenuItem('System', 'Stats', 'resource limits', 'open-rlimits')

visidata/features/random_sample.py

@@ -0,0 +1,6 @@
+from visidata import vd, Sheet
+
+Sheet.addCommand(None, 'random-rows', 'nrows=int(input("random number to filter: ", value=nRows)); vs=copy(sheet); vs.name=name+"_sample"; vs.rows=sample(rows, nrows or nRows); vd.push(vs)', 'open duplicate sheet with a random population subset of N rows')
+Sheet.addCommand(None, 'select-random', 'nrows=int(input("random number to select: ", value=nRows)); select(sample(rows, nrows or nRows))', 'select random sample of N rows')
+
+vd.addMenuItems('Row > Select > random sample > select-random')

visidata/{regex.py → features/regex.py}

@@ -5,17 +5,11 @@ from visidata import asyncthread, options, vd
 from visidata import VisiData, BaseSheet, Sheet, Column, Progress
 
 
-@Sheet.api
-def setSubst(sheet, cols, rows):
-    if not rows:
-        vd.warning('no %s selected' % sheet.rowtype)
-        return
-    modified = 'column' if len(cols) == 1 else 'columns'
-    rex = vd.input("transform %s by regex: " % modified, type="regex-subst")
-    setValuesFromRegex(cols, rows, rex)
+@VisiData.lazy_property
+def help_regex(vd):
+    return vd.getHelpPane('regex', module='visidata')
 
 
-vd.option('regex_flags', 'I', 'flags to pass to re.compile() [AILMSUX]', replay=True)
 vd.option('regex_maxsplit', 0, 'maxsplit to pass to regex.split', replay=True)
 
 @VisiData.api
@@ -24,6 +18,8 @@ def makeRegexSplitter(vd, regex, origcol):
 
 @VisiData.api
 def makeRegexMatcher(vd, regex, origcol):
+    if not regex.groups:
+        vd.fail('specify a capture group')  #1778
     def _regexMatcher(row):
         m = regex.search(origcol.getDisplayValue(row))
         if m:
@@ -52,12 +48,9 @@ def addRegexColumns(vs, regexMaker, origcol, regexstr):
     cols = {}
     ncols = 0  # number of new columns added already
     for r in Progress(vs.getSampleRows()):
-        try:
-            m = func(r)
-            if not m:
-                continue
-        except Exception as e:
-            vd.exceptionCaught(e)
+        m = vd.callNoExceptions(func, r)
+        if not m:
+            continue
 
         if isinstance(m, dict):
             for name in m:
@@ -74,12 +67,15 @@ def addRegexColumns(vs, regexMaker, origcol, regexstr):
         else:
             raise TypeError("addRegexColumns() expects a dict, list, or tuple from regexMaker, but got a "+type(m).__name__)
 
+    if not cols:
+        vd.warning("no regex matches found, didn't add column")
+        return
+
     vs.addColumnAtCursor(*cols.values())
 
 
 @VisiData.api
-def regexTransform(vd, origcol, instr):
-    before, after = vd.parse_sed_transform(instr)
+def regexTransform(vd, origcol, before='', after=''):
     return lambda col,row,origcol=origcol,before=before,after=after,flags=origcol.sheet.regex_flags(): re.sub(before, after, origcol.getDisplayValue(row), flags=flags)
 
 
@@ -104,27 +100,47 @@ def indexWithEscape(s, char, escape_char='\\'):
     return None
 
 
+@Sheet.api
 @asyncthread
-def setValuesFromRegex(cols, rows, rex):
-    transforms = [vd.regexTransform(col, rex) for col in cols]
+def setValuesFromRegex(sheet, cols, rows, before='', after=''):
+    transforms = [vd.regexTransform(col, before=before, after=after) for col in cols]
     vd.addUndoSetValues(cols, rows)
     for r in Progress(rows, 'replacing'):
         for col, transform in zip(cols, transforms):
-            col.setValueSafe(r, transform(col, r))
+            vd.callNoExceptions(col.setValue, r, transform(col, r))
+
     for col in cols:
         col.recalc()
 
 
-@BaseSheet.api
-def regex_flags(sheet):
-    'Return flags to pass to regex functions from options'
-    return sum(getattr(re, f.upper()) for f in options.regex_flags)
+@VisiData.api
+def inputRegex(vd, prompt, type='regex', **kwargs):
+    return vd.input(prompt, type=type, help=vd.help_regex, **kwargs)
+
 
 
-Sheet.addCommand(':', 'split-col', 'addRegexColumns(makeRegexSplitter, cursorCol, input("split regex: ", type="regex-split"))', 'Add new columns from regex split')
-Sheet.addCommand(';', 'capture-col', 'addRegexColumns(makeRegexMatcher, cursorCol, input("capture regex: ", type="regex-capture"))', 'add new column from capture groups of regex; requires example row')
-Sheet.addCommand('', 'addcol-split', 'addColumnAtCursor(RegexColumn(makeRegexSplitter, cursorCol, input("split regex: ", type="regex-split")))', 'Add column split by regex')
-Sheet.addCommand('', 'addcol-capture', 'addColumnAtCursor(RegexColumn(makeRegexMatcher, cursorCol, input("capture regex: ", type="regex-capture")))', 'Add column captured by regex')
-Sheet.addCommand('*', 'addcol-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, input("transform column by regex: ", type="regex-subst"))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)')
-Sheet.addCommand('g*', 'setcol-subst', 'setSubst([cursorCol], someSelectedRows)', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)')
-Sheet.addCommand('gz*', 'setcol-subst-all', 'setSubst(visibleCols, someSelectedRows)', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)')
+@VisiData.api
+def inputRegexSubst(vd, prompt):
+    'Input regex transform via oneliner (separated with `/`).  Return parsed transformer as dict(before=, after=).'
+    return vd.inputMultiple(before=dict(type='regex', prompt='search: ', help=prompt),
+                            after=dict(type='regex-replace', prompt='replace: ', help=prompt))
+
+
+Sheet.addCommand(':', 'addcol-split', 'addColumnAtCursor(RegexColumn(makeRegexSplitter, cursorCol, inputRegex("split regex: ", type="regex-split")))', 'Add column split by regex')
+Sheet.addCommand(';', 'addcol-capture', 'addColumnAtCursor(RegexColumn(makeRegexMatcher, cursorCol, inputRegex("capture regex: ", type="regex-capture")))', 'Add column captured by regex')
+
+Sheet.addCommand('*', 'addcol-regex-subst', 'addColumnAtCursor(Column(cursorCol.name + "_re", getter=regexTransform(cursorCol, **inputRegexSubst("regex transform column"))))', 'add column derived from current column, replacing regex with subst (may include \1 backrefs)')
+Sheet.addCommand('g*', 'setcol-regex-subst', 'setValuesFromRegex([cursorCol], someSelectedRows, **inputRegexSubst("regex transform column"))', 'regex/subst - modify selected rows in current column, replacing regex with subst, (may include backreferences \\1 etc)')
+Sheet.addCommand('gz*', 'setcol-regex-subst-all', 'setValuesFromRegex(visibleCols, someSelectedRows, **inputRegexSubst(f"regex transform {nVisibleCols} columns"))', 'modify selected rows in all visible columns, replacing regex with subst (may include \\1 backrefs)')
+
+
+vd.addMenuItems('''
+    Edit > Modify > selected cells > regex substitution > setcol-regex-subst
+    Column > Add column > capture by regex > addcol-capture
+    Column > Add column > split by regex > addcol-split
+    Column > Add column > subst by regex > addcol-regex-subst
+    Row > Select > by regex > current column > select-col-regex
+    Row > Select > by regex > all columns > select-cols-regex
+    Row > Unselect > by regex > current column > unselect-col-regex
+    Row > Unselect > by regex > all columns > unselect-cols-regex
+''')