visidata 2.11.1__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/themes/light.py

@@ -0,0 +1,17 @@
+'Light-mode theme using 256-colors.'
+
+from visidata import vd
+
+vd.themes['light'] = dict(
+        color_default      = 'black on white',  # the default fg and bg colors
+        color_key_col      = '20 blue',   # color of key columns
+        color_edit_cell    = '234 black',     # cell color to use when editing cell
+        color_selected_row = '164 magenta',  # color of selected rows
+        color_note_row     = '164 magenta',  # color of row note on left edge
+        color_note_type    = '88 red',  # color of cell note for non-str types in anytype columns
+        color_warning      = '202 11 yellow',
+        color_add_pending  = '34 green',
+        color_change_pending  = '166 yellow',
+        plot_colors = '20 red magenta black 28 88 94 99 106'
+)
+

visidata/threads.py

@@ -5,16 +5,17 @@ import functools
 import cProfile
 import threading
 import collections
+import subprocess
+import curses
 
 from visidata import VisiData, vd, options, globalCommand, Sheet, EscapeException
-from visidata import ColumnAttr, Column
-from visidata import *
+from visidata import ColumnAttr, Column, BaseSheet, ItemColumn
 
 
-vd.option('profile', False, 'enable profiling on threads')
-vd.option('min_memory_mb', 0, 'minimum memory to continue loading and async processing')
+vd.option('profile', False, 'enable profiling on threads', max_help=-1)
+vd.option('min_memory_mb', 0, 'minimum memory to continue loading and async processing', max_help=-1)
 
-vd.option('color_working', 'green', 'color of system running smoothly')
+vd.theme_option('color_working', '118 5', 'color of system running smoothly')
 
 BaseSheet.init('currentThreads', list)
 
@@ -115,29 +116,32 @@ def elapsed_s(t):
 
 @VisiData.api
 def checkMemoryUsage(vd):
-    min_mem = options.min_memory_mb
     threads = vd.unfinishedThreads
     if not threads:
-        return None
-    ret = ''
-    attr = 'color_working'
-    if min_mem:
-        try:
-            freestats = subprocess.run('free --total --mega'.split(), check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip().splitlines()
-        except FileNotFoundError as e:
-            if options.debug:
-                vd.exceptionCaught(e)
-            options.min_memory_mb = 0
-            vd.warning('disabling min_memory_mb: "free" not installed')
-            return '', attr
-        tot_m, used_m, free_m = map(int, freestats[-1].split()[1:])
-        ret = '[%dMB] ' % free_m + ret
-        if free_m < min_mem:
-            attr = 'color_warning'
-            vd.warning('%dMB free < %dMB minimum, stopping threads' % (free_m, min_mem))
-            vd.cancelThread(*vd.unfinishedThreads)
-            curses.flash()
-    return ret, attr
+        return ''
+
+    min_mem = vd.options.min_memory_mb
+    if not min_mem:
+        return ''
+
+    try:
+        freestats = subprocess.run('free --total --mega'.split(), check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.strip().splitlines()
+    except FileNotFoundError as e:
+        if vd.options.debug:
+            vd.exceptionCaught(e)
+        vd.options.min_memory_mb = 0
+        vd.warning('disabling min_memory_mb: "free" not installed')
+        return ''
+    tot_m, used_m, free_m = map(int, freestats[-1].split()[1:])
+    ret = f'  [{free_m}MB] '
+    if free_m < min_mem:
+        attr = '[:warning]'
+        vd.warning(f'{free_m}MB free < {min_mem}MB minimum, stopping threads')
+        vd.cancelThread(*vd.unfinishedThreads)
+        curses.flash()
+    else:
+        attr = '[:working]'
+    return attr + ret + '[/]'
 
 
 # for progress bar
@@ -170,17 +174,34 @@ def _annotate_thread(t, endTime=None):
     return t
 
 # all long-running threads, including main and finished
-VisiData.init('threads', lambda: [_annotate_thread(threading.current_thread(), 0)])
+vd.threads = [_annotate_thread(threading.current_thread(), 0)]
 
 @VisiData.api
-def execAsync(self, func, *args, sheet=None, **kwargs):
-    'Execute ``func(*args, **kwargs)`` in a separate thread.'
+def execSync(vd, func, *args, sheet=None, **kwargs):
+    'Execute ``func(*args, **kwargs)`` in this thread (synchronously). A drop-in substitute for vd.execAsync.'
+    vd.callNoExceptions(func, *args, **kwargs)
+    t = threading.current_thread()
+    t.sheet = sheet or vd.activeSheet
+    return t
 
-    thread = threading.Thread(target=_toplevelTryFunc, daemon=True, args=(func,)+args, kwargs=kwargs)
-    self.threads.append(_annotate_thread(thread))
+@VisiData.api
+def execAsync(vd, func, *args, **kwargs):
+    '''Execute ``func(*args, **kwargs)`` in a separate thread.  `sheet` is a
+    special kwarg to indicate which sheet the thread should be associated with;
+    by default, uses vd.activeSheet.  If `sheet` explicitly given as None, the thread
+    will be ignored by vd.sync and thread status indicators.
+    '''
+
+    if 'sheet' not in kwargs:
+        sheet = vd.activeSheet
+    else:
+        sheet = kwargs.pop('sheet')
+
+    if sheet is not None and (sheet.lastCommandThreads and threading.current_thread() not in sheet.lastCommandThreads):
+        vd.fail(f'still running **{sheet.lastCommandThreads[-1].name}** from previous command')
 
-    if sheet is None:
-        sheet = self.activeSheet
+    thread = threading.Thread(target=_toplevelTryFunc, daemon=True, args=(func,)+args, kwargs=kwargs)
+    vd.threads.append(_annotate_thread(thread))
 
     if sheet is not None:
         sheet.currentThreads.append(thread)
@@ -190,7 +211,7 @@ def execAsync(self, func, *args, sheet=None, **kwargs):
 
     return thread
 
-def _toplevelTryFunc(func, *args, status=vd.status, **kwargs):
+def _toplevelTryFunc(func, *args, **kwargs):
   with ThreadProfiler(threading.current_thread()) as prof:
     t = threading.current_thread()
     t.name = func.__name__
@@ -198,8 +219,7 @@ def _toplevelTryFunc(func, *args, status=vd.status, **kwargs):
         t.status = func(*args, **kwargs)
     except EscapeException as e:  # user aborted
         t.status = 'aborted by user'
-        if status:
-            status('%s aborted' % t.name, priority=2)
+        vd.warning(f'{t.name} aborted')
     except Exception as e:
         t.exception = e
         t.status = 'exception'
@@ -208,6 +228,18 @@ def _toplevelTryFunc(func, *args, status=vd.status, **kwargs):
     if t.sheet:
         t.sheet.currentThreads.remove(t)
 
+def asyncignore(func):
+    'Decorator like `@asyncthread` but without attaching to a sheet, so no sheet.threadStatus will show it.'
+    @functools.wraps(func)
+    def _execAsync(*args, **kwargs):
+        @functools.wraps(func)
+        def _func(*args, **kwargs):
+            func(*args, **kwargs)
+
+        return vd.execAsync(_func, *args, **kwargs, sheet=None)
+
+    return _execAsync
+
 def asyncsingle(func):
     '''Function decorator like `@asyncthread` but as a singleton.  When called, `func(...)` spawns a new thread, and cancels any previous thread still running *func*.
     ``vd.sync()`` does not wait for unfinished asyncsingle threads.
@@ -233,7 +265,7 @@ def asyncsingle(func):
 @VisiData.property
 def unfinishedThreads(self):
     'A list of unfinished threads (those without a recorded `endTime`).'
-    return [t for t in self.threads if getattr(t, 'endTime', None) is None]
+    return [t for t in self.threads if getattr(t, 'endTime', None) is None and getattr(t, 'sheet', None) is not None]
 
 @VisiData.api
 def checkForFinishedThreads(self):
@@ -251,8 +283,9 @@ def sync(self, *joiningThreads):
     while True:
         deads = set()  # dead threads
         threads = joiningThreads or set(self.unfinishedThreads)
-        threads -= set([threading.current_thread(), getattr(vd, 'drawThread', None)])
+        threads -= set([threading.current_thread(), getattr(vd, 'drawThread', None), getattr(vd, 'outputProgressThread', None)])
         threads -= deads
+        threads -= set([None])
         for t in threads:
             try:
                 if not t.is_alive() or t not in threading.enumerate() or getattr(t, 'noblock', False) is True:
@@ -272,7 +305,7 @@ min_thread_time_s = 0.10 # only keep threads that take longer than this number o
 @VisiData.api
 def open_pyprof(vd, p):
     import pstats
-    return ProfileStatsSheet(p.name, source=pstats.Stats(p.given).stats)
+    return ProfileStatsSheet(p.base_stem, source=pstats.Stats(p.given).stats)
 
 
 @VisiData.api
@@ -308,10 +341,19 @@ class ThreadProfiler:
             # remove very-short-lived async actions
             if elapsed_s(self.thread) < min_thread_time_s:
                 vd.threads.remove(self.thread)
+            else:
+                if vd.options.profile:
+                    self.thread.profile.dump_stats(f'{self.thread.name}.pyprof')
 
 
 class ProfileSheet(Sheet):
     rowtype = 'callsites' # rowdef: profiler_entry
+    guide = '''
+        # Profile Sheet
+        - `z Ctrl+S` to save as pyprof file
+        - `Ctrl+O` to open current function in $EDITOR
+        - `Enter` to open list of calls from current function
+    '''
     columns = [
         Column('funcname', getter=lambda col,row: codestr(row.code)),
         Column('filename', getter=lambda col,row: os.path.split(row.code.co_filename)[-1] if not isinstance(row.code, str) else ''),
@@ -404,4 +446,10 @@ vd.addGlobals({
     'Progress': Progress,
     'asynccache': asynccache,
     'asyncsingle': asyncsingle,
+    'asyncignore': asyncignore,
 })
+
+vd.addMenuItems('''
+    System > Threads sheet > threads-all
+    System > Toggle profiling > toggle-profile
+''')

visidata/tuiwin.py

@@ -0,0 +1,22 @@
+from visidata import VisiData, vd
+
+vd._parentscrs = {}  # scr -> parentscr
+
+
+@VisiData.api
+def subwindow(vd, scr, x, y, w, h):
+    'Return subwindow with its (0,0) at (x,y) relative to parent scr.  Replacement for scr.derwin() to track parent scr.'
+    newscr = scr.derwin(h, w, y, x)
+    vd._parentscrs[newscr] = scr
+    return newscr
+
+
+@VisiData.api
+def getrootxy(vd, scr):  # like scr.getparyx() but for all ancestor scrs
+    px, py = 0, 0
+    while scr in vd._parentscrs:
+        dy, dx = scr.getparyx()
+        if dy > 0: py += dy
+        if dx > 0: px += dx
+        scr = vd._parentscrs[scr]
+    return px, py

visidata/type_currency.py

@@ -1,6 +1,7 @@
-from visidata import vd, Sheet
+from visidata import vd, Sheet, Column
 
-vd.option('disp_currency_fmt', '%.02f', 'default fmtstr to format for currency values', replay=True)
+vd.option('disp_currency_fmt', '%.02f', 'default fmtstr to format for currency values', replay=True, help=vd.help_float_fmt)
+vd.theme_option('color_currency_neg', 'red', 'color for negative values in currency displayer', replay=True)
 
 
 floatchars='+-0123456789.'
@@ -13,4 +14,22 @@ def currency(*args):
     return float(*args)
 
 
-Sheet.addCommand('$', 'type-currency', 'cursorCol.type = currency', 'set type of current column to currency')
+@Column.api
+def displayer_currency(col, dw, width=None):
+    text = dw.text
+
+    if isinstance(dw.typedval, (int, float)):
+        if dw.typedval < 0:
+            text = f'({dw.text[1:]})'.rjust(width-1)
+            yield ('currency_neg', '')
+        else:
+            text = text.rjust(width-2)
+
+    yield ('', text)
+
+
+Sheet.addCommand('$', 'type-currency', 'cursorCol.type=currency', 'set type of current column to currency')
+
+vd.addMenuItems('''
+    Column > Type as > dirty float > type-currency
+''')

visidata/type_date.py

@@ -1,16 +1,33 @@
 import datetime
 
-from visidata import vd, Sheet
-
-try:
-    from dateutil.parser import parse as date_parse
-except ImportError:
-    def date_parse(r=''):
+from visidata import VisiData, vd, Sheet
+
+@VisiData.lazy_property
+def date_parse(vd):
+    try:
+        from dateutil.parser import parse
+        return parse
+    except ImportError:
         vd.warning('install python-dateutil for date type')
-        return r
+        return str
+
+vd.help_date = '''
+- RFC3339: `%Y-%m-%d %H:%M:%S.%f %z`
+- `%A`  Weekday as locale’s full name.
+- `%w`  Weekday as a decimal number, where 0 is Sunday and 6 is Saturday.
+- `%d`  Day of the month as a zero-padded decimal number.
+- `%b`  Month as locale’s abbreviated name.
+- `%B`  Month as locale’s full name.
+- `%p`  Locale’s equivalent of either AM or PM.
+- `%c`  Locale’s appropriate date and time representation.
+- `%x`  Locale’s appropriate date representation.
+- `%X`  Locale’s appropriate time representation.
+- `%Z`  Time zone name (empty string if the object is naive).
 
+See [:onclick https://strftime.org]Python strftime()[/] for a full list of format codes.
+'''
 
-vd.option('disp_date_fmt','%Y-%m-%d', 'default fmtstr to strftime for date values', replay=True)
+vd.option('disp_date_fmt','%Y-%m-%d', 'default fmtstr passed to strftime for date values', replay=True, help=vd.help_date)
 
 
 @vd.numericType('@', '', formatter=lambda fmtstr,val: val.strftime(fmtstr or vd.options.disp_date_fmt))
@@ -28,7 +45,7 @@ class date(datetime.datetime):
         if isinstance(s, int) or isinstance(s, float):
             r = datetime.datetime.fromtimestamp(s)
         elif isinstance(s, str):
-            r = date_parse(s)
+            r = vd.date_parse(s)
         elif isinstance(s, (datetime.datetime, datetime.date)):
             r = s
         else:
@@ -109,3 +126,8 @@ vd.addGlobals(
 
 
 Sheet.addCommand('@', 'type-date', 'cursorCol.type = date', 'set type of current column to date')
+Sheet.addCommand('', 'type-datetime', 'cursorCol.type=date; cursorCol.fmtstr="%Y-%m-%d %H:%M:%S"', 'set type of current column to datetime')
+
+vd.addMenuItems('''
+    Column > Type as > date > type-date
+''')

visidata/type_floatsi.py

@@ -22,7 +22,7 @@ def floatsi(*args):
     if not args:
         return 0.0
     if not isinstance(args[0], str):
-        return args[0]
+        return float(args[0])
 
     s=args[0].strip()
     for i, p in enumerate(vd.si_prefixes):
@@ -33,3 +33,7 @@ def floatsi(*args):
 
 
 Sheet.addCommand('z%', 'type-floatsi', 'cursorCol.type = floatsi', 'set type of current column to SI float')
+
+vd.addMenuItems('''
+    Column > Type as > SI float > type-floatsi
+''')

visidata/undo.py

@@ -18,13 +18,14 @@ def isUndoableCommand(longname):
 @VisiData.api
 def addUndo(vd, undofunc, *args, **kwargs):
     'On undo of latest command, call ``undofunc(*args, **kwargs)``.'
-    if options.undo:
+    if vd.options.undo:
         # occurs when VisiData is just starting up
         if getattr(vd, 'activeCommand', UNLOADED) is UNLOADED:
             return
         r = vd.modifyCommand
         # some special commands, like open-file, do not have an undofuncs set
-        if not r or not isUndoableCommand(r.longname):
+        # do not set undofuncs for non-logged commands
+        if not r or not isUndoableCommand(r.longname) or not vd.activeCommand or not vd.isLoggableCommand(vd.activeCommand.longname):
             return
         if not r.undofuncs:
             r.undofuncs = []
@@ -33,16 +34,17 @@ def addUndo(vd, undofunc, *args, **kwargs):
 
 @VisiData.api
 def undo(vd, sheet):
-    if not options.undo:
+    if not vd.options.undo:
         vd.fail("options.undo not enabled")
 
     # don't allow undo of first command on a sheet, which is always the command that created the sheet.
-    for cmdlogrow in sheet.cmdlog_sheet.rows[:0:-1]:
+    for i, cmdlogrow in enumerate(sheet.cmdlog_sheet.rows[:0:-1]):
         if cmdlogrow.undofuncs:
             for undofunc, args, kwargs, in cmdlogrow.undofuncs[::-1]:
                 undofunc(*args, **kwargs)
             sheet.undone.append(cmdlogrow)
-            sheet.cmdlog_sheet.rows.remove(cmdlogrow)
+            row_idx = len(sheet.cmdlog_sheet.rows)-1 - i
+            del sheet.cmdlog_sheet.rows[row_idx]
 
             vd.clearCaches()  # undofunc can invalidate the drawcache
 
@@ -113,3 +115,13 @@ def addUndoColNames(vd, cols):
 
 BaseSheet.addCommand('U', 'undo-last', 'vd.undo(sheet)', 'Undo the most recent change (options.undo must be enabled)')
 BaseSheet.addCommand('R', 'redo-last', 'vd.redo(sheet)', 'Redo the most recent undo (options.undo must be enabled)')
+
+vd.addGlobals(
+    undoAttrFunc=undoAttrFunc,
+    Fanout=Fanout,
+    undoAttrCopyFunc=undoAttrCopyFunc)
+
+vd.addMenuItems('''
+    Edit > Undo > undo-last
+    Edit > Redo > redo-last
+''')

visidata/utils.py

@@ -1,10 +1,11 @@
+from contextlib import contextmanager
 import operator
 import string
 import re
 
 'Various helper classes and functions.'
 
-__all__ = ['AlwaysDict', 'AttrDict', 'moveListItem', 'namedlist', 'classproperty', 'cleanName', 'MissingAttrFormatter']
+__all__ = ['AlwaysDict', 'AttrDict', 'DefaultAttrDict', 'moveListItem', 'namedlist', 'classproperty', 'MissingAttrFormatter', 'getitem', 'setitem', 'getitemdef', 'getitemdeep', 'setitemdeep', 'getattrdeep', 'setattrdeep', 'ExplodingMock', 'ScopedSetattr']
 
 
 class AlwaysDict(dict):
@@ -36,6 +37,24 @@ class AttrDict(dict):
         return self.keys()
 
 
+class DefaultAttrDict(dict):
+    'Augment a dict with more convenient .attr syntax.  not-present keys store new DefaultAttrDict.  like a recursive defaultdict.'
+    def __getattr__(self, k):
+        if k not in self:
+            if k.startswith("__"):
+                raise AttributeError from e
+            self[k] = DefaultAttrDict()
+        return self[k]
+
+    def __setattr__(self, k, v):
+        self[k] = v
+
+    def __dir__(self):
+        return self.keys()
+
+
+
+
 class classproperty(property):
     def __get__(self, cls, obj):
         return classmethod(self.fget).__get__(None, obj or cls)()
@@ -50,34 +69,75 @@ def moveListItem(L, fromidx, toidx):
     return toidx
 
 
-def cleanName(s):
-    s = re.sub(r'[^\w\d_]', '_', s)  # replace non-alphanum chars with _
-    s = re.sub(r'_+', '_', s)  # replace runs of _ with a single _
-    s = s.strip('_')
-    return s
+def setitem(r, i, v):  # function needed for use in lambda
+    r[i] = v
+    return True
+
+def getitem(o, k, default=None):
+    return default if o is None else o[k]
+
+def getitemdef(o, k, default=None):
+    try:
+        return default if o is None else o[k]
+    except Exception:
+        return default
+
 
+def getattrdeep(obj, attr, *default, getter=getattr):
+    try:
+        'Return dotted attr (like "a.b.c") from obj, or default if any of the components are missing.'
+        if not isinstance(attr, str):
+            return getter(obj, attr, *default)
+
+        try:  # if attribute exists, return toplevel value, even if dotted
+            if attr in obj:
+                return getter(obj, attr)
+        except RecursionError:  #1696
+            raise
+        except Exception as e:
+            pass
+
+        attrs = attr.split('.')
+        for a in attrs[:-1]:
+            obj = getter(obj, a)
+
+        return getter(obj, attrs[-1])
+    except Exception as e:
+        if not default: raise
+        return default[0]
 
-class OnExit:
-    '"with OnExit(func, ...):" calls func(...) when the context is exited'
-    def __init__(self, func, *args, **kwargs):
-        self.func = func
-        self.args = args
-        self.kwargs = kwargs
 
-    def __enter__(self):
-        return self
+def setattrdeep(obj, attr, val, getter=getattr, setter=setattr):
+    'Set dotted attr (like "a.b.c") on obj to val.'
+    if not isinstance(attr, str):
+        return setter(obj, attr, val)
 
-    def __exit__(self, exc_type, exc_value, exc_traceback):
+    try:  # if attribute exists, overwrite toplevel value, even if dotted
+        getter(obj, attr)
+        return setter(obj, attr, val)
+    except Exception as e:
+        pass
+
+    attrs = attr.split('.')
+    for a in attrs[:-1]:
         try:
-            self.func(*self.args, **self.kwargs)
+            obj = getter(obj, a)
         except Exception as e:
-            vd.exceptionCaught(e)
+            obj = obj[a] = type(obj)()  # assume homogeneous nesting
+
+    setter(obj, attrs[-1], val)
 
 
-def itemsetter(i):
-    def g(obj, v):
-        obj[i] = v
-    return g
+def getitemdeep(obj, k, *default):
+    if not isinstance(k, str):
+        try:
+            return obj[k]
+        except IndexError:
+            pass
+    return getattrdeep(obj, k, *default, getter=getitem)
+
+def setitemdeep(obj, k, val):
+    return setattrdeep(obj, k, val, getter=getitemdef, setter=setitem)
 
 
 def namedlist(objname, fieldnames):
@@ -111,11 +171,24 @@ def namedlist(objname, fieldnames):
 
     return NamedListTemplate
 
+
+class ExplodingMock:
+    'A mock object that raises an exception for everything except conversion to True/False.'
+    def __init__(self, msg):
+        self.__msg = msg
+
+    def __getattr__(self, k):
+        raise Exception(self.__msg)
+
+    def __bool__(self):
+        return False
+
+
 class MissingAttrFormatter(string.Formatter):
     "formats {} fields with `''`, that would normally result in a raised KeyError or AttributeError; intended for user customisable format strings."
-    def get_field(self, field_name, *args, **kwargs):
+    def get_field(self, field_name, args, kwargs):
         try:
-            return super().get_field(field_name, *args, **kwargs)
+            return super().get_field(field_name, args, kwargs)
         except (KeyError, AttributeError):
             return (None, field_name)
 
@@ -126,3 +199,13 @@ class MissingAttrFormatter(string.Formatter):
         elif not value:
             return str(value)
         return super().format_field(value, format_spec)
+
+
+@contextmanager
+def ScopedSetattr(obj, attrname, val):
+    oldval = getattr(obj, attrname)
+    try:
+        setattr(obj, attrname, val)
+        yield
+    finally:
+        setattr(obj, attrname, oldval)