visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/pivot.py

@@ -1,5 +1,8 @@
 import collections
-from visidata import *
+from copy import copy
+from visidata import ScopedSetattr, Column, Sheet, asyncthread, Progress, forward, wrapply, INPROGRESS
+from visidata import vlen, vd, date, setitem, anytype
+import visidata
 
 
 # discrete_keys = tuple of formatted discrete keys that group the row
@@ -8,10 +11,10 @@ from visidata import *
 # pivotrows is { pivot_values: list(source.rows in group with pivot_values) }
 PivotGroupRow = collections.namedtuple('PivotGroupRow', 'discrete_keys numeric_key sourcerows pivotrows'.split())
 
-def Pivot(source, groupByCols, pivotCols):
+def makePivot(source, groupByCols, pivotCols):
     return PivotSheet('',
-            groupByCols,
-            pivotCols,
+            groupByCols=groupByCols,
+            pivotCols=pivotCols,
             source=source)
 
 def makeErrorKey(col):
@@ -23,6 +26,8 @@ def makeErrorKey(col):
 def formatRange(col, numeric_key):
     a, b = numeric_key
     nankey = makeErrorKey(col)
+    if b is None:
+        return a
     if a is nankey and b is nankey:
         return '#ERR'
     elif a == b:
@@ -44,31 +49,38 @@ class RangeColumn(Column):
         return formatRange(self.origcol, typedval)
 
 
-def AggrColumn(aggcol, aggregator):
+class AggrColumn(Column):
+    def calcValue(col, row):
+        if col.sheet.loading:
+            return visidata.INPROGRESS
+        return col.aggregator(col.origCol, row.sourcerows)
+
+
+def makeAggrColumn(aggcol, aggregator):
     aggname = '%s_%s' % (aggcol.name, aggregator.name)
 
-    return Column(aggname,
+    return AggrColumn(aggname,
                   type=aggregator.type or aggcol.type,
                   fmtstr=aggcol.fmtstr,
-                  getter=lambda col,row,agg=aggregator: agg(col.origCol, row.sourcerows),
                   origCol=aggcol,
-                  )
+                  aggregator=aggregator)
 
 
 class PivotSheet(Sheet):
     'Summarize key columns in pivot table and display as new sheet.'
     rowtype = 'grouped rows'  # rowdef: PivotGroupRow
-    def __init__(self, name, groupByCols, pivotCols, **kwargs):
-        super().__init__(name, **kwargs)
 
-        self.pivotCols = pivotCols  # whose values become columns
-        self.groupByCols = groupByCols  # whose values become rows
+    def __init__(self, *names, groupByCols=[], pivotCols=[], **kwargs):
+        super().__init__(*names,
+                pivotCols=pivotCols, # whose values become columns
+                groupByCols=groupByCols,  # whose values become rows
+                **kwargs)
 
     def isNumericRange(self, col):
         return vd.isNumeric(col) and self.source.options.numeric_binning
 
-    def initCols(self):
-        self.columns = []
+    def resetCols(self):
+        super().resetCols()
 
         # add key columns (grouped by)
         for colnum, c in enumerate(self.groupByCols):
@@ -102,12 +114,10 @@ class PivotSheet(Sheet):
         vs.rows = row.pivotrows.get(col.aggvalue, [])
         return vs
 
-    def reload(self):
-        self.initCols()
-
+    def loader(self):
         # two different threads for better interactive display
-        self.addAggregateCols()
-        self.groupRows()
+        vd.sync(self.addAggregateCols(),
+                self.groupRows())
 
     @asyncthread
     def addAggregateCols(self):
@@ -129,7 +139,7 @@ class PivotSheet(Sheet):
         if not self.pivotCols:
             for aggcol, aggregatorlist in aggcols.items():
                 for aggregator in aggregatorlist:
-                    c = AggrColumn(aggcol, aggregator)
+                    c = makeAggrColumn(aggcol, aggregator)
                     self.addColumn(c)
 
         # add pivoted columns
@@ -176,6 +186,7 @@ class PivotSheet(Sheet):
 
     @asyncthread
     def groupRows(self, rowfunc=None):
+      with ScopedSetattr(self, 'loading', True):
         self.rows = []
 
         discreteCols = [c for c in self.groupByCols if not self.isNumericRange(c)]
@@ -190,13 +201,16 @@ class PivotSheet(Sheet):
         if numericCols:
             nbins = self.source.options.histogram_bins or int(len(self.source.rows) ** (1./2))
             vals = tuple(numericCols[0].getValues(self.source.rows))
-            minval = min(vals)
-            maxval = max(vals)
+            minval = min(vals) if vals else 0
+            maxval = max(vals) if vals else 0
             width = (maxval - minval)/nbins
 
             if width == 0:
-                # only one value (and maybe errors)
-                numericBins = [(minval, maxval)]
+                if vals:
+                    # only one value
+                    numericBins = [(minval, maxval)]
+                else:
+                    numericBins = []
             elif (numericCols[0].type in (int, vlen) and nbins > (maxval - minval)) or (width == 1):
                 # (more bins than int vals) or (if bins are of width 1), just use the vals as bins
                 degenerateBinning = True
@@ -226,25 +240,29 @@ class PivotSheet(Sheet):
             if numericCols:
                 try:
                     val = numericCols[0].getValue(sourcerow)
-                    if val is not None:
-                        val = numericCols[0].type(val)
-                    if not width:
-                        binidx = 0
-                    elif degenerateBinning:
-                        # in degenerate binning, each val has its own bin
-                        binidx = numericBins.index((val, val))
+                    val = wrapply(numericCols[0].type, val)
+                    if not val:
+                        groupRow = numericGroupRows.get(str(val), None)
                     else:
-                        binidx = int((val-minval)//width)
-                    groupRow = numericGroupRows[formatRange(numericCols[0], numericBins[min(binidx, nbins-1)])]
+                        if not width:
+                            binidx = 0
+                        elif degenerateBinning:
+                            # in degenerate binning, each val has its own bin
+                            binidx = numericBins.index((val, val))
+                        else:
+                            binidx = int((val-minval)//width)
+                        groupRow = numericGroupRows[formatRange(numericCols[0], numericBins[min(binidx, nbins-1)])]
                 except Exception as e:
-                    # leave in main/error bin
-                    pass
+                    vd.exceptionCaught(e)
 
             # add the main bin if no numeric bin (error, or no numeric cols)
             if groupRow is None:
-                nankey = makeErrorKey(numericCols[0]) if numericCols else 0
-                groupRow = PivotGroupRow(discreteKeys, (nankey, nankey), [], {})
-                groups[formattedDiscreteKeys] = (numericGroupRows, groupRow)
+                if numericCols:
+                    groupRow = PivotGroupRow(discreteKeys, val, [], {})
+                    numericGroupRows[str(val)] = groupRow
+                else:
+                    groupRow = PivotGroupRow(discreteKeys, (0, 0), [], {})
+                    groups[formattedDiscreteKeys] = (numericGroupRows, groupRow)
                 self.addRow(groupRow)
 
             # add the sourcerow to its all bin
@@ -261,24 +279,33 @@ class PivotSheet(Sheet):
             if rowfunc:
                 rowfunc(groupRow)
 
+    def afterLoad(self):
+        super().afterLoad()
+
         # automatically add cache to all columns now that everything is binned
         for c in self.nonKeyVisibleCols:
-            c.setCache(True)
+            if isinstance(c, AggrColumn):
+                c.setCache(True)
 
 
 @PivotSheet.api
 def addcol_aggr(sheet, col):
     hasattr(col, 'origCol') or vd.fail('not an aggregation column')
-    for agg in vd.chooseMany(vd.aggregator_choices):
-        sheet.addColumnAtCursor(AggrColumn(col.origCol, vd.aggregators[agg]))
+    for agg in vd.chooseAggregators():
+        sheet.addColumnAtCursor(makeAggrColumn(col.origCol, vd.aggregators[agg]))
 
 
-Sheet.addCommand('W', 'pivot', 'vd.push(Pivot(sheet, keyCols, [cursorCol]))', 'open Pivot Table: group rows by key column and summarize current column')
+Sheet.addCommand('W', 'pivot', 'vd.push(makePivot(sheet, keyCols, [cursorCol]))', 'open Pivot Table: group rows by key column and summarize current column')
 
 PivotSheet.addCommand('', 'addcol-aggr', 'addcol_aggr(cursorCol)', 'add aggregation column from source of current column')
-vd.addMenuItem('Column', 'Add column', 'aggregator', 'addcol-aggr')
 
-vd.addGlobals({
-    'Pivot': Pivot,
-    'PivotGroupRow': PivotGroupRow,
-})
+vd.addGlobals(
+    makePivot=makePivot,
+    PivotSheet=PivotSheet,
+    PivotGroupRow=PivotGroupRow,
+)
+
+vd.addMenuItems('''
+    Column > Add column > aggregator > addcol-aggr
+    Data > Pivot > pivot
+''')

visidata/plugins.py

@@ -5,225 +5,98 @@ import re
 import shutil
 import importlib
 import subprocess
-import urllib
+import urllib.error
 
-from visidata import VisiData, vd, Path, CellColorizer, JsonLinesSheet, AttrDict, Column, Progress, ExpectedException, BaseSheet, asyncsingle, asyncthread
+from visidata import VisiData, vd, Path, CellColorizer, Sheet, AttrDict, ItemColumn, Column, Progress, ExpectedException, BaseSheet, asyncthread
 
 
-vd.option('plugins_url', 'https://visidata.org/plugins/plugins.jsonl', 'source of plugins sheet')
 vd.option('plugins_autoload', True, 'do not autoload plugins if False')
 
 
-@VisiData.lazy_property
-def pluginsSheet(p):
-    return PluginsSheet('plugins_global')
-
-def _plugin_path(plugin):
-    return Path(os.path.join(vd.options.visidata_dir, "plugins", plugin.name+".py"))
-
-def _plugin_init():
+@VisiData.property
+def pluginConfig(self):
     return Path(os.path.join(vd.options.visidata_dir, "plugins", "__init__.py"))
 
-def _plugin_import(plugin):
-    return "import " + _plugin_import_name(plugin)
 
-def _plugin_import_name(plugin):
+@VisiData.property
+def pluginConfigLines(self):
+    return Path(self.pluginConfig).open(mode='r', encoding='utf-8').readlines()
+
+def _plugin_import_name(self, plugin):
     if not plugin.url:
         return 'visidata.plugins.'+plugin.name
     if 'git+' in plugin.url:
         return plugin.name
     return "plugins."+plugin.name
 
-def _plugin_in_import_list(plugin):
-    with Path(_plugin_init()).open_text(mode='r', encoding='utf-8') as fprc:
-        r = re.compile(r'^{}\W'.format(_plugin_import(plugin)))
-        for line in fprc.readlines():
-            if r.match(line):
-                return True
 
-def _installedStatus(col, plugin):
-    return '*' if importlib.util.find_spec(_plugin_import_name(plugin)) else ''
+@VisiData.api
+def enablePlugin(vd, plugin:str):
+    with vd.pluginConfig.open(mode='a', encoding='utf-8') as fprc:
+        print(f'import {plugin}', file=fprc)
+        importlib.import_module(plugin)
+        vd.status(f'{plugin} plugin enabled')
 
-def _loadedVersion(plugin):
-    name = _plugin_import_name(plugin)
-    if name not in sys.modules:
-        return ''
-    mod = sys.modules[name]
-    return getattr(mod, '__version__', 'unknown version installed')
+@VisiData.api
+def removePlugin(vd, plugin:str):
+    path = vd.pluginConfig
+    pathbackup = path.with_suffix(path.suffix + '.bak')
+    try:
+        shutil.copyfile(path, pathbackup)
+
+        # Copy lines from the backup init file into its replacement, skipping lines that import the removed plugin.
+        #
+        # By matching from the start of a line through a word boundary, we avoid removing commented lines or inadvertently removing
+        # plugins with similar names.
+
+        r = re.compile(f'^import {plugin}\\W')
+        nonimports = [line for line in vd.pluginConfigLines if not r.match(line)]
+        if len(nonimports) == len(vd.pluginConfigLines):
+            vd.fail("plugin not in import list")
 
-def _checkHash(data, sha):
-    import hashlib
-    return hashlib.sha256(data.strip().encode('utf-8')).hexdigest() == sha
+        with path.open(mode='w', encoding='utf-8') as new:
+            new.writelines(nonimports)
 
-def _pluginColorizer(s,c,r,v):
-    if not r: return None
-    ver = _loadedVersion(r)
-    if not ver: return None
-    if not r.latest_ver: return None
-    if ver != r.latest_ver: return 'color_warning'
-    return 'color_working'
+        sys.modules.pop(plugin)
+        importlib.invalidate_caches()
+        vd.warning(f'"{plugin}" plugin removed')
+    except FileNotFoundError:
+        vd.debug("no {vd.pluginConfig} found")
 
 
-@VisiData.api
-def pipinstall(vd, deps):
-    'Install *deps*, a list of pypi modules to install via pip into the plugins-deps directory. Return True if successful (no error).'
-    p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',
-                        '--target', str(Path(vd.options.visidata_dir)/"plugins-deps"),
-                      ] + deps,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-    out, err = p.communicate()
-    vd.status(out.decode())
-    if err:
-        vd.warning(err.decode())
-        return False
-    return True
-
-
-class PluginsSheet(JsonLinesSheet):
+@VisiData.lazy_property
+def pluginsSheet(p):
+    return PluginsSheet('plugins')
+
+
+class PluginsSheet(Sheet):
     rowtype = "plugins"  # rowdef: AttrDict of json dict
     colorizers = [
-        CellColorizer(3, None, _pluginColorizer)
+            CellColorizer(3, 'color_working', lambda s,c,r,v: r and r.installed)
     ]
-
+    columns = [
+        ItemColumn('name'),
+        ItemColumn('installed', width=8),
+        ItemColumn('description', width=60),
+    ]
+    nKeys = 1
     def iterload(self):
-        for r in JsonLinesSheet.iterload(self):
-            yield AttrDict(r)
-
-    @asyncsingle
-    def reload(self):
-        try:
-            self.source = vd.urlcache(vd.options.plugins_url or vd.fail(), days=1)  # for VisiDataMetaSheet.reload()
-        except urllib.error.URLError as e:
-            vd.debug(e)
-            return
-
-        super().reload.__wrapped__(self)
-        self.addColumn(Column('available', width=0, getter=_installedStatus), index=1)
-        self.addColumn(Column('installed', width=8, getter=lambda c,r: _loadedVersion(r)), index=2)
-        self.column('description').width = 40
-        self.setKeys([self.column("name")])
-
+        import pkgutil
+        import ast
+        # enumerate installed plugins
         for name, mod in sys.modules.items():
-            if name.startswith('visidata.plugins.'):
-                self.addRow(AttrDict(name='.'.join(name.split('.')[2:]),
-                                 description=getattr(mod, '__description__', mod.__doc__),
-                                 maintainer=getattr(mod, '__author__', None),
-                                 latest_release='',
-                                 latest_ver='',
-                                 url=''
-                                 ))
-
-        for r in Progress(self.rows):
-            for funcname in (r.provides or '').split():
-                func = lambda *args, **kwargs: vd.fail('this requires the %s plugin' % r.name)
-                vd.addGlobals({funcname: func})
-                setattr(vd, funcname, func)
-
-        # check for plugins with newer versions
-        def is_stale(r):
-            v = _loadedVersion(r)
-            return v and r.latest_ver and v != r.latest_ver
-
-        stale_plugins = list(filter(is_stale, self.rows))
-        if len(stale_plugins) > 0:
-            vd.warning(f'update available for {len(stale_plugins)} plugins')
-
-    def installPlugin(self, plugin):
-        # pip3 install requirements
-        initpath = _plugin_init()
-        os.makedirs(initpath.parent, exist_ok=True)
-        if not initpath.exists():
-            initpath.touch()
-
-        outpath = _plugin_path(plugin)
-        overwrite = True
-        if outpath.exists():
-            try:
-                vd.confirm("plugin path already exists, overwrite? ")
-            except ExpectedException:
-                overwrite = False
-                if _plugin_in_import_list(plugin):
-                    vd.fail("plugin already loaded")
-                else:
-                    self._loadPlugin(plugin)
-        if overwrite:
-            self._install(plugin)
-
-    @asyncthread
-    def _install(self, plugin):
-        outpath = _plugin_path(plugin)
-
-        if "git+" in plugin.url:
-            p = subprocess.Popen([sys.executable, '-m', 'pip', 'install',
-                                '--upgrade', plugin.url],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE)
-            out, err = p.communicate()
-            vd.status(out.decode())
-            if err:
-                vd.warning(err.decode())
-            if p.returncode != 0:
-                vd.fail('pip install failed')
-        else:
-            with vd.urlcache(plugin.url, days=0).open_text(encoding='utf-8') as pyfp:
-                contents = pyfp.read()
-                if plugin.sha256:
-                    if not _checkHash(contents, plugin.sha256):
-                        vd.error('%s plugin SHA256 does not match!' % plugin.name)
-                else:
-                    vd.warning('no SHA256 provided for %s plugin, not validating' % plugin.name)
-                with outpath.open_text(mode='w', encoding='utf-8') as outfp:
-                    outfp.write(contents)
-
-        if plugin.pydeps:
-            vd.pipinstall(plugin.pydeps.split())
-
-        vd.status('%s plugin installed' % plugin.name)
-
-        if _plugin_in_import_list(plugin):
-            vd.warning("plugin already loaded")
-        else:
-            self._loadPlugin(plugin)
-
-
-    def _loadPlugin(self, plugin):
-        with Path(_plugin_init()).open_text(mode='a', encoding='utf-8') as fprc:
-            print(_plugin_import(plugin), file=fprc)
-            importlib.import_module(_plugin_import_name(plugin))
-            vd.status('%s plugin loaded' % plugin.name)
-
-
-    def removePluginIfExists(self, plugin):
-        self.removePlugin(plugin)
-
-    def removePlugin(self, plugin):
-        if not _plugin_in_import_list(plugin):
-            vd.fail("plugin not in import list")
-
-        initpath = Path(_plugin_init())
-        oldinitpath = Path(initpath.with_suffix(initpath.suffix + '.bak'))
-        try:
-            shutil.copyfile(initpath, oldinitpath)
-
-            # Copy lines from the backup init file into its replacement, skipping lines that import the removed plugin.
-            #
-            # By matching from the start of a line through a word boundary, we avoid removing commented lines or inadvertently removing
-            # plugins with similar names.
-            with oldinitpath.open_text(encoding='utf-8') as old, initpath.open_text(mode='w', encoding='utf-8') as new:
-                r = re.compile(r'^{}\W'.format(_plugin_import(plugin)))
-                new.writelines(line for line in old.readlines() if not r.match(line))
-
-            if os.path.exists(_plugin_path(plugin)):
-                os.unlink(_plugin_path(plugin))
-            sys.modules.pop(_plugin_import_name(plugin))
-            importlib.invalidate_caches()
-            vd.warning('{0} plugin uninstalled'.format(plugin['name']))
-        except FileNotFoundError:
-            vd.warning("no plugins/__init__.py found")
+            if name.startswith(('visidata.plugins.', 'visidata.experimental.')):
+                yield AttrDict(name=name, # '.'.join(name.split('.')[2:]),
+                               description=getattr(mod, '__description__', mod.__doc__),
+                               installed=getattr(mod, '__version__', 'yes'),
+                               maintainer=getattr(mod, '__author__', None))
 
 
 BaseSheet.addCommand(None, 'open-plugins', 'vd.push(vd.pluginsSheet)', 'Open Plugins Sheet to manage supported plugins')
 
-PluginsSheet.addCommand('a', 'add-plugin', 'installPlugin(cursorRow)', 'Install and enable current plugin')
-PluginsSheet.addCommand('d', 'delete-plugin', 'removePluginIfExists(cursorRow)', 'Disable current plugin')
+PluginsSheet.addCommand('a', 'add-plugin', 'enablePlugin(cursorRow.name); reload_rows()', 'Enable current plugin by adding to imports')
+PluginsSheet.addCommand('d', 'delete-plugin', 'removePlugin(cursorRow.name); reload_rows()', 'Disable current plugin by removing from imports')
+
+vd.addMenuItems('''
+    System > Plugins Sheet > open-plugins
+''')