visidata 3.1.1__py3-none-any.whl → 3.2__py3-none-any.whl

visidata/graph.py

@@ -72,6 +72,8 @@ class InvertedCanvas(Canvas):
 
 # provides axis labels, legend
 class GraphSheet(InvertedCanvas):
+    rowtype = 'points'
+
     def __init__(self, *names, **kwargs):
         self.ylabel_maxw = 0
         super().__init__(*names, **kwargs)
@@ -145,7 +147,7 @@ class GraphSheet(InvertedCanvas):
             for char_x in range(0, self.plotwidth//2):
                 has_x_line = char_x in self.reflines_char_x.keys()
                 if has_x_line or has_y_line:
-                    cattr = colors.color_refline
+                    cattr = colors.color_graph_refline
                     if has_x_line:
                         ch = self.reflines_char_x[char_x]
                         # where two lines cross, draw the vertical line, not the horizontal one
@@ -268,11 +270,11 @@ class GraphSheet(InvertedCanvas):
             txt = tick + txt
         else:
             right_margin = self.plotwidth - 1 - self.plotviewBox.xmax
-            if (len(txt)+len(tick))*2 <= right_margin:
+            if (dispwidth(txt)+dispwidth(tick))*2 <= right_margin:
                 txt = tick + txt
             else:
                 # shift rightmost label to be left of its tick
-                x -= len(txt)*2
+                x -= dispwidth(txt)*2
                 if len(tick) == 0:
                     x += 1
                 txt = txt + tick

visidata/guides/AggregatorsSheet.md

@@ -0,0 +1,84 @@
+---
+sheet: AggregatorSheet
+---
+# Aggregations like sum, mean, and distinct
+
+Aggregators provide summary statistics for grouped rows.
+
+The current aggregators include:
+
+   min           smallest value in the group
+   max           largest value in the group
+   avg/mean      average value of the group
+   mode          most frequently appearing value in group
+   median        median value in the group
+   q3/q4/q5/q10  add quantile aggregators to group (e.g. q4 adds p25, p50, p75)
+   sum           total summation of all numbers in the group
+   distinct      number of distinct values in the group
+   count         number of values in the group
+   keymax        key of the row with the largest value in the group
+   list          gathers values in column into a list
+   stdev         standard deviation of values
+
+## View a one-off aggregation of a column
+
+- {help.commands.memo-aggregate} 
+
+## Create an aggregator column
+
+Aggregated columns appear in the **Frequency Table** and **Pivot Table** (grouped sheets).  Aggregated values will also appear at the bottom of their columns in the source sheet.
+
+- {help.commands.aggregate-col}
+
+Then aggregate the sheet with one of the grouping commands:
+
+- {help.commands.freq-col}
+- {help.commands.pivot}
+
+Aggregators can be viewed and modified on the **Columns Sheet** in the `aggregators` column.
+
+- {help.commands.columns-sheet}
+
+## The Describe Sheet
+
+To get a predefined set of summary statistics for every column in the sheet, use the **Describe Sheet* :
+
+- {help.commands.describe-all}
+
+## Examples
+
+Sample input sheet **sales**:
+
+   date        color  price
+   ----------  -----  -----
+   2024-09-01  R      30
+   2024-09-02  B      28
+   2024-09-03  R      100
+   2024-09-03  B      33
+   2024-09-03  B      99
+
+1. Move to the `price` column
+2. Set it to currency: [:keys]$[/key]
+3. Quickly show average price
+    - [:keys]z+[/] (`memo-aggregate`) then enter 'avg'
+4. Add an `sum` aggregator column:
+    - Press [:keys]+[/] (`aggregate-column`) then enter 'sum'
+5. Move to the date column [:keys]gh[/]
+6. Generate a **Frequency Table** by `date`
+    - [:keys]Shift-F[/] (`freq`)
+
+
+   date        count  price_sum
+   ----------  -----  ---------
+   2024-09-03  3      232.00
+   2024-09-01  1      30.00
+   2024-09-02  1      28.00
+
+## Creating new aggregator functions
+
+To add a new aggregator to compute the range of the grouped values (max - min), add the following to `.visidatarc`:
+
+[:code]vd.aggregator('range', lambda values: max(values) - min(values), 'range of values')[/]
+
+The `values` parameter is a list of typed values from the column, with the function returning the aggregated value.
+The new aggregator will now be available the next time VisiData is started.

visidata/guides/MacrosSheet.md

@@ -6,7 +6,7 @@ The basic usage is:
     2. Execute a series of commands.
     3. `m` again to complete the recording, and prompt for the keystroke or longname to bind it to.
 
-The macro will then be executed everytime the provided keystroke or longname are used. Note: the Alt+keys and the function keys are left unbound; overriding other keys may conflict with existing bindings, now or in the future.
+The macro will then be executed every time the provided keystroke or longname are used. Note: the Alt+keys and the function keys are left unbound; overriding other keys may conflict with existing bindings, now or in the future.
 
 Executing a macro will the series of commands starting on the current row and column on the current sheet.
 

visidata/guides/RankGuide.md

@@ -0,0 +1,51 @@
+# Ranking
+
+Ranking assigns numeric ranks to rows based on column values. VisiData provides two ranking approaches: sheet-wide ranking and group-based ranking.
+
+## Sheet-wide ranking
+
+[:keys]addcol-sheetrank[/] ranks all rows across the entire sheet.
+
+Navigate to the column to rank by and execute [:keys]addcol-sheetrank[/]. A new column appears with ranks, where 1 indicates the best value.
+
+**Example:**
+```
+Name   | Salary | Salary_rank
+Alice  | 95000  | 1
+Bob    | 85000  | 2
+Carol  | 70000  | 3
+```
+
+## Group-based ranking
+
+[:keys]addcol-aggregate[/] with [:code]rank[/] aggregator ranks rows within groups defined by key columns.
+
+1. Set key columns with [:keys]![/] (defines groups)
+2. Navigate to the column to rank by
+3. Execute [:keys]addcol-aggregate[/] 
+4. Select [:code]rank[/] aggregator
+
+**Example with Department as key column:**
+```
+Name   | Department  | Salary | Salary_rank
+Alice  | Engineering | 95000  | 1
+Bob    | Engineering | 85000  | 2
+Carol  | Sales       | 70000  | 1
+Dave   | Sales       | 65000  | 2
+```
+
+Alice and Carol both receive rank 1 as the highest earners in their respective departments.
+
+## Sort direction
+
+Ranking follows the current sort direction of the column:
+- Ascending sort: lower values get better ranks
+- Descending sort: higher values get better ranks
+
+## Usage patterns
+
+**Global comparison:** Use [:keys]addcol-sheetrank[/] to find overall leaders across all data.
+
+**Category comparison:** Use [:keys]addcol-aggregate[/] + [:code]rank[/] to find leaders within each group defined by key columns.
+
+**Multiple groupings:** Set multiple key columns before group ranking for complex categorization.

visidata/guides/TypesSheet.md

@@ -17,7 +17,7 @@ VisiData pre-set defaults for formatting types:
 - `currency` removes non-numeric characters and parses the remainder as `float`.
 - `date` parses dates into date object (shown as ISO8601).
 - `vlen` formats the cell value to the length of the content
-- `float` uses the decimal seperator, keeping two significant digits.
+- `float` uses the decimal separator, keeping two significant digits.
 
 Change float precision with:
 - {help.commands.setcol-precision-less}

visidata/guides/WindowFunctionGuide.md

@@ -0,0 +1,49 @@
+---
+sheet: Sheet
+---
+# Create a window over consecutive rows
+
+Window functions enable computations that relate the current window to surrounding rows, like cumulative sum, rolling averages or lead/lag computations.
+
+{help.commands.addcol-window}
+
+With large window sizes, [:code]g'[/] (`freeze-sheet`) to calculate all cells and copy the entire sheet into a new source sheet, which will conserve CPU.
+
+## Examples
+
+   date        color  price
+   ----------  -----  -----
+   2024-09-01  R      30
+   2024-09-02  B      28
+   2024-09-03  R      100
+   2024-09-03  B      33
+   2024-09-03  B      99
+
+
+1. [:keys]#[/] (`type-int`) on the **price** column to type as int.
+2. [:keys]w[/] (`addcol-window`) on the **price** column, followed by `1 2`, to create a window consisting of 4 rows: 1 row before the current row, and 2 rows after.
+3. To create a moving average of the values in the window, add a new column with a python expression: [:keys]=[/] (`addcol-expr`)
+followed by `sum(price_window)/len(price_window)`
+
+date            color   price   price_window            sum(price_window)/len(price_window)
+----------      -----   -----   -------------------     -----------------------------------
+2024-09-01      R       38      [4] ; 38; 28; 100       41.5
+2024-09-02      B       28      [4] 38; 28; 100; 33     49.75
+2024-09-03      R       100     [4] 28; 100; 33; 99     65.0
+2024-09-03      B       33      [4] 100; 33; 99;        58.0
+2024-09-03      B       99      [4] 33; 99; ;           33.0
+
+
+## Workflows
+
+### Create a cumulative sum 
+
+1. Set the before window size to the total number of rows in the table, and the after rows to 0. In the above example that would be `w 5 0` (`addcol-window`).
+2. Add an expression ([:keys]=[/] (`addcol-expr`) of `sum(window)` where `window` is the name of the window function column.
+
+### Compute the change between rows
+
+1. `w 1 0` on the `foo` column to create a window function of size 1 before and 0 after.
+2. Add a python expression. The window function column is 'foo_window':
+    `=foo_window[1] - foo_window[0] if len(foo_window) > 1 else None`
+

visidata/help.py

@@ -135,7 +135,7 @@ class HelpPane:
         self.scr.erase()
         self.scr.box()
         self.amgr.draw(self.scr, y=1, x=2, **kwargs)
-        self.scr.refresh()
+        self.scr.noutrefresh()
 
 
 @VisiData.api
@@ -176,13 +176,12 @@ BaseSheet.bindkey('gKEY_BACKSPACE', 'sysopen-help')
 HelpSheet.addCommand(None, 'exec-command', 'quit(sheet); draw_all(); activeStack[0].execCommand(cursorRow.longname)', 'execute command on undersheet')
 BaseSheet.addCommand(None, 'open-tutorial-visidata', 'launchBrowser("https://jsvine.github.io/intro-to-visidata/")', 'open https://jsvine.github.io/intro-to-visidata/')
 
-vd.addMenuItem("Help", "VisiData tutorial", 'open-tutorial-visidata')
-vd.addMenuItem("Help", 'Sheet commands', 'help-commands')
-vd.addMenuItem("Help", 'All commands', 'help-commands-all')
 
 vd.addGlobals(HelpSheet=HelpSheet)
 
 vd.addMenuItems('''
+    Help > VisiData tutorial > open-tutorial-visidata
+    Help > All commands > help-commands-all
     Help > Quick reference > sysopen-help
     Help > Command list > help-commands
 ''')

visidata/indexsheet.py

@@ -99,7 +99,7 @@ BaseSheet.addCommand('g<', 'open-source-prev', 'vd.replace(openSource(source.nex
 IndexSheet.addCommand('g^R', 'reload-selected', 'reloadSheets(selectedRows or rows)', 'reload all selected sheets')
 
 # when diving into a sheet, remove the index unless it is precious
-SheetsSheet.addCommand('gC', 'columns-selected', 'vd.push(ColumnsSheet("all_columns", source=selectedRows))', 'open Columns Sheet with all visible columns from selected sheets')
+IndexSheet.addCommand('gC', 'columns-selected', 'vd.push(ColumnsSheet("all_columns", source=selectedRows))', 'open Columns Sheet with all visible columns from selected sheets')
 IndexSheet.addCommand('^C', 'cancel-row', 'cancelThread(*cursorRow.currentThreads)', 'abort async thread for current sheet')
 IndexSheet.addCommand('gz^C', 'cancel-rows', 'for vs in selectedRows: cancelThread(*vs.currentThreads)', 'abort async threads for selected sheets')
 SheetsSheet.addCommand('Enter', 'open-row', 'dest=cursorRow; vd.sheets.remove(sheet) if not sheet.precious else None; vd.push(openRow(dest))', 'open sheet referenced in current row')

visidata/loaders/_pandas.py

@@ -28,7 +28,7 @@ def save_dta(vd, p, *sheets):
     vs = sheets[0]
 
     columns = [col.name for col in vs.visibleCols]
-    
+
     # Get data types
     types = list()
     dispvals = next(vs.iterdispvals(format=True))
@@ -154,6 +154,8 @@ class PandasSheet(Sheet):
                 readfunc = self.read_tsv
             elif filetype == 'jsonl':
                 readfunc = partial(pd.read_json, lines=True)
+            elif filetype == 'hdf5':
+                readfunc = partial(pd.read_hdf, lines=True)
             else:
                 readfunc = getattr(pd, 'read_'+filetype) or vd.error('no pandas.read_'+filetype)
             # readfunc() handles binary and text open()

visidata/loaders/archive.py

@@ -5,18 +5,21 @@ import datetime
 from visidata.loaders import unzip_http
 
 from visidata import vd, VisiData, asyncthread, Sheet, Progress, Menu, options
-from visidata import ColumnAttr, Column, Path
+from visidata import ColumnAttr, Column, Path, filesize
 from visidata.type_date import date
 
 @VisiData.api
 def guess_zip(vd, p):
     if not p.is_url() and zipfile.is_zipfile(p.open_bytes()):
-        return dict(filetype='zip')
+        return dict(filetype='zip', _likelihood=10)
 
 @VisiData.api
 def guess_tar(vd, p):
+    # an empty file will pass is_tarfile(), but can't be opened by tarfile.open()
+    if filesize(p) == 0:
+        return None
     if tarfile.is_tarfile(p.open_bytes()):
-        return dict(filetype='tar')
+        return dict(filetype='tar', _likelihood=10)
 
 @VisiData.api
 def open_zip(vd, p):

visidata/loaders/csv.py

@@ -26,7 +26,11 @@ def guess_csv(vd, p):
 
         for csvopt in dir(dialect):
             if not csvopt.startswith('_'):
-                r['csv_'+csvopt] = getattr(dialect, csvopt)
+                v = getattr(dialect, csvopt)
+                optname = 'csv_'+csvopt
+                r[optname] = v
+                if vd.options.get(optname) != v:
+                    vd.warning(f'guessed option {optname}={v}')
 
         return r
 

visidata/loaders/eml.py

@@ -6,6 +6,8 @@ from visidata import VisiData, vd, Column, TableSheet, vlen
 def open_eml(vd, p):
     return EmailSheet(p.base_stem, source=p)
 
+open_mhtml = open_eml
+
 class EmailSheet(TableSheet):
     rowtype = 'parts'  # rowdef: sub-Messages
     columns = [

visidata/loaders/f5log.py

@@ -14,7 +14,7 @@ Regex: (?:/Common/)(?P<site>[^-]+)-(?P<vstype>[^-]+)-(?P<application>[^-]+)
 
 /Common/newyork-www-banking1
 
-... | site    | vstype | appliction | ...
+... | site    | vstype | application | ...
 ... | newyork | www    | banking1   | ...
 
 Adding to .visidatarc
@@ -313,7 +313,7 @@ class F5LogSheet(Sheet):
             cmd_data = msg[cmd_data_loc + 1 :]
             # split the message and the command
             msg, cmd = msg[:cmd_data_loc].rsplit(" ", maxsplit=1)
-            # strip off the trailling " -" from the message
+            # strip off the trailing " -" from the message
             msg = msg[:-2]
             object = cmd_data.split('"', maxsplit=2)
             if len(object) == 3:

visidata/loaders/fec.py

@@ -106,22 +106,19 @@ class DiveSheet(Sheet):
                     self.addRow(item)
 
                 except Exception as e:
-                    vd.warning("Can't dive on lists with heterogenous item types.")
+                    vd.warning("Can't dive on lists with heterogeneous item types.")
                     return False
 
     def openRow(self, row):
         if self.is_keyvalue:
             cell = row["value"]
-            name = vd.joinSheetnames(self.name, row["key"])
-
             if isinstance(cell, (list, dict)):
-                vs = self.__class__(name, source = cell)
+                vs = self.__class__(self.name, row["key"], source = cell)
             else:
                 vd.warning("Nothing to dive into.")
                 return
         else:
-            name = vd.joinSheetnames(self.name, "row")
-            vs = self.__class__(name, source = self.row)
+            vs = self.__class__(self.name, "row", source = self.row)
 
         success = vs.reload()
         if success == False:
@@ -174,7 +171,7 @@ class FECScheduleSheet(Sheet):
 
         for schedule_name in self.source.keys():
             vs = FECItemizationSheet(
-                vd.joinSheetnames(self.name, schedule_name),
+                self.name, schedule_name,
                 schedule_name = schedule_name,
                 source = self.source[schedule_name],
                 size = len(self.source[schedule_name]),
@@ -225,7 +222,7 @@ class FECFiling(Sheet):
                 ] else dict
 
             vs = cls(
-                vd.joinSheetnames(self.name, component_name),
+                self.name, component_name,
                 component_name = component_name,
                 source = source_cls(),
                 size = 0,
@@ -270,7 +267,7 @@ class FECFiling(Sheet):
                 if form_type not in sheet_row.source:
                     sheet_row.source[form_type] = [ ] 
                     subsheet = FECItemizationSheet(
-                        vd.joinSheetnames(sheet_row.name, form_type),
+                        sheet_row.name, form_type,
                         schedule_name = form_type,
                         source = [ ],
                         size = 0,

visidata/loaders/fixed_width.py

@@ -110,3 +110,5 @@ def save_fixed(vd, p, *vsheets):
                     for col, val in dispvals.items():
                         fp.write(('{0:%s%s.%s} ' % ('>' if vd.isNumeric(col) else '<', widths[col], widths[col])).format(val))
                     fp.write('\n')
+
+FixedWidthColumnsSheet.options.null_value = ''    # the file format cannot contain None, so use empty string instead

visidata/loaders/hdf5.py

@@ -1,4 +1,5 @@
-from visidata import VisiData, vd, Sheet, Path, Column, ItemColumn, BaseSheet
+from visidata import VisiData, vd, Sheet, Path, Column, ItemColumn, BaseSheet, anytype
+from itertools import chain
 
 @VisiData.api
 def open_h5(vd, p):
@@ -6,8 +7,11 @@ def open_h5(vd, p):
 
 VisiData.open_hdf5 = VisiData.open_h5
 
+vd.option('hdf5_matrix_enumerate', False, 'enumerate matrix rows and columns')
+
 class Hdf5ObjSheet(Sheet):
     'Support sheets in HDF5 format.'
+
     def iterload(self):
         h5py = vd.importExternal('h5py')
         source = self.source
@@ -26,25 +30,39 @@ class Hdf5ObjSheet(Sheet):
             for k, v in source.items():
                 yield Hdf5ObjSheet(self.name, k, source=v)
         elif isinstance(source, h5py.Dataset):
-            if len(source.shape) == 1:
+            if len(source.shape)==1:
                 if source.dtype.names:
-                    for i, colname in enumerate(source.dtype.names):
-                        self.addColumn(ItemColumn(colname, colname), index=i)
+                    for i, (colname, fmt, *_) in enumerate(source.dtype.descr):
+                        if not colname:
+                            colname = f"col{i}"
+                        ctype = _guess_type(fmt)
+                        self.addColumn(ItemColumn(colname, i, type=ctype))
                     yield from source  # copy
                 else:
                     self.addColumn(ItemColumn(source.name, 0))
                     for v in source:
                         yield [v]
-            elif len(source.shape) == 2:  # matrix
+            elif len(source.shape)==2:
+                matrix_enumerate = bool(self.options.hdf5_matrix_enumerate)
+
                 ncols = source.shape[1]
-                for i in range(ncols):
-                    self.addColumn(ItemColumn('', i, width=8), index=i)
-                self.recalc()
-                yield from source  # copy
+                ctype = _guess_type(source.dtype.descr[0][1])
+
+                if matrix_enumerate:
+                    self.addColumn(ItemColumn("row", 0, width=8, keycol=1, type=int), index=0)
+                    for i in range(ncols):
+                        self.addColumn(ItemColumn(f'col{i}', i+1, width=8, type=ctype), index=i+1)
+                    self.recalc()
+                    yield from list(list((chain((i,), row))) for i, row in enumerate(source))
+                else:
+                    for i in range(ncols):
+                        self.addColumn(ItemColumn('', i, width=8, type=ctype), index=i)
+                    self.recalc()
+                    yield from source  # copy
             else:
                 vd.fail('too many dimensions in shape %s' % str(source.shape))
         else:
-            vd.fail('unknown h5 object type %s' % type(source))
+            vd.fail(f"too many dimensions in shape {source.shape}")
 
 
     def openRow(self, row):
@@ -59,5 +77,11 @@ class Hdf5ObjSheet(Sheet):
         if isinstance(row, numpy.ndarray):
             return NpySheet(None, npy=row)
 
+def _guess_type(fmt):
+    if 'i' in fmt or 'u' in fmt:
+        return int
+    elif 'f' in fmt:
+        return float
+    return anytype
 
 Hdf5ObjSheet.addCommand('A', 'dive-metadata', 'vd.push(SheetDict(cursorRow.name + "_attrs", source=cursorRow.attrs))', 'open metadata sheet for object referenced in current row')

visidata/loaders/npy.py

@@ -1,4 +1,5 @@
-from visidata import VisiData, vd, Sheet, date, anytype, options, Column, Progress, ColumnItem, vlen, PyobjSheet, TypedWrapper
+from visidata import VisiData, vd, Sheet, date, anytype, options, Column, ItemColumn, Progress, vlen, PyobjSheet, TypedWrapper
+from itertools import chain
 
 'Loaders for .npy and .npz.  Save to .npy.  Depends on the zip loader.'
 
@@ -11,44 +12,74 @@ def open_npz(vd, p):
     return NpzSheet(p.base_stem, source=p)
 
 vd.option('npy_allow_pickle', False, 'numpy allow unpickling objects (unsafe)')
+vd.option('npy_matrix_enumerate', False, 'enumerate matrix rows and columns')
 
 class NpySheet(Sheet):
     def iterload(self):
         numpy = vd.importExternal('numpy')
         if not hasattr(self, 'npy'):
-            self.npy = numpy.load(str(self.source), encoding='bytes', **self.options.getall('npy_'))
+            self.npy = numpy.load(str(self.source), encoding='bytes', allow_pickle=bool(self.options.npy_allow_pickle))
         self.reloadCols()
-        yield from Progress(self.npy, total=len(self.npy))
+        transpose = len(self.npy.shape)==1 and not bool(self.npy.dtype.names)
+        if transpose:
+            source = self.npy[:,None]
+        else:
+            source = self.npy
+
+        nrows = len(self.npy)
+
+        if self.options.npy_matrix_enumerate:
+            source = list(list((chain((i,), row))) for i, row in enumerate(source))
+
+        yield from Progress(source, nrows)
+
 
     def reloadCols(self):
         self.columns = []
-        for i, (name, fmt, *shape) in enumerate(self.npy.dtype.descr):
-            if not name:
-                continue
-            if shape:
-                t = anytype
-            elif 'M' in fmt:
-                self.addColumn(Column(name, type=date, getter=lambda c,r,i=i: str(r[i])))
-                continue
-            elif 'i' in fmt:
-                t = int
-            elif 'f' in fmt:
-                t = float
+        if len(self.npy.shape)==1:
+            for i, (colname, fmt, *shape) in enumerate(self.npy.dtype.descr):
+                if not colname:
+                    colname = f"col{i}"
+                ctype = _guess_type(shape, fmt)
+                if ctype=="time":
+                    self.addColumn(Column(colname, type=date, getter=lambda c,r,i=i: str(r[i])))
+                    continue
+                self.addColumn(ItemColumn(colname, i, type=ctype))
+        elif len(self.npy.shape)==2:
+            ncols = self.npy.shape[1]
+            ctype = _guess_type(None, self.npy.dtype.descr[0][1])
+
+            if self.options.npy_matrix_enumerate:
+                self.addColumn(ItemColumn("row", 0, width=8, keycol=1, type=int), index=0)
+                for i in range(ncols):
+                    self.addColumn(ItemColumn(f'col{i}', i+1, width=8, type=ctype), index=i+1)
             else:
-                t = anytype
-            self.addColumn(ColumnItem(name, i, type=t))
-
+                for i in range(ncols):
+                    self.addColumn(ItemColumn('', i, width=8, type=ctype), index=i)
+        else:
+            vd.fail(f"too many dimensions in shape {self.npy.shape}")
+
+def _guess_type(shape, fmt):
+    if shape:
+        return anytype
+    elif 'M' in fmt:
+        return "time"
+    elif 'i' in fmt or 'u' in fmt:
+        return int
+    elif 'f' in fmt:
+        return float
+    return anytype
 
 class NpzSheet(vd.ZipSheet):
     # rowdef: tuple(tablename, table)
     columns = [
-        ColumnItem('name', 0),
-        ColumnItem('length', 1, type=vlen),
+        ItemColumn('name', 0),
+        ItemColumn('length', 1, type=vlen),
     ]
 
     def iterload(self):
         numpy = vd.importExternal('numpy')
-        self.npz = numpy.load(str(self.source), encoding='bytes', **self.options.getall('npy_'))
+        self.npz = numpy.load(str(self.source), encoding='bytes', allow_pickle=bool(self.options.npy_allow_pickle))
         yield from Progress(self.npz.items())
 
     def openRow(self, row):
@@ -74,7 +105,7 @@ def save_npy(vd, p, sheet):
         elif col.type in vd.numericTypes:
             dt = 'f8'
         else: #  if col.type in (str, anytype):
-            width = col.getMaxWidth(sheet.rows)
+            width = col.getMaxDataWidth(sheet.rows)
             dt = 'U'+str(width)
         dtype.append((col.name, dt))
 
@@ -94,4 +125,4 @@ def save_npy(vd, p, sheet):
 
     arr = np.array(data, dtype=dtype)
     with p.open_bytes(mode='w') as outf:
-        np.save(outf, arr, **sheet.options.getall('npy_'))
+        np.save(outf, arr, allow_pickle=bool(sheet.options.npy_allow_pickle))

visidata/loaders/orgmode.py

@@ -55,8 +55,8 @@ def encode_date(dt=None):
 
 
 class OrgContentsColumn(Column):
-    def setValue(self, row, v):
-        super().setValue(row, v)
+    def setValue(self, row, v, setModified=True):
+        super().setValue(row, v, setModified=setModified)
         orgmode_parse_into(row, v)
 
     def putValue(self, row, v):
@@ -88,6 +88,7 @@ def sectionize(lines):
 
 def orgmode_parse(all_lines):
     root = parent = OrgSheet().newRow()
+    root.orig_contents = ''
     for linenum, lines in sectionize(all_lines):
         section = OrgSheet().newRow()
 

visidata/loaders/pandas_freqtbl.py

@@ -83,6 +83,10 @@ class PandasFreqTableSheet(PivotSheet):
         self.source._selectByILoc(row.sourcerows.mask_iloc, selected=False)
         return super().unselectRow(row)
 
+    def addUndoSelection(self):
+        self.source.addUndoSelection()
+        super().addUndoSelection()
+
     def updateLargest(self, grouprow):
         self.largest = max(self.largest, len(grouprow.sourcerows))
 

visidata/loaders/psv.py

@@ -0,0 +1,13 @@
+from visidata import VisiData, TsvSheet
+
+
+@VisiData.api
+def open_psv(vd, p):
+    return PsvSheet(p.name, source=p)
+
+
+class PsvSheet(TsvSheet):
+    pass
+
+
+PsvSheet.options.delimiter = '|'