visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- visidata/__init__.py +72 -91
- visidata/_input.py +263 -44
- visidata/_open.py +84 -29
- visidata/_types.py +22 -4
- visidata/_urlcache.py +17 -4
- visidata/aggregators.py +65 -25
- visidata/apps/__init__.py +0 -0
- visidata/apps/vdsql/__about__.py +8 -0
- visidata/apps/vdsql/__init__.py +5 -0
- visidata/apps/vdsql/__main__.py +27 -0
- visidata/apps/vdsql/_ibis.py +748 -0
- visidata/apps/vdsql/bigquery.py +61 -0
- visidata/apps/vdsql/clickhouse.py +53 -0
- visidata/apps/vdsql/setup.py +40 -0
- visidata/apps/vdsql/snowflake.py +67 -0
- visidata/apps/vgit/__init__.py +13 -0
- visidata/apps/vgit/__main__.py +3 -0
- visidata/apps/vgit/abort.py +23 -0
- visidata/apps/vgit/blame.py +76 -0
- visidata/apps/vgit/branch.py +153 -0
- visidata/apps/vgit/config.py +95 -0
- visidata/apps/vgit/diff.py +169 -0
- visidata/apps/vgit/gitsheet.py +161 -0
- visidata/apps/vgit/grep.py +37 -0
- visidata/apps/vgit/log.py +81 -0
- visidata/apps/vgit/main.py +55 -0
- visidata/apps/vgit/remote.py +57 -0
- visidata/apps/vgit/repos.py +71 -0
- visidata/apps/vgit/setup.py +37 -0
- visidata/apps/vgit/stash.py +69 -0
- visidata/apps/vgit/status.py +204 -0
- visidata/apps/vgit/statusbar.py +34 -0
- visidata/basesheet.py +59 -50
- visidata/canvas.py +251 -99
- visidata/choose.py +15 -11
- visidata/clean_names.py +29 -0
- visidata/clipboard.py +84 -18
- visidata/cliptext.py +220 -46
- visidata/cmdlog.py +89 -114
- visidata/color.py +142 -56
- visidata/column.py +134 -131
- visidata/ddw/input.ddw +74 -79
- visidata/ddw/regex.ddw +57 -0
- visidata/ddwplay.py +33 -14
- visidata/deprecated.py +77 -3
- visidata/desktop/visidata.desktop +7 -0
- visidata/editor.py +12 -6
- visidata/errors.py +5 -1
- visidata/experimental/__init__.py +0 -0
- visidata/experimental/diff_sheet.py +29 -0
- visidata/experimental/digit_autoedit.py +6 -0
- visidata/experimental/gdrive.py +89 -0
- visidata/experimental/google.py +37 -0
- visidata/experimental/gsheets.py +79 -0
- visidata/experimental/live_search.py +37 -0
- visidata/experimental/liveupdate.py +45 -0
- visidata/experimental/mark.py +133 -0
- visidata/experimental/noahs_tapestry/__init__.py +1 -0
- visidata/experimental/noahs_tapestry/tapestry.py +147 -0
- visidata/experimental/rownum.py +73 -0
- visidata/experimental/slide_cells.py +26 -0
- visidata/expr.py +8 -4
- visidata/extensible.py +32 -6
- visidata/features/__init__.py +0 -0
- visidata/features/addcol_audiometadata.py +42 -0
- visidata/features/addcol_histogram.py +34 -0
- visidata/features/canvas_save_svg.py +69 -0
- visidata/features/change_precision.py +46 -0
- visidata/features/cmdpalette.py +163 -0
- visidata/features/colorbrewer.py +363 -0
- visidata/{colorsheet.py → features/colorsheet.py} +17 -16
- visidata/features/command_server.py +105 -0
- visidata/features/currency_to_usd.py +70 -0
- visidata/{customdate.py → features/customdate.py} +2 -0
- visidata/features/dedupe.py +132 -0
- visidata/{describe.py → features/describe.py} +17 -15
- visidata/features/errors_guide.py +26 -0
- visidata/features/expand_cols.py +202 -0
- visidata/{fill.py → features/fill.py} +4 -2
- visidata/{freeze.py → features/freeze.py} +11 -6
- visidata/features/graph_seaborn.py +79 -0
- visidata/features/helloworld.py +10 -0
- visidata/features/hint_types.py +17 -0
- visidata/{incr.py → features/incr.py} +5 -0
- visidata/{join.py → features/join.py} +107 -53
- visidata/features/known_cols.py +21 -0
- visidata/features/layout.py +62 -0
- visidata/{melt.py → features/melt.py} +33 -21
- visidata/features/normcol.py +118 -0
- visidata/features/open_config.py +7 -0
- visidata/features/open_syspaste.py +18 -0
- visidata/features/ping.py +157 -0
- visidata/features/procmgr.py +208 -0
- visidata/features/random_sample.py +6 -0
- visidata/{regex.py → features/regex.py} +47 -31
- visidata/features/reload_every.py +55 -0
- visidata/features/rename_col_cascade.py +30 -0
- visidata/features/scroll_context.py +60 -0
- visidata/features/select_equal_selected.py +11 -0
- visidata/features/setcol_fake.py +65 -0
- visidata/{slide.py → features/slide.py} +75 -21
- visidata/features/sparkline.py +48 -0
- visidata/features/status_source.py +20 -0
- visidata/{sysedit.py → features/sysedit.py} +2 -1
- visidata/features/sysopen_mailcap.py +46 -0
- visidata/features/term_extras.py +13 -0
- visidata/{transpose.py → features/transpose.py} +5 -4
- visidata/features/type_ipaddr.py +73 -0
- visidata/features/type_url.py +11 -0
- visidata/{unfurl.py → features/unfurl.py} +9 -9
- visidata/{window.py → features/window.py} +2 -2
- visidata/form.py +50 -21
- visidata/freqtbl.py +81 -33
- visidata/fuzzymatch.py +414 -0
- visidata/graph.py +105 -33
- visidata/guide.py +180 -0
- visidata/help.py +75 -44
- visidata/hint.py +39 -0
- visidata/indexsheet.py +109 -0
- visidata/input_history.py +55 -0
- visidata/interface.py +58 -0
- visidata/keys.py +17 -16
- visidata/loaders/__init__.py +9 -0
- visidata/loaders/_pandas.py +61 -21
- visidata/loaders/api_airtable.py +70 -0
- visidata/loaders/api_bitio.py +102 -0
- visidata/loaders/api_matrix.py +148 -0
- visidata/loaders/api_reddit.py +306 -0
- visidata/loaders/api_zulip.py +249 -0
- visidata/loaders/archive.py +41 -7
- visidata/loaders/arrow.py +7 -7
- visidata/loaders/conll.py +49 -0
- visidata/loaders/csv.py +25 -7
- visidata/loaders/eml.py +3 -4
- visidata/loaders/f5log.py +1204 -0
- visidata/loaders/fec.py +325 -0
- visidata/loaders/fixed_width.py +3 -5
- visidata/loaders/frictionless.py +3 -3
- visidata/loaders/geojson.py +8 -5
- visidata/loaders/google.py +48 -0
- visidata/loaders/graphviz.py +4 -4
- visidata/loaders/hdf5.py +4 -4
- visidata/loaders/html.py +48 -10
- visidata/loaders/http.py +84 -30
- visidata/loaders/imap.py +20 -10
- visidata/loaders/jrnl.py +52 -0
- visidata/loaders/json.py +83 -29
- visidata/loaders/jsonla.py +74 -0
- visidata/loaders/lsv.py +15 -11
- visidata/loaders/mailbox.py +40 -0
- visidata/loaders/markdown.py +1 -3
- visidata/loaders/mbtiles.py +4 -5
- visidata/loaders/mysql.py +11 -13
- visidata/loaders/npy.py +7 -7
- visidata/loaders/odf.py +4 -1
- visidata/loaders/orgmode.py +428 -0
- visidata/loaders/pandas_freqtbl.py +14 -20
- visidata/loaders/parquet.py +62 -6
- visidata/loaders/pcap.py +3 -3
- visidata/loaders/pdf.py +4 -3
- visidata/loaders/png.py +19 -13
- visidata/loaders/postgres.py +9 -8
- visidata/loaders/rec.py +7 -3
- visidata/loaders/s3.py +342 -0
- visidata/loaders/sas.py +5 -5
- visidata/loaders/scrape.py +186 -0
- visidata/loaders/shp.py +6 -5
- visidata/loaders/spss.py +5 -6
- visidata/loaders/sqlite.py +68 -28
- visidata/loaders/texttables.py +1 -1
- visidata/loaders/toml.py +60 -0
- visidata/loaders/tsv.py +61 -19
- visidata/loaders/ttf.py +19 -7
- visidata/loaders/unzip_http.py +6 -5
- visidata/loaders/usv.py +1 -1
- visidata/loaders/vcf.py +16 -16
- visidata/loaders/vds.py +10 -7
- visidata/loaders/vdx.py +30 -5
- visidata/loaders/xlsb.py +8 -1
- visidata/loaders/xlsx.py +145 -25
- visidata/loaders/xml.py +6 -3
- visidata/loaders/xword.py +4 -4
- visidata/loaders/yaml.py +15 -5
- visidata/macos.py +1 -1
- visidata/macros.py +130 -41
- visidata/main.py +119 -94
- visidata/mainloop.py +101 -154
- visidata/man/parse_options.py +2 -2
- visidata/man/vd.1 +302 -147
- visidata/man/vd.txt +291 -151
- visidata/memory.py +3 -3
- visidata/menu.py +104 -423
- visidata/metasheets.py +59 -141
- visidata/modify.py +79 -23
- visidata/motd.py +3 -3
- visidata/mouse.py +137 -0
- visidata/movement.py +43 -35
- visidata/optionssheet.py +99 -0
- visidata/path.py +131 -43
- visidata/pivot.py +74 -47
- visidata/plugins.py +65 -192
- visidata/pyobj.py +50 -201
- visidata/rename_col.py +20 -0
- visidata/save.py +42 -20
- visidata/search.py +54 -10
- visidata/selection.py +84 -5
- visidata/settings.py +162 -24
- visidata/sheets.py +229 -257
- visidata/shell.py +51 -21
- visidata/sidebar.py +162 -0
- visidata/sort.py +11 -4
- visidata/statusbar.py +113 -104
- visidata/stored_list.py +43 -0
- visidata/stored_prop.py +38 -0
- visidata/tests/conftest.py +3 -3
- visidata/tests/test_cliptext.py +39 -0
- visidata/tests/test_commands.py +62 -7
- visidata/tests/test_edittext.py +2 -2
- visidata/tests/test_features.py +17 -0
- visidata/tests/test_menu.py +14 -0
- visidata/tests/test_path.py +13 -4
- visidata/text_source.py +53 -0
- visidata/textsheet.py +10 -3
- visidata/theme.py +44 -0
- visidata/themes/__init__.py +0 -0
- visidata/themes/ascii8.py +84 -0
- visidata/themes/asciimono.py +84 -0
- visidata/themes/light.py +17 -0
- visidata/threads.py +87 -39
- visidata/tuiwin.py +22 -0
- visidata/type_currency.py +22 -3
- visidata/type_date.py +31 -9
- visidata/type_floatsi.py +5 -1
- visidata/undo.py +18 -6
- visidata/utils.py +106 -23
- visidata/vdobj.py +28 -17
- visidata/windows.py +10 -0
- visidata/wrappers.py +9 -3
- visidata-3.0.data/data/share/applications/visidata.desktop +7 -0
- {visidata-2.11.dev0.data → visidata-3.0.data}/data/share/man/man1/vd.1 +302 -147
- {visidata-2.11.dev0.data → visidata-3.0.data}/data/share/man/man1/visidata.1 +302 -147
- visidata-3.0.data/scripts/vd2to3.vdx +9 -0
- {visidata-2.11.dev0.dist-info → visidata-3.0.dist-info}/METADATA +13 -11
- visidata-3.0.dist-info/RECORD +257 -0
- {visidata-2.11.dev0.dist-info → visidata-3.0.dist-info}/WHEEL +1 -1
- {visidata-2.11.dev0.dist-info → visidata-3.0.dist-info}/entry_points.txt +0 -1
- visidata/layout.py +0 -44
- visidata/misc.py +0 -5
- visidata-2.11.dev0.dist-info/RECORD +0 -142
- /visidata/{repeat.py → features/repeat.py} +0 -0
- {visidata-2.11.dev0.data → visidata-3.0.data}/scripts/vd +0 -0
- {visidata-2.11.dev0.dist-info → visidata-3.0.dist-info}/LICENSE.gpl3 +0 -0
- {visidata-2.11.dev0.dist-info → visidata-3.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,132 @@
|
|
1
|
+
"""
|
2
|
+
# Usage
|
3
|
+
|
4
|
+
Duplicates are determined by the sheet's key columns.
|
5
|
+
|
6
|
+
If no key columns are specified, then a duplicate row is one where the values
|
7
|
+
of *all non-hidden* columns are exactly the same as a row that occurs earlier
|
8
|
+
in the sheet.
|
9
|
+
|
10
|
+
If key columns *are* specified, then duplicates are detected based on the
|
11
|
+
values in just those columns.
|
12
|
+
|
13
|
+
## Commands
|
14
|
+
|
15
|
+
- `select-duplicate-rows` sets the selection status in VisiData to `selected`
|
16
|
+
for each row in the active sheet that is a duplicate of a prior row.
|
17
|
+
|
18
|
+
- `dedupe-rows` pushes a new sheet in which only non-duplicate rows in the
|
19
|
+
active sheet are included.
|
20
|
+
"""
|
21
|
+
|
22
|
+
|
23
|
+
__author__ = "Jeremy Singer-Vine <jsvine@gmail.com>"
|
24
|
+
|
25
|
+
from copy import copy
|
26
|
+
|
27
|
+
from visidata import Sheet, BaseSheet, asyncthread, Progress, vd
|
28
|
+
|
29
|
+
|
30
|
+
def gen_identify_duplicates(sheet):
|
31
|
+
"""
|
32
|
+
Takes a sheet, and returns a generator yielding a tuple for each row
|
33
|
+
encountered. The tuple's structure is `(row_object, is_dupe)`, where
|
34
|
+
is_dupe is True/False.
|
35
|
+
|
36
|
+
See note in Usage section above regarding how duplicates are determined.
|
37
|
+
"""
|
38
|
+
|
39
|
+
keyCols = sheet.keyCols
|
40
|
+
|
41
|
+
cols_to_check = None
|
42
|
+
if len(keyCols) == 0:
|
43
|
+
vd.warning("No key cols specified. Using all columns.")
|
44
|
+
cols_to_check = sheet.visibleCols
|
45
|
+
else:
|
46
|
+
cols_to_check = sheet.keyCols
|
47
|
+
|
48
|
+
seen = set()
|
49
|
+
for r in sheet.rows:
|
50
|
+
vals = tuple(col.getValue(r) for col in cols_to_check)
|
51
|
+
is_dupe = vals in seen
|
52
|
+
if not is_dupe:
|
53
|
+
seen.add(vals)
|
54
|
+
yield (r, is_dupe)
|
55
|
+
|
56
|
+
|
57
|
+
@Sheet.api
|
58
|
+
@asyncthread
|
59
|
+
def select_duplicate_rows(sheet, duplicates=True):
|
60
|
+
"""
|
61
|
+
Given a sheet, sets the selection status in VisiData to `selected` for each
|
62
|
+
row that is a duplicate of a prior row.
|
63
|
+
|
64
|
+
If `duplicates = False`, then the behavior is reversed; sets the selection
|
65
|
+
status to `selected` for each row that is *not* a duplicate.
|
66
|
+
"""
|
67
|
+
before = len(sheet.selectedRows)
|
68
|
+
|
69
|
+
gen = gen_identify_duplicates(sheet)
|
70
|
+
prog = Progress(gen, gerund="selecting", total=sheet.nRows)
|
71
|
+
|
72
|
+
for row, is_dupe in prog:
|
73
|
+
if is_dupe == duplicates:
|
74
|
+
sheet.selectRow(row)
|
75
|
+
|
76
|
+
sel_count = len(sheet.selectedRows) - before
|
77
|
+
|
78
|
+
more_str = " more" if before > 0 else ""
|
79
|
+
|
80
|
+
vd.status(f"selected {sel_count}{more_str} {sheet.rowtype}")
|
81
|
+
|
82
|
+
|
83
|
+
@Sheet.api
|
84
|
+
def dedupe_rows(sheet):
|
85
|
+
"""
|
86
|
+
Given a sheet, pushes a new sheet in which only non-duplicate rows are
|
87
|
+
included.
|
88
|
+
"""
|
89
|
+
vs = copy(sheet)
|
90
|
+
vs.name += "_deduped"
|
91
|
+
|
92
|
+
@asyncthread
|
93
|
+
def _reload(self=vs):
|
94
|
+
self.rows = []
|
95
|
+
gen = gen_identify_duplicates(sheet)
|
96
|
+
prog = Progress(gen, gerund="deduplicating", total=sheet.nRows)
|
97
|
+
for row, is_dupe in prog:
|
98
|
+
if not is_dupe:
|
99
|
+
self.addRow(row)
|
100
|
+
|
101
|
+
vs.reload = _reload
|
102
|
+
return vs
|
103
|
+
|
104
|
+
|
105
|
+
# Add longname-commands to VisiData to execute these methods
|
106
|
+
BaseSheet.addCommand(None, "select-duplicate-rows", "sheet.select_duplicate_rows()", "select each row that is a duplicate of a prior row")
|
107
|
+
BaseSheet.addCommand(None, "dedupe-rows", "vd.push(sheet.dedupe_rows())", "open new sheet in which only non-duplicate rows in the active sheet are included")
|
108
|
+
|
109
|
+
vd.addMenuItems('''
|
110
|
+
Row > Select > duplicate rows > select-duplicate-rows
|
111
|
+
Data > Deduplicate rows > dedupe-rows
|
112
|
+
''')
|
113
|
+
|
114
|
+
"""
|
115
|
+
# Changelog
|
116
|
+
|
117
|
+
## 0.2.0 - 2021-09-22
|
118
|
+
|
119
|
+
Use `vd.warning(...)` instead of `warning(...)`
|
120
|
+
|
121
|
+
## 0.1.0 - 2020-10-09
|
122
|
+
|
123
|
+
Revised for compatibility with VisiData 2.x
|
124
|
+
|
125
|
+
## 0.0.1 - 2019-01-01
|
126
|
+
|
127
|
+
Internal change, no external effects: Migrates from ._selectedRows to .selectedRows.
|
128
|
+
|
129
|
+
## 0.0.0 - 2018-12-30
|
130
|
+
|
131
|
+
Initial release.
|
132
|
+
"""
|
@@ -2,12 +2,10 @@ from copy import copy
|
|
2
2
|
from statistics import mode, median, mean, stdev
|
3
3
|
|
4
4
|
from visidata import vd, Column, ColumnAttr, vlen, RowColorizer, asyncthread, Progress, wrapply
|
5
|
-
from visidata import BaseSheet, TableSheet, ColumnsSheet
|
5
|
+
from visidata import BaseSheet, TableSheet, ColumnsSheet, SheetsSheet
|
6
6
|
|
7
|
-
__all__ = ['DescribeSheet']
|
8
7
|
|
9
|
-
|
10
|
-
vd.option('describe_aggrs', 'mean stdev', 'numeric aggregators to calculate on Describe sheet')
|
8
|
+
vd.option('describe_aggrs', 'mean stdev', 'numeric aggregators to calculate on Describe sheet', help=vd.help_aggregators)
|
11
9
|
|
12
10
|
|
13
11
|
@Column.api
|
@@ -24,12 +22,19 @@ def isError(col, row):
|
|
24
22
|
|
25
23
|
class DescribeColumn(Column):
|
26
24
|
def __init__(self, name, **kwargs):
|
25
|
+
kwargs.setdefault('width', 10)
|
27
26
|
super().__init__(name, getter=lambda col,srccol: col.sheet.describeData[srccol].get(col.expr, ''), expr=name, **kwargs)
|
28
27
|
|
29
28
|
|
30
29
|
# rowdef: Column from source sheet
|
31
30
|
class DescribeSheet(ColumnsSheet):
|
32
31
|
# rowtype = 'columns'
|
32
|
+
guide = '''
|
33
|
+
# Describe Sheet
|
34
|
+
This `Describe Sheet` shows a few basic metrics over data in {sheet.displaySource}, with each column represented by a row.
|
35
|
+
|
36
|
+
For example, row {sheet.cursorRowIndex} describes the _{sheet.cursorRow.name}_ column, showing its minimum value, maximum value, mean, median, and other measures.
|
37
|
+
'''
|
33
38
|
precious = True
|
34
39
|
columns = [
|
35
40
|
ColumnAttr('sheet', 'sheet', width=0),
|
@@ -49,17 +54,11 @@ class DescribeSheet(ColumnsSheet):
|
|
49
54
|
]
|
50
55
|
nKeys = 2
|
51
56
|
|
52
|
-
|
53
|
-
|
54
|
-
super().reload()
|
57
|
+
def loader(self):
|
58
|
+
super().loader()
|
55
59
|
self.rows = [c for c in self.rows if not c.hidden]
|
56
60
|
self.describeData = { col: {} for col in self.rows }
|
57
|
-
|
58
|
-
self.columns = []
|
59
|
-
for c in type(self).columns:
|
60
|
-
self.addColumn(c)
|
61
|
-
|
62
|
-
self.setKeys(self.columns[:self.nKeys])
|
61
|
+
self.resetCols()
|
63
62
|
|
64
63
|
for aggrname in vd.options.describe_aggrs.split():
|
65
64
|
self.addColumn(DescribeColumn(aggrname, type=float))
|
@@ -94,8 +93,8 @@ class DescribeSheet(ColumnsSheet):
|
|
94
93
|
for func in [min, max, sum, median]: # use type
|
95
94
|
d[func.__name__] = self.calcStatistic(d, func, vals)
|
96
95
|
for aggrname in vd.options.describe_aggrs.split():
|
97
|
-
|
98
|
-
d[
|
96
|
+
aggr = vd.aggregators[aggrname].funcValues
|
97
|
+
d[aggrname] = self.calcStatistic(d, aggr, vals)
|
99
98
|
|
100
99
|
def calcStatistic(self, d, func, *args, **kwargs):
|
101
100
|
r = wrapply(func, *args, **kwargs)
|
@@ -115,8 +114,11 @@ class DescribeSheet(ColumnsSheet):
|
|
115
114
|
|
116
115
|
TableSheet.addCommand('I', 'describe-sheet', 'vd.push(DescribeSheet(sheet.name+"_describe", source=[sheet]))', 'open Describe Sheet with descriptive statistics for all visible columns')
|
117
116
|
BaseSheet.addCommand('gI', 'describe-all', 'vd.push(DescribeSheet("describe_all", source=vd.stackedSheets))', 'open Describe Sheet with description statistics for all visible columns from all sheets')
|
117
|
+
SheetsSheet.addCommand('gI', 'describe-selected', 'vd.push(DescribeSheet("describe_all", source=selectedRows))', 'open Describe Sheet with all visible columns from selected sheets')
|
118
118
|
|
119
119
|
DescribeSheet.addCommand('zs', 'select-cell', 'cursorRow.sheet.select(cursorValue)', 'select rows on source sheet which are being described in current cell')
|
120
120
|
DescribeSheet.addCommand('zu', 'unselect-cell', 'cursorRow.sheet.unselect(cursorValue)', 'unselect rows on source sheet which are being described in current cell')
|
121
121
|
|
122
|
+
vd.addMenuItems('Data > Statistics > describe-sheet')
|
123
|
+
|
122
124
|
vd.addGlobals({'DescribeSheet':DescribeSheet})
|
@@ -0,0 +1,26 @@
|
|
1
|
+
from visidata import GuideSheet, vd
|
2
|
+
|
3
|
+
class ErrorsGuide(GuideSheet):
|
4
|
+
guide_text='''# What was that error?
|
5
|
+
|
6
|
+
Status messages include [:warning]warnings[/] and [:error]errors[/].
|
7
|
+
|
8
|
+
A command may issue a [:warning]warning[/] status and continue running.
|
9
|
+
A command that [:warning]fails[/] or [:error]errors[/] is aborted.
|
10
|
+
|
11
|
+
## Investigating errors further
|
12
|
+
|
13
|
+
If a Python Exception like [:error]RuntimeError[/] appears in the sidebar:
|
14
|
+
|
15
|
+
- {help.commands.error_recent}
|
16
|
+
- {help.commands.errors_all}
|
17
|
+
|
18
|
+
If [:note_type]{vd.options.note_format_exc}[/] or [:error]{vd.options.note_getter_exc}[/] appear inside a cell, it indicates an error happened during calculation, type-conversion, or formatting. When the cursor is on an error cell:
|
19
|
+
|
20
|
+
- {help.commands.error_cell}
|
21
|
+
'''
|
22
|
+
|
23
|
+
|
24
|
+
|
25
|
+
vd.addGuide('ErrorsSheet', ErrorsGuide)
|
26
|
+
|
@@ -0,0 +1,202 @@
|
|
1
|
+
import math
|
2
|
+
import random
|
3
|
+
import os.path
|
4
|
+
from functools import singledispatch
|
5
|
+
|
6
|
+
from visidata import vd, Sheet, asyncthread, Progress, Column, VisiData, deduceType, anytype, getitemdef, ColumnsSheet
|
7
|
+
|
8
|
+
|
9
|
+
@Sheet.api
|
10
|
+
def getSampleRows(sheet):
|
11
|
+
'Return list of sample rows, including the cursor row as the first row.'
|
12
|
+
|
13
|
+
# do not include cursorRow in sample
|
14
|
+
ret = sheet.rows[:sheet.cursorRowIndex] + sheet.rows[sheet.cursorRowIndex+1:]
|
15
|
+
|
16
|
+
n = sheet.options.default_sample_size
|
17
|
+
if n != 0 and n < sheet.nRows:
|
18
|
+
vd.aside(f'sampling {n} rows')
|
19
|
+
ret = random.sample(ret, n)
|
20
|
+
|
21
|
+
return [sheet.cursorRow] + ret
|
22
|
+
|
23
|
+
|
24
|
+
@Sheet.api
|
25
|
+
def expandCols(sheet, cols, rows=None, depth=0):
|
26
|
+
'expand all visible columns of containers to the given depth (0=fully)'
|
27
|
+
ret = []
|
28
|
+
if not rows:
|
29
|
+
rows = sheet.getSampleRows()
|
30
|
+
|
31
|
+
for col in cols:
|
32
|
+
newcols = col.expand(rows)
|
33
|
+
if depth != 1: # countdown not yet complete, or negative (indefinite)
|
34
|
+
ret.extend(sheet.expandCols(newcols, rows, depth-1))
|
35
|
+
return ret
|
36
|
+
|
37
|
+
@singledispatch
|
38
|
+
def _createExpandedColumns(sampleValue, col, rows):
|
39
|
+
'''By default, a column is not expandable. Supported container types for
|
40
|
+
sampleValue trigger alternate, type-specific expansions.'''
|
41
|
+
return []
|
42
|
+
|
43
|
+
@_createExpandedColumns.register(dict)
|
44
|
+
def _(sampleValue, col, vals):
|
45
|
+
'''Build a set of columns to add, using the first occurrence of each key to
|
46
|
+
determine column type'''
|
47
|
+
newcols = {}
|
48
|
+
|
49
|
+
for val in Progress(vals, 'expanding'):
|
50
|
+
if not isinstance(val, dict): # allow mixed-use columns
|
51
|
+
continue
|
52
|
+
colsToAdd = set(val).difference(newcols)
|
53
|
+
colsToAdd and newcols.update({
|
54
|
+
k: deduceType(v)
|
55
|
+
for k, v in val.items()
|
56
|
+
if k in colsToAdd
|
57
|
+
})
|
58
|
+
|
59
|
+
return [
|
60
|
+
ExpandedColumn(col.sheet.options.fmt_expand_dict % (col.name, k), type=v, origCol=col, expr=k)
|
61
|
+
for k, v in newcols.items()
|
62
|
+
]
|
63
|
+
|
64
|
+
def _createExpandedColumnsNamedTuple(col, val):
|
65
|
+
return [
|
66
|
+
ExpandedColumn(col.sheet.options.fmt_expand_dict % (col.name, k), type=colType, origCol=col, expr=i)
|
67
|
+
for i, (k, colType) in enumerate(zip(val._fields, (deduceType(v) for v in val)))
|
68
|
+
]
|
69
|
+
|
70
|
+
@_createExpandedColumns.register(list)
|
71
|
+
@_createExpandedColumns.register(tuple)
|
72
|
+
def _(sampleValue, col, vals):
|
73
|
+
'''Use the longest sequence to determine the number of columns we need to
|
74
|
+
create, and their presumed types. Ignore strings and exceptions. '''
|
75
|
+
def lenNoExceptions(v):
|
76
|
+
try:
|
77
|
+
if isinstance(v, str):
|
78
|
+
return 0
|
79
|
+
return len(v)
|
80
|
+
except Exception as e:
|
81
|
+
return 0
|
82
|
+
|
83
|
+
if hasattr(sampleValue, '_fields'): # looks like a namedtuple
|
84
|
+
return _createExpandedColumnsNamedTuple(col, vals[0])
|
85
|
+
|
86
|
+
longestSeq = max(vals, key=lenNoExceptions)
|
87
|
+
colTypes = [deduceType(v) for v in longestSeq]
|
88
|
+
return [
|
89
|
+
ExpandedColumn(col.sheet.options.fmt_expand_list % (col.name, k), type=colType, origCol=col, expr=k)
|
90
|
+
for k, colType in enumerate(colTypes)
|
91
|
+
]
|
92
|
+
|
93
|
+
|
94
|
+
@Column.api
|
95
|
+
def expand(col, rows):
|
96
|
+
isNull = col.sheet.isNullFunc()
|
97
|
+
nonNulls = [
|
98
|
+
col.getTypedValue(row)
|
99
|
+
for row in rows
|
100
|
+
if not isNull(col.getValue(row))
|
101
|
+
]
|
102
|
+
|
103
|
+
if not nonNulls:
|
104
|
+
return []
|
105
|
+
|
106
|
+
# The type of the first non-null value for col determines if and how the
|
107
|
+
# column can be expanded.
|
108
|
+
expandedCols = _createExpandedColumns(nonNulls[0], col, nonNulls)
|
109
|
+
|
110
|
+
idx = col.sheet.columns.index(col)
|
111
|
+
|
112
|
+
for i, c in enumerate(expandedCols):
|
113
|
+
col.sheet.addColumn(c, index=idx+i+1)
|
114
|
+
if expandedCols:
|
115
|
+
col.hide()
|
116
|
+
return expandedCols
|
117
|
+
|
118
|
+
|
119
|
+
@VisiData.api
|
120
|
+
class ExpandedColumn(Column):
|
121
|
+
def calcValue(self, row):
|
122
|
+
return getitemdef(self.origCol.getValue(row), self.expr)
|
123
|
+
|
124
|
+
def setValue(self, row, value):
|
125
|
+
self.origCol.getValue(row)[self.expr] = value
|
126
|
+
|
127
|
+
|
128
|
+
@Sheet.api
|
129
|
+
@asyncthread
|
130
|
+
def contract_cols(sheet, cols, depth=1): # depth == 0 means contract all the way
|
131
|
+
'Remove any columns in cols with .origCol, and also remove others in sheet.columns which share those .origCol. The inverse of expand.'
|
132
|
+
vd.addUndo(setattr, sheet, 'columns', sheet.columns)
|
133
|
+
for i in range(depth or 10000):
|
134
|
+
colsToClose = [c for c in cols if getattr(c, "origCol", None)]
|
135
|
+
|
136
|
+
if not colsToClose:
|
137
|
+
break
|
138
|
+
|
139
|
+
origCols = set(c.origCol for c in colsToClose)
|
140
|
+
for col in origCols:
|
141
|
+
col.width = sheet.options.default_width
|
142
|
+
|
143
|
+
sheet.columns = [col for col in sheet.columns if getattr(col, 'origCol', None) not in origCols]
|
144
|
+
|
145
|
+
|
146
|
+
@Sheet.api
|
147
|
+
@asyncthread
|
148
|
+
def expand_cols_deep(sheet, cols, rows=None, depth=0): # depth == 0 means drill all the way
|
149
|
+
return sheet.expandCols(cols, rows=rows, depth=depth)
|
150
|
+
|
151
|
+
|
152
|
+
@ColumnsSheet.api
|
153
|
+
def contract_source_cols(sheet, cols):
|
154
|
+
prefix = os.path.commonprefix([c.name for c in cols])
|
155
|
+
ret = ColumnGroup(prefix or 'group', prefix=prefix, sourceCols=cols)
|
156
|
+
for c in cols:
|
157
|
+
c.origCol = ret
|
158
|
+
for vs in sheet.source:
|
159
|
+
vd.addUndo(setattr, vs, 'columns', vs.columns)
|
160
|
+
vs.columns[:] = [c for c in vs.columns if c not in cols]
|
161
|
+
return ret
|
162
|
+
|
163
|
+
|
164
|
+
class ColumnGroup(Column):
|
165
|
+
def calcValue(self, row):
|
166
|
+
return {c.name[len(self.prefix):]:c.getValue(row) for c in self.sourceCols}
|
167
|
+
|
168
|
+
def expand(self, rows):
|
169
|
+
idx = self.sheet.columns.index(self)
|
170
|
+
|
171
|
+
for i, c in enumerate(self.sourceCols):
|
172
|
+
self.sheet.addColumn(c, index=idx+i+1)
|
173
|
+
|
174
|
+
self.hide()
|
175
|
+
|
176
|
+
return self.sourceCols
|
177
|
+
|
178
|
+
|
179
|
+
Sheet.addCommand('(', 'expand-col', 'expand_cols_deep([cursorCol], depth=1)', 'expand current column of containers one level')
|
180
|
+
Sheet.addCommand('g(', 'expand-cols', 'expand_cols_deep(visibleCols, depth=1)', 'expand all visible columns of containers one level')
|
181
|
+
Sheet.addCommand('z(', 'expand-col-depth', 'expand_cols_deep([cursorCol], depth=int(input("expand depth=", value=0)))', 'expand current column of containers to given depth (0=fully)')
|
182
|
+
Sheet.addCommand('gz(', 'expand-cols-depth', 'expand_cols_deep(visibleCols, depth=int(input("expand depth=", value=0)))', 'expand all visible columns of containers to given depth (0=fully)')
|
183
|
+
|
184
|
+
Sheet.addCommand(')', 'contract-col', 'contract_cols([cursorCol])', 'remove current column and siblings from sheet columns and unhide parent')
|
185
|
+
Sheet.addCommand('g)', 'contract-cols', 'contract_cols(visibleCols)', 'remove all child columns and unhide toplevel parents')
|
186
|
+
Sheet.addCommand('z)', 'contract-col-depth', 'contract_cols([cursorCol], depth=int(input("contract depth=", value=0)))', 'remove current column and siblings from sheet columns and unhide parent')
|
187
|
+
Sheet.addCommand('gz)', 'contract-cols-depth', 'contract_cols(visibleCols, depth=int(input("contract depth=", value=0)))', 'remove all child columns and unhide toplevel parents')
|
188
|
+
|
189
|
+
ColumnsSheet.addCommand(')', 'contract-source-cols', 'source[0].addColumn(contract_source_cols(someSelectedRows), index=cursorRowIndex)', 'contract selected columns into column group') #1702
|
190
|
+
|
191
|
+
|
192
|
+
vd.addMenuItems('''
|
193
|
+
Column > Expand > one level > expand-col
|
194
|
+
Column > Expand > to depth N > expand-col-depth
|
195
|
+
Column > Expand > all columns one level > expand-cols
|
196
|
+
Column > Expand > all columns to depth > expand-cols-depth
|
197
|
+
Column > Contract > one level > contract-col
|
198
|
+
Column > Contract > N levels > contract-col-depth
|
199
|
+
Column > Contract > all columns one level > contract-cols
|
200
|
+
Column > Contract > all columns N levels > contract-cols-depth
|
201
|
+
Column > Contract > selected columns on source sheet > contract-source-cols
|
202
|
+
''')
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from visidata import
|
1
|
+
from visidata import vd, VisiData, asyncthread, Sheet, Progress
|
2
2
|
|
3
3
|
|
4
4
|
@VisiData.api
|
@@ -17,7 +17,7 @@ def fillNullValues(vd, col, rows):
|
|
17
17
|
val = e
|
18
18
|
|
19
19
|
if isNull(val):
|
20
|
-
if lastval and (id(r) in rowsToFill):
|
20
|
+
if not isNull(lastval) and (id(r) in rowsToFill):
|
21
21
|
oldvals.append((col,r,val))
|
22
22
|
col.setValue(r, lastval)
|
23
23
|
n += 1
|
@@ -34,3 +34,5 @@ def fillNullValues(vd, col, rows):
|
|
34
34
|
|
35
35
|
|
36
36
|
Sheet.addCommand('f', 'setcol-fill', 'fillNullValues(cursorCol, someSelectedRows)', 'fills null cells in selected rows of current column with contents of non-null cells up the current column')
|
37
|
+
|
38
|
+
vd.addMenuItems('Column > Fill > setcol-fill')
|
@@ -1,5 +1,6 @@
|
|
1
|
-
from visidata import *
|
2
1
|
import collections
|
2
|
+
from visidata import Column, Sheet, VisiData, ColumnItem, Progress, TypedExceptionWrapper, SettableColumn
|
3
|
+
from visidata import asyncthread, vd
|
3
4
|
|
4
5
|
|
5
6
|
@Column.api
|
@@ -35,6 +36,7 @@ class StaticSheet(Sheet):
|
|
35
36
|
def __init__(self, source):
|
36
37
|
super().__init__(source.name + "'", source=source)
|
37
38
|
|
39
|
+
def resetCols(self):
|
38
40
|
self.columns = []
|
39
41
|
for i, col in enumerate(self.source.visibleCols):
|
40
42
|
colcopy = ColumnItem(col.name)
|
@@ -44,12 +46,12 @@ class StaticSheet(Sheet):
|
|
44
46
|
if col in self.source.keyCols:
|
45
47
|
self.setKeys([colcopy])
|
46
48
|
|
47
|
-
|
48
|
-
def reload(self):
|
49
|
-
self.rows = []
|
49
|
+
def iterload(self):
|
50
50
|
for r in Progress(self.source.rows, 'calculating'):
|
51
51
|
row = []
|
52
|
-
|
52
|
+
yield row
|
53
|
+
|
54
|
+
# now fill out row
|
53
55
|
for col in self.source.visibleCols:
|
54
56
|
val = col.getTypedValue(r)
|
55
57
|
if isinstance(val, TypedExceptionWrapper):
|
@@ -58,7 +60,10 @@ class StaticSheet(Sheet):
|
|
58
60
|
row.append(val)
|
59
61
|
|
60
62
|
|
61
|
-
Sheet.addCommand("'", 'freeze-col', 'sheet.addColumnAtCursor(
|
63
|
+
Sheet.addCommand("'", 'freeze-col', 'sheet.addColumnAtCursor(freeze_col(cursorCol))', 'add a frozen copy of current column with all cells evaluated')
|
62
64
|
Sheet.addCommand("g'", 'freeze-sheet', 'vd.push(StaticSheet(sheet)); status("pushed frozen copy of "+name)', 'open a frozen copy of current sheet with all visible columns evaluated')
|
63
65
|
Sheet.addCommand("z'", 'cache-col', 'cursorCol.resetCache()', 'add/reset cache for current column')
|
64
66
|
Sheet.addCommand("gz'", 'cache-cols', 'for c in visibleCols: c.resetCache()', 'add/reset cache for all visible columns')
|
67
|
+
|
68
|
+
vd.addMenuItem('Column', 'Freeze', 'freeze-col')
|
69
|
+
vd.addMenuItem('File', 'Freeze', 'freeze-sheet')
|
@@ -0,0 +1,79 @@
|
|
1
|
+
'''
|
2
|
+
Add plot-column-ext and plot-numerics-ext to Sheet, and plot-ext to GraphSheet, to open current graph in new matplotlib window.
|
3
|
+
'''
|
4
|
+
|
5
|
+
from visidata import vd, VisiData, Sheet, GraphSheet, Progress, asyncthread
|
6
|
+
|
7
|
+
@VisiData.api
|
8
|
+
@asyncthread
|
9
|
+
def plot_seaborn(vd, rows, xcols, ycols):
|
10
|
+
vd.status(f'plotting {len(rows)} rows using matplotlib')
|
11
|
+
import multiprocessing
|
12
|
+
mp = multiprocessing.Process(target=ext_plot_seaborn, args=(rows, xcols, ycols))
|
13
|
+
mp.start()
|
14
|
+
|
15
|
+
|
16
|
+
def ext_plot_seaborn(rows, xcols, ycols):
|
17
|
+
pd = vd.importExternal('pandas')
|
18
|
+
plt = vd.importExternal('matplotlib.pyplot', 'matplotlib')
|
19
|
+
sns = vd.importExternal('seaborn')
|
20
|
+
|
21
|
+
# Set the default theme
|
22
|
+
sns.set()
|
23
|
+
|
24
|
+
plt.figure(figsize=(10, 6))
|
25
|
+
plt.title('')
|
26
|
+
plt.xticks(rotation=15)
|
27
|
+
|
28
|
+
nerrors = 0
|
29
|
+
nplotted = 0
|
30
|
+
|
31
|
+
x_array = []
|
32
|
+
y_array = []
|
33
|
+
cat_array = []
|
34
|
+
|
35
|
+
catcols = [c for c in xcols if not vd.isNumeric(c)]
|
36
|
+
numcols = vd.numericCols(xcols)
|
37
|
+
for rownum, row in enumerate(Progress(rows, 'plotting')):
|
38
|
+
for ycol in ycols:
|
39
|
+
try:
|
40
|
+
if catcols:
|
41
|
+
k = tuple(c.getValue(row) for c in catcols)
|
42
|
+
if len(catcols) == 1:
|
43
|
+
k = k[0]
|
44
|
+
else:
|
45
|
+
k = ycol.name
|
46
|
+
|
47
|
+
graph_x = numcols[0].type(numcols[0].getValue(row)) if numcols else rownum
|
48
|
+
graph_y = ycol.type(ycol.getValue(row))
|
49
|
+
|
50
|
+
x_array.append(graph_x)
|
51
|
+
y_array.append(graph_y)
|
52
|
+
cat_array.append(k)
|
53
|
+
|
54
|
+
nplotted += 1
|
55
|
+
except Exception:
|
56
|
+
nerrors += 1
|
57
|
+
if options.debug:
|
58
|
+
raise
|
59
|
+
|
60
|
+
sns.scatterplot(
|
61
|
+
x=x_array,
|
62
|
+
y=y_array,
|
63
|
+
hue=cat_array,
|
64
|
+
# hue_order=df.tag.value_counts().iloc[:top].index,
|
65
|
+
# data=tmpdf,
|
66
|
+
s=5,
|
67
|
+
linewidth=0,
|
68
|
+
).legend().set_title = (None)
|
69
|
+
|
70
|
+
plt.show()
|
71
|
+
|
72
|
+
|
73
|
+
Sheet.addCommand('', 'plot-column-ext', 'plot_seaborn(rows, keyCols, numericCols([cursorCol]))', 'plot current numeric column on y-axis vs key columns on x-axis using matplotlib/seaborn')
|
74
|
+
Sheet.addCommand('', 'plot-numerics-ext', 'plot_seaborn(rows, keyCols, numericCols(nonKeyVisibleCols))', 'plot a graph of all visible numeric columns using matplotlib/seaborn')
|
75
|
+
GraphSheet.addCommand('', 'plot-ext', 'plot_seaborn(sourceRows, xcols, ycols)', 'replot current graph using matplotlib/seaborn')
|
76
|
+
|
77
|
+
vd.addMenuItem('Plot', 'Graph', 'using matplotlib', 'current column', 'plot-column-ext')
|
78
|
+
vd.addMenuItem('Plot', 'Graph', 'using matplotlib', 'all numeric columns', 'plot-numerics-ext')
|
79
|
+
vd.addMenuItem('Plot', 'Graph', 'replot using matplotlib', 'plot-ext')
|
@@ -0,0 +1,10 @@
|
|
1
|
+
'''
|
2
|
+
Hello world minimal plugin. Press F2 to show options.hello_world on the status line.
|
3
|
+
|
4
|
+
.visidatarc: `import plugins.hello`
|
5
|
+
'''
|
6
|
+
from visidata import vd, BaseSheet
|
7
|
+
|
8
|
+
vd.option('hello_world', '¡Hola mundo!', 'shown by the hello-world command')
|
9
|
+
|
10
|
+
BaseSheet.addCommand('KEY_F(2)', 'hello-world', 'status(options.hello_world)', 'print greeting to status')
|
@@ -0,0 +1,17 @@
|
|
1
|
+
from visidata import Sheet, anytype
|
2
|
+
|
3
|
+
@Sheet.api
|
4
|
+
def hint_type_int(sheet):
|
5
|
+
c = sheet.cursorCol
|
6
|
+
if c.type is anytype:
|
7
|
+
v = c.getTypedValue(sheet.cursorRow)
|
8
|
+
if isinstance(v, int) or int(v) is not None:
|
9
|
+
return 2, '[:onclick type-int]Set column type to integer[/] with `#`.'
|
10
|
+
|
11
|
+
@Sheet.api
|
12
|
+
def hint_type_float(sheet):
|
13
|
+
c = sheet.cursorCol
|
14
|
+
if c.type is anytype:
|
15
|
+
v = c.getTypedValue(sheet.cursorRow)
|
16
|
+
if isinstance(v, float) or float(v) is not None:
|
17
|
+
return 1, '[:onclick type-float]Set column type to floating point[/] with `%`.'
|
@@ -24,3 +24,8 @@ Sheet.addCommand('i', 'addcol-incr', 'c=SettableColumn(type=int); addColumnAtCur
|
|
24
24
|
Sheet.addCommand('gi', 'setcol-incr', 'cursorCol.setValues(selectedRows, *numrange(sheet.nSelectedRows))', 'set current column for selected rows to incremental values')
|
25
25
|
Sheet.addCommand('zi', 'addcol-incr-step', 'n=num(input("interval step: ")); c=SettableColumn(type=type(n)); addColumnAtCursor(c); c.setValues(rows, *numrange(nRows, step=n))', 'add column with incremental values times given step')
|
26
26
|
Sheet.addCommand('gzi', 'setcol-incr-step', 'n=num(input("interval step: ")); cursorCol.setValues(selectedRows, *numrange(nSelectedRows, n))', 'set current column for selected rows to incremental values times given step')
|
27
|
+
|
28
|
+
vd.addMenuItems('''
|
29
|
+
Column > Add column > increment > addcol-incr
|
30
|
+
Edit > Modify > selected cells > increment > setcol-incr
|
31
|
+
''')
|