visidata 2.11.1__py3-none-any.whl → 3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- visidata/__init__.py +72 -91
- visidata/_input.py +259 -42
- visidata/_open.py +84 -29
- visidata/_types.py +21 -3
- visidata/_urlcache.py +17 -4
- visidata/aggregators.py +65 -25
- visidata/apps/__init__.py +0 -0
- visidata/apps/vdsql/__about__.py +8 -0
- visidata/apps/vdsql/__init__.py +5 -0
- visidata/apps/vdsql/__main__.py +27 -0
- visidata/apps/vdsql/_ibis.py +748 -0
- visidata/apps/vdsql/bigquery.py +61 -0
- visidata/apps/vdsql/clickhouse.py +53 -0
- visidata/apps/vdsql/setup.py +40 -0
- visidata/apps/vdsql/snowflake.py +67 -0
- visidata/apps/vgit/__init__.py +13 -0
- {vgit → visidata/apps/vgit}/blame.py +5 -2
- {vgit → visidata/apps/vgit}/branch.py +31 -16
- {vgit → visidata/apps/vgit}/config.py +3 -3
- visidata/apps/vgit/diff.py +169 -0
- visidata/apps/vgit/gitsheet.py +161 -0
- {vgit → visidata/apps/vgit}/grep.py +6 -5
- visidata/apps/vgit/log.py +81 -0
- {vgit → visidata/apps/vgit}/main.py +18 -5
- {vgit → visidata/apps/vgit}/remote.py +8 -4
- visidata/apps/vgit/repos.py +71 -0
- {vgit → visidata/apps/vgit}/setup.py +6 -4
- visidata/apps/vgit/stash.py +69 -0
- visidata/apps/vgit/status.py +204 -0
- {vgit → visidata/apps/vgit}/statusbar.py +2 -0
- visidata/basesheet.py +59 -50
- visidata/canvas.py +208 -93
- visidata/choose.py +6 -6
- visidata/clean_names.py +29 -0
- visidata/clipboard.py +73 -17
- visidata/cliptext.py +220 -46
- visidata/cmdlog.py +88 -114
- visidata/color.py +142 -56
- visidata/column.py +121 -129
- visidata/ddw/input.ddw +74 -79
- visidata/ddw/regex.ddw +57 -0
- visidata/ddwplay.py +33 -14
- visidata/deprecated.py +77 -3
- visidata/desktop/visidata.desktop +7 -0
- visidata/editor.py +12 -6
- visidata/errors.py +5 -1
- visidata/experimental/__init__.py +0 -0
- visidata/experimental/diff_sheet.py +29 -0
- visidata/experimental/digit_autoedit.py +6 -0
- visidata/experimental/gdrive.py +89 -0
- visidata/experimental/google.py +37 -0
- visidata/experimental/gsheets.py +79 -0
- visidata/experimental/live_search.py +37 -0
- visidata/experimental/liveupdate.py +45 -0
- visidata/experimental/mark.py +133 -0
- visidata/experimental/noahs_tapestry/__init__.py +1 -0
- visidata/experimental/noahs_tapestry/tapestry.py +147 -0
- visidata/experimental/rownum.py +73 -0
- visidata/experimental/slide_cells.py +26 -0
- visidata/expr.py +8 -4
- visidata/extensible.py +30 -5
- visidata/features/__init__.py +0 -0
- visidata/features/addcol_audiometadata.py +42 -0
- visidata/features/addcol_histogram.py +34 -0
- visidata/features/canvas_save_svg.py +69 -0
- visidata/features/change_precision.py +46 -0
- visidata/features/cmdpalette.py +163 -0
- visidata/features/colorbrewer.py +363 -0
- visidata/{colorsheet.py → features/colorsheet.py} +17 -16
- visidata/features/command_server.py +105 -0
- visidata/features/currency_to_usd.py +70 -0
- visidata/{customdate.py → features/customdate.py} +2 -0
- visidata/features/dedupe.py +132 -0
- visidata/{describe.py → features/describe.py} +17 -15
- visidata/features/errors_guide.py +26 -0
- visidata/features/expand_cols.py +202 -0
- visidata/{fill.py → features/fill.py} +3 -1
- visidata/{freeze.py → features/freeze.py} +11 -6
- visidata/features/graph_seaborn.py +79 -0
- visidata/features/helloworld.py +10 -0
- visidata/features/hint_types.py +17 -0
- visidata/{incr.py → features/incr.py} +5 -0
- visidata/{join.py → features/join.py} +107 -53
- visidata/features/known_cols.py +21 -0
- visidata/features/layout.py +62 -0
- visidata/{melt.py → features/melt.py} +32 -21
- visidata/features/normcol.py +118 -0
- visidata/features/open_config.py +7 -0
- visidata/features/open_syspaste.py +18 -0
- visidata/features/ping.py +157 -0
- visidata/features/procmgr.py +208 -0
- visidata/features/random_sample.py +6 -0
- visidata/{regex.py → features/regex.py} +47 -31
- visidata/features/reload_every.py +55 -0
- visidata/features/rename_col_cascade.py +30 -0
- visidata/features/scroll_context.py +60 -0
- visidata/features/select_equal_selected.py +11 -0
- visidata/features/setcol_fake.py +65 -0
- visidata/{slide.py → features/slide.py} +75 -21
- visidata/features/sparkline.py +48 -0
- visidata/features/status_source.py +20 -0
- visidata/{sysedit.py → features/sysedit.py} +2 -1
- visidata/features/sysopen_mailcap.py +46 -0
- visidata/features/term_extras.py +13 -0
- visidata/{transpose.py → features/transpose.py} +5 -4
- visidata/features/type_ipaddr.py +73 -0
- visidata/features/type_url.py +11 -0
- visidata/{unfurl.py → features/unfurl.py} +9 -9
- visidata/{window.py → features/window.py} +2 -2
- visidata/form.py +50 -21
- visidata/freqtbl.py +81 -33
- visidata/fuzzymatch.py +414 -0
- visidata/graph.py +105 -33
- visidata/guide.py +180 -0
- visidata/help.py +75 -44
- visidata/hint.py +39 -0
- visidata/indexsheet.py +109 -0
- visidata/input_history.py +55 -0
- visidata/interface.py +58 -0
- visidata/keys.py +17 -16
- visidata/loaders/__init__.py +9 -0
- visidata/loaders/_pandas.py +61 -21
- visidata/loaders/api_airtable.py +70 -0
- visidata/loaders/api_bitio.py +102 -0
- visidata/loaders/api_matrix.py +148 -0
- visidata/loaders/api_reddit.py +306 -0
- visidata/loaders/api_zulip.py +249 -0
- visidata/loaders/archive.py +41 -7
- visidata/loaders/arrow.py +7 -7
- visidata/loaders/conll.py +49 -0
- visidata/loaders/csv.py +25 -7
- visidata/loaders/eml.py +3 -4
- visidata/loaders/f5log.py +1204 -0
- visidata/loaders/fec.py +325 -0
- visidata/loaders/fixed_width.py +2 -4
- visidata/loaders/frictionless.py +3 -3
- visidata/loaders/geojson.py +8 -5
- visidata/loaders/google.py +48 -0
- visidata/loaders/graphviz.py +4 -4
- visidata/loaders/hdf5.py +4 -4
- visidata/loaders/html.py +48 -10
- visidata/loaders/http.py +84 -30
- visidata/loaders/imap.py +20 -10
- visidata/loaders/jrnl.py +52 -0
- visidata/loaders/json.py +83 -29
- visidata/loaders/jsonla.py +74 -0
- visidata/loaders/lsv.py +15 -11
- visidata/loaders/mailbox.py +40 -0
- visidata/loaders/markdown.py +1 -3
- visidata/loaders/mbtiles.py +4 -5
- visidata/loaders/mysql.py +11 -13
- visidata/loaders/npy.py +7 -7
- visidata/loaders/odf.py +4 -1
- visidata/loaders/orgmode.py +428 -0
- visidata/loaders/pandas_freqtbl.py +14 -20
- visidata/loaders/parquet.py +62 -6
- visidata/loaders/pcap.py +3 -3
- visidata/loaders/pdf.py +4 -3
- visidata/loaders/png.py +19 -13
- visidata/loaders/postgres.py +9 -8
- visidata/loaders/rec.py +7 -3
- visidata/loaders/s3.py +342 -0
- visidata/loaders/sas.py +5 -5
- visidata/loaders/scrape.py +186 -0
- visidata/loaders/shp.py +6 -5
- visidata/loaders/spss.py +5 -6
- visidata/loaders/sqlite.py +68 -28
- visidata/loaders/texttables.py +1 -1
- visidata/loaders/toml.py +60 -0
- visidata/loaders/tsv.py +61 -19
- visidata/loaders/ttf.py +19 -7
- visidata/loaders/unzip_http.py +6 -5
- visidata/loaders/usv.py +1 -1
- visidata/loaders/vcf.py +16 -16
- visidata/loaders/vds.py +10 -7
- visidata/loaders/vdx.py +30 -5
- visidata/loaders/xlsb.py +8 -1
- visidata/loaders/xlsx.py +145 -25
- visidata/loaders/xml.py +6 -3
- visidata/loaders/xword.py +4 -4
- visidata/loaders/yaml.py +15 -5
- visidata/macros.py +129 -42
- visidata/main.py +119 -94
- visidata/mainloop.py +101 -155
- visidata/man/parse_options.py +2 -2
- visidata/man/vd.1 +301 -148
- visidata/man/vd.txt +290 -153
- visidata/memory.py +3 -3
- visidata/menu.py +104 -423
- visidata/metasheets.py +59 -141
- visidata/modify.py +78 -23
- visidata/motd.py +3 -3
- visidata/mouse.py +137 -0
- visidata/movement.py +43 -35
- visidata/optionssheet.py +99 -0
- visidata/path.py +113 -32
- visidata/pivot.py +73 -47
- visidata/plugins.py +65 -192
- visidata/pyobj.py +50 -201
- visidata/rename_col.py +20 -0
- visidata/save.py +37 -20
- visidata/search.py +54 -10
- visidata/selection.py +84 -5
- visidata/settings.py +162 -25
- visidata/sheets.py +229 -257
- visidata/shell.py +51 -21
- visidata/sidebar.py +162 -0
- visidata/sort.py +11 -4
- visidata/statusbar.py +113 -104
- visidata/stored_list.py +43 -0
- visidata/stored_prop.py +38 -0
- visidata/tests/conftest.py +3 -3
- visidata/tests/test_cliptext.py +39 -0
- visidata/tests/test_commands.py +62 -7
- visidata/tests/test_edittext.py +2 -2
- visidata/tests/test_features.py +17 -0
- visidata/tests/test_menu.py +14 -0
- visidata/tests/test_path.py +13 -4
- visidata/text_source.py +53 -0
- visidata/textsheet.py +10 -3
- visidata/theme.py +44 -0
- visidata/themes/__init__.py +0 -0
- visidata/themes/ascii8.py +84 -0
- visidata/themes/asciimono.py +84 -0
- visidata/themes/light.py +17 -0
- visidata/threads.py +87 -39
- visidata/tuiwin.py +22 -0
- visidata/type_currency.py +22 -3
- visidata/type_date.py +31 -9
- visidata/type_floatsi.py +5 -1
- visidata/undo.py +17 -5
- visidata/utils.py +106 -23
- visidata/vdobj.py +28 -17
- visidata/windows.py +10 -0
- visidata/wrappers.py +9 -3
- visidata-3.0.data/data/share/applications/visidata.desktop +7 -0
- {visidata-2.11.1.data → visidata-3.0.data}/data/share/man/man1/vd.1 +301 -148
- {visidata-2.11.1.data → visidata-3.0.data}/data/share/man/man1/visidata.1 +301 -148
- visidata-3.0.data/scripts/vd2to3.vdx +9 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/METADATA +12 -8
- visidata-3.0.dist-info/RECORD +257 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/WHEEL +1 -1
- vgit/__init__.py +0 -1
- vgit/gitsheet.py +0 -164
- visidata/layout.py +0 -44
- visidata/misc.py +0 -5
- visidata-2.11.1.data/scripts/vgit +0 -9
- visidata-2.11.1.dist-info/RECORD +0 -155
- {vgit → visidata/apps/vgit}/__main__.py +0 -0
- {vgit → visidata/apps/vgit}/abort.py +0 -0
- /visidata/{repeat.py → features/repeat.py} +0 -0
- {visidata-2.11.1.data → visidata-3.0.data}/scripts/vd +0 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/LICENSE.gpl3 +0 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/entry_points.txt +0 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/top_level.txt +0 -0
visidata/keys.py
CHANGED
@@ -8,6 +8,8 @@ visidata.vd.prettykeys_trdict = {
|
|
8
8
|
'^J': 'Enter',
|
9
9
|
'^M': 'Enter',
|
10
10
|
'^I': 'Tab',
|
11
|
+
'KEY_BTAB': 'Shift+Tab',
|
12
|
+
'^@': 'Ctrl+Space',
|
11
13
|
'KEY_UP': 'Up',
|
12
14
|
'KEY_DOWN': 'Down',
|
13
15
|
'KEY_LEFT': 'Left',
|
@@ -30,6 +32,8 @@ visidata.vd.prettykeys_trdict = {
|
|
30
32
|
'kNXT5': 'Ctrl+PgDn',
|
31
33
|
'KEY_IC5': 'Ctrl+Ins',
|
32
34
|
'KEY_DC5': 'Ctrl+Del',
|
35
|
+
'kDC5': 'Ctrl+Del',
|
36
|
+
'KEY_SDC': 'Shift+Del',
|
33
37
|
|
34
38
|
'KEY_IC': 'Ins',
|
35
39
|
'KEY_DC': 'Del',
|
@@ -47,24 +51,21 @@ visidata.vd.prettykeys_trdict = {
|
|
47
51
|
'BUTTON1_PRESSED': 'LeftClick',
|
48
52
|
'BUTTON2_PRESSED': 'MiddleClick',
|
49
53
|
'BUTTON3_PRESSED': 'RightClick',
|
50
|
-
'BUTTON4_PRESSED': '
|
51
|
-
'BUTTON5_PRESSED': '
|
52
|
-
'REPORT_MOUSE_POSITION': '
|
53
|
-
'2097152': '
|
54
|
-
'KEY_F(1)': 'F1',
|
55
|
-
'KEY_F(2)': 'F2',
|
56
|
-
'KEY_F(3)': 'F3',
|
57
|
-
'KEY_F(4)': 'F4',
|
58
|
-
'KEY_F(5)': 'F5',
|
59
|
-
'KEY_F(6)': 'F6',
|
60
|
-
'KEY_F(7)': 'F7',
|
61
|
-
'KEY_F(8)': 'F8',
|
62
|
-
'KEY_F(9)': 'F9',
|
63
|
-
'KEY_F(10)': 'F10',
|
64
|
-
'KEY_F(11)': 'F11',
|
65
|
-
'KEY_F(12)': 'F12',
|
54
|
+
'BUTTON4_PRESSED': 'ScrollUp',
|
55
|
+
'BUTTON5_PRESSED': 'ScrollDown',
|
56
|
+
'REPORT_MOUSE_POSITION': 'ScrollDown',
|
57
|
+
'2097152': 'ScrollDown',
|
66
58
|
}
|
67
59
|
|
60
|
+
for i in range(1, 13):
|
61
|
+
d = visidata.vd.prettykeys_trdict
|
62
|
+
d[f'KEY_F({i})'] = f'F{i}'
|
63
|
+
d[f'KEY_F({i+12})'] = f'Shift+F{i}'
|
64
|
+
d[f'KEY_F({i+24})'] = f'Ctrl+F{i}'
|
65
|
+
d[f'KEY_F({i+36})'] = f'Ctrl+Shift+F{i}'
|
66
|
+
d[f'KEY_F({i+48})'] = f'Alt+F{i}'
|
67
|
+
d[f'KEY_F({i+60})'] = f'Alt+Shift+F{i}'
|
68
|
+
|
68
69
|
|
69
70
|
@visidata.VisiData.api
|
70
71
|
def prettykeys(vd, key):
|
visidata/loaders/__init__.py
CHANGED
@@ -0,0 +1,9 @@
|
|
1
|
+
from visidata import vd, IndexSheet
|
2
|
+
|
3
|
+
|
4
|
+
vd.option('load_lazy', False, 'load subsheets always (False) or lazily (True)')
|
5
|
+
vd.option('skip', 0, 'skip N rows before header', replay=True)
|
6
|
+
vd.option('header', 1, 'parse first N rows as column names', replay=True)
|
7
|
+
|
8
|
+
IndexSheet.options.header = 0
|
9
|
+
IndexSheet.options.skip = 0
|
visidata/loaders/_pandas.py
CHANGED
@@ -4,23 +4,57 @@ from visidata import VisiData, vd, Sheet, date, anytype, Path, options, Column,
|
|
4
4
|
|
5
5
|
@VisiData.api
|
6
6
|
def open_pandas(vd, p):
|
7
|
-
return PandasSheet(p.
|
7
|
+
return PandasSheet(p.base_stem, source=p)
|
8
8
|
|
9
9
|
@VisiData.api
|
10
10
|
def open_dta(vd, p):
|
11
|
-
return PandasSheet(p.
|
11
|
+
return PandasSheet(p.base_stem, source=p, filetype='stata')
|
12
12
|
|
13
13
|
VisiData.open_stata = VisiData.open_pandas
|
14
14
|
|
15
15
|
for ft in 'feather gbq orc pickle sas stata'.split():
|
16
16
|
funcname ='open_'+ft
|
17
17
|
if not getattr(VisiData, funcname, None):
|
18
|
-
setattr(VisiData, funcname, lambda vd,p,ft=ft: PandasSheet(p.
|
18
|
+
setattr(VisiData, funcname, lambda vd,p,ft=ft: PandasSheet(p.base_stem, source=p, filetype=ft))
|
19
19
|
|
20
|
+
@VisiData.api
|
21
|
+
@asyncthread
|
22
|
+
def save_dta(vd, p, *sheets):
|
23
|
+
import pandas as pd
|
24
|
+
import numpy as np
|
25
|
+
|
26
|
+
# STATA is a one-sheet software
|
27
|
+
# Save only the first sheet
|
28
|
+
vs = sheets[0]
|
29
|
+
|
30
|
+
columns = [col.name for col in vs.visibleCols]
|
31
|
+
|
32
|
+
# Get data types
|
33
|
+
types = list()
|
34
|
+
dispvals = next(vs.iterdispvals(format=True))
|
35
|
+
for col,_ in dispvals.items():
|
36
|
+
if col.type in [bool, int, float]:
|
37
|
+
types.append(col.type)
|
38
|
+
elif vd.isNumeric(col):
|
39
|
+
types.append(float)
|
40
|
+
else:
|
41
|
+
types.append(str)
|
42
|
+
|
43
|
+
# Populate numpy array
|
44
|
+
data = np.empty((vs.nRows, len(columns)), dtype=object)
|
45
|
+
for r_i, dispvals in enumerate(vs.iterdispvals(format=True)):
|
46
|
+
for c_i, v in enumerate(dispvals.values()):
|
47
|
+
data[r_i, c_i] = v
|
48
|
+
|
49
|
+
# Convert to pandas DataFrame and save
|
50
|
+
dtype = {col:t for col,t in zip(columns, types)}
|
51
|
+
df = pd.DataFrame(data, columns=columns)
|
52
|
+
df = df.astype(dtype)
|
53
|
+
df.to_stata(p, version=118, write_index=False)
|
20
54
|
|
21
55
|
class DataFrameAdapter:
|
22
56
|
def __init__(self, df):
|
23
|
-
|
57
|
+
pd = vd.importExternal('pandas')
|
24
58
|
if not isinstance(df, pd.DataFrame):
|
25
59
|
vd.fail('%s is not a dataframe' % type(df).__name__)
|
26
60
|
|
@@ -57,7 +91,7 @@ class PandasSheet(Sheet):
|
|
57
91
|
'''
|
58
92
|
|
59
93
|
def dtype_to_type(self, dtype):
|
60
|
-
|
94
|
+
np = vd.importExternal('numpy')
|
61
95
|
# Find the underlying numpy dtype for any pandas extension dtypes
|
62
96
|
dtype = getattr(dtype, 'numpy_dtype', dtype)
|
63
97
|
try:
|
@@ -74,7 +108,7 @@ class PandasSheet(Sheet):
|
|
74
108
|
|
75
109
|
def read_tsv(self, path, **kwargs):
|
76
110
|
'Partial function for reading TSV files using pd.read_csv'
|
77
|
-
|
111
|
+
pd = vd.importExternal('pandas')
|
78
112
|
return pd.read_csv(path, sep='\t', **kwargs)
|
79
113
|
|
80
114
|
@property
|
@@ -111,7 +145,7 @@ class PandasSheet(Sheet):
|
|
111
145
|
|
112
146
|
@asyncthread
|
113
147
|
def reload(self):
|
114
|
-
|
148
|
+
pd = vd.importExternal('pandas')
|
115
149
|
if isinstance(self.source, pd.DataFrame):
|
116
150
|
df = self.source
|
117
151
|
elif isinstance(self.source, Path):
|
@@ -124,6 +158,12 @@ class PandasSheet(Sheet):
|
|
124
158
|
readfunc = getattr(pd, 'read_'+filetype) or vd.error('no pandas.read_'+filetype)
|
125
159
|
# readfunc() handles binary and text open()
|
126
160
|
df = readfunc(self.source, **options.getall('pandas_'+filetype+'_'))
|
161
|
+
# some read methods (html, for example) return a list of dataframes
|
162
|
+
if isinstance(df, list):
|
163
|
+
for idx, inner_df in enumerate(df[1:], start=1):
|
164
|
+
vd.push(PandasSheet(f'{self.name}[{idx}]', source=inner_df))
|
165
|
+
df = df[0]
|
166
|
+
self.name += '[0]'
|
127
167
|
if (filetype == 'pickle') and not isinstance(df, pd.DataFrame):
|
128
168
|
vd.fail('pandas loader can only unpickle dataframes')
|
129
169
|
else:
|
@@ -169,7 +209,7 @@ class PandasSheet(Sheet):
|
|
169
209
|
self.rows.sort_values(by=by_cols, ascending=ascending, inplace=True)
|
170
210
|
|
171
211
|
def _checkSelectedIndex(self):
|
172
|
-
|
212
|
+
pd = vd.importExternal('pandas')
|
173
213
|
if self._selectedMask.index is not self.df.index:
|
174
214
|
# DataFrame was modified inplace, so the selection is no longer valid
|
175
215
|
vd.status('pd.DataFrame.index updated, clearing {} selected rows'
|
@@ -224,7 +264,7 @@ class PandasSheet(Sheet):
|
|
224
264
|
self.unselectRow(row)
|
225
265
|
|
226
266
|
def clearSelected(self):
|
227
|
-
|
267
|
+
pd = vd.importExternal('pandas')
|
228
268
|
self._selectedMask = pd.Series(False, index=self.df.index)
|
229
269
|
|
230
270
|
def selectByIndex(self, start=None, end=None):
|
@@ -251,7 +291,7 @@ class PandasSheet(Sheet):
|
|
251
291
|
matching rows to the selection. If unselect is True, remove from the
|
252
292
|
active selection instead.
|
253
293
|
'''
|
254
|
-
|
294
|
+
pd = vd.importExternal('pandas')
|
255
295
|
case_sensitive = 'I' not in vd.options.regex_flags
|
256
296
|
masks = pd.DataFrame([
|
257
297
|
self.df[col.expr].astype(str).str.contains(pat=regex, case=case_sensitive, regex=True)
|
@@ -278,13 +318,13 @@ class PandasSheet(Sheet):
|
|
278
318
|
DataFrame's dtypes.
|
279
319
|
'''
|
280
320
|
|
281
|
-
|
321
|
+
pd = vd.importExternal('pandas')
|
282
322
|
return pd.DataFrame({
|
283
323
|
col: [None] * n for col in self.df.columns
|
284
324
|
}).astype(self.df.dtypes.to_dict(), errors='ignore')
|
285
325
|
|
286
326
|
def addRows(self, rows, index=None, undo=True):
|
287
|
-
|
327
|
+
pd = vd.importExternal('pandas')
|
288
328
|
if index is None:
|
289
329
|
self.df = self.df.append(pd.DataFrame(rows))
|
290
330
|
else:
|
@@ -296,7 +336,7 @@ class PandasSheet(Sheet):
|
|
296
336
|
vd.addUndo(self._deleteRows, range(index, index + len(rows)))
|
297
337
|
|
298
338
|
def _deleteRows(self, which):
|
299
|
-
|
339
|
+
pd = vd.importExternal('pandas')
|
300
340
|
self.df.drop(which, inplace=True)
|
301
341
|
self.df.index = pd.RangeIndex(self.nRows)
|
302
342
|
self._checkSelectedIndex()
|
@@ -306,7 +346,7 @@ class PandasSheet(Sheet):
|
|
306
346
|
vd.addUndo(self._deleteRows, index or self.nRows - 1)
|
307
347
|
|
308
348
|
def delete_row(self, rowidx):
|
309
|
-
|
349
|
+
pd = vd.importExternal('pandas')
|
310
350
|
oldrow = self.df.iloc[rowidx:rowidx+1]
|
311
351
|
|
312
352
|
# Use to_dict() here to work around an edge case when applying undos.
|
@@ -321,7 +361,7 @@ class PandasSheet(Sheet):
|
|
321
361
|
|
322
362
|
def deleteBy(self, by):
|
323
363
|
'''Delete rows for which func(row) is true. Returns number of deleted rows.'''
|
324
|
-
|
364
|
+
pd = vd.importExternal('pandas')
|
325
365
|
nRows = self.nRows
|
326
366
|
vd.addUndo(setattr, self, 'df', self.df.copy())
|
327
367
|
self.df = self.df[~by]
|
@@ -353,17 +393,17 @@ PandasSheet.addCommand(None, 'unselect-before', 'unselectByIndex(end=cursorRowIn
|
|
353
393
|
PandasSheet.addCommand(None, 'stoggle-after', 'toggleByIndex(start=cursorRowIndex)', 'toggle selection of rows from cursor to bottom')
|
354
394
|
PandasSheet.addCommand(None, 'select-after', 'selectByIndex(start=cursorRowIndex)', 'select all rows from cursor to bottom')
|
355
395
|
PandasSheet.addCommand(None, 'unselect-after', 'unselectByIndex(start=cursorRowIndex)', 'unselect all rows from cursor to bottom')
|
356
|
-
PandasSheet.addCommand(None, 'random-rows', 'nrows=int(input("random number to select: ", value=nRows)); vs=copy(sheet); vs.name=name+"_sample"; vs.rows=DataFrameAdapter(sheet.df.sample(nrows or nRows)); vd.push(vs)', 'open duplicate sheet with a random population subset of N rows')
|
396
|
+
PandasSheet.addCommand(None, 'random-rows', 'nrows=int(input("random number to select: ", value=nRows)); vs=copy(sheet); vs.name=name+"_sample"; vs.rows=DataFrameAdapter(sheet.df.sample(nrows or nRows)); vd.push(vs)', 'open duplicate sheet with a random population subset of N rows')
|
357
397
|
|
358
398
|
# Handle the regex selection family of commands through a single method,
|
359
399
|
# since the core logic is shared
|
360
|
-
PandasSheet.addCommand('|', 'select-col-regex', 'selectByRegex(regex=
|
361
|
-
PandasSheet.addCommand('\\', 'unselect-col-regex', 'selectByRegex(regex=
|
362
|
-
PandasSheet.addCommand('g|', 'select-cols-regex', 'selectByRegex(regex=
|
363
|
-
PandasSheet.addCommand('g\\', 'unselect-cols-regex', 'selectByRegex(regex=
|
400
|
+
PandasSheet.addCommand('|', 'select-col-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=[cursorCol])', 'select rows matching regex in current column')
|
401
|
+
PandasSheet.addCommand('\\', 'unselect-col-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=[cursorCol], unselect=True)', 'unselect rows matching regex in current column')
|
402
|
+
PandasSheet.addCommand('g|', 'select-cols-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=visibleCols)', 'select rows matching regex in any visible column')
|
403
|
+
PandasSheet.addCommand('g\\', 'unselect-cols-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=visibleCols, unselect=True)', 'unselect rows matching regex in any visible column')
|
364
404
|
|
365
405
|
# Override with a pandas/dataframe-aware implementation
|
366
|
-
PandasSheet.addCommand('"', 'dup-selected', 'vs=PandasSheet(sheet.name, "selectedref", source=selectedRows.df); vd.push(vs)', 'open duplicate sheet with only selected rows')
|
406
|
+
PandasSheet.addCommand('"', 'dup-selected', 'vs=PandasSheet(sheet.name, "selectedref", source=selectedRows.df); vd.push(vs)', 'open duplicate sheet with only selected rows')
|
367
407
|
|
368
408
|
vd.addGlobals({
|
369
409
|
'PandasSheet': PandasSheet,
|
@@ -0,0 +1,70 @@
|
|
1
|
+
import re
|
2
|
+
import os
|
3
|
+
|
4
|
+
from visidata import vd, date, asyncthread, VisiData, Progress, Sheet, Column, ItemColumn, deduceType, TypedWrapper, setitem
|
5
|
+
|
6
|
+
|
7
|
+
vd.option('airtable_auth_token', '', 'Airtable API key from https://airtable.com/account')
|
8
|
+
|
9
|
+
airtable_regex = r'^https://airtable.com/(app[A-Za-z0-9]+)/(tbl[A-Za-z0-9]+)/?(viw[A-z0-9]+)?'
|
10
|
+
|
11
|
+
@VisiData.api
|
12
|
+
def guessurl_airtable(vd, p, response):
|
13
|
+
m = re.search(airtable_regex, p.given)
|
14
|
+
if m:
|
15
|
+
return dict(filetype='airtable', _likelihood=10)
|
16
|
+
|
17
|
+
|
18
|
+
@VisiData.api
|
19
|
+
def open_airtable(vd, p):
|
20
|
+
pyairtable = vd.importExternal('pyairtable')
|
21
|
+
|
22
|
+
token = os.environ.get('AIRTABLE_AUTH_TOKEN') or vd.options.airtable_auth_token
|
23
|
+
if not token:
|
24
|
+
vd.requireOptions('airtable_auth_token', help='https://support.airtable.com/docs/creating-and-using-api-keys-and-access-tokens')
|
25
|
+
|
26
|
+
m = re.search(airtable_regex, p.given)
|
27
|
+
if not m:
|
28
|
+
vd.fail('invalid airtable url')
|
29
|
+
|
30
|
+
app, tbl, viw = m.groups()
|
31
|
+
return AirtableSheet('airtable', source=p,
|
32
|
+
airtable_auth_token=token,
|
33
|
+
airtable_base=app,
|
34
|
+
airtable_table=tbl,
|
35
|
+
airtable_view=viw)
|
36
|
+
|
37
|
+
|
38
|
+
class AirtableSheet(Sheet):
|
39
|
+
guide = '''
|
40
|
+
# Airtable
|
41
|
+
This sheet is a read-only download of all records in a table at _airtable.com_.
|
42
|
+
'''
|
43
|
+
rowtype = 'records' # rowdef: dict
|
44
|
+
|
45
|
+
columns = [
|
46
|
+
ItemColumn('id', 'id', type=str, width=0),
|
47
|
+
ItemColumn('createdTime', 'createdTime', type=date, width=0)
|
48
|
+
]
|
49
|
+
|
50
|
+
def iterload(self):
|
51
|
+
self.fields = set()
|
52
|
+
|
53
|
+
for page in self.api.iterate(self.airtable_base, self.airtable_table, view=self.airtable_view):
|
54
|
+
for row in page:
|
55
|
+
yield row
|
56
|
+
|
57
|
+
for field, value in row['fields'].items():
|
58
|
+
if field not in self.fields:
|
59
|
+
col = ItemColumn('fields.'+field, type=deduceType(value))
|
60
|
+
self.addColumn(col)
|
61
|
+
self.fields.add(field)
|
62
|
+
|
63
|
+
def newRow(self):
|
64
|
+
return AttrDict(fields=AttrDict())
|
65
|
+
|
66
|
+
|
67
|
+
@AirtableSheet.lazy_property
|
68
|
+
def api(self):
|
69
|
+
import pyairtable
|
70
|
+
return pyairtable.Api(self.airtable_auth_token)
|
@@ -0,0 +1,102 @@
|
|
1
|
+
|
2
|
+
from visidata import *
|
3
|
+
|
4
|
+
vd.option('bitio_api_key', '', 'API key')
|
5
|
+
|
6
|
+
@VisiData.api
|
7
|
+
def new_bitio(vd, p):
|
8
|
+
vd.importExternal('bitdotio')
|
9
|
+
vd.requireOptions('bitio_api_key', help='https://docs.bit.io/docs/connecting-via-the-api')
|
10
|
+
return BitioReposSheet(p.name, source=p)
|
11
|
+
|
12
|
+
vd.openhttp_bitio = vd.new_bitio
|
13
|
+
|
14
|
+
@VisiData.lazy_property
|
15
|
+
def bitio_client(vd):
|
16
|
+
import bitdotio
|
17
|
+
return bitdotio.bitdotio(vd.options.bitio_api_key)
|
18
|
+
|
19
|
+
@VisiData.api
|
20
|
+
def bitio_api(vd, path, method, **kwargs):
|
21
|
+
t = vd.bitio_client.api_client.call_api(path, method,
|
22
|
+
header_params={'Accept': 'application/json',
|
23
|
+
'Authorization': 'Bearer '+vd.bitio_client.access_token,
|
24
|
+
'Content-Type': 'application/json'}, async_req=True, body=kwargs)
|
25
|
+
if not t.successful():
|
26
|
+
vd.warning(resp['Reason'])
|
27
|
+
return t.get()
|
28
|
+
|
29
|
+
|
30
|
+
class BitioReposSheet(Sheet):
|
31
|
+
rowtype = 'repos'
|
32
|
+
columns = [
|
33
|
+
AttrColumn('name'),
|
34
|
+
AttrColumn('creator'),
|
35
|
+
AttrColumn('owner'),
|
36
|
+
AttrColumn('bytes', type=int, width=0),
|
37
|
+
AttrColumn('collaborators'),
|
38
|
+
AttrColumn('description'),
|
39
|
+
AttrColumn('documentation'),
|
40
|
+
AttrColumn('endpoints'),
|
41
|
+
AttrColumn('is_private'),
|
42
|
+
AttrColumn('license'),
|
43
|
+
AttrColumn('query_count'),
|
44
|
+
AttrColumn('stars'),
|
45
|
+
AttrColumn('tables'),
|
46
|
+
AttrColumn('url'),
|
47
|
+
AttrColumn('watchers'),
|
48
|
+
]
|
49
|
+
defer = True
|
50
|
+
def iterload(self):
|
51
|
+
yield from vd.bitio_client.list_repos(self.source.name)
|
52
|
+
|
53
|
+
|
54
|
+
@asyncthread
|
55
|
+
def putChanges(self):
|
56
|
+
adds, mods, dels = self.getDeferredChanges()
|
57
|
+
for row in dels.values():
|
58
|
+
vd.bitio_api(f'/users/{self.source.name}/repos/{row.name}/', 'DELETE')
|
59
|
+
|
60
|
+
for row, rowmods in Progress(list(mods.values()), gerund="updating"):
|
61
|
+
kwargs = {col.name: val for col, val in rowmods.items()}
|
62
|
+
vd.bitio_api(f'/users/{self.source.name}/repos/{row.name}/', 'PATCH', **kwargs)
|
63
|
+
|
64
|
+
def openRow(self, row):
|
65
|
+
return BitioRepoSheet(self.source.name, row.name, source=row)
|
66
|
+
|
67
|
+
|
68
|
+
class BitioRepoSheet(Sheet):
|
69
|
+
rowtype = 'tables'
|
70
|
+
columns = [
|
71
|
+
AttrColumn('url'),
|
72
|
+
AttrColumn('current_name'),
|
73
|
+
AttrColumn('description'),
|
74
|
+
AttrColumn('columns'),
|
75
|
+
AttrColumn('num_records', type=int),
|
76
|
+
AttrColumn('bytes', type=int),
|
77
|
+
AttrColumn('repo'),
|
78
|
+
AttrColumn('documentation'),
|
79
|
+
]
|
80
|
+
def iterload(self):
|
81
|
+
yield from vd.bitio_client.list_tables(self.source.owner.split('/')[-2], self.source.name)
|
82
|
+
|
83
|
+
def openRow(self, row):
|
84
|
+
username = row.repo.split('/')[-4]
|
85
|
+
repo = row.repo.split('/')[-2]
|
86
|
+
return BitioTable(username, repo, source=row)
|
87
|
+
|
88
|
+
|
89
|
+
class BitioTable(Sheet):
|
90
|
+
def iterload(self):
|
91
|
+
username = self.source.repo.split('/')[-4]
|
92
|
+
repo = self.source.repo.split('/')[-2]
|
93
|
+
conn = vd.bitio_client.get_connection()
|
94
|
+
with conn.cursor() as cur:
|
95
|
+
cur.execute(f'SELECT * FROM "{username}/{repo}"."{self.source.current_name}"')
|
96
|
+
r = cur.fetchone()
|
97
|
+
if r:
|
98
|
+
yield r
|
99
|
+
self.columns = []
|
100
|
+
for c in vd.postgresGetColumns(cur):
|
101
|
+
self.addColumn(c)
|
102
|
+
yield from cur
|
@@ -0,0 +1,148 @@
|
|
1
|
+
'''
|
2
|
+
# Matrix Chat Sheet
|
3
|
+
A list of messages from [:underline]{sheet.sourcename}[/].
|
4
|
+
|
5
|
+
- `Enter` to push a sheet with all the messages from [:underline]{sheet.cursorRow.room.display_name}[/].
|
6
|
+
- `a` to send a message to [:underline]{sheet.cursorRow.room.display_name}[/].
|
7
|
+
'''
|
8
|
+
|
9
|
+
from visidata import vd, VisiData, Sheet, Column, ItemColumn, date, asyncthread, AttrDict, vlen, Path
|
10
|
+
|
11
|
+
|
12
|
+
vd.option('matrix_token', '', 'matrix API token')
|
13
|
+
vd.option('matrix_user_id', '', 'matrix user ID associated with token')
|
14
|
+
vd.option('matrix_device_id', 'VisiData', 'device ID associated with matrix login')
|
15
|
+
|
16
|
+
vd.matrix_client = None
|
17
|
+
|
18
|
+
|
19
|
+
@VisiData.api
|
20
|
+
def openhttp_matrix(vd, p):
|
21
|
+
vd.importExternal('matrix_client')
|
22
|
+
|
23
|
+
if not vd.options.matrix_token:
|
24
|
+
from matrix_client.client import MatrixClient
|
25
|
+
|
26
|
+
username = vd.input(f'{p.given} username: ', record=False)
|
27
|
+
password = vd.input('password: ', record=False, display=False)
|
28
|
+
|
29
|
+
vd.matrix_client = MatrixClient(p.given)
|
30
|
+
matrix_token = vd.matrix_client.login(username, password, device_id=vd.options.matrix_device_id)
|
31
|
+
|
32
|
+
vd.setPersistentOptions(matrix_user_id=username, matrix_token=matrix_token)
|
33
|
+
|
34
|
+
vd.timeouts_before_idle = -1
|
35
|
+
return MatrixSheet(p.base_stem, source=p)
|
36
|
+
|
37
|
+
vd.open_matrix = vd.openhttp_matrix
|
38
|
+
|
39
|
+
|
40
|
+
class MatrixRoomsSheet(Sheet):
|
41
|
+
def iterload(self):
|
42
|
+
yield from vd.matrix_client.get_rooms().values()
|
43
|
+
|
44
|
+
|
45
|
+
class MatrixSheet(Sheet):
|
46
|
+
help = __doc__
|
47
|
+
columns = [
|
48
|
+
Column('room', width=12, getter=lambda c,r: r.room and r.room.display_name),
|
49
|
+
ItemColumn('sender', width=0),
|
50
|
+
Column('sender', width=10, getter=lambda c,r: r.sender.split(':')[0][1:]),
|
51
|
+
Column('timestamp', width=16, type=date, getter=lambda c,r: r and r.origin_server_ts/1000, fmtstr='%Y-%m-%d %H:%m'),
|
52
|
+
ItemColumn('type', width=15),
|
53
|
+
ItemColumn('content', width=0),
|
54
|
+
ItemColumn('content.body', width=50),
|
55
|
+
ItemColumn('received', type=vlen, width=0),
|
56
|
+
ItemColumn('content.msgtype'),
|
57
|
+
]
|
58
|
+
|
59
|
+
@property
|
60
|
+
def sourcename(self):
|
61
|
+
from matrix_client.room import Room
|
62
|
+
if isinstance(self.source, Room):
|
63
|
+
return self.source.display_name or self.source.room_id
|
64
|
+
else:
|
65
|
+
return str(self.source)
|
66
|
+
|
67
|
+
@asyncthread
|
68
|
+
def reload(self):
|
69
|
+
from matrix_client.client import MatrixClient
|
70
|
+
from matrix_client.room import Room
|
71
|
+
if not vd.matrix_client:
|
72
|
+
vd.matrix_client = MatrixClient(self.source.given,
|
73
|
+
token=self.options.matrix_token,
|
74
|
+
user_id=self.options.matrix_user_id)
|
75
|
+
|
76
|
+
if isinstance(self.source, Room):
|
77
|
+
self.add_room(self.source)
|
78
|
+
self.get_room_messages(self.source)
|
79
|
+
return
|
80
|
+
|
81
|
+
for room in vd.matrix_client.get_rooms().values():
|
82
|
+
self.add_room(room)
|
83
|
+
room.backfill_previous_messages(limit=1)
|
84
|
+
|
85
|
+
vd.matrix_client.add_listener(self.global_event)
|
86
|
+
vd.matrix_client.add_ephemeral_listener(self.global_event)
|
87
|
+
|
88
|
+
vd.matrix_client.start_listener_thread(exception_handler=vd.exceptionCaught)
|
89
|
+
|
90
|
+
vd.matrix_client._sync()
|
91
|
+
|
92
|
+
vd.matrix_client.listen_for_events() # vd.matrix_client.sync(full_state=True)
|
93
|
+
|
94
|
+
def add_room(self, room):
|
95
|
+
room.add_listener(self.room_event)
|
96
|
+
room.add_ephemeral_listener(self.room_event)
|
97
|
+
room.add_state_listener(self.global_event)
|
98
|
+
|
99
|
+
@asyncthread
|
100
|
+
def get_room_messages(self, room):
|
101
|
+
while room.prev_batch:
|
102
|
+
ret = vd.matrix_client.api.get_room_messages(room.room_id, room.prev_batch, direction='b', limit=100)
|
103
|
+
for r in ret['chunk']:
|
104
|
+
r['room'] = room
|
105
|
+
self.addRow(r)
|
106
|
+
|
107
|
+
if 'end' not in ret or ret['end'] == room.prev_batch:
|
108
|
+
break
|
109
|
+
room.prev_batch = ret['end']
|
110
|
+
|
111
|
+
def addRow(self, r, **kwargs):
|
112
|
+
r = AttrDict(r)
|
113
|
+
if r.event_id not in self.event_index:
|
114
|
+
super().addRow(r, **kwargs)
|
115
|
+
self.event_index[r.event_id] = r
|
116
|
+
|
117
|
+
def global_event(self, chunk):
|
118
|
+
self.addRow(chunk)
|
119
|
+
|
120
|
+
def room_event(self, room, chunk):
|
121
|
+
ev = AttrDict(chunk)
|
122
|
+
t = chunk['type']
|
123
|
+
if t == 'm.receipt':
|
124
|
+
for msgid, content in chunk['content'].items():
|
125
|
+
msg = self.event_index.setdefault(msgid, {})
|
126
|
+
if 'received' not in msg:
|
127
|
+
msg['received'] = {}
|
128
|
+
|
129
|
+
for t, c in content.items():
|
130
|
+
assert t == 'm.read'
|
131
|
+
for userid, v in c.items():
|
132
|
+
msg['received'][(t, userid)] = v['ts']/1000
|
133
|
+
return
|
134
|
+
|
135
|
+
chunk['room'] = room
|
136
|
+
self.addRow(chunk)
|
137
|
+
|
138
|
+
def add_message(self, text):
|
139
|
+
vd.matrix_client.send_text(text)
|
140
|
+
|
141
|
+
def openRow(self, row):
|
142
|
+
return MatrixSheet(row.room.display_name, source=row.room, last_id=row.event_id)
|
143
|
+
|
144
|
+
|
145
|
+
MatrixSheet.init('event_index', dict)
|
146
|
+
|
147
|
+
MatrixSheet.addCommand('a', 'add-row', 'cursorRow.room.send_text(input(cursorRow.room.display_name+"> "))', 'send chat message to current room')
|
148
|
+
vd.addMenuItems('Row > Add > send matrix message > add-msg')
|