visidata 2.11.1__py3-none-any.whl → 3.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. visidata/__init__.py +72 -91
  2. visidata/_input.py +259 -42
  3. visidata/_open.py +84 -29
  4. visidata/_types.py +21 -3
  5. visidata/_urlcache.py +17 -4
  6. visidata/aggregators.py +78 -25
  7. visidata/apps/__init__.py +0 -0
  8. visidata/apps/vdsql/__about__.py +8 -0
  9. visidata/apps/vdsql/__init__.py +5 -0
  10. visidata/apps/vdsql/__main__.py +27 -0
  11. visidata/apps/vdsql/_ibis.py +748 -0
  12. visidata/apps/vdsql/bigquery.py +61 -0
  13. visidata/apps/vdsql/clickhouse.py +53 -0
  14. visidata/apps/vdsql/setup.py +40 -0
  15. visidata/apps/vdsql/snowflake.py +67 -0
  16. visidata/apps/vgit/__init__.py +13 -0
  17. {vgit → visidata/apps/vgit}/blame.py +5 -2
  18. {vgit → visidata/apps/vgit}/branch.py +31 -16
  19. {vgit → visidata/apps/vgit}/config.py +3 -3
  20. visidata/apps/vgit/diff.py +169 -0
  21. visidata/apps/vgit/gitsheet.py +161 -0
  22. {vgit → visidata/apps/vgit}/grep.py +6 -5
  23. visidata/apps/vgit/log.py +81 -0
  24. {vgit → visidata/apps/vgit}/main.py +18 -5
  25. {vgit → visidata/apps/vgit}/remote.py +8 -4
  26. visidata/apps/vgit/repos.py +71 -0
  27. {vgit → visidata/apps/vgit}/setup.py +6 -4
  28. visidata/apps/vgit/stash.py +69 -0
  29. visidata/apps/vgit/status.py +204 -0
  30. {vgit → visidata/apps/vgit}/statusbar.py +2 -0
  31. visidata/basesheet.py +63 -51
  32. visidata/canvas.py +208 -93
  33. visidata/choose.py +6 -6
  34. visidata/clean_names.py +29 -0
  35. visidata/clipboard.py +73 -17
  36. visidata/cliptext.py +220 -46
  37. visidata/cmdlog.py +88 -114
  38. visidata/color.py +142 -56
  39. visidata/column.py +121 -129
  40. visidata/ddw/input.ddw +74 -79
  41. visidata/ddw/regex.ddw +57 -0
  42. visidata/ddwplay.py +33 -14
  43. visidata/deprecated.py +77 -3
  44. visidata/desktop/visidata.desktop +7 -0
  45. visidata/editor.py +12 -6
  46. visidata/errors.py +6 -2
  47. visidata/experimental/__init__.py +0 -0
  48. visidata/experimental/diff_sheet.py +29 -0
  49. visidata/experimental/digit_autoedit.py +6 -0
  50. visidata/experimental/gdrive.py +89 -0
  51. visidata/experimental/google.py +37 -0
  52. visidata/experimental/gsheets.py +79 -0
  53. visidata/experimental/live_search.py +37 -0
  54. visidata/experimental/liveupdate.py +45 -0
  55. visidata/experimental/mark.py +133 -0
  56. visidata/experimental/noahs_tapestry/__init__.py +1 -0
  57. visidata/experimental/noahs_tapestry/tapestry.py +147 -0
  58. visidata/experimental/rownum.py +73 -0
  59. visidata/experimental/slide_cells.py +26 -0
  60. visidata/expr.py +8 -4
  61. visidata/extensible.py +22 -4
  62. visidata/features/__init__.py +0 -0
  63. visidata/features/addcol_audiometadata.py +42 -0
  64. visidata/features/addcol_histogram.py +34 -0
  65. visidata/features/canvas_save_svg.py +69 -0
  66. visidata/features/change_precision.py +46 -0
  67. visidata/features/cmdpalette.py +197 -0
  68. visidata/features/colorbrewer.py +363 -0
  69. visidata/{colorsheet.py → features/colorsheet.py} +17 -16
  70. visidata/features/command_server.py +105 -0
  71. visidata/features/currency_to_usd.py +70 -0
  72. visidata/{customdate.py → features/customdate.py} +2 -0
  73. visidata/features/dedupe.py +132 -0
  74. visidata/{describe.py → features/describe.py} +17 -15
  75. visidata/features/errors_guide.py +26 -0
  76. visidata/features/expand_cols.py +202 -0
  77. visidata/{fill.py → features/fill.py} +3 -1
  78. visidata/{freeze.py → features/freeze.py} +11 -6
  79. visidata/features/graph_seaborn.py +79 -0
  80. visidata/features/helloworld.py +10 -0
  81. visidata/features/hint_types.py +17 -0
  82. visidata/{incr.py → features/incr.py} +5 -0
  83. visidata/{join.py → features/join.py} +107 -53
  84. visidata/features/known_cols.py +21 -0
  85. visidata/features/layout.py +62 -0
  86. visidata/{melt.py → features/melt.py} +32 -21
  87. visidata/features/normcol.py +118 -0
  88. visidata/features/open_config.py +7 -0
  89. visidata/features/open_syspaste.py +18 -0
  90. visidata/features/ping.py +157 -0
  91. visidata/features/procmgr.py +208 -0
  92. visidata/features/random_sample.py +6 -0
  93. visidata/{regex.py → features/regex.py} +47 -31
  94. visidata/features/reload_every.py +55 -0
  95. visidata/features/rename_col_cascade.py +30 -0
  96. visidata/features/scroll_context.py +60 -0
  97. visidata/features/select_equal_selected.py +11 -0
  98. visidata/features/setcol_fake.py +65 -0
  99. visidata/{slide.py → features/slide.py} +77 -21
  100. visidata/features/sparkline.py +48 -0
  101. visidata/features/status_source.py +20 -0
  102. visidata/{sysedit.py → features/sysedit.py} +2 -1
  103. visidata/features/sysopen_mailcap.py +46 -0
  104. visidata/features/term_extras.py +13 -0
  105. visidata/{transpose.py → features/transpose.py} +5 -4
  106. visidata/features/type_ipaddr.py +73 -0
  107. visidata/features/type_url.py +11 -0
  108. visidata/{unfurl.py → features/unfurl.py} +9 -9
  109. visidata/{window.py → features/window.py} +2 -2
  110. visidata/form.py +50 -21
  111. visidata/freqtbl.py +81 -33
  112. visidata/fuzzymatch.py +414 -0
  113. visidata/graph.py +105 -33
  114. visidata/guide.py +200 -0
  115. visidata/help.py +75 -44
  116. visidata/hint.py +39 -0
  117. visidata/indexsheet.py +109 -0
  118. visidata/input_history.py +55 -0
  119. visidata/interface.py +58 -0
  120. visidata/keys.py +20 -16
  121. visidata/loaders/__init__.py +9 -0
  122. visidata/loaders/_pandas.py +61 -21
  123. visidata/loaders/api_airtable.py +70 -0
  124. visidata/loaders/api_bitio.py +102 -0
  125. visidata/loaders/api_matrix.py +148 -0
  126. visidata/loaders/api_reddit.py +306 -0
  127. visidata/loaders/api_zulip.py +249 -0
  128. visidata/loaders/archive.py +41 -7
  129. visidata/loaders/arrow.py +7 -7
  130. visidata/loaders/conll.py +49 -0
  131. visidata/loaders/csv.py +25 -7
  132. visidata/loaders/eml.py +3 -4
  133. visidata/loaders/f5log.py +1204 -0
  134. visidata/loaders/fec.py +325 -0
  135. visidata/loaders/fixed_width.py +2 -4
  136. visidata/loaders/frictionless.py +3 -3
  137. visidata/loaders/geojson.py +8 -5
  138. visidata/loaders/google.py +48 -0
  139. visidata/loaders/graphviz.py +4 -4
  140. visidata/loaders/hdf5.py +4 -4
  141. visidata/loaders/html.py +54 -12
  142. visidata/loaders/http.py +84 -30
  143. visidata/loaders/imap.py +20 -10
  144. visidata/loaders/jrnl.py +52 -0
  145. visidata/loaders/json.py +83 -29
  146. visidata/loaders/jsonla.py +74 -0
  147. visidata/loaders/lsv.py +15 -11
  148. visidata/loaders/mailbox.py +40 -0
  149. visidata/loaders/markdown.py +1 -3
  150. visidata/loaders/mbtiles.py +4 -5
  151. visidata/loaders/mysql.py +11 -13
  152. visidata/loaders/npy.py +7 -7
  153. visidata/loaders/odf.py +4 -1
  154. visidata/loaders/orgmode.py +428 -0
  155. visidata/loaders/pandas_freqtbl.py +14 -20
  156. visidata/loaders/parquet.py +62 -6
  157. visidata/loaders/pcap.py +3 -3
  158. visidata/loaders/pdf.py +4 -3
  159. visidata/loaders/png.py +19 -13
  160. visidata/loaders/postgres.py +9 -8
  161. visidata/loaders/rec.py +7 -3
  162. visidata/loaders/s3.py +342 -0
  163. visidata/loaders/sas.py +5 -5
  164. visidata/loaders/scrape.py +186 -0
  165. visidata/loaders/shp.py +6 -5
  166. visidata/loaders/spss.py +5 -6
  167. visidata/loaders/sqlite.py +68 -28
  168. visidata/loaders/texttables.py +1 -1
  169. visidata/loaders/toml.py +60 -0
  170. visidata/loaders/tsv.py +61 -19
  171. visidata/loaders/ttf.py +19 -7
  172. visidata/loaders/unzip_http.py +6 -5
  173. visidata/loaders/usv.py +1 -1
  174. visidata/loaders/vcf.py +16 -16
  175. visidata/loaders/vds.py +10 -7
  176. visidata/loaders/vdx.py +30 -5
  177. visidata/loaders/xlsb.py +8 -1
  178. visidata/loaders/xlsx.py +145 -25
  179. visidata/loaders/xml.py +6 -3
  180. visidata/loaders/xword.py +4 -4
  181. visidata/loaders/yaml.py +15 -5
  182. visidata/macros.py +129 -42
  183. visidata/main.py +119 -94
  184. visidata/mainloop.py +101 -155
  185. visidata/man/parse_options.py +2 -2
  186. visidata/man/vd.1 +302 -149
  187. visidata/man/vd.txt +291 -154
  188. visidata/memory.py +3 -3
  189. visidata/menu.py +104 -423
  190. visidata/metasheets.py +59 -141
  191. visidata/modify.py +78 -23
  192. visidata/motd.py +3 -3
  193. visidata/mouse.py +137 -0
  194. visidata/movement.py +43 -35
  195. visidata/optionssheet.py +99 -0
  196. visidata/path.py +113 -32
  197. visidata/pivot.py +73 -47
  198. visidata/plugins.py +65 -192
  199. visidata/pyobj.py +55 -205
  200. visidata/rename_col.py +20 -0
  201. visidata/save.py +37 -20
  202. visidata/search.py +54 -10
  203. visidata/selection.py +84 -5
  204. visidata/settings.py +162 -25
  205. visidata/sheets.py +239 -260
  206. visidata/shell.py +51 -21
  207. visidata/sidebar.py +162 -0
  208. visidata/sort.py +11 -4
  209. visidata/statusbar.py +114 -104
  210. visidata/stored_list.py +43 -0
  211. visidata/stored_prop.py +38 -0
  212. visidata/tests/benchmark.csv +52 -0
  213. visidata/tests/conftest.py +3 -3
  214. visidata/tests/test_cliptext.py +39 -0
  215. visidata/tests/test_commands.py +65 -7
  216. visidata/tests/test_edittext.py +2 -2
  217. visidata/tests/test_features.py +28 -0
  218. visidata/tests/test_menu.py +14 -0
  219. visidata/tests/test_path.py +13 -4
  220. visidata/text_source.py +53 -0
  221. visidata/textsheet.py +10 -3
  222. visidata/theme.py +44 -0
  223. visidata/themes/__init__.py +0 -0
  224. visidata/themes/ascii8.py +84 -0
  225. visidata/themes/asciimono.py +84 -0
  226. visidata/themes/light.py +17 -0
  227. visidata/threads.py +89 -40
  228. visidata/tuiwin.py +22 -0
  229. visidata/type_currency.py +22 -3
  230. visidata/type_date.py +31 -9
  231. visidata/type_floatsi.py +5 -1
  232. visidata/undo.py +17 -5
  233. visidata/utils.py +106 -23
  234. visidata/vdobj.py +28 -17
  235. visidata/windows.py +10 -0
  236. visidata/wrappers.py +9 -3
  237. visidata-3.0.1.data/data/share/applications/visidata.desktop +7 -0
  238. {visidata-2.11.1.data → visidata-3.0.1.data}/data/share/man/man1/vd.1 +302 -149
  239. {visidata-2.11.1.data → visidata-3.0.1.data}/data/share/man/man1/visidata.1 +302 -149
  240. visidata-3.0.1.data/scripts/vd2to3.vdx +9 -0
  241. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/METADATA +12 -8
  242. visidata-3.0.1.dist-info/RECORD +258 -0
  243. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/WHEEL +1 -1
  244. vgit/__init__.py +0 -1
  245. vgit/gitsheet.py +0 -164
  246. visidata/layout.py +0 -44
  247. visidata/misc.py +0 -5
  248. visidata-2.11.1.data/scripts/vgit +0 -9
  249. visidata-2.11.1.dist-info/RECORD +0 -155
  250. {vgit → visidata/apps/vgit}/__main__.py +0 -0
  251. {vgit → visidata/apps/vgit}/abort.py +0 -0
  252. /visidata/{repeat.py → features/repeat.py} +0 -0
  253. {visidata-2.11.1.data → visidata-3.0.1.data}/scripts/vd +0 -0
  254. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/LICENSE.gpl3 +0 -0
  255. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/entry_points.txt +0 -0
  256. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/top_level.txt +0 -0
visidata/keys.py CHANGED
@@ -8,6 +8,8 @@ visidata.vd.prettykeys_trdict = {
8
8
  '^J': 'Enter',
9
9
  '^M': 'Enter',
10
10
  '^I': 'Tab',
11
+ 'KEY_BTAB': 'Shift+Tab',
12
+ '^@': 'Ctrl+Space',
11
13
  'KEY_UP': 'Up',
12
14
  'KEY_DOWN': 'Down',
13
15
  'KEY_LEFT': 'Left',
@@ -30,6 +32,8 @@ visidata.vd.prettykeys_trdict = {
30
32
  'kNXT5': 'Ctrl+PgDn',
31
33
  'KEY_IC5': 'Ctrl+Ins',
32
34
  'KEY_DC5': 'Ctrl+Del',
35
+ 'kDC5': 'Ctrl+Del',
36
+ 'KEY_SDC': 'Shift+Del',
33
37
 
34
38
  'KEY_IC': 'Ins',
35
39
  'KEY_DC': 'Del',
@@ -44,27 +48,27 @@ visidata.vd.prettykeys_trdict = {
44
48
  'KEY_SNEXT': 'Shift+PgDn',
45
49
 
46
50
  'KEY_BACKSPACE': 'Bksp',
51
+ 'BUTTON1_RELEASED': 'LeftBtnUp',
52
+ 'BUTTON2_RELEASED': 'MiddleBtnUp',
53
+ 'BUTTON3_RELEASED': 'RightBtnUp',
47
54
  'BUTTON1_PRESSED': 'LeftClick',
48
55
  'BUTTON2_PRESSED': 'MiddleClick',
49
56
  'BUTTON3_PRESSED': 'RightClick',
50
- 'BUTTON4_PRESSED': 'ScrollwheelUp',
51
- 'BUTTON5_PRESSED': 'ScrollwheelDown',
52
- 'REPORT_MOUSE_POSITION': 'ScrollwheelDown',
53
- '2097152': 'ScrollwheelDown',
54
- 'KEY_F(1)': 'F1',
55
- 'KEY_F(2)': 'F2',
56
- 'KEY_F(3)': 'F3',
57
- 'KEY_F(4)': 'F4',
58
- 'KEY_F(5)': 'F5',
59
- 'KEY_F(6)': 'F6',
60
- 'KEY_F(7)': 'F7',
61
- 'KEY_F(8)': 'F8',
62
- 'KEY_F(9)': 'F9',
63
- 'KEY_F(10)': 'F10',
64
- 'KEY_F(11)': 'F11',
65
- 'KEY_F(12)': 'F12',
57
+ 'BUTTON4_PRESSED': 'ScrollUp',
58
+ 'BUTTON5_PRESSED': 'ScrollDown',
59
+ 'REPORT_MOUSE_POSITION': 'ScrollDown',
60
+ '2097152': 'ScrollDown',
66
61
  }
67
62
 
63
+ for i in range(1, 13):
64
+ d = visidata.vd.prettykeys_trdict
65
+ d[f'KEY_F({i})'] = f'F{i}'
66
+ d[f'KEY_F({i+12})'] = f'Shift+F{i}'
67
+ d[f'KEY_F({i+24})'] = f'Ctrl+F{i}'
68
+ d[f'KEY_F({i+36})'] = f'Ctrl+Shift+F{i}'
69
+ d[f'KEY_F({i+48})'] = f'Alt+F{i}'
70
+ d[f'KEY_F({i+60})'] = f'Alt+Shift+F{i}'
71
+
68
72
 
69
73
  @visidata.VisiData.api
70
74
  def prettykeys(vd, key):
@@ -0,0 +1,9 @@
1
+ from visidata import vd, IndexSheet
2
+
3
+
4
+ vd.option('load_lazy', False, 'load subsheets always (False) or lazily (True)')
5
+ vd.option('skip', 0, 'skip N rows before header', replay=True)
6
+ vd.option('header', 1, 'parse first N rows as column names', replay=True)
7
+
8
+ IndexSheet.options.header = 0
9
+ IndexSheet.options.skip = 0
@@ -4,23 +4,57 @@ from visidata import VisiData, vd, Sheet, date, anytype, Path, options, Column,
4
4
 
5
5
  @VisiData.api
6
6
  def open_pandas(vd, p):
7
- return PandasSheet(p.name, source=p)
7
+ return PandasSheet(p.base_stem, source=p)
8
8
 
9
9
  @VisiData.api
10
10
  def open_dta(vd, p):
11
- return PandasSheet(p.name, source=p, filetype='stata')
11
+ return PandasSheet(p.base_stem, source=p, filetype='stata')
12
12
 
13
13
  VisiData.open_stata = VisiData.open_pandas
14
14
 
15
15
  for ft in 'feather gbq orc pickle sas stata'.split():
16
16
  funcname ='open_'+ft
17
17
  if not getattr(VisiData, funcname, None):
18
- setattr(VisiData, funcname, lambda vd,p,ft=ft: PandasSheet(p.name, source=p, filetype=ft))
18
+ setattr(VisiData, funcname, lambda vd,p,ft=ft: PandasSheet(p.base_stem, source=p, filetype=ft))
19
19
 
20
+ @VisiData.api
21
+ @asyncthread
22
+ def save_dta(vd, p, *sheets):
23
+ import pandas as pd
24
+ import numpy as np
25
+
26
+ # STATA is a one-sheet software
27
+ # Save only the first sheet
28
+ vs = sheets[0]
29
+
30
+ columns = [col.name for col in vs.visibleCols]
31
+
32
+ # Get data types
33
+ types = list()
34
+ dispvals = next(vs.iterdispvals(format=True))
35
+ for col,_ in dispvals.items():
36
+ if col.type in [bool, int, float]:
37
+ types.append(col.type)
38
+ elif vd.isNumeric(col):
39
+ types.append(float)
40
+ else:
41
+ types.append(str)
42
+
43
+ # Populate numpy array
44
+ data = np.empty((vs.nRows, len(columns)), dtype=object)
45
+ for r_i, dispvals in enumerate(vs.iterdispvals(format=True)):
46
+ for c_i, v in enumerate(dispvals.values()):
47
+ data[r_i, c_i] = v
48
+
49
+ # Convert to pandas DataFrame and save
50
+ dtype = {col:t for col,t in zip(columns, types)}
51
+ df = pd.DataFrame(data, columns=columns)
52
+ df = df.astype(dtype)
53
+ df.to_stata(p, version=118, write_index=False)
20
54
 
21
55
  class DataFrameAdapter:
22
56
  def __init__(self, df):
23
- import pandas as pd
57
+ pd = vd.importExternal('pandas')
24
58
  if not isinstance(df, pd.DataFrame):
25
59
  vd.fail('%s is not a dataframe' % type(df).__name__)
26
60
 
@@ -57,7 +91,7 @@ class PandasSheet(Sheet):
57
91
  '''
58
92
 
59
93
  def dtype_to_type(self, dtype):
60
- import numpy as np
94
+ np = vd.importExternal('numpy')
61
95
  # Find the underlying numpy dtype for any pandas extension dtypes
62
96
  dtype = getattr(dtype, 'numpy_dtype', dtype)
63
97
  try:
@@ -74,7 +108,7 @@ class PandasSheet(Sheet):
74
108
 
75
109
  def read_tsv(self, path, **kwargs):
76
110
  'Partial function for reading TSV files using pd.read_csv'
77
- import pandas as pd
111
+ pd = vd.importExternal('pandas')
78
112
  return pd.read_csv(path, sep='\t', **kwargs)
79
113
 
80
114
  @property
@@ -111,7 +145,7 @@ class PandasSheet(Sheet):
111
145
 
112
146
  @asyncthread
113
147
  def reload(self):
114
- import pandas as pd
148
+ pd = vd.importExternal('pandas')
115
149
  if isinstance(self.source, pd.DataFrame):
116
150
  df = self.source
117
151
  elif isinstance(self.source, Path):
@@ -124,6 +158,12 @@ class PandasSheet(Sheet):
124
158
  readfunc = getattr(pd, 'read_'+filetype) or vd.error('no pandas.read_'+filetype)
125
159
  # readfunc() handles binary and text open()
126
160
  df = readfunc(self.source, **options.getall('pandas_'+filetype+'_'))
161
+ # some read methods (html, for example) return a list of dataframes
162
+ if isinstance(df, list):
163
+ for idx, inner_df in enumerate(df[1:], start=1):
164
+ vd.push(PandasSheet(f'{self.name}[{idx}]', source=inner_df))
165
+ df = df[0]
166
+ self.name += '[0]'
127
167
  if (filetype == 'pickle') and not isinstance(df, pd.DataFrame):
128
168
  vd.fail('pandas loader can only unpickle dataframes')
129
169
  else:
@@ -169,7 +209,7 @@ class PandasSheet(Sheet):
169
209
  self.rows.sort_values(by=by_cols, ascending=ascending, inplace=True)
170
210
 
171
211
  def _checkSelectedIndex(self):
172
- import pandas as pd
212
+ pd = vd.importExternal('pandas')
173
213
  if self._selectedMask.index is not self.df.index:
174
214
  # DataFrame was modified inplace, so the selection is no longer valid
175
215
  vd.status('pd.DataFrame.index updated, clearing {} selected rows'
@@ -224,7 +264,7 @@ class PandasSheet(Sheet):
224
264
  self.unselectRow(row)
225
265
 
226
266
  def clearSelected(self):
227
- import pandas as pd
267
+ pd = vd.importExternal('pandas')
228
268
  self._selectedMask = pd.Series(False, index=self.df.index)
229
269
 
230
270
  def selectByIndex(self, start=None, end=None):
@@ -251,7 +291,7 @@ class PandasSheet(Sheet):
251
291
  matching rows to the selection. If unselect is True, remove from the
252
292
  active selection instead.
253
293
  '''
254
- import pandas as pd
294
+ pd = vd.importExternal('pandas')
255
295
  case_sensitive = 'I' not in vd.options.regex_flags
256
296
  masks = pd.DataFrame([
257
297
  self.df[col.expr].astype(str).str.contains(pat=regex, case=case_sensitive, regex=True)
@@ -278,13 +318,13 @@ class PandasSheet(Sheet):
278
318
  DataFrame's dtypes.
279
319
  '''
280
320
 
281
- import pandas as pd
321
+ pd = vd.importExternal('pandas')
282
322
  return pd.DataFrame({
283
323
  col: [None] * n for col in self.df.columns
284
324
  }).astype(self.df.dtypes.to_dict(), errors='ignore')
285
325
 
286
326
  def addRows(self, rows, index=None, undo=True):
287
- import pandas as pd
327
+ pd = vd.importExternal('pandas')
288
328
  if index is None:
289
329
  self.df = self.df.append(pd.DataFrame(rows))
290
330
  else:
@@ -296,7 +336,7 @@ class PandasSheet(Sheet):
296
336
  vd.addUndo(self._deleteRows, range(index, index + len(rows)))
297
337
 
298
338
  def _deleteRows(self, which):
299
- import pandas as pd
339
+ pd = vd.importExternal('pandas')
300
340
  self.df.drop(which, inplace=True)
301
341
  self.df.index = pd.RangeIndex(self.nRows)
302
342
  self._checkSelectedIndex()
@@ -306,7 +346,7 @@ class PandasSheet(Sheet):
306
346
  vd.addUndo(self._deleteRows, index or self.nRows - 1)
307
347
 
308
348
  def delete_row(self, rowidx):
309
- import pandas as pd
349
+ pd = vd.importExternal('pandas')
310
350
  oldrow = self.df.iloc[rowidx:rowidx+1]
311
351
 
312
352
  # Use to_dict() here to work around an edge case when applying undos.
@@ -321,7 +361,7 @@ class PandasSheet(Sheet):
321
361
 
322
362
  def deleteBy(self, by):
323
363
  '''Delete rows for which func(row) is true. Returns number of deleted rows.'''
324
- import pandas as pd
364
+ pd = vd.importExternal('pandas')
325
365
  nRows = self.nRows
326
366
  vd.addUndo(setattr, self, 'df', self.df.copy())
327
367
  self.df = self.df[~by]
@@ -353,17 +393,17 @@ PandasSheet.addCommand(None, 'unselect-before', 'unselectByIndex(end=cursorRowIn
353
393
  PandasSheet.addCommand(None, 'stoggle-after', 'toggleByIndex(start=cursorRowIndex)', 'toggle selection of rows from cursor to bottom')
354
394
  PandasSheet.addCommand(None, 'select-after', 'selectByIndex(start=cursorRowIndex)', 'select all rows from cursor to bottom')
355
395
  PandasSheet.addCommand(None, 'unselect-after', 'unselectByIndex(start=cursorRowIndex)', 'unselect all rows from cursor to bottom')
356
- PandasSheet.addCommand(None, 'random-rows', 'nrows=int(input("random number to select: ", value=nRows)); vs=copy(sheet); vs.name=name+"_sample"; vs.rows=DataFrameAdapter(sheet.df.sample(nrows or nRows)); vd.push(vs)', 'open duplicate sheet with a random population subset of N rows'),
396
+ PandasSheet.addCommand(None, 'random-rows', 'nrows=int(input("random number to select: ", value=nRows)); vs=copy(sheet); vs.name=name+"_sample"; vs.rows=DataFrameAdapter(sheet.df.sample(nrows or nRows)); vd.push(vs)', 'open duplicate sheet with a random population subset of N rows')
357
397
 
358
398
  # Handle the regex selection family of commands through a single method,
359
399
  # since the core logic is shared
360
- PandasSheet.addCommand('|', 'select-col-regex', 'selectByRegex(regex=input("select regex: ", type="regex", defaultLast=True), columns=[cursorCol])', 'select rows matching regex in current column')
361
- PandasSheet.addCommand('\\', 'unselect-col-regex', 'selectByRegex(regex=input("select regex: ", type="regex", defaultLast=True), columns=[cursorCol], unselect=True)', 'unselect rows matching regex in current column')
362
- PandasSheet.addCommand('g|', 'select-cols-regex', 'selectByRegex(regex=input("select regex: ", type="regex", defaultLast=True), columns=visibleCols)', 'select rows matching regex in any visible column')
363
- PandasSheet.addCommand('g\\', 'unselect-cols-regex', 'selectByRegex(regex=input("select regex: ", type="regex", defaultLast=True), columns=visibleCols, unselect=True)', 'unselect rows matching regex in any visible column')
400
+ PandasSheet.addCommand('|', 'select-col-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=[cursorCol])', 'select rows matching regex in current column')
401
+ PandasSheet.addCommand('\\', 'unselect-col-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=[cursorCol], unselect=True)', 'unselect rows matching regex in current column')
402
+ PandasSheet.addCommand('g|', 'select-cols-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=visibleCols)', 'select rows matching regex in any visible column')
403
+ PandasSheet.addCommand('g\\', 'unselect-cols-regex', 'selectByRegex(regex=inputRegex("select regex: ", defaultLast=True), columns=visibleCols, unselect=True)', 'unselect rows matching regex in any visible column')
364
404
 
365
405
  # Override with a pandas/dataframe-aware implementation
366
- PandasSheet.addCommand('"', 'dup-selected', 'vs=PandasSheet(sheet.name, "selectedref", source=selectedRows.df); vd.push(vs)', 'open duplicate sheet with only selected rows'),
406
+ PandasSheet.addCommand('"', 'dup-selected', 'vs=PandasSheet(sheet.name, "selectedref", source=selectedRows.df); vd.push(vs)', 'open duplicate sheet with only selected rows')
367
407
 
368
408
  vd.addGlobals({
369
409
  'PandasSheet': PandasSheet,
@@ -0,0 +1,70 @@
1
+ import re
2
+ import os
3
+
4
+ from visidata import vd, date, asyncthread, VisiData, Progress, Sheet, Column, ItemColumn, deduceType, TypedWrapper, setitem
5
+
6
+
7
+ vd.option('airtable_auth_token', '', 'Airtable API key from https://airtable.com/account')
8
+
9
+ airtable_regex = r'^https://airtable.com/(app[A-Za-z0-9]+)/(tbl[A-Za-z0-9]+)/?(viw[A-z0-9]+)?'
10
+
11
+ @VisiData.api
12
+ def guessurl_airtable(vd, p, response):
13
+ m = re.search(airtable_regex, p.given)
14
+ if m:
15
+ return dict(filetype='airtable', _likelihood=10)
16
+
17
+
18
+ @VisiData.api
19
+ def open_airtable(vd, p):
20
+ pyairtable = vd.importExternal('pyairtable')
21
+
22
+ token = os.environ.get('AIRTABLE_AUTH_TOKEN') or vd.options.airtable_auth_token
23
+ if not token:
24
+ vd.requireOptions('airtable_auth_token', help='https://support.airtable.com/docs/creating-and-using-api-keys-and-access-tokens')
25
+
26
+ m = re.search(airtable_regex, p.given)
27
+ if not m:
28
+ vd.fail('invalid airtable url')
29
+
30
+ app, tbl, viw = m.groups()
31
+ return AirtableSheet('airtable', source=p,
32
+ airtable_auth_token=token,
33
+ airtable_base=app,
34
+ airtable_table=tbl,
35
+ airtable_view=viw)
36
+
37
+
38
+ class AirtableSheet(Sheet):
39
+ guide = '''
40
+ # Airtable
41
+ This sheet is a read-only download of all records in a table at _airtable.com_.
42
+ '''
43
+ rowtype = 'records' # rowdef: dict
44
+
45
+ columns = [
46
+ ItemColumn('id', 'id', type=str, width=0),
47
+ ItemColumn('createdTime', 'createdTime', type=date, width=0)
48
+ ]
49
+
50
+ def iterload(self):
51
+ self.fields = set()
52
+
53
+ for page in self.api.iterate(self.airtable_base, self.airtable_table, view=self.airtable_view):
54
+ for row in page:
55
+ yield row
56
+
57
+ for field, value in row['fields'].items():
58
+ if field not in self.fields:
59
+ col = ItemColumn('fields.'+field, type=deduceType(value))
60
+ self.addColumn(col)
61
+ self.fields.add(field)
62
+
63
+ def newRow(self):
64
+ return AttrDict(fields=AttrDict())
65
+
66
+
67
+ @AirtableSheet.lazy_property
68
+ def api(self):
69
+ import pyairtable
70
+ return pyairtable.Api(self.airtable_auth_token)
@@ -0,0 +1,102 @@
1
+
2
+ from visidata import *
3
+
4
+ vd.option('bitio_api_key', '', 'API key')
5
+
6
+ @VisiData.api
7
+ def new_bitio(vd, p):
8
+ vd.importExternal('bitdotio')
9
+ vd.requireOptions('bitio_api_key', help='https://docs.bit.io/docs/connecting-via-the-api')
10
+ return BitioReposSheet(p.name, source=p)
11
+
12
+ vd.openhttp_bitio = vd.new_bitio
13
+
14
+ @VisiData.lazy_property
15
+ def bitio_client(vd):
16
+ import bitdotio
17
+ return bitdotio.bitdotio(vd.options.bitio_api_key)
18
+
19
+ @VisiData.api
20
+ def bitio_api(vd, path, method, **kwargs):
21
+ t = vd.bitio_client.api_client.call_api(path, method,
22
+ header_params={'Accept': 'application/json',
23
+ 'Authorization': 'Bearer '+vd.bitio_client.access_token,
24
+ 'Content-Type': 'application/json'}, async_req=True, body=kwargs)
25
+ if not t.successful():
26
+ vd.warning(resp['Reason'])
27
+ return t.get()
28
+
29
+
30
+ class BitioReposSheet(Sheet):
31
+ rowtype = 'repos'
32
+ columns = [
33
+ AttrColumn('name'),
34
+ AttrColumn('creator'),
35
+ AttrColumn('owner'),
36
+ AttrColumn('bytes', type=int, width=0),
37
+ AttrColumn('collaborators'),
38
+ AttrColumn('description'),
39
+ AttrColumn('documentation'),
40
+ AttrColumn('endpoints'),
41
+ AttrColumn('is_private'),
42
+ AttrColumn('license'),
43
+ AttrColumn('query_count'),
44
+ AttrColumn('stars'),
45
+ AttrColumn('tables'),
46
+ AttrColumn('url'),
47
+ AttrColumn('watchers'),
48
+ ]
49
+ defer = True
50
+ def iterload(self):
51
+ yield from vd.bitio_client.list_repos(self.source.name)
52
+
53
+
54
+ @asyncthread
55
+ def putChanges(self):
56
+ adds, mods, dels = self.getDeferredChanges()
57
+ for row in dels.values():
58
+ vd.bitio_api(f'/users/{self.source.name}/repos/{row.name}/', 'DELETE')
59
+
60
+ for row, rowmods in Progress(list(mods.values()), gerund="updating"):
61
+ kwargs = {col.name: val for col, val in rowmods.items()}
62
+ vd.bitio_api(f'/users/{self.source.name}/repos/{row.name}/', 'PATCH', **kwargs)
63
+
64
+ def openRow(self, row):
65
+ return BitioRepoSheet(self.source.name, row.name, source=row)
66
+
67
+
68
+ class BitioRepoSheet(Sheet):
69
+ rowtype = 'tables'
70
+ columns = [
71
+ AttrColumn('url'),
72
+ AttrColumn('current_name'),
73
+ AttrColumn('description'),
74
+ AttrColumn('columns'),
75
+ AttrColumn('num_records', type=int),
76
+ AttrColumn('bytes', type=int),
77
+ AttrColumn('repo'),
78
+ AttrColumn('documentation'),
79
+ ]
80
+ def iterload(self):
81
+ yield from vd.bitio_client.list_tables(self.source.owner.split('/')[-2], self.source.name)
82
+
83
+ def openRow(self, row):
84
+ username = row.repo.split('/')[-4]
85
+ repo = row.repo.split('/')[-2]
86
+ return BitioTable(username, repo, source=row)
87
+
88
+
89
+ class BitioTable(Sheet):
90
+ def iterload(self):
91
+ username = self.source.repo.split('/')[-4]
92
+ repo = self.source.repo.split('/')[-2]
93
+ conn = vd.bitio_client.get_connection()
94
+ with conn.cursor() as cur:
95
+ cur.execute(f'SELECT * FROM "{username}/{repo}"."{self.source.current_name}"')
96
+ r = cur.fetchone()
97
+ if r:
98
+ yield r
99
+ self.columns = []
100
+ for c in vd.postgresGetColumns(cur):
101
+ self.addColumn(c)
102
+ yield from cur
@@ -0,0 +1,148 @@
1
+ '''
2
+ # Matrix Chat Sheet
3
+ A list of messages from [:underline]{sheet.sourcename}[/].
4
+
5
+ - `Enter` to push a sheet with all the messages from [:underline]{sheet.cursorRow.room.display_name}[/].
6
+ - `a` to send a message to [:underline]{sheet.cursorRow.room.display_name}[/].
7
+ '''
8
+
9
+ from visidata import vd, VisiData, Sheet, Column, ItemColumn, date, asyncthread, AttrDict, vlen, Path
10
+
11
+
12
+ vd.option('matrix_token', '', 'matrix API token')
13
+ vd.option('matrix_user_id', '', 'matrix user ID associated with token')
14
+ vd.option('matrix_device_id', 'VisiData', 'device ID associated with matrix login')
15
+
16
+ vd.matrix_client = None
17
+
18
+
19
+ @VisiData.api
20
+ def openhttp_matrix(vd, p):
21
+ vd.importExternal('matrix_client')
22
+
23
+ if not vd.options.matrix_token:
24
+ from matrix_client.client import MatrixClient
25
+
26
+ username = vd.input(f'{p.given} username: ', record=False)
27
+ password = vd.input('password: ', record=False, display=False)
28
+
29
+ vd.matrix_client = MatrixClient(p.given)
30
+ matrix_token = vd.matrix_client.login(username, password, device_id=vd.options.matrix_device_id)
31
+
32
+ vd.setPersistentOptions(matrix_user_id=username, matrix_token=matrix_token)
33
+
34
+ vd.timeouts_before_idle = -1
35
+ return MatrixSheet(p.base_stem, source=p)
36
+
37
+ vd.open_matrix = vd.openhttp_matrix
38
+
39
+
40
+ class MatrixRoomsSheet(Sheet):
41
+ def iterload(self):
42
+ yield from vd.matrix_client.get_rooms().values()
43
+
44
+
45
+ class MatrixSheet(Sheet):
46
+ help = __doc__
47
+ columns = [
48
+ Column('room', width=12, getter=lambda c,r: r.room and r.room.display_name),
49
+ ItemColumn('sender', width=0),
50
+ Column('sender', width=10, getter=lambda c,r: r.sender.split(':')[0][1:]),
51
+ Column('timestamp', width=16, type=date, getter=lambda c,r: r and r.origin_server_ts/1000, fmtstr='%Y-%m-%d %H:%m'),
52
+ ItemColumn('type', width=15),
53
+ ItemColumn('content', width=0),
54
+ ItemColumn('content.body', width=50),
55
+ ItemColumn('received', type=vlen, width=0),
56
+ ItemColumn('content.msgtype'),
57
+ ]
58
+
59
+ @property
60
+ def sourcename(self):
61
+ from matrix_client.room import Room
62
+ if isinstance(self.source, Room):
63
+ return self.source.display_name or self.source.room_id
64
+ else:
65
+ return str(self.source)
66
+
67
+ @asyncthread
68
+ def reload(self):
69
+ from matrix_client.client import MatrixClient
70
+ from matrix_client.room import Room
71
+ if not vd.matrix_client:
72
+ vd.matrix_client = MatrixClient(self.source.given,
73
+ token=self.options.matrix_token,
74
+ user_id=self.options.matrix_user_id)
75
+
76
+ if isinstance(self.source, Room):
77
+ self.add_room(self.source)
78
+ self.get_room_messages(self.source)
79
+ return
80
+
81
+ for room in vd.matrix_client.get_rooms().values():
82
+ self.add_room(room)
83
+ room.backfill_previous_messages(limit=1)
84
+
85
+ vd.matrix_client.add_listener(self.global_event)
86
+ vd.matrix_client.add_ephemeral_listener(self.global_event)
87
+
88
+ vd.matrix_client.start_listener_thread(exception_handler=vd.exceptionCaught)
89
+
90
+ vd.matrix_client._sync()
91
+
92
+ vd.matrix_client.listen_for_events() # vd.matrix_client.sync(full_state=True)
93
+
94
+ def add_room(self, room):
95
+ room.add_listener(self.room_event)
96
+ room.add_ephemeral_listener(self.room_event)
97
+ room.add_state_listener(self.global_event)
98
+
99
+ @asyncthread
100
+ def get_room_messages(self, room):
101
+ while room.prev_batch:
102
+ ret = vd.matrix_client.api.get_room_messages(room.room_id, room.prev_batch, direction='b', limit=100)
103
+ for r in ret['chunk']:
104
+ r['room'] = room
105
+ self.addRow(r)
106
+
107
+ if 'end' not in ret or ret['end'] == room.prev_batch:
108
+ break
109
+ room.prev_batch = ret['end']
110
+
111
+ def addRow(self, r, **kwargs):
112
+ r = AttrDict(r)
113
+ if r.event_id not in self.event_index:
114
+ super().addRow(r, **kwargs)
115
+ self.event_index[r.event_id] = r
116
+
117
+ def global_event(self, chunk):
118
+ self.addRow(chunk)
119
+
120
+ def room_event(self, room, chunk):
121
+ ev = AttrDict(chunk)
122
+ t = chunk['type']
123
+ if t == 'm.receipt':
124
+ for msgid, content in chunk['content'].items():
125
+ msg = self.event_index.setdefault(msgid, {})
126
+ if 'received' not in msg:
127
+ msg['received'] = {}
128
+
129
+ for t, c in content.items():
130
+ assert t == 'm.read'
131
+ for userid, v in c.items():
132
+ msg['received'][(t, userid)] = v['ts']/1000
133
+ return
134
+
135
+ chunk['room'] = room
136
+ self.addRow(chunk)
137
+
138
+ def add_message(self, text):
139
+ vd.matrix_client.send_text(text)
140
+
141
+ def openRow(self, row):
142
+ return MatrixSheet(row.room.display_name, source=row.room, last_id=row.event_id)
143
+
144
+
145
+ MatrixSheet.init('event_index', dict)
146
+
147
+ MatrixSheet.addCommand('a', 'add-row', 'cursorRow.room.send_text(input(cursorRow.room.display_name+"> "))', 'send chat message to current room')
148
+ vd.addMenuItems('Row > Add > send matrix message > add-msg')