visidata 2.11.1__py3-none-any.whl → 3.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. visidata/__init__.py +72 -91
  2. visidata/_input.py +259 -42
  3. visidata/_open.py +84 -29
  4. visidata/_types.py +21 -3
  5. visidata/_urlcache.py +17 -4
  6. visidata/aggregators.py +78 -25
  7. visidata/apps/__init__.py +0 -0
  8. visidata/apps/vdsql/__about__.py +8 -0
  9. visidata/apps/vdsql/__init__.py +5 -0
  10. visidata/apps/vdsql/__main__.py +27 -0
  11. visidata/apps/vdsql/_ibis.py +748 -0
  12. visidata/apps/vdsql/bigquery.py +61 -0
  13. visidata/apps/vdsql/clickhouse.py +53 -0
  14. visidata/apps/vdsql/setup.py +40 -0
  15. visidata/apps/vdsql/snowflake.py +67 -0
  16. visidata/apps/vgit/__init__.py +13 -0
  17. {vgit → visidata/apps/vgit}/blame.py +5 -2
  18. {vgit → visidata/apps/vgit}/branch.py +31 -16
  19. {vgit → visidata/apps/vgit}/config.py +3 -3
  20. visidata/apps/vgit/diff.py +169 -0
  21. visidata/apps/vgit/gitsheet.py +161 -0
  22. {vgit → visidata/apps/vgit}/grep.py +6 -5
  23. visidata/apps/vgit/log.py +81 -0
  24. {vgit → visidata/apps/vgit}/main.py +18 -5
  25. {vgit → visidata/apps/vgit}/remote.py +8 -4
  26. visidata/apps/vgit/repos.py +71 -0
  27. {vgit → visidata/apps/vgit}/setup.py +6 -4
  28. visidata/apps/vgit/stash.py +69 -0
  29. visidata/apps/vgit/status.py +204 -0
  30. {vgit → visidata/apps/vgit}/statusbar.py +2 -0
  31. visidata/basesheet.py +63 -51
  32. visidata/canvas.py +208 -93
  33. visidata/choose.py +6 -6
  34. visidata/clean_names.py +29 -0
  35. visidata/clipboard.py +73 -17
  36. visidata/cliptext.py +220 -46
  37. visidata/cmdlog.py +88 -114
  38. visidata/color.py +142 -56
  39. visidata/column.py +121 -129
  40. visidata/ddw/input.ddw +74 -79
  41. visidata/ddw/regex.ddw +57 -0
  42. visidata/ddwplay.py +33 -14
  43. visidata/deprecated.py +77 -3
  44. visidata/desktop/visidata.desktop +7 -0
  45. visidata/editor.py +12 -6
  46. visidata/errors.py +6 -2
  47. visidata/experimental/__init__.py +0 -0
  48. visidata/experimental/diff_sheet.py +29 -0
  49. visidata/experimental/digit_autoedit.py +6 -0
  50. visidata/experimental/gdrive.py +89 -0
  51. visidata/experimental/google.py +37 -0
  52. visidata/experimental/gsheets.py +79 -0
  53. visidata/experimental/live_search.py +37 -0
  54. visidata/experimental/liveupdate.py +45 -0
  55. visidata/experimental/mark.py +133 -0
  56. visidata/experimental/noahs_tapestry/__init__.py +1 -0
  57. visidata/experimental/noahs_tapestry/tapestry.py +147 -0
  58. visidata/experimental/rownum.py +73 -0
  59. visidata/experimental/slide_cells.py +26 -0
  60. visidata/expr.py +8 -4
  61. visidata/extensible.py +22 -4
  62. visidata/features/__init__.py +0 -0
  63. visidata/features/addcol_audiometadata.py +42 -0
  64. visidata/features/addcol_histogram.py +34 -0
  65. visidata/features/canvas_save_svg.py +69 -0
  66. visidata/features/change_precision.py +46 -0
  67. visidata/features/cmdpalette.py +197 -0
  68. visidata/features/colorbrewer.py +363 -0
  69. visidata/{colorsheet.py → features/colorsheet.py} +17 -16
  70. visidata/features/command_server.py +105 -0
  71. visidata/features/currency_to_usd.py +70 -0
  72. visidata/{customdate.py → features/customdate.py} +2 -0
  73. visidata/features/dedupe.py +132 -0
  74. visidata/{describe.py → features/describe.py} +17 -15
  75. visidata/features/errors_guide.py +26 -0
  76. visidata/features/expand_cols.py +202 -0
  77. visidata/{fill.py → features/fill.py} +3 -1
  78. visidata/{freeze.py → features/freeze.py} +11 -6
  79. visidata/features/graph_seaborn.py +79 -0
  80. visidata/features/helloworld.py +10 -0
  81. visidata/features/hint_types.py +17 -0
  82. visidata/{incr.py → features/incr.py} +5 -0
  83. visidata/{join.py → features/join.py} +107 -53
  84. visidata/features/known_cols.py +21 -0
  85. visidata/features/layout.py +62 -0
  86. visidata/{melt.py → features/melt.py} +32 -21
  87. visidata/features/normcol.py +118 -0
  88. visidata/features/open_config.py +7 -0
  89. visidata/features/open_syspaste.py +18 -0
  90. visidata/features/ping.py +157 -0
  91. visidata/features/procmgr.py +208 -0
  92. visidata/features/random_sample.py +6 -0
  93. visidata/{regex.py → features/regex.py} +47 -31
  94. visidata/features/reload_every.py +55 -0
  95. visidata/features/rename_col_cascade.py +30 -0
  96. visidata/features/scroll_context.py +60 -0
  97. visidata/features/select_equal_selected.py +11 -0
  98. visidata/features/setcol_fake.py +65 -0
  99. visidata/{slide.py → features/slide.py} +77 -21
  100. visidata/features/sparkline.py +48 -0
  101. visidata/features/status_source.py +20 -0
  102. visidata/{sysedit.py → features/sysedit.py} +2 -1
  103. visidata/features/sysopen_mailcap.py +46 -0
  104. visidata/features/term_extras.py +13 -0
  105. visidata/{transpose.py → features/transpose.py} +5 -4
  106. visidata/features/type_ipaddr.py +73 -0
  107. visidata/features/type_url.py +11 -0
  108. visidata/{unfurl.py → features/unfurl.py} +9 -9
  109. visidata/{window.py → features/window.py} +2 -2
  110. visidata/form.py +50 -21
  111. visidata/freqtbl.py +81 -33
  112. visidata/fuzzymatch.py +414 -0
  113. visidata/graph.py +105 -33
  114. visidata/guide.py +200 -0
  115. visidata/help.py +75 -44
  116. visidata/hint.py +39 -0
  117. visidata/indexsheet.py +109 -0
  118. visidata/input_history.py +55 -0
  119. visidata/interface.py +58 -0
  120. visidata/keys.py +20 -16
  121. visidata/loaders/__init__.py +9 -0
  122. visidata/loaders/_pandas.py +61 -21
  123. visidata/loaders/api_airtable.py +70 -0
  124. visidata/loaders/api_bitio.py +102 -0
  125. visidata/loaders/api_matrix.py +148 -0
  126. visidata/loaders/api_reddit.py +306 -0
  127. visidata/loaders/api_zulip.py +249 -0
  128. visidata/loaders/archive.py +41 -7
  129. visidata/loaders/arrow.py +7 -7
  130. visidata/loaders/conll.py +49 -0
  131. visidata/loaders/csv.py +25 -7
  132. visidata/loaders/eml.py +3 -4
  133. visidata/loaders/f5log.py +1204 -0
  134. visidata/loaders/fec.py +325 -0
  135. visidata/loaders/fixed_width.py +2 -4
  136. visidata/loaders/frictionless.py +3 -3
  137. visidata/loaders/geojson.py +8 -5
  138. visidata/loaders/google.py +48 -0
  139. visidata/loaders/graphviz.py +4 -4
  140. visidata/loaders/hdf5.py +4 -4
  141. visidata/loaders/html.py +54 -12
  142. visidata/loaders/http.py +84 -30
  143. visidata/loaders/imap.py +20 -10
  144. visidata/loaders/jrnl.py +52 -0
  145. visidata/loaders/json.py +83 -29
  146. visidata/loaders/jsonla.py +74 -0
  147. visidata/loaders/lsv.py +15 -11
  148. visidata/loaders/mailbox.py +40 -0
  149. visidata/loaders/markdown.py +1 -3
  150. visidata/loaders/mbtiles.py +4 -5
  151. visidata/loaders/mysql.py +11 -13
  152. visidata/loaders/npy.py +7 -7
  153. visidata/loaders/odf.py +4 -1
  154. visidata/loaders/orgmode.py +428 -0
  155. visidata/loaders/pandas_freqtbl.py +14 -20
  156. visidata/loaders/parquet.py +62 -6
  157. visidata/loaders/pcap.py +3 -3
  158. visidata/loaders/pdf.py +4 -3
  159. visidata/loaders/png.py +19 -13
  160. visidata/loaders/postgres.py +9 -8
  161. visidata/loaders/rec.py +7 -3
  162. visidata/loaders/s3.py +342 -0
  163. visidata/loaders/sas.py +5 -5
  164. visidata/loaders/scrape.py +186 -0
  165. visidata/loaders/shp.py +6 -5
  166. visidata/loaders/spss.py +5 -6
  167. visidata/loaders/sqlite.py +68 -28
  168. visidata/loaders/texttables.py +1 -1
  169. visidata/loaders/toml.py +60 -0
  170. visidata/loaders/tsv.py +61 -19
  171. visidata/loaders/ttf.py +19 -7
  172. visidata/loaders/unzip_http.py +6 -5
  173. visidata/loaders/usv.py +1 -1
  174. visidata/loaders/vcf.py +16 -16
  175. visidata/loaders/vds.py +10 -7
  176. visidata/loaders/vdx.py +30 -5
  177. visidata/loaders/xlsb.py +8 -1
  178. visidata/loaders/xlsx.py +145 -25
  179. visidata/loaders/xml.py +6 -3
  180. visidata/loaders/xword.py +4 -4
  181. visidata/loaders/yaml.py +15 -5
  182. visidata/macros.py +129 -42
  183. visidata/main.py +119 -94
  184. visidata/mainloop.py +101 -155
  185. visidata/man/parse_options.py +2 -2
  186. visidata/man/vd.1 +302 -149
  187. visidata/man/vd.txt +291 -154
  188. visidata/memory.py +3 -3
  189. visidata/menu.py +104 -423
  190. visidata/metasheets.py +59 -141
  191. visidata/modify.py +78 -23
  192. visidata/motd.py +3 -3
  193. visidata/mouse.py +137 -0
  194. visidata/movement.py +43 -35
  195. visidata/optionssheet.py +99 -0
  196. visidata/path.py +113 -32
  197. visidata/pivot.py +73 -47
  198. visidata/plugins.py +65 -192
  199. visidata/pyobj.py +55 -205
  200. visidata/rename_col.py +20 -0
  201. visidata/save.py +37 -20
  202. visidata/search.py +54 -10
  203. visidata/selection.py +84 -5
  204. visidata/settings.py +162 -25
  205. visidata/sheets.py +239 -260
  206. visidata/shell.py +51 -21
  207. visidata/sidebar.py +162 -0
  208. visidata/sort.py +11 -4
  209. visidata/statusbar.py +114 -104
  210. visidata/stored_list.py +43 -0
  211. visidata/stored_prop.py +38 -0
  212. visidata/tests/benchmark.csv +52 -0
  213. visidata/tests/conftest.py +3 -3
  214. visidata/tests/test_cliptext.py +39 -0
  215. visidata/tests/test_commands.py +65 -7
  216. visidata/tests/test_edittext.py +2 -2
  217. visidata/tests/test_features.py +28 -0
  218. visidata/tests/test_menu.py +14 -0
  219. visidata/tests/test_path.py +13 -4
  220. visidata/text_source.py +53 -0
  221. visidata/textsheet.py +10 -3
  222. visidata/theme.py +44 -0
  223. visidata/themes/__init__.py +0 -0
  224. visidata/themes/ascii8.py +84 -0
  225. visidata/themes/asciimono.py +84 -0
  226. visidata/themes/light.py +17 -0
  227. visidata/threads.py +89 -40
  228. visidata/tuiwin.py +22 -0
  229. visidata/type_currency.py +22 -3
  230. visidata/type_date.py +31 -9
  231. visidata/type_floatsi.py +5 -1
  232. visidata/undo.py +17 -5
  233. visidata/utils.py +106 -23
  234. visidata/vdobj.py +28 -17
  235. visidata/windows.py +10 -0
  236. visidata/wrappers.py +9 -3
  237. visidata-3.0.1.data/data/share/applications/visidata.desktop +7 -0
  238. {visidata-2.11.1.data → visidata-3.0.1.data}/data/share/man/man1/vd.1 +302 -149
  239. {visidata-2.11.1.data → visidata-3.0.1.data}/data/share/man/man1/visidata.1 +302 -149
  240. visidata-3.0.1.data/scripts/vd2to3.vdx +9 -0
  241. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/METADATA +12 -8
  242. visidata-3.0.1.dist-info/RECORD +258 -0
  243. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/WHEEL +1 -1
  244. vgit/__init__.py +0 -1
  245. vgit/gitsheet.py +0 -164
  246. visidata/layout.py +0 -44
  247. visidata/misc.py +0 -5
  248. visidata-2.11.1.data/scripts/vgit +0 -9
  249. visidata-2.11.1.dist-info/RECORD +0 -155
  250. {vgit → visidata/apps/vgit}/__main__.py +0 -0
  251. {vgit → visidata/apps/vgit}/abort.py +0 -0
  252. /visidata/{repeat.py → features/repeat.py} +0 -0
  253. {visidata-2.11.1.data → visidata-3.0.1.data}/scripts/vd +0 -0
  254. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/LICENSE.gpl3 +0 -0
  255. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/entry_points.txt +0 -0
  256. {visidata-2.11.1.dist-info → visidata-3.0.1.dist-info}/top_level.txt +0 -0
@@ -5,11 +5,11 @@ import sqlite3
5
5
 
6
6
  @VisiData.api
7
7
  def open_pbf(vd, p):
8
- return PbfSheet(p.name, source=p)
8
+ return PbfSheet(p.base_stem, source=p)
9
9
 
10
10
  @VisiData.api
11
11
  def open_mbtiles(vd, p):
12
- return MbtilesSheet(p.name, source=p)
12
+ return MbtilesSheet(p.base_stem, source=p)
13
13
 
14
14
  def getListDepth(L):
15
15
  if not isinstance(L, list):
@@ -36,7 +36,7 @@ class MbtilesSheet(Sheet):
36
36
  ]
37
37
 
38
38
  def getTile(self, zoom_level, tile_col, tile_row):
39
- import mapbox_vector_tile
39
+ mapbox_vector_tile = vd.importExternal('mapbox_vector_tile', 'mapbox-vector-tile')
40
40
 
41
41
  con = sqlite3.connect(str(self.source))
42
42
  tile_data = con.execute('''
@@ -128,11 +128,10 @@ class PbfCanvas(InvertedCanvas):
128
128
  if disptext:
129
129
  self.label(textx, texty, disptext, attr, row)
130
130
 
131
-
132
131
  self.refresh()
133
132
 
134
133
 
135
134
  PbfSheet.addCommand('.', 'plot-row', 'vd.push(PbfCanvas(name+"_map", source=sheet, sourceRows=[cursorRow], textCol=cursorCol))', 'plot blocks in current row')
136
135
  PbfSheet.addCommand('g.', 'plot-rows', 'vd.push(PbfCanvas(name+"_map", source=sheet, sourceRows=rows, textCol=cursorCol))', 'plot selected blocks')
137
136
  MbtilesSheet.addCommand('.', 'plot-row', 'vd.push(getPlot(cursorRow))', 'plot tiles in current row')
138
- MbtilesSheet.addCommand('g.', 'plot-selected', 'vd.push(getPlot(*selectedRows))', 'plot selected tiles'),
137
+ MbtilesSheet.addCommand('g.', 'plot-selected', 'vd.push(getPlot(*selectedRows))', 'plot selected tiles')
visidata/loaders/mysql.py CHANGED
@@ -1,6 +1,7 @@
1
1
  from contextlib import contextmanager
2
+ from urllib.parse import urlparse, unquote
2
3
 
3
- from visidata import VisiData, vd, Sheet, anytype, asyncthread, urlparse, ColumnItem
4
+ from visidata import VisiData, vd, Sheet, anytype, asyncthread, ColumnItem
4
5
 
5
6
  def codeToType(type_code, colname):
6
7
  import MySQLdb as mysql
@@ -41,7 +42,7 @@ class SQL:
41
42
  database=self.url.path[1:],
42
43
  host=self.url.hostname,
43
44
  port=self.url.port or 3306,
44
- password=self.url.password,
45
+ password=unquote(self.url.password),
45
46
  use_unicode=True,
46
47
  charset='utf8',
47
48
  cursorclass=cursors.SSCursor) ## if SSCursor is not used mysql will first fetch ALL data, and only then visualize it
@@ -71,7 +72,7 @@ def cursorToColumns(cur, sheet):
71
72
  class MyTablesSheet(Sheet):
72
73
  rowtype = 'tables'
73
74
 
74
- def reload(self):
75
+ def iterload(self):
75
76
  qstr = f'''
76
77
  select
77
78
  t.table_name,
@@ -96,16 +97,15 @@ class MyTablesSheet(Sheet):
96
97
  '''
97
98
 
98
99
  with self.sql.cur(qstr) as cur:
99
- self.rows = []
100
100
  # try to get first row to make cur.description available
101
101
  r = cur.fetchone()
102
102
  if r:
103
- self.addRow(r)
103
+ yield r
104
104
  cursorToColumns(cur, self)
105
105
  self.setKeys(self.columns[0:1]) # table_name is the key
106
106
 
107
107
  for r in cur:
108
- self.addRow(r)
108
+ yield r
109
109
 
110
110
  def openRow(self, row):
111
111
  return MyTable(self.name+"."+row[0], source=row[0], sql=self.sql)
@@ -113,22 +113,20 @@ class MyTablesSheet(Sheet):
113
113
 
114
114
  # rowdef: tuple of values as returned by fetchone()
115
115
  class MyTable(Sheet):
116
- @asyncthread
117
- def reload(self):
116
+ def iterload(self):
118
117
  with self.sql.cur("SELECT * FROM " + self.source) as cur:
119
- self.rows = []
120
118
  r = cur.fetchone()
121
119
  if r is None:
122
120
  return
123
-
124
- self.addRow(r)
121
+
122
+ yield r
125
123
  cursorToColumns(cur, self)
126
124
  while True:
127
125
  try:
128
126
  r = cur.fetchone()
129
127
  if r is None:
130
128
  break
131
-
132
- self.addRow(r)
129
+
130
+ yield r
133
131
  except UnicodeDecodeError as e:
134
132
  vd.exceptionCaught(e)
visidata/loaders/npy.py CHANGED
@@ -1,20 +1,20 @@
1
- from visidata import VisiData, vd, Sheet, date, anytype, options, Column, Progress, ColumnItem, vlen, PyobjSheet, TypedWrapper, InferColumnsSheet
1
+ from visidata import VisiData, vd, Sheet, date, anytype, options, Column, Progress, ColumnItem, vlen, PyobjSheet, TypedWrapper
2
2
 
3
3
  'Loaders for .npy and .npz. Save to .npy. Depends on the zip loader.'
4
4
 
5
5
  @VisiData.api
6
6
  def open_npy(vd, p):
7
- return NpySheet(p.name, source=p)
7
+ return NpySheet(p.base_stem, source=p)
8
8
 
9
9
  @VisiData.api
10
10
  def open_npz(vd, p):
11
- return NpzSheet(p.name, source=p)
11
+ return NpzSheet(p.base_stem, source=p)
12
12
 
13
13
  vd.option('npy_allow_pickle', False, 'numpy allow unpickling objects (unsafe)')
14
14
 
15
15
  class NpySheet(Sheet):
16
16
  def iterload(self):
17
- import numpy
17
+ numpy = vd.importExternal('numpy')
18
18
  if not hasattr(self, 'npy'):
19
19
  self.npy = numpy.load(str(self.source), encoding='bytes', **self.options.getall('npy_'))
20
20
  self.reloadCols()
@@ -47,12 +47,12 @@ class NpzSheet(vd.ZipSheet):
47
47
  ]
48
48
 
49
49
  def iterload(self):
50
- import numpy
50
+ numpy = vd.importExternal('numpy')
51
51
  self.npz = numpy.load(str(self.source), encoding='bytes', **self.options.getall('npy_'))
52
52
  yield from Progress(self.npz.items())
53
53
 
54
54
  def openRow(self, row):
55
- import numpy
55
+ numpy = vd.importExternal('numpy')
56
56
  tablename, tbl = row
57
57
  if isinstance(tbl, numpy.ndarray):
58
58
  return NpySheet(tablename, npy=tbl)
@@ -62,7 +62,7 @@ class NpzSheet(vd.ZipSheet):
62
62
 
63
63
  @VisiData.api
64
64
  def save_npy(vd, p, sheet):
65
- import numpy as np
65
+ np = vd.importExternal('numpy')
66
66
 
67
67
  dtype = []
68
68
 
visidata/loaders/odf.py CHANGED
@@ -3,11 +3,12 @@ from visidata import vd, VisiData, Sheet, IndexSheet, SequenceSheet
3
3
 
4
4
  @VisiData.api
5
5
  def open_ods(vd, p):
6
- return OdsIndexSheet(p.name, source=p)
6
+ return OdsIndexSheet(p.base_stem, source=p)
7
7
 
8
8
 
9
9
  class OdsIndexSheet(IndexSheet):
10
10
  def iterload(self):
11
+ vd.importExternal('odf', 'odfpy')
11
12
  import odf.opendocument
12
13
  import odf.table
13
14
  self.doc = odf.opendocument.load(self.source)
@@ -16,6 +17,7 @@ class OdsIndexSheet(IndexSheet):
16
17
 
17
18
 
18
19
  def _get_cell_string_value(cell, text_s):
20
+ vd.importExternal('odf', 'odfpy')
19
21
  from odf.element import Element
20
22
  from odf.namespaces import TEXTNS
21
23
 
@@ -34,6 +36,7 @@ def _get_cell_string_value(cell, text_s):
34
36
 
35
37
  class OdsSheet(SequenceSheet):
36
38
  def iterload(self):
39
+ vd.importExternal('odf', 'odfpy')
37
40
  import odf.table
38
41
  import odf.text
39
42
  from odf.namespaces import TABLENS, OFFICENS
@@ -0,0 +1,428 @@
1
+ '''
2
+ View sections in one or more .org files. Easily edit sections within the system editor.
3
+
4
+ ## Supported syntax
5
+
6
+ - `--- [comment]` at start of line starts a new section
7
+ - any number of `#` or `*` (followed by a space) leads a headline
8
+ - `#+key: value` adds metadata to next element (headline, table, section, )
9
+ - orgmode style tags: `:tag1:tag2:`
10
+ - `[[tagname]]` links to that set of tags
11
+ - `[[url]]` links to external url
12
+ - orgmode and markdown links
13
+ - `[[url][linktext]]`
14
+ - `[linktext](url)`
15
+ - all other markup/orgmode/whatever is ignored and passed through
16
+
17
+ ## Usage
18
+
19
+ - `vd file.org`
20
+ - or `find orgfiles/ -name '*.org' | vd -f forg`
21
+ - or `vd orgfiles/ -f orgdir`
22
+ '''
23
+
24
+ import collections
25
+ import datetime
26
+ import os
27
+ import re
28
+
29
+ from visidata import vd, VisiData, Column, Sheet, ItemColumn, vlen, asyncthread, Path, AttrDict, date
30
+
31
+
32
+ @VisiData.api
33
+ def open_org(vd, p):
34
+ return OrgSheet(p.base_stem, source=p, filetype='org')
35
+
36
+
37
+ @VisiData.api
38
+ def open_forg(vd, p):
39
+ return OrgSheet(p.base_stem, source=p, filetype='forg')
40
+
41
+
42
+ @VisiData.api
43
+ def open_orgdir(vd, p):
44
+ return OrgSheet(p.base_stem, source=p, filetype='orgdir')
45
+
46
+
47
+ def encode_date(dt=None):
48
+ if not dt:
49
+ dt = datetime.datetime.now()
50
+ elif isinstance(dt, str):
51
+ dt = datetime.datetime.fromisoformat(dt)
52
+
53
+ s = '123456789abcdefghijklmnopqrstuvwxyz'
54
+ return '%02d%s%s' % (dt.year % 100, s[dt.month-1], s[dt.day-1])
55
+
56
+
57
+ class OrgContentsColumn(Column):
58
+ def setValue(self, row, v):
59
+ super().setValue(row, v)
60
+ orgmode_parse_into(row, v)
61
+
62
+ def putValue(self, row, v):
63
+ self.sheet.save(row)
64
+
65
+
66
+ def sectionize(lines):
67
+ 'Generate (startinglinenum, contentlines) for each section. First section may not have leading * or # but all others will.'
68
+ startinglinenum = 0
69
+ prev_contents = contents = []
70
+ for linenum, line in enumerate(lines):
71
+ line = line.rstrip('\n')
72
+
73
+ if line and line[0] in '#*':
74
+ if contents:
75
+ yield startinglinenum, contents
76
+ prev_contents = contents
77
+ contents = []
78
+ startinglinenum = linenum
79
+ else:
80
+ if not line and not contents:
81
+ prev_contents.append(line)
82
+
83
+ contents.append(line)
84
+
85
+ if contents:
86
+ yield startinglinenum, contents
87
+
88
+
89
+ def orgmode_parse(all_lines):
90
+ root = parent = OrgSheet().newRow()
91
+ for linenum, lines in sectionize(all_lines):
92
+ section = OrgSheet().newRow()
93
+
94
+ section.contents = ''
95
+ for i, line in enumerate(lines):
96
+ if not line and not lines[i-1]:
97
+ continue
98
+ section.contents += line + '\n'
99
+ section.orig_contents = section.contents
100
+
101
+ section.linenum = linenum+1
102
+ for line in lines:
103
+ section.tags.extend(re.findall(r':([\S:]+?):', line))
104
+ links = re.findall(r'\[.*?\]\(.*?\)', line)
105
+ if links:
106
+ section.links.extend(links)
107
+
108
+ title = orgmode_parse_title(lines[0])
109
+ if not title:
110
+ root = parent = section
111
+ continue
112
+
113
+ section.update(title)
114
+
115
+ while parent and section.level <= parent.level:
116
+ parent = parent.parent
117
+
118
+ parent.children.append(section)
119
+ section.parent = parent
120
+ parent = section
121
+
122
+ return root
123
+
124
+ def _replace(node, newnode):
125
+ node.update(newnode) # must replace insides of same row object
126
+ for c in node.children:
127
+ c.parent = node
128
+ return node
129
+
130
+ def orgmode_parse_into(toprow, text):
131
+ row = orgmode_parse(text.splitlines())
132
+ if not row.title and len(row.children) == 1:
133
+ row = row.children[0]
134
+ row.parent = toprow.parent
135
+ toprow = _replace(toprow, row)
136
+
137
+ return toprow
138
+
139
+
140
+ def orgmode_to_string(section, prestars=''):
141
+ ret = ''
142
+
143
+ # if section.title:
144
+ # ret += prestars[len(section.stars):] + (section.title or ' ') + '\n'
145
+ ret += section.contents.rstrip() or ''
146
+ ret += '\n\n'
147
+ ret += ''.join(orgmode_to_string(c, prestars+(section.stars or '')) for c in section.children).rstrip() + '\n'
148
+
149
+ return ret
150
+
151
+
152
+ def orgmode_parse_title(line):
153
+ m = re.match(r'^(?P<stars>[*#]+)\s*(?P<keyword>(TODO|FEEDBACK|VERIFY|DONE|DELEGATED))?\s*(?P<prio>\[#[A-z]\])?\s*(?P<title>.*)', line)
154
+ if not m:
155
+ assert not line or line[0] not in '#*', line
156
+ return {}
157
+
158
+ return dict(stars=m.group('stars'),
159
+ level=len(m.group('stars')),
160
+ keyword=m.group('keyword') or '',
161
+ prio=m.group('prio') or '',
162
+ title=line)
163
+
164
+
165
+ class OrgSheet(Sheet):
166
+ guide = '''# Orgmode Sheet (experimental)
167
+ A list of orgmode sections from _{sheet.source}_.
168
+
169
+ - `Enter` to expand current section
170
+ - `z Enter` to contract current section
171
+ - `ga` to combine selected sections into a new entry
172
+ - `Ctrl+O` to edit section in system editor (edits source directly)
173
+ - `g Ctrl+S` to save all org files
174
+ '''
175
+ defer = True
176
+ columns = [
177
+ Column('path', getter=lambda c,r: _root(r).path, width=0),
178
+ Column('id', getter=lambda c,r: _root(r).path.stem),
179
+ ItemColumn('title', width=40),
180
+ # ItemColumn('date', width=0, type=date),
181
+ ItemColumn('tags', width=10, type=lambda v: ' '.join(v)),
182
+ ItemColumn('links', type=vlen),
183
+ # ItemColumn('parent', width=0),
184
+ ItemColumn('children', type=vlen),
185
+ ItemColumn('linenum', width=0, type=int),
186
+ Column('to_string', width=0, getter=lambda c,r: orgmode_to_string(r)),
187
+ OrgContentsColumn('orig_contents', width=0, getter=lambda c,r: r.orig_contents),
188
+ ]
189
+
190
+ def __init__(self, *args, **kwargs):
191
+ super().__init__(*args, **kwargs)
192
+ self.opened_rows = set()
193
+ # self.open_max=0 # all levels closed after this point
194
+ # self.close_max=0 # all levels open after this point
195
+
196
+ def isSelectedParents(self, row):
197
+ return super().isSelected(row) or row.parent and self.isSelectedParents(row.parent)
198
+
199
+ def isSelected(self, row):
200
+ return self.isSelectedParents(row)
201
+
202
+ def refreshRows(self):
203
+ self.rows = list(self._deepiter(self.sourceRows))
204
+
205
+ def _deepiter(self, objlist, depth=1):
206
+ for obj in objlist:
207
+ if not obj.parent and obj.children:
208
+ # ignore toplevel file, dive into subrows
209
+ yield from self._deepiter(obj.children, depth)
210
+ else:
211
+ yield obj
212
+ if id(obj) in self.opened_rows:
213
+ yield from self._deepiter(obj.children, depth-1)
214
+
215
+ def openRows(self, rows):
216
+ for row in rows:
217
+ self.opened_rows.add(id(row))
218
+ self.refreshRows()
219
+
220
+ def closeRows(self, rows=None):
221
+ if rows is None:
222
+ for row in rows:
223
+ self.opened_rows.remove(id(row))
224
+ else:
225
+ self.opened_rows.clear()
226
+ self.refreshRows()
227
+
228
+ def newRow(self):
229
+ return AttrDict(title='', contents='', tags=[], children=[], links=[], level=0, linenum=0)
230
+
231
+ def iterload(self):
232
+ self.rows = []
233
+ def _walkfiles(p):
234
+ basepath = str(p)
235
+ for folder, subdirs, files in os.walk(basepath):
236
+ subfolder = folder[len(basepath)+1:]
237
+ if subfolder.startswith('.'): continue
238
+ if subfolder in ['.', '..']: continue
239
+
240
+ fpath = Path(folder)
241
+ yield fpath
242
+
243
+ for fn in files:
244
+ yield fpath/fn
245
+
246
+ if self.filetype == 'orgdir':
247
+ basepath = str(self.source)
248
+ for p in _walkfiles(self.source):
249
+ if p.base_stem.startswith('.'): continue
250
+ if p.ext in ['org', 'md']:
251
+ yield self.parse_orgmd(p)
252
+ elif self.filetype == 'forg':
253
+ for fn in self.source.open():
254
+ yield self.parse_orgmd(Path(fn.rstrip()))
255
+ elif self.filetype == 'org':
256
+ yield self.parse_orgmd(self.source)
257
+
258
+ self.sourceRows = self.rows
259
+ self.refreshRows()
260
+
261
+ def parse_orgmd(self, path):
262
+ # row.file_string = open(path).read()
263
+ row = orgmode_parse(open(path).readlines())
264
+ st = path.stat()
265
+ if st:
266
+ mtime = st.st_mtime
267
+
268
+ row.path = path
269
+ row.date = mtime
270
+ return row
271
+
272
+ def draw(self, scr):
273
+ super().draw(scr)
274
+
275
+ @asyncthread
276
+ def putChanges(self):
277
+ adds, mods, dels = self.getDeferredChanges()
278
+
279
+ saveset = {} # path:_root(row)
280
+ for r in adds.values():
281
+ saveset[_root(r).path] = _root(r)
282
+ for r, _ in mods.values():
283
+ saveset[_root(r).path] = _root(r)
284
+
285
+ for row in addset.values():
286
+ self.save(row)
287
+
288
+ self.commitAdds()
289
+
290
+ self.commitMods()
291
+
292
+ for row in dels.values():
293
+ row.path.rename('%s-%s' % (row.path, encode_date()))
294
+
295
+ self.commitDeletes()
296
+
297
+ def save_all(self):
298
+ for row in self.sourceRows:
299
+ self.save(row)
300
+ vd.status('saved %s org files' % len(self.sourceRows))
301
+
302
+ def save(self, row):
303
+ try:
304
+ row.path.rename('%s-%s' % (row.path, encode_date())) # backup
305
+ except FileNotFoundError:
306
+ pass
307
+
308
+ with row.path.open(mode='w') as f:
309
+ print(orgmode_to_string(row).rstrip(), file=f)
310
+
311
+
312
+ @OrgSheet.api
313
+ def paste_into(sheet, row, sourcerows, cols):
314
+ row.children.extend(sourcerows)
315
+ for r in sourcerows:
316
+ r.parent.children.remove(r)
317
+ r.parent = row
318
+ sheet.refreshRows()
319
+
320
+
321
+ @OrgSheet.api
322
+ def paste_data_into(sheet, row, sourcerows, cols):
323
+ body = row.body or ''
324
+ for r in sourcerows:
325
+ data = vd.encode_json(r, cols)
326
+ row.contents += f':{cols[0].sheet.name}:{data}\n'
327
+
328
+
329
+ @OrgSheet.api
330
+ def combine_rows(sheet, rows):
331
+ newrow = sheet.newRow()
332
+ newrow.date = datetime.datetime.today()
333
+ orgid = vd.cleanName(rows[0].title)
334
+ newrow.path = Path((orgid or encode_date())+'.org')
335
+ for r in rows:
336
+ hdr = sheet.newRow()
337
+ if hdr.title:
338
+ hdr.update(orgmode_parse_title(hdr.title))
339
+ hdr.children = list(r.children)
340
+ newrow.children.append(hdr)
341
+ # newrow.title = ' '
342
+ newrow.file_string = orgmode_to_string(newrow, '*')
343
+ return newrow
344
+
345
+
346
+ def _root(row):
347
+ while row and row.parent:
348
+ row = row.parent
349
+ return row
350
+
351
+
352
+ @OrgSheet.api
353
+ def sysopen_row(sheet, row):
354
+ root = _root(row)
355
+ if root.path.exists():
356
+ vd.launchEditor(root.path, '+%s'%row['linenum'])
357
+ else:
358
+ orgmode_parse_into(row, vd.launchExternalEditor(root.file_string))
359
+
360
+ @VisiData.api
361
+ def save_org(vd, p, *vsheets):
362
+ with p.open(mode='w', encoding=vsheets[0].options.save_encoding) as fp:
363
+ for vs in vsheets:
364
+ if isinstance(vs, OrgSheet):
365
+ for row in vs.rows:
366
+ print(orgmode_to_string(row).strip(), file=fp)
367
+ else:
368
+ vd.warning('not implemented')
369
+
370
+
371
+ @OrgSheet.api
372
+ def sysopen_rows(sheet, rows):
373
+ ret = ''
374
+ for r in rows:
375
+ s = orgmode_to_string(r).strip()
376
+ ret += '{::%s::}\n%s\n\n' % (id(r), s)
377
+
378
+ ret = vd.launchExternalEditor(ret + "{::}")
379
+
380
+ idrows = {id(r):r for r in rows}
381
+
382
+ idtexts = {}
383
+ for idtag, text in re.findall(r'\{::(\d+)::\}(.*?)(?={::)', ret, re.DOTALL):
384
+ idtexts[int(idtag)] = text
385
+
386
+ # reparse all existing
387
+ for rowid, row in idrows.items():
388
+ text = idtexts.get(rowid, '').strip()
389
+ if text:
390
+ orgmode_parse_into(row, text)
391
+ else:
392
+ # sometimes, top levels are missing parents
393
+ # this needs to be debugged
394
+ if row.parent:
395
+ row.parent.children.remove(row)
396
+
397
+ lastrow = None
398
+ # find new rows to add
399
+ for rowid, text in idtexts.items():
400
+ if rowid not in idrows:
401
+ section = orgmode_parse(text.splitlines())
402
+ while section.level <= lastrow.level:
403
+ lastrow = lastrow.parent
404
+
405
+ if lastrow:
406
+ sourceRows.append(section)
407
+ else:
408
+ sheet.addRow(section)
409
+
410
+ sheet.refreshRows()
411
+
412
+ OrgSheet.addCommand('^O', 'sysopen-row', 'sysopen_row(cursorRow)', 'open current file in external $EDITOR')
413
+ OrgSheet.addCommand('g^O', 'sysopen-rows', 'sysopen_rows(selectedRows)', 'open selected files in external $EDITOR')
414
+ OrgSheet.addCommand('^J', 'expand-row', 'openRows([cursorRow]); sheet.cursorRowIndex += 1')
415
+ OrgSheet.addCommand('z^J', 'close-row', 'closeRows([cursorRow]); sheet.cursorRowIndex += 1')
416
+ OrgSheet.addCommand('g^J', 'expand-selected', 'openRows(selectedRows)')
417
+ OrgSheet.addCommand('gz^J', 'close-selected', 'closeRows(selectedRows)')
418
+ OrgSheet.addCommand('ga', 'combine-selected', 'addRows([combine_rows(selectedRows)], index=cursorRowIndex); cursorDown(1)', 'combine selected rows into new org entry')
419
+
420
+ OrgSheet.addCommand('zp', 'paste-data', 'paste_data_into(cursorRow, vd.memory.cliprows, vd.memory.clipcols)', 'move clipboard rows to children of current row')
421
+ OrgSheet.addCommand('p', 'paste-sections', 'paste_data_into(cursorRow, vd.memory.cliprows, vd.memory.clipcols)', 'move clipboard rows to children of current row')
422
+ OrgSheet.addCommand('g^S', 'save-all', 'save_all()', 'save all org files')
423
+
424
+
425
+ if __name__ == '__main__':
426
+ for fn in sys.argv[1:]:
427
+ sect = orgmode_parse(Path(fn))
428
+ print(orgmode_to_string(sect).strip())
@@ -1,4 +1,4 @@
1
- from visidata import vd, Sheet, options, Column, asyncthread, Progress, PivotGroupRow, ENTER
1
+ from visidata import vd, Sheet, options, Column, asyncthread, Progress, PivotGroupRow, ENTER, HistogramColumn
2
2
 
3
3
  from visidata.loaders._pandas import PandasSheet
4
4
  from visidata.pivot import PivotSheet
@@ -11,8 +11,8 @@ class DataFrameRowSliceAdapter:
11
11
  by visidata's selectRow implementation.
12
12
  """
13
13
  def __init__(self, df, mask):
14
- import pandas as pd
15
- import numpy as np
14
+ pd = vd.importExternal('pandas')
15
+ np = vd.importExternal('numpy')
16
16
  if not isinstance(df, pd.DataFrame):
17
17
  vd.fail('%s is not a dataframe' % type(df).__name__)
18
18
  if not isinstance(mask, pd.Series):
@@ -62,15 +62,15 @@ class DataFrameRowSliceIter:
62
62
  self.index += 1
63
63
  return row
64
64
 
65
+ def makePandasFreqTable(sheet, *groupByCols):
66
+ fqcolname = '%s_freq' % '-'.join(col.name for col in groupByCols)
67
+ return PandasFreqTableSheet(sheet.name, fqcolname, groupByCols=groupByCols, source=sheet)
68
+
69
+
65
70
  class PandasFreqTableSheet(PivotSheet):
66
71
  'Generate frequency-table sheet on currently selected column.'
67
72
  rowtype = 'bins' # rowdef FreqRow(keys, sourcerows)
68
73
 
69
- def __init__(self, sheet, *groupByCols):
70
- fqcolname = '%s_%s_freq' % (sheet.name, '-'.join(col.name for col in groupByCols))
71
- super().__init__(fqcolname, groupByCols, [], source=sheet)
72
- self.largest = 1
73
-
74
74
  def selectRow(self, row):
75
75
  # Select all entries in the bin on the source sheet.
76
76
  # Use the internally defined _selectByLoc to avoid
@@ -86,8 +86,7 @@ class PandasFreqTableSheet(PivotSheet):
86
86
  def updateLargest(self, grouprow):
87
87
  self.largest = max(self.largest, len(grouprow.sourcerows))
88
88
 
89
- @asyncthread
90
- def reload(self):
89
+ def loader(self):
91
90
  'Generate frequency table then reverse-sort by length.'
92
91
  import pandas as pd
93
92
 
@@ -95,8 +94,6 @@ class PandasFreqTableSheet(PivotSheet):
95
94
  # (e.g. as a histogram). We currently don't provide support for this
96
95
  # for PandasSheet, although we could implement it with a pd.Grouper
97
96
  # that operates similarly to pd.cut.
98
- super().initCols()
99
-
100
97
  df = self.source.df.copy()
101
98
 
102
99
  # Implementation (special case): for one row, this degenerates
@@ -141,9 +138,7 @@ class PandasFreqTableSheet(PivotSheet):
141
138
  getter=lambda col,row: len(row.sourcerows)),
142
139
  Column('percent', type=float,
143
140
  getter=lambda col,row: len(row.sourcerows)*100/df.shape[0]),
144
- Column('histogram', type=str,
145
- getter=lambda col,row: options.disp_histogram*(options.disp_histolen*len(row.sourcerows)//value_counts.max()),
146
- width=options.disp_histolen+2),
141
+ HistogramColumn('histogram', type=str, width=self.options.default_width*2)
147
142
  ]:
148
143
  self.addColumn(c)
149
144
 
@@ -174,11 +169,10 @@ def expand_source_rows(sheet, row):
174
169
  vd.fail("no source rows")
175
170
  return PandasSheet(sheet.name, vd.valueNames(row.discrete_keys, row.numeric_key), source=row.sourcerows)
176
171
 
177
- PandasSheet.addCommand('F', 'freq-col', 'vd.push(PandasFreqTableSheet(sheet, cursorCol))', 'open Frequency Table grouped on current column, with aggregations of other columns')
178
- PandasSheet.addCommand('gF', 'freq-keys', 'vd.push(PandasFreqTableSheet(sheet, *keyCols))', 'open Frequency Table grouped by all key columns on source sheet, with aggregations of other columns')
172
+ PandasSheet.addCommand('F', 'freq-col', 'vd.push(makePandasFreqTable(sheet, cursorCol))', 'open Frequency Table grouped on current column, with aggregations of other columns')
173
+ PandasSheet.addCommand('gF', 'freq-keys', 'vd.push(makePandasFreqTable(sheet, *keyCols))', 'open Frequency Table grouped by all key columns on source sheet, with aggregations of other columns')
179
174
 
175
+ PandasFreqTableSheet.init('largest', lambda: 1)
180
176
  PandasFreqTableSheet.options.numeric_binning = False
181
177
 
182
- vd.addGlobals({
183
- 'PandasFreqTableSheet': PandasFreqTableSheet,
184
- })
178
+ vd.addGlobals(makePandasFreqTable=makePandasFreqTable)