visidata 2.11.1__py3-none-any.whl → 3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- visidata/__init__.py +72 -91
- visidata/_input.py +259 -42
- visidata/_open.py +84 -29
- visidata/_types.py +21 -3
- visidata/_urlcache.py +17 -4
- visidata/aggregators.py +65 -25
- visidata/apps/__init__.py +0 -0
- visidata/apps/vdsql/__about__.py +8 -0
- visidata/apps/vdsql/__init__.py +5 -0
- visidata/apps/vdsql/__main__.py +27 -0
- visidata/apps/vdsql/_ibis.py +748 -0
- visidata/apps/vdsql/bigquery.py +61 -0
- visidata/apps/vdsql/clickhouse.py +53 -0
- visidata/apps/vdsql/setup.py +40 -0
- visidata/apps/vdsql/snowflake.py +67 -0
- visidata/apps/vgit/__init__.py +13 -0
- {vgit → visidata/apps/vgit}/blame.py +5 -2
- {vgit → visidata/apps/vgit}/branch.py +31 -16
- {vgit → visidata/apps/vgit}/config.py +3 -3
- visidata/apps/vgit/diff.py +169 -0
- visidata/apps/vgit/gitsheet.py +161 -0
- {vgit → visidata/apps/vgit}/grep.py +6 -5
- visidata/apps/vgit/log.py +81 -0
- {vgit → visidata/apps/vgit}/main.py +18 -5
- {vgit → visidata/apps/vgit}/remote.py +8 -4
- visidata/apps/vgit/repos.py +71 -0
- {vgit → visidata/apps/vgit}/setup.py +6 -4
- visidata/apps/vgit/stash.py +69 -0
- visidata/apps/vgit/status.py +204 -0
- {vgit → visidata/apps/vgit}/statusbar.py +2 -0
- visidata/basesheet.py +59 -50
- visidata/canvas.py +208 -93
- visidata/choose.py +6 -6
- visidata/clean_names.py +29 -0
- visidata/clipboard.py +73 -17
- visidata/cliptext.py +220 -46
- visidata/cmdlog.py +88 -114
- visidata/color.py +142 -56
- visidata/column.py +121 -129
- visidata/ddw/input.ddw +74 -79
- visidata/ddw/regex.ddw +57 -0
- visidata/ddwplay.py +33 -14
- visidata/deprecated.py +77 -3
- visidata/desktop/visidata.desktop +7 -0
- visidata/editor.py +12 -6
- visidata/errors.py +5 -1
- visidata/experimental/__init__.py +0 -0
- visidata/experimental/diff_sheet.py +29 -0
- visidata/experimental/digit_autoedit.py +6 -0
- visidata/experimental/gdrive.py +89 -0
- visidata/experimental/google.py +37 -0
- visidata/experimental/gsheets.py +79 -0
- visidata/experimental/live_search.py +37 -0
- visidata/experimental/liveupdate.py +45 -0
- visidata/experimental/mark.py +133 -0
- visidata/experimental/noahs_tapestry/__init__.py +1 -0
- visidata/experimental/noahs_tapestry/tapestry.py +147 -0
- visidata/experimental/rownum.py +73 -0
- visidata/experimental/slide_cells.py +26 -0
- visidata/expr.py +8 -4
- visidata/extensible.py +30 -5
- visidata/features/__init__.py +0 -0
- visidata/features/addcol_audiometadata.py +42 -0
- visidata/features/addcol_histogram.py +34 -0
- visidata/features/canvas_save_svg.py +69 -0
- visidata/features/change_precision.py +46 -0
- visidata/features/cmdpalette.py +163 -0
- visidata/features/colorbrewer.py +363 -0
- visidata/{colorsheet.py → features/colorsheet.py} +17 -16
- visidata/features/command_server.py +105 -0
- visidata/features/currency_to_usd.py +70 -0
- visidata/{customdate.py → features/customdate.py} +2 -0
- visidata/features/dedupe.py +132 -0
- visidata/{describe.py → features/describe.py} +17 -15
- visidata/features/errors_guide.py +26 -0
- visidata/features/expand_cols.py +202 -0
- visidata/{fill.py → features/fill.py} +3 -1
- visidata/{freeze.py → features/freeze.py} +11 -6
- visidata/features/graph_seaborn.py +79 -0
- visidata/features/helloworld.py +10 -0
- visidata/features/hint_types.py +17 -0
- visidata/{incr.py → features/incr.py} +5 -0
- visidata/{join.py → features/join.py} +107 -53
- visidata/features/known_cols.py +21 -0
- visidata/features/layout.py +62 -0
- visidata/{melt.py → features/melt.py} +32 -21
- visidata/features/normcol.py +118 -0
- visidata/features/open_config.py +7 -0
- visidata/features/open_syspaste.py +18 -0
- visidata/features/ping.py +157 -0
- visidata/features/procmgr.py +208 -0
- visidata/features/random_sample.py +6 -0
- visidata/{regex.py → features/regex.py} +47 -31
- visidata/features/reload_every.py +55 -0
- visidata/features/rename_col_cascade.py +30 -0
- visidata/features/scroll_context.py +60 -0
- visidata/features/select_equal_selected.py +11 -0
- visidata/features/setcol_fake.py +65 -0
- visidata/{slide.py → features/slide.py} +75 -21
- visidata/features/sparkline.py +48 -0
- visidata/features/status_source.py +20 -0
- visidata/{sysedit.py → features/sysedit.py} +2 -1
- visidata/features/sysopen_mailcap.py +46 -0
- visidata/features/term_extras.py +13 -0
- visidata/{transpose.py → features/transpose.py} +5 -4
- visidata/features/type_ipaddr.py +73 -0
- visidata/features/type_url.py +11 -0
- visidata/{unfurl.py → features/unfurl.py} +9 -9
- visidata/{window.py → features/window.py} +2 -2
- visidata/form.py +50 -21
- visidata/freqtbl.py +81 -33
- visidata/fuzzymatch.py +414 -0
- visidata/graph.py +105 -33
- visidata/guide.py +180 -0
- visidata/help.py +75 -44
- visidata/hint.py +39 -0
- visidata/indexsheet.py +109 -0
- visidata/input_history.py +55 -0
- visidata/interface.py +58 -0
- visidata/keys.py +17 -16
- visidata/loaders/__init__.py +9 -0
- visidata/loaders/_pandas.py +61 -21
- visidata/loaders/api_airtable.py +70 -0
- visidata/loaders/api_bitio.py +102 -0
- visidata/loaders/api_matrix.py +148 -0
- visidata/loaders/api_reddit.py +306 -0
- visidata/loaders/api_zulip.py +249 -0
- visidata/loaders/archive.py +41 -7
- visidata/loaders/arrow.py +7 -7
- visidata/loaders/conll.py +49 -0
- visidata/loaders/csv.py +25 -7
- visidata/loaders/eml.py +3 -4
- visidata/loaders/f5log.py +1204 -0
- visidata/loaders/fec.py +325 -0
- visidata/loaders/fixed_width.py +2 -4
- visidata/loaders/frictionless.py +3 -3
- visidata/loaders/geojson.py +8 -5
- visidata/loaders/google.py +48 -0
- visidata/loaders/graphviz.py +4 -4
- visidata/loaders/hdf5.py +4 -4
- visidata/loaders/html.py +48 -10
- visidata/loaders/http.py +84 -30
- visidata/loaders/imap.py +20 -10
- visidata/loaders/jrnl.py +52 -0
- visidata/loaders/json.py +83 -29
- visidata/loaders/jsonla.py +74 -0
- visidata/loaders/lsv.py +15 -11
- visidata/loaders/mailbox.py +40 -0
- visidata/loaders/markdown.py +1 -3
- visidata/loaders/mbtiles.py +4 -5
- visidata/loaders/mysql.py +11 -13
- visidata/loaders/npy.py +7 -7
- visidata/loaders/odf.py +4 -1
- visidata/loaders/orgmode.py +428 -0
- visidata/loaders/pandas_freqtbl.py +14 -20
- visidata/loaders/parquet.py +62 -6
- visidata/loaders/pcap.py +3 -3
- visidata/loaders/pdf.py +4 -3
- visidata/loaders/png.py +19 -13
- visidata/loaders/postgres.py +9 -8
- visidata/loaders/rec.py +7 -3
- visidata/loaders/s3.py +342 -0
- visidata/loaders/sas.py +5 -5
- visidata/loaders/scrape.py +186 -0
- visidata/loaders/shp.py +6 -5
- visidata/loaders/spss.py +5 -6
- visidata/loaders/sqlite.py +68 -28
- visidata/loaders/texttables.py +1 -1
- visidata/loaders/toml.py +60 -0
- visidata/loaders/tsv.py +61 -19
- visidata/loaders/ttf.py +19 -7
- visidata/loaders/unzip_http.py +6 -5
- visidata/loaders/usv.py +1 -1
- visidata/loaders/vcf.py +16 -16
- visidata/loaders/vds.py +10 -7
- visidata/loaders/vdx.py +30 -5
- visidata/loaders/xlsb.py +8 -1
- visidata/loaders/xlsx.py +145 -25
- visidata/loaders/xml.py +6 -3
- visidata/loaders/xword.py +4 -4
- visidata/loaders/yaml.py +15 -5
- visidata/macros.py +129 -42
- visidata/main.py +119 -94
- visidata/mainloop.py +101 -155
- visidata/man/parse_options.py +2 -2
- visidata/man/vd.1 +301 -148
- visidata/man/vd.txt +290 -153
- visidata/memory.py +3 -3
- visidata/menu.py +104 -423
- visidata/metasheets.py +59 -141
- visidata/modify.py +78 -23
- visidata/motd.py +3 -3
- visidata/mouse.py +137 -0
- visidata/movement.py +43 -35
- visidata/optionssheet.py +99 -0
- visidata/path.py +113 -32
- visidata/pivot.py +73 -47
- visidata/plugins.py +65 -192
- visidata/pyobj.py +50 -201
- visidata/rename_col.py +20 -0
- visidata/save.py +37 -20
- visidata/search.py +54 -10
- visidata/selection.py +84 -5
- visidata/settings.py +162 -25
- visidata/sheets.py +229 -257
- visidata/shell.py +51 -21
- visidata/sidebar.py +162 -0
- visidata/sort.py +11 -4
- visidata/statusbar.py +113 -104
- visidata/stored_list.py +43 -0
- visidata/stored_prop.py +38 -0
- visidata/tests/conftest.py +3 -3
- visidata/tests/test_cliptext.py +39 -0
- visidata/tests/test_commands.py +62 -7
- visidata/tests/test_edittext.py +2 -2
- visidata/tests/test_features.py +17 -0
- visidata/tests/test_menu.py +14 -0
- visidata/tests/test_path.py +13 -4
- visidata/text_source.py +53 -0
- visidata/textsheet.py +10 -3
- visidata/theme.py +44 -0
- visidata/themes/__init__.py +0 -0
- visidata/themes/ascii8.py +84 -0
- visidata/themes/asciimono.py +84 -0
- visidata/themes/light.py +17 -0
- visidata/threads.py +87 -39
- visidata/tuiwin.py +22 -0
- visidata/type_currency.py +22 -3
- visidata/type_date.py +31 -9
- visidata/type_floatsi.py +5 -1
- visidata/undo.py +17 -5
- visidata/utils.py +106 -23
- visidata/vdobj.py +28 -17
- visidata/windows.py +10 -0
- visidata/wrappers.py +9 -3
- visidata-3.0.data/data/share/applications/visidata.desktop +7 -0
- {visidata-2.11.1.data → visidata-3.0.data}/data/share/man/man1/vd.1 +301 -148
- {visidata-2.11.1.data → visidata-3.0.data}/data/share/man/man1/visidata.1 +301 -148
- visidata-3.0.data/scripts/vd2to3.vdx +9 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/METADATA +12 -8
- visidata-3.0.dist-info/RECORD +257 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/WHEEL +1 -1
- vgit/__init__.py +0 -1
- vgit/gitsheet.py +0 -164
- visidata/layout.py +0 -44
- visidata/misc.py +0 -5
- visidata-2.11.1.data/scripts/vgit +0 -9
- visidata-2.11.1.dist-info/RECORD +0 -155
- {vgit → visidata/apps/vgit}/__main__.py +0 -0
- {vgit → visidata/apps/vgit}/abort.py +0 -0
- /visidata/{repeat.py → features/repeat.py} +0 -0
- {visidata-2.11.1.data → visidata-3.0.data}/scripts/vd +0 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/LICENSE.gpl3 +0 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/entry_points.txt +0 -0
- {visidata-2.11.1.dist-info → visidata-3.0.dist-info}/top_level.txt +0 -0
visidata/loaders/mbtiles.py
CHANGED
@@ -5,11 +5,11 @@ import sqlite3
|
|
5
5
|
|
6
6
|
@VisiData.api
|
7
7
|
def open_pbf(vd, p):
|
8
|
-
return PbfSheet(p.
|
8
|
+
return PbfSheet(p.base_stem, source=p)
|
9
9
|
|
10
10
|
@VisiData.api
|
11
11
|
def open_mbtiles(vd, p):
|
12
|
-
return MbtilesSheet(p.
|
12
|
+
return MbtilesSheet(p.base_stem, source=p)
|
13
13
|
|
14
14
|
def getListDepth(L):
|
15
15
|
if not isinstance(L, list):
|
@@ -36,7 +36,7 @@ class MbtilesSheet(Sheet):
|
|
36
36
|
]
|
37
37
|
|
38
38
|
def getTile(self, zoom_level, tile_col, tile_row):
|
39
|
-
|
39
|
+
mapbox_vector_tile = vd.importExternal('mapbox_vector_tile', 'mapbox-vector-tile')
|
40
40
|
|
41
41
|
con = sqlite3.connect(str(self.source))
|
42
42
|
tile_data = con.execute('''
|
@@ -128,11 +128,10 @@ class PbfCanvas(InvertedCanvas):
|
|
128
128
|
if disptext:
|
129
129
|
self.label(textx, texty, disptext, attr, row)
|
130
130
|
|
131
|
-
|
132
131
|
self.refresh()
|
133
132
|
|
134
133
|
|
135
134
|
PbfSheet.addCommand('.', 'plot-row', 'vd.push(PbfCanvas(name+"_map", source=sheet, sourceRows=[cursorRow], textCol=cursorCol))', 'plot blocks in current row')
|
136
135
|
PbfSheet.addCommand('g.', 'plot-rows', 'vd.push(PbfCanvas(name+"_map", source=sheet, sourceRows=rows, textCol=cursorCol))', 'plot selected blocks')
|
137
136
|
MbtilesSheet.addCommand('.', 'plot-row', 'vd.push(getPlot(cursorRow))', 'plot tiles in current row')
|
138
|
-
MbtilesSheet.addCommand('g.', 'plot-selected', 'vd.push(getPlot(*selectedRows))', 'plot selected tiles')
|
137
|
+
MbtilesSheet.addCommand('g.', 'plot-selected', 'vd.push(getPlot(*selectedRows))', 'plot selected tiles')
|
visidata/loaders/mysql.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
from contextlib import contextmanager
|
2
|
+
from urllib.parse import urlparse, unquote
|
2
3
|
|
3
|
-
from visidata import VisiData, vd, Sheet, anytype, asyncthread,
|
4
|
+
from visidata import VisiData, vd, Sheet, anytype, asyncthread, ColumnItem
|
4
5
|
|
5
6
|
def codeToType(type_code, colname):
|
6
7
|
import MySQLdb as mysql
|
@@ -41,7 +42,7 @@ class SQL:
|
|
41
42
|
database=self.url.path[1:],
|
42
43
|
host=self.url.hostname,
|
43
44
|
port=self.url.port or 3306,
|
44
|
-
password=self.url.password,
|
45
|
+
password=unquote(self.url.password),
|
45
46
|
use_unicode=True,
|
46
47
|
charset='utf8',
|
47
48
|
cursorclass=cursors.SSCursor) ## if SSCursor is not used mysql will first fetch ALL data, and only then visualize it
|
@@ -71,7 +72,7 @@ def cursorToColumns(cur, sheet):
|
|
71
72
|
class MyTablesSheet(Sheet):
|
72
73
|
rowtype = 'tables'
|
73
74
|
|
74
|
-
def
|
75
|
+
def iterload(self):
|
75
76
|
qstr = f'''
|
76
77
|
select
|
77
78
|
t.table_name,
|
@@ -96,16 +97,15 @@ class MyTablesSheet(Sheet):
|
|
96
97
|
'''
|
97
98
|
|
98
99
|
with self.sql.cur(qstr) as cur:
|
99
|
-
self.rows = []
|
100
100
|
# try to get first row to make cur.description available
|
101
101
|
r = cur.fetchone()
|
102
102
|
if r:
|
103
|
-
|
103
|
+
yield r
|
104
104
|
cursorToColumns(cur, self)
|
105
105
|
self.setKeys(self.columns[0:1]) # table_name is the key
|
106
106
|
|
107
107
|
for r in cur:
|
108
|
-
|
108
|
+
yield r
|
109
109
|
|
110
110
|
def openRow(self, row):
|
111
111
|
return MyTable(self.name+"."+row[0], source=row[0], sql=self.sql)
|
@@ -113,22 +113,20 @@ class MyTablesSheet(Sheet):
|
|
113
113
|
|
114
114
|
# rowdef: tuple of values as returned by fetchone()
|
115
115
|
class MyTable(Sheet):
|
116
|
-
|
117
|
-
def reload(self):
|
116
|
+
def iterload(self):
|
118
117
|
with self.sql.cur("SELECT * FROM " + self.source) as cur:
|
119
|
-
self.rows = []
|
120
118
|
r = cur.fetchone()
|
121
119
|
if r is None:
|
122
120
|
return
|
123
|
-
|
124
|
-
|
121
|
+
|
122
|
+
yield r
|
125
123
|
cursorToColumns(cur, self)
|
126
124
|
while True:
|
127
125
|
try:
|
128
126
|
r = cur.fetchone()
|
129
127
|
if r is None:
|
130
128
|
break
|
131
|
-
|
132
|
-
|
129
|
+
|
130
|
+
yield r
|
133
131
|
except UnicodeDecodeError as e:
|
134
132
|
vd.exceptionCaught(e)
|
visidata/loaders/npy.py
CHANGED
@@ -1,20 +1,20 @@
|
|
1
|
-
from visidata import VisiData, vd, Sheet, date, anytype, options, Column, Progress, ColumnItem, vlen, PyobjSheet, TypedWrapper
|
1
|
+
from visidata import VisiData, vd, Sheet, date, anytype, options, Column, Progress, ColumnItem, vlen, PyobjSheet, TypedWrapper
|
2
2
|
|
3
3
|
'Loaders for .npy and .npz. Save to .npy. Depends on the zip loader.'
|
4
4
|
|
5
5
|
@VisiData.api
|
6
6
|
def open_npy(vd, p):
|
7
|
-
return NpySheet(p.
|
7
|
+
return NpySheet(p.base_stem, source=p)
|
8
8
|
|
9
9
|
@VisiData.api
|
10
10
|
def open_npz(vd, p):
|
11
|
-
return NpzSheet(p.
|
11
|
+
return NpzSheet(p.base_stem, source=p)
|
12
12
|
|
13
13
|
vd.option('npy_allow_pickle', False, 'numpy allow unpickling objects (unsafe)')
|
14
14
|
|
15
15
|
class NpySheet(Sheet):
|
16
16
|
def iterload(self):
|
17
|
-
|
17
|
+
numpy = vd.importExternal('numpy')
|
18
18
|
if not hasattr(self, 'npy'):
|
19
19
|
self.npy = numpy.load(str(self.source), encoding='bytes', **self.options.getall('npy_'))
|
20
20
|
self.reloadCols()
|
@@ -47,12 +47,12 @@ class NpzSheet(vd.ZipSheet):
|
|
47
47
|
]
|
48
48
|
|
49
49
|
def iterload(self):
|
50
|
-
|
50
|
+
numpy = vd.importExternal('numpy')
|
51
51
|
self.npz = numpy.load(str(self.source), encoding='bytes', **self.options.getall('npy_'))
|
52
52
|
yield from Progress(self.npz.items())
|
53
53
|
|
54
54
|
def openRow(self, row):
|
55
|
-
|
55
|
+
numpy = vd.importExternal('numpy')
|
56
56
|
tablename, tbl = row
|
57
57
|
if isinstance(tbl, numpy.ndarray):
|
58
58
|
return NpySheet(tablename, npy=tbl)
|
@@ -62,7 +62,7 @@ class NpzSheet(vd.ZipSheet):
|
|
62
62
|
|
63
63
|
@VisiData.api
|
64
64
|
def save_npy(vd, p, sheet):
|
65
|
-
|
65
|
+
np = vd.importExternal('numpy')
|
66
66
|
|
67
67
|
dtype = []
|
68
68
|
|
visidata/loaders/odf.py
CHANGED
@@ -3,11 +3,12 @@ from visidata import vd, VisiData, Sheet, IndexSheet, SequenceSheet
|
|
3
3
|
|
4
4
|
@VisiData.api
|
5
5
|
def open_ods(vd, p):
|
6
|
-
return OdsIndexSheet(p.
|
6
|
+
return OdsIndexSheet(p.base_stem, source=p)
|
7
7
|
|
8
8
|
|
9
9
|
class OdsIndexSheet(IndexSheet):
|
10
10
|
def iterload(self):
|
11
|
+
vd.importExternal('odf', 'odfpy')
|
11
12
|
import odf.opendocument
|
12
13
|
import odf.table
|
13
14
|
self.doc = odf.opendocument.load(self.source)
|
@@ -16,6 +17,7 @@ class OdsIndexSheet(IndexSheet):
|
|
16
17
|
|
17
18
|
|
18
19
|
def _get_cell_string_value(cell, text_s):
|
20
|
+
vd.importExternal('odf', 'odfpy')
|
19
21
|
from odf.element import Element
|
20
22
|
from odf.namespaces import TEXTNS
|
21
23
|
|
@@ -34,6 +36,7 @@ def _get_cell_string_value(cell, text_s):
|
|
34
36
|
|
35
37
|
class OdsSheet(SequenceSheet):
|
36
38
|
def iterload(self):
|
39
|
+
vd.importExternal('odf', 'odfpy')
|
37
40
|
import odf.table
|
38
41
|
import odf.text
|
39
42
|
from odf.namespaces import TABLENS, OFFICENS
|
@@ -0,0 +1,428 @@
|
|
1
|
+
'''
|
2
|
+
View sections in one or more .org files. Easily edit sections within the system editor.
|
3
|
+
|
4
|
+
## Supported syntax
|
5
|
+
|
6
|
+
- `--- [comment]` at start of line starts a new section
|
7
|
+
- any number of `#` or `*` (followed by a space) leads a headline
|
8
|
+
- `#+key: value` adds metadata to next element (headline, table, section, )
|
9
|
+
- orgmode style tags: `:tag1:tag2:`
|
10
|
+
- `[[tagname]]` links to that set of tags
|
11
|
+
- `[[url]]` links to external url
|
12
|
+
- orgmode and markdown links
|
13
|
+
- `[[url][linktext]]`
|
14
|
+
- `[linktext](url)`
|
15
|
+
- all other markup/orgmode/whatever is ignored and passed through
|
16
|
+
|
17
|
+
## Usage
|
18
|
+
|
19
|
+
- `vd file.org`
|
20
|
+
- or `find orgfiles/ -name '*.org' | vd -f forg`
|
21
|
+
- or `vd orgfiles/ -f orgdir`
|
22
|
+
'''
|
23
|
+
|
24
|
+
import collections
|
25
|
+
import datetime
|
26
|
+
import os
|
27
|
+
import re
|
28
|
+
|
29
|
+
from visidata import vd, VisiData, Column, Sheet, ItemColumn, vlen, asyncthread, Path, AttrDict, date
|
30
|
+
|
31
|
+
|
32
|
+
@VisiData.api
|
33
|
+
def open_org(vd, p):
|
34
|
+
return OrgSheet(p.base_stem, source=p, filetype='org')
|
35
|
+
|
36
|
+
|
37
|
+
@VisiData.api
|
38
|
+
def open_forg(vd, p):
|
39
|
+
return OrgSheet(p.base_stem, source=p, filetype='forg')
|
40
|
+
|
41
|
+
|
42
|
+
@VisiData.api
|
43
|
+
def open_orgdir(vd, p):
|
44
|
+
return OrgSheet(p.base_stem, source=p, filetype='orgdir')
|
45
|
+
|
46
|
+
|
47
|
+
def encode_date(dt=None):
|
48
|
+
if not dt:
|
49
|
+
dt = datetime.datetime.now()
|
50
|
+
elif isinstance(dt, str):
|
51
|
+
dt = datetime.datetime.fromisoformat(dt)
|
52
|
+
|
53
|
+
s = '123456789abcdefghijklmnopqrstuvwxyz'
|
54
|
+
return '%02d%s%s' % (dt.year % 100, s[dt.month-1], s[dt.day-1])
|
55
|
+
|
56
|
+
|
57
|
+
class OrgContentsColumn(Column):
|
58
|
+
def setValue(self, row, v):
|
59
|
+
super().setValue(row, v)
|
60
|
+
orgmode_parse_into(row, v)
|
61
|
+
|
62
|
+
def putValue(self, row, v):
|
63
|
+
self.sheet.save(row)
|
64
|
+
|
65
|
+
|
66
|
+
def sectionize(lines):
|
67
|
+
'Generate (startinglinenum, contentlines) for each section. First section may not have leading * or # but all others will.'
|
68
|
+
startinglinenum = 0
|
69
|
+
prev_contents = contents = []
|
70
|
+
for linenum, line in enumerate(lines):
|
71
|
+
line = line.rstrip('\n')
|
72
|
+
|
73
|
+
if line and line[0] in '#*':
|
74
|
+
if contents:
|
75
|
+
yield startinglinenum, contents
|
76
|
+
prev_contents = contents
|
77
|
+
contents = []
|
78
|
+
startinglinenum = linenum
|
79
|
+
else:
|
80
|
+
if not line and not contents:
|
81
|
+
prev_contents.append(line)
|
82
|
+
|
83
|
+
contents.append(line)
|
84
|
+
|
85
|
+
if contents:
|
86
|
+
yield startinglinenum, contents
|
87
|
+
|
88
|
+
|
89
|
+
def orgmode_parse(all_lines):
|
90
|
+
root = parent = OrgSheet().newRow()
|
91
|
+
for linenum, lines in sectionize(all_lines):
|
92
|
+
section = OrgSheet().newRow()
|
93
|
+
|
94
|
+
section.contents = ''
|
95
|
+
for i, line in enumerate(lines):
|
96
|
+
if not line and not lines[i-1]:
|
97
|
+
continue
|
98
|
+
section.contents += line + '\n'
|
99
|
+
section.orig_contents = section.contents
|
100
|
+
|
101
|
+
section.linenum = linenum+1
|
102
|
+
for line in lines:
|
103
|
+
section.tags.extend(re.findall(r':([\S:]+?):', line))
|
104
|
+
links = re.findall(r'\[.*?\]\(.*?\)', line)
|
105
|
+
if links:
|
106
|
+
section.links.extend(links)
|
107
|
+
|
108
|
+
title = orgmode_parse_title(lines[0])
|
109
|
+
if not title:
|
110
|
+
root = parent = section
|
111
|
+
continue
|
112
|
+
|
113
|
+
section.update(title)
|
114
|
+
|
115
|
+
while parent and section.level <= parent.level:
|
116
|
+
parent = parent.parent
|
117
|
+
|
118
|
+
parent.children.append(section)
|
119
|
+
section.parent = parent
|
120
|
+
parent = section
|
121
|
+
|
122
|
+
return root
|
123
|
+
|
124
|
+
def _replace(node, newnode):
|
125
|
+
node.update(newnode) # must replace insides of same row object
|
126
|
+
for c in node.children:
|
127
|
+
c.parent = node
|
128
|
+
return node
|
129
|
+
|
130
|
+
def orgmode_parse_into(toprow, text):
|
131
|
+
row = orgmode_parse(text.splitlines())
|
132
|
+
if not row.title and len(row.children) == 1:
|
133
|
+
row = row.children[0]
|
134
|
+
row.parent = toprow.parent
|
135
|
+
toprow = _replace(toprow, row)
|
136
|
+
|
137
|
+
return toprow
|
138
|
+
|
139
|
+
|
140
|
+
def orgmode_to_string(section, prestars=''):
|
141
|
+
ret = ''
|
142
|
+
|
143
|
+
# if section.title:
|
144
|
+
# ret += prestars[len(section.stars):] + (section.title or ' ') + '\n'
|
145
|
+
ret += section.contents.rstrip() or ''
|
146
|
+
ret += '\n\n'
|
147
|
+
ret += ''.join(orgmode_to_string(c, prestars+(section.stars or '')) for c in section.children).rstrip() + '\n'
|
148
|
+
|
149
|
+
return ret
|
150
|
+
|
151
|
+
|
152
|
+
def orgmode_parse_title(line):
|
153
|
+
m = re.match(r'^(?P<stars>[*#]+)\s*(?P<keyword>(TODO|FEEDBACK|VERIFY|DONE|DELEGATED))?\s*(?P<prio>\[#[A-z]\])?\s*(?P<title>.*)', line)
|
154
|
+
if not m:
|
155
|
+
assert not line or line[0] not in '#*', line
|
156
|
+
return {}
|
157
|
+
|
158
|
+
return dict(stars=m.group('stars'),
|
159
|
+
level=len(m.group('stars')),
|
160
|
+
keyword=m.group('keyword') or '',
|
161
|
+
prio=m.group('prio') or '',
|
162
|
+
title=line)
|
163
|
+
|
164
|
+
|
165
|
+
class OrgSheet(Sheet):
|
166
|
+
guide = '''# Orgmode Sheet (experimental)
|
167
|
+
A list of orgmode sections from _{sheet.source}_.
|
168
|
+
|
169
|
+
- `Enter` to expand current section
|
170
|
+
- `z Enter` to contract current section
|
171
|
+
- `ga` to combine selected sections into a new entry
|
172
|
+
- `Ctrl+O` to edit section in system editor (edits source directly)
|
173
|
+
- `g Ctrl+S` to save all org files
|
174
|
+
'''
|
175
|
+
defer = True
|
176
|
+
columns = [
|
177
|
+
Column('path', getter=lambda c,r: _root(r).path, width=0),
|
178
|
+
Column('id', getter=lambda c,r: _root(r).path.stem),
|
179
|
+
ItemColumn('title', width=40),
|
180
|
+
# ItemColumn('date', width=0, type=date),
|
181
|
+
ItemColumn('tags', width=10, type=lambda v: ' '.join(v)),
|
182
|
+
ItemColumn('links', type=vlen),
|
183
|
+
# ItemColumn('parent', width=0),
|
184
|
+
ItemColumn('children', type=vlen),
|
185
|
+
ItemColumn('linenum', width=0, type=int),
|
186
|
+
Column('to_string', width=0, getter=lambda c,r: orgmode_to_string(r)),
|
187
|
+
OrgContentsColumn('orig_contents', width=0, getter=lambda c,r: r.orig_contents),
|
188
|
+
]
|
189
|
+
|
190
|
+
def __init__(self, *args, **kwargs):
|
191
|
+
super().__init__(*args, **kwargs)
|
192
|
+
self.opened_rows = set()
|
193
|
+
# self.open_max=0 # all levels closed after this point
|
194
|
+
# self.close_max=0 # all levels open after this point
|
195
|
+
|
196
|
+
def isSelectedParents(self, row):
|
197
|
+
return super().isSelected(row) or row.parent and self.isSelectedParents(row.parent)
|
198
|
+
|
199
|
+
def isSelected(self, row):
|
200
|
+
return self.isSelectedParents(row)
|
201
|
+
|
202
|
+
def refreshRows(self):
|
203
|
+
self.rows = list(self._deepiter(self.sourceRows))
|
204
|
+
|
205
|
+
def _deepiter(self, objlist, depth=1):
|
206
|
+
for obj in objlist:
|
207
|
+
if not obj.parent and obj.children:
|
208
|
+
# ignore toplevel file, dive into subrows
|
209
|
+
yield from self._deepiter(obj.children, depth)
|
210
|
+
else:
|
211
|
+
yield obj
|
212
|
+
if id(obj) in self.opened_rows:
|
213
|
+
yield from self._deepiter(obj.children, depth-1)
|
214
|
+
|
215
|
+
def openRows(self, rows):
|
216
|
+
for row in rows:
|
217
|
+
self.opened_rows.add(id(row))
|
218
|
+
self.refreshRows()
|
219
|
+
|
220
|
+
def closeRows(self, rows=None):
|
221
|
+
if rows is None:
|
222
|
+
for row in rows:
|
223
|
+
self.opened_rows.remove(id(row))
|
224
|
+
else:
|
225
|
+
self.opened_rows.clear()
|
226
|
+
self.refreshRows()
|
227
|
+
|
228
|
+
def newRow(self):
|
229
|
+
return AttrDict(title='', contents='', tags=[], children=[], links=[], level=0, linenum=0)
|
230
|
+
|
231
|
+
def iterload(self):
|
232
|
+
self.rows = []
|
233
|
+
def _walkfiles(p):
|
234
|
+
basepath = str(p)
|
235
|
+
for folder, subdirs, files in os.walk(basepath):
|
236
|
+
subfolder = folder[len(basepath)+1:]
|
237
|
+
if subfolder.startswith('.'): continue
|
238
|
+
if subfolder in ['.', '..']: continue
|
239
|
+
|
240
|
+
fpath = Path(folder)
|
241
|
+
yield fpath
|
242
|
+
|
243
|
+
for fn in files:
|
244
|
+
yield fpath/fn
|
245
|
+
|
246
|
+
if self.filetype == 'orgdir':
|
247
|
+
basepath = str(self.source)
|
248
|
+
for p in _walkfiles(self.source):
|
249
|
+
if p.base_stem.startswith('.'): continue
|
250
|
+
if p.ext in ['org', 'md']:
|
251
|
+
yield self.parse_orgmd(p)
|
252
|
+
elif self.filetype == 'forg':
|
253
|
+
for fn in self.source.open():
|
254
|
+
yield self.parse_orgmd(Path(fn.rstrip()))
|
255
|
+
elif self.filetype == 'org':
|
256
|
+
yield self.parse_orgmd(self.source)
|
257
|
+
|
258
|
+
self.sourceRows = self.rows
|
259
|
+
self.refreshRows()
|
260
|
+
|
261
|
+
def parse_orgmd(self, path):
|
262
|
+
# row.file_string = open(path).read()
|
263
|
+
row = orgmode_parse(open(path).readlines())
|
264
|
+
st = path.stat()
|
265
|
+
if st:
|
266
|
+
mtime = st.st_mtime
|
267
|
+
|
268
|
+
row.path = path
|
269
|
+
row.date = mtime
|
270
|
+
return row
|
271
|
+
|
272
|
+
def draw(self, scr):
|
273
|
+
super().draw(scr)
|
274
|
+
|
275
|
+
@asyncthread
|
276
|
+
def putChanges(self):
|
277
|
+
adds, mods, dels = self.getDeferredChanges()
|
278
|
+
|
279
|
+
saveset = {} # path:_root(row)
|
280
|
+
for r in adds.values():
|
281
|
+
saveset[_root(r).path] = _root(r)
|
282
|
+
for r, _ in mods.values():
|
283
|
+
saveset[_root(r).path] = _root(r)
|
284
|
+
|
285
|
+
for row in addset.values():
|
286
|
+
self.save(row)
|
287
|
+
|
288
|
+
self.commitAdds()
|
289
|
+
|
290
|
+
self.commitMods()
|
291
|
+
|
292
|
+
for row in dels.values():
|
293
|
+
row.path.rename('%s-%s' % (row.path, encode_date()))
|
294
|
+
|
295
|
+
self.commitDeletes()
|
296
|
+
|
297
|
+
def save_all(self):
|
298
|
+
for row in self.sourceRows:
|
299
|
+
self.save(row)
|
300
|
+
vd.status('saved %s org files' % len(self.sourceRows))
|
301
|
+
|
302
|
+
def save(self, row):
|
303
|
+
try:
|
304
|
+
row.path.rename('%s-%s' % (row.path, encode_date())) # backup
|
305
|
+
except FileNotFoundError:
|
306
|
+
pass
|
307
|
+
|
308
|
+
with row.path.open(mode='w') as f:
|
309
|
+
print(orgmode_to_string(row).rstrip(), file=f)
|
310
|
+
|
311
|
+
|
312
|
+
@OrgSheet.api
|
313
|
+
def paste_into(sheet, row, sourcerows, cols):
|
314
|
+
row.children.extend(sourcerows)
|
315
|
+
for r in sourcerows:
|
316
|
+
r.parent.children.remove(r)
|
317
|
+
r.parent = row
|
318
|
+
sheet.refreshRows()
|
319
|
+
|
320
|
+
|
321
|
+
@OrgSheet.api
|
322
|
+
def paste_data_into(sheet, row, sourcerows, cols):
|
323
|
+
body = row.body or ''
|
324
|
+
for r in sourcerows:
|
325
|
+
data = vd.encode_json(r, cols)
|
326
|
+
row.contents += f':{cols[0].sheet.name}:{data}\n'
|
327
|
+
|
328
|
+
|
329
|
+
@OrgSheet.api
|
330
|
+
def combine_rows(sheet, rows):
|
331
|
+
newrow = sheet.newRow()
|
332
|
+
newrow.date = datetime.datetime.today()
|
333
|
+
orgid = vd.cleanName(rows[0].title)
|
334
|
+
newrow.path = Path((orgid or encode_date())+'.org')
|
335
|
+
for r in rows:
|
336
|
+
hdr = sheet.newRow()
|
337
|
+
if hdr.title:
|
338
|
+
hdr.update(orgmode_parse_title(hdr.title))
|
339
|
+
hdr.children = list(r.children)
|
340
|
+
newrow.children.append(hdr)
|
341
|
+
# newrow.title = ' '
|
342
|
+
newrow.file_string = orgmode_to_string(newrow, '*')
|
343
|
+
return newrow
|
344
|
+
|
345
|
+
|
346
|
+
def _root(row):
|
347
|
+
while row and row.parent:
|
348
|
+
row = row.parent
|
349
|
+
return row
|
350
|
+
|
351
|
+
|
352
|
+
@OrgSheet.api
|
353
|
+
def sysopen_row(sheet, row):
|
354
|
+
root = _root(row)
|
355
|
+
if root.path.exists():
|
356
|
+
vd.launchEditor(root.path, '+%s'%row['linenum'])
|
357
|
+
else:
|
358
|
+
orgmode_parse_into(row, vd.launchExternalEditor(root.file_string))
|
359
|
+
|
360
|
+
@VisiData.api
|
361
|
+
def save_org(vd, p, *vsheets):
|
362
|
+
with p.open(mode='w', encoding=vsheets[0].options.save_encoding) as fp:
|
363
|
+
for vs in vsheets:
|
364
|
+
if isinstance(vs, OrgSheet):
|
365
|
+
for row in vs.rows:
|
366
|
+
print(orgmode_to_string(row).strip(), file=fp)
|
367
|
+
else:
|
368
|
+
vd.warning('not implemented')
|
369
|
+
|
370
|
+
|
371
|
+
@OrgSheet.api
|
372
|
+
def sysopen_rows(sheet, rows):
|
373
|
+
ret = ''
|
374
|
+
for r in rows:
|
375
|
+
s = orgmode_to_string(r).strip()
|
376
|
+
ret += '{::%s::}\n%s\n\n' % (id(r), s)
|
377
|
+
|
378
|
+
ret = vd.launchExternalEditor(ret + "{::}")
|
379
|
+
|
380
|
+
idrows = {id(r):r for r in rows}
|
381
|
+
|
382
|
+
idtexts = {}
|
383
|
+
for idtag, text in re.findall(r'\{::(\d+)::\}(.*?)(?={::)', ret, re.DOTALL):
|
384
|
+
idtexts[int(idtag)] = text
|
385
|
+
|
386
|
+
# reparse all existing
|
387
|
+
for rowid, row in idrows.items():
|
388
|
+
text = idtexts.get(rowid, '').strip()
|
389
|
+
if text:
|
390
|
+
orgmode_parse_into(row, text)
|
391
|
+
else:
|
392
|
+
# sometimes, top levels are missing parents
|
393
|
+
# this needs to be debugged
|
394
|
+
if row.parent:
|
395
|
+
row.parent.children.remove(row)
|
396
|
+
|
397
|
+
lastrow = None
|
398
|
+
# find new rows to add
|
399
|
+
for rowid, text in idtexts.items():
|
400
|
+
if rowid not in idrows:
|
401
|
+
section = orgmode_parse(text.splitlines())
|
402
|
+
while section.level <= lastrow.level:
|
403
|
+
lastrow = lastrow.parent
|
404
|
+
|
405
|
+
if lastrow:
|
406
|
+
sourceRows.append(section)
|
407
|
+
else:
|
408
|
+
sheet.addRow(section)
|
409
|
+
|
410
|
+
sheet.refreshRows()
|
411
|
+
|
412
|
+
OrgSheet.addCommand('^O', 'sysopen-row', 'sysopen_row(cursorRow)', 'open current file in external $EDITOR')
|
413
|
+
OrgSheet.addCommand('g^O', 'sysopen-rows', 'sysopen_rows(selectedRows)', 'open selected files in external $EDITOR')
|
414
|
+
OrgSheet.addCommand('^J', 'expand-row', 'openRows([cursorRow]); sheet.cursorRowIndex += 1')
|
415
|
+
OrgSheet.addCommand('z^J', 'close-row', 'closeRows([cursorRow]); sheet.cursorRowIndex += 1')
|
416
|
+
OrgSheet.addCommand('g^J', 'expand-selected', 'openRows(selectedRows)')
|
417
|
+
OrgSheet.addCommand('gz^J', 'close-selected', 'closeRows(selectedRows)')
|
418
|
+
OrgSheet.addCommand('ga', 'combine-selected', 'addRows([combine_rows(selectedRows)], index=cursorRowIndex); cursorDown(1)', 'combine selected rows into new org entry')
|
419
|
+
|
420
|
+
OrgSheet.addCommand('zp', 'paste-data', 'paste_data_into(cursorRow, vd.memory.cliprows, vd.memory.clipcols)', 'move clipboard rows to children of current row')
|
421
|
+
OrgSheet.addCommand('p', 'paste-sections', 'paste_data_into(cursorRow, vd.memory.cliprows, vd.memory.clipcols)', 'move clipboard rows to children of current row')
|
422
|
+
OrgSheet.addCommand('g^S', 'save-all', 'save_all()', 'save all org files')
|
423
|
+
|
424
|
+
|
425
|
+
if __name__ == '__main__':
|
426
|
+
for fn in sys.argv[1:]:
|
427
|
+
sect = orgmode_parse(Path(fn))
|
428
|
+
print(orgmode_to_string(sect).strip())
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from visidata import vd, Sheet, options, Column, asyncthread, Progress, PivotGroupRow, ENTER
|
1
|
+
from visidata import vd, Sheet, options, Column, asyncthread, Progress, PivotGroupRow, ENTER, HistogramColumn
|
2
2
|
|
3
3
|
from visidata.loaders._pandas import PandasSheet
|
4
4
|
from visidata.pivot import PivotSheet
|
@@ -11,8 +11,8 @@ class DataFrameRowSliceAdapter:
|
|
11
11
|
by visidata's selectRow implementation.
|
12
12
|
"""
|
13
13
|
def __init__(self, df, mask):
|
14
|
-
|
15
|
-
|
14
|
+
pd = vd.importExternal('pandas')
|
15
|
+
np = vd.importExternal('numpy')
|
16
16
|
if not isinstance(df, pd.DataFrame):
|
17
17
|
vd.fail('%s is not a dataframe' % type(df).__name__)
|
18
18
|
if not isinstance(mask, pd.Series):
|
@@ -62,15 +62,15 @@ class DataFrameRowSliceIter:
|
|
62
62
|
self.index += 1
|
63
63
|
return row
|
64
64
|
|
65
|
+
def makePandasFreqTable(sheet, *groupByCols):
|
66
|
+
fqcolname = '%s_freq' % '-'.join(col.name for col in groupByCols)
|
67
|
+
return PandasFreqTableSheet(sheet.name, fqcolname, groupByCols=groupByCols, source=sheet)
|
68
|
+
|
69
|
+
|
65
70
|
class PandasFreqTableSheet(PivotSheet):
|
66
71
|
'Generate frequency-table sheet on currently selected column.'
|
67
72
|
rowtype = 'bins' # rowdef FreqRow(keys, sourcerows)
|
68
73
|
|
69
|
-
def __init__(self, sheet, *groupByCols):
|
70
|
-
fqcolname = '%s_%s_freq' % (sheet.name, '-'.join(col.name for col in groupByCols))
|
71
|
-
super().__init__(fqcolname, groupByCols, [], source=sheet)
|
72
|
-
self.largest = 1
|
73
|
-
|
74
74
|
def selectRow(self, row):
|
75
75
|
# Select all entries in the bin on the source sheet.
|
76
76
|
# Use the internally defined _selectByLoc to avoid
|
@@ -86,8 +86,7 @@ class PandasFreqTableSheet(PivotSheet):
|
|
86
86
|
def updateLargest(self, grouprow):
|
87
87
|
self.largest = max(self.largest, len(grouprow.sourcerows))
|
88
88
|
|
89
|
-
|
90
|
-
def reload(self):
|
89
|
+
def loader(self):
|
91
90
|
'Generate frequency table then reverse-sort by length.'
|
92
91
|
import pandas as pd
|
93
92
|
|
@@ -95,8 +94,6 @@ class PandasFreqTableSheet(PivotSheet):
|
|
95
94
|
# (e.g. as a histogram). We currently don't provide support for this
|
96
95
|
# for PandasSheet, although we could implement it with a pd.Grouper
|
97
96
|
# that operates similarly to pd.cut.
|
98
|
-
super().initCols()
|
99
|
-
|
100
97
|
df = self.source.df.copy()
|
101
98
|
|
102
99
|
# Implementation (special case): for one row, this degenerates
|
@@ -141,9 +138,7 @@ class PandasFreqTableSheet(PivotSheet):
|
|
141
138
|
getter=lambda col,row: len(row.sourcerows)),
|
142
139
|
Column('percent', type=float,
|
143
140
|
getter=lambda col,row: len(row.sourcerows)*100/df.shape[0]),
|
144
|
-
|
145
|
-
getter=lambda col,row: options.disp_histogram*(options.disp_histolen*len(row.sourcerows)//value_counts.max()),
|
146
|
-
width=options.disp_histolen+2),
|
141
|
+
HistogramColumn('histogram', type=str, width=self.options.default_width*2)
|
147
142
|
]:
|
148
143
|
self.addColumn(c)
|
149
144
|
|
@@ -174,11 +169,10 @@ def expand_source_rows(sheet, row):
|
|
174
169
|
vd.fail("no source rows")
|
175
170
|
return PandasSheet(sheet.name, vd.valueNames(row.discrete_keys, row.numeric_key), source=row.sourcerows)
|
176
171
|
|
177
|
-
PandasSheet.addCommand('F', 'freq-col', 'vd.push(
|
178
|
-
PandasSheet.addCommand('gF', 'freq-keys', 'vd.push(
|
172
|
+
PandasSheet.addCommand('F', 'freq-col', 'vd.push(makePandasFreqTable(sheet, cursorCol))', 'open Frequency Table grouped on current column, with aggregations of other columns')
|
173
|
+
PandasSheet.addCommand('gF', 'freq-keys', 'vd.push(makePandasFreqTable(sheet, *keyCols))', 'open Frequency Table grouped by all key columns on source sheet, with aggregations of other columns')
|
179
174
|
|
175
|
+
PandasFreqTableSheet.init('largest', lambda: 1)
|
180
176
|
PandasFreqTableSheet.options.numeric_binning = False
|
181
177
|
|
182
|
-
vd.addGlobals(
|
183
|
-
'PandasFreqTableSheet': PandasFreqTableSheet,
|
184
|
-
})
|
178
|
+
vd.addGlobals(makePandasFreqTable=makePandasFreqTable)
|