visidata 3.1.1__py3-none-any.whl → 3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- visidata/__init__.py +2 -2
- visidata/_input.py +106 -58
- visidata/_open.py +10 -7
- visidata/_types.py +2 -2
- visidata/aggregators.py +125 -16
- visidata/apps/vdsql/_ibis.py +8 -13
- visidata/basesheet.py +4 -3
- visidata/canvas.py +11 -7
- visidata/clipboard.py +11 -2
- visidata/cliptext.py +68 -23
- visidata/cmdlog.py +5 -1
- visidata/column.py +48 -33
- visidata/ddwplay.py +2 -2
- visidata/deprecated.py +96 -63
- visidata/errors.py +41 -5
- visidata/{features → experimental}/helloworld.py +1 -1
- visidata/experimental/liveupdate.py +1 -1
- visidata/expr.py +1 -0
- visidata/extensible.py +4 -0
- visidata/features/cmdpalette.py +64 -25
- visidata/features/describe.py +2 -2
- visidata/features/expand_cols.py +7 -5
- visidata/features/freeze.py +14 -2
- visidata/features/go_col.py +3 -3
- visidata/features/graph_zoom_y.py +47 -0
- visidata/features/incr.py +7 -3
- visidata/features/join.py +23 -12
- visidata/features/layout.py +8 -4
- visidata/features/melt.py +1 -0
- visidata/features/rank.py +103 -0
- visidata/features/reload_every.py +11 -8
- visidata/features/sysedit.py +14 -4
- visidata/features/transpose.py +1 -0
- visidata/features/window.py +12 -0
- visidata/form.py +10 -9
- visidata/freqtbl.py +47 -3
- visidata/fuzzymatch.py +11 -7
- visidata/graph.py +5 -3
- visidata/guides/AggregatorsSheet.md +84 -0
- visidata/guides/CommandsSheet.md +1 -0
- visidata/guides/MacrosSheet.md +1 -1
- visidata/guides/RankGuide.md +51 -0
- visidata/guides/TypesSheet.md +1 -1
- visidata/guides/WindowFunctionGuide.md +49 -0
- visidata/help.py +23 -6
- visidata/indexsheet.py +1 -1
- visidata/loaders/_pandas.py +3 -1
- visidata/loaders/archive.py +33 -6
- visidata/loaders/csv.py +12 -1
- visidata/loaders/eml.py +2 -0
- visidata/loaders/f5log.py +2 -2
- visidata/loaders/fec.py +6 -9
- visidata/loaders/fixed_width.py +2 -0
- visidata/loaders/hdf5.py +34 -10
- visidata/loaders/npy.py +54 -23
- visidata/loaders/orgmode.py +3 -2
- visidata/loaders/pandas_freqtbl.py +4 -0
- visidata/loaders/psv.py +13 -0
- visidata/loaders/sqlite.py +1 -1
- visidata/loaders/vds.py +3 -4
- visidata/macros.py +5 -4
- visidata/main.py +21 -11
- visidata/mainloop.py +8 -5
- visidata/man/parse_options.py +3 -2
- visidata/man/vd.1 +38 -17
- visidata/man/vd.txt +47 -17
- visidata/menu.py +10 -10
- visidata/metasheets.py +3 -3
- visidata/mouse.py +3 -0
- visidata/movement.py +6 -3
- visidata/pyobj.py +17 -9
- visidata/save.py +10 -2
- visidata/selection.py +29 -18
- visidata/settings.py +9 -5
- visidata/sheets.py +124 -48
- visidata/shell.py +2 -2
- visidata/sidebar.py +11 -8
- visidata/sort.py +89 -11
- visidata/statusbar.py +10 -9
- visidata/tests/test_cliptext.py +164 -0
- visidata/tests/test_commands.py +6 -2
- visidata/tests/test_menu.py +1 -1
- visidata/textsheet.py +34 -8
- visidata/themes/ascii8.py +2 -2
- visidata/themes/light.py +5 -0
- visidata/threads.py +38 -8
- visidata/utils.py +15 -1
- visidata/vendor/__init__.py +0 -0
- {visidata-3.1.1.data → visidata-3.3.data}/data/share/man/man1/vd.1 +38 -17
- {visidata-3.1.1.data → visidata-3.3.data}/data/share/man/man1/visidata.1 +38 -17
- {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/METADATA +62 -15
- {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/RECORD +98 -92
- {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/WHEEL +1 -1
- {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/entry_points.txt +1 -0
- visidata-3.1.1.data/scripts/vd +0 -6
- {visidata-3.1.1.data → visidata-3.3.data}/data/share/applications/visidata.desktop +0 -0
- {visidata-3.1.1.data → visidata-3.3.data}/scripts/vd2to3.vdx +0 -0
- {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/LICENSE.gpl3 +0 -0
- {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,84 @@
|
|
1
|
+
---
|
2
|
+
sheet: AggregatorSheet
|
3
|
+
---
|
4
|
+
# Aggregations like sum, mean, and distinct
|
5
|
+
|
6
|
+
Aggregators provide summary statistics for grouped rows.
|
7
|
+
|
8
|
+
The current aggregators include:
|
9
|
+
|
10
|
+
min smallest value in the group
|
11
|
+
max largest value in the group
|
12
|
+
avg/mean average value of the group
|
13
|
+
mode most frequently appearing value in group
|
14
|
+
median median value in the group
|
15
|
+
q3/q4/q5/q10 add quantile aggregators to group (e.g. q4 adds p25, p50, p75)
|
16
|
+
sum total summation of all numbers in the group
|
17
|
+
distinct number of distinct values in the group
|
18
|
+
count number of values in the group
|
19
|
+
keymax key of the row with the largest value in the group
|
20
|
+
list gathers values in column into a list
|
21
|
+
stdev standard deviation of values
|
22
|
+
|
23
|
+
## View a one-off aggregation of a column
|
24
|
+
|
25
|
+
- {help.commands.memo-aggregate}
|
26
|
+
|
27
|
+
## Create an aggregator column
|
28
|
+
|
29
|
+
Aggregated columns appear in the **Frequency Table** and **Pivot Table** (grouped sheets). Aggregated values will also appear at the bottom of their columns in the source sheet.
|
30
|
+
|
31
|
+
- {help.commands.aggregate-col}
|
32
|
+
|
33
|
+
Then aggregate the sheet with one of the grouping commands:
|
34
|
+
|
35
|
+
- {help.commands.freq-col}
|
36
|
+
- {help.commands.pivot}
|
37
|
+
|
38
|
+
Aggregators can be viewed and modified on the **Columns Sheet** in the `aggregators` column.
|
39
|
+
|
40
|
+
- {help.commands.columns-sheet}
|
41
|
+
|
42
|
+
## The Describe Sheet
|
43
|
+
|
44
|
+
To get a predefined set of summary statistics for every column in the sheet, use the **Describe Sheet* :
|
45
|
+
|
46
|
+
- {help.commands.describe-all}
|
47
|
+
|
48
|
+
## Examples
|
49
|
+
|
50
|
+
Sample input sheet **sales**:
|
51
|
+
|
52
|
+
date color price
|
53
|
+
---------- ----- -----
|
54
|
+
2024-09-01 R 30
|
55
|
+
2024-09-02 B 28
|
56
|
+
2024-09-03 R 100
|
57
|
+
2024-09-03 B 33
|
58
|
+
2024-09-03 B 99
|
59
|
+
|
60
|
+
1. Move to the `price` column
|
61
|
+
2. Set it to currency: [:keys]$[/key]
|
62
|
+
3. Quickly show average price
|
63
|
+
- [:keys]z+[/] (`memo-aggregate`) then enter 'avg'
|
64
|
+
4. Add an `sum` aggregator column:
|
65
|
+
- Press [:keys]+[/] (`aggregate-column`) then enter 'sum'
|
66
|
+
5. Move to the date column [:keys]gh[/]
|
67
|
+
6. Generate a **Frequency Table** by `date`
|
68
|
+
- [:keys]Shift-F[/] (`freq`)
|
69
|
+
|
70
|
+
|
71
|
+
date count price_sum
|
72
|
+
---------- ----- ---------
|
73
|
+
2024-09-03 3 232.00
|
74
|
+
2024-09-01 1 30.00
|
75
|
+
2024-09-02 1 28.00
|
76
|
+
|
77
|
+
## Creating new aggregator functions
|
78
|
+
|
79
|
+
To add a new aggregator to compute the range of the grouped values (max - min), add the following to `.visidatarc`:
|
80
|
+
|
81
|
+
[:code]vd.aggregator('range', lambda values: max(values) - min(values), 'range of values')[/]
|
82
|
+
|
83
|
+
The `values` parameter is a list of typed values from the column, with the function returning the aggregated value.
|
84
|
+
The new aggregator will now be available the next time VisiData is started.
|
visidata/guides/CommandsSheet.md
CHANGED
@@ -8,6 +8,7 @@ Start typing a command longname or keyword in its helpstring.
|
|
8
8
|
|
9
9
|
- [:code]Enter[/] to execute top command.
|
10
10
|
- [:code]Tab[/] to highlight top command and provide a numeric jumplist.
|
11
|
+
- [:code]PgUp[/]/[:code]PgDn[/] to scroll through commands.
|
11
12
|
|
12
13
|
When a command is highlighted:
|
13
14
|
|
visidata/guides/MacrosSheet.md
CHANGED
@@ -6,7 +6,7 @@ The basic usage is:
|
|
6
6
|
2. Execute a series of commands.
|
7
7
|
3. `m` again to complete the recording, and prompt for the keystroke or longname to bind it to.
|
8
8
|
|
9
|
-
The macro will then be executed
|
9
|
+
The macro will then be executed every time the provided keystroke or longname are used. Note: the Alt+keys and the function keys are left unbound; overriding other keys may conflict with existing bindings, now or in the future.
|
10
10
|
|
11
11
|
Executing a macro will the series of commands starting on the current row and column on the current sheet.
|
12
12
|
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# Ranking
|
2
|
+
|
3
|
+
Ranking assigns numeric ranks to rows based on column values. VisiData provides two ranking approaches: sheet-wide ranking and group-based ranking.
|
4
|
+
|
5
|
+
## Sheet-wide ranking
|
6
|
+
|
7
|
+
[:keys]addcol-sheetrank[/] ranks all rows across the entire sheet.
|
8
|
+
|
9
|
+
Navigate to the column to rank by and execute [:keys]addcol-sheetrank[/]. A new column appears with ranks, where 1 indicates the best value.
|
10
|
+
|
11
|
+
**Example:**
|
12
|
+
```
|
13
|
+
Name | Salary | Salary_rank
|
14
|
+
Alice | 95000 | 1
|
15
|
+
Bob | 85000 | 2
|
16
|
+
Carol | 70000 | 3
|
17
|
+
```
|
18
|
+
|
19
|
+
## Group-based ranking
|
20
|
+
|
21
|
+
[:keys]addcol-aggregate[/] with [:code]rank[/] aggregator ranks rows within groups defined by key columns.
|
22
|
+
|
23
|
+
1. Set key columns with [:keys]![/] (defines groups)
|
24
|
+
2. Navigate to the column to rank by
|
25
|
+
3. Execute [:keys]addcol-aggregate[/]
|
26
|
+
4. Select [:code]rank[/] aggregator
|
27
|
+
|
28
|
+
**Example with Department as key column:**
|
29
|
+
```
|
30
|
+
Name | Department | Salary | Salary_rank
|
31
|
+
Alice | Engineering | 95000 | 1
|
32
|
+
Bob | Engineering | 85000 | 2
|
33
|
+
Carol | Sales | 70000 | 1
|
34
|
+
Dave | Sales | 65000 | 2
|
35
|
+
```
|
36
|
+
|
37
|
+
Alice and Carol both receive rank 1 as the highest earners in their respective departments.
|
38
|
+
|
39
|
+
## Sort direction
|
40
|
+
|
41
|
+
Ranking follows the current sort direction of the column:
|
42
|
+
- Ascending sort: lower values get better ranks
|
43
|
+
- Descending sort: higher values get better ranks
|
44
|
+
|
45
|
+
## Usage patterns
|
46
|
+
|
47
|
+
**Global comparison:** Use [:keys]addcol-sheetrank[/] to find overall leaders across all data.
|
48
|
+
|
49
|
+
**Category comparison:** Use [:keys]addcol-aggregate[/] + [:code]rank[/] to find leaders within each group defined by key columns.
|
50
|
+
|
51
|
+
**Multiple groupings:** Set multiple key columns before group ranking for complex categorization.
|
visidata/guides/TypesSheet.md
CHANGED
@@ -17,7 +17,7 @@ VisiData pre-set defaults for formatting types:
|
|
17
17
|
- `currency` removes non-numeric characters and parses the remainder as `float`.
|
18
18
|
- `date` parses dates into date object (shown as ISO8601).
|
19
19
|
- `vlen` formats the cell value to the length of the content
|
20
|
-
- `float` uses the decimal
|
20
|
+
- `float` uses the decimal separator, keeping two significant digits.
|
21
21
|
|
22
22
|
Change float precision with:
|
23
23
|
- {help.commands.setcol-precision-less}
|
@@ -0,0 +1,49 @@
|
|
1
|
+
---
|
2
|
+
sheet: Sheet
|
3
|
+
---
|
4
|
+
# Create a window over consecutive rows
|
5
|
+
|
6
|
+
Window functions enable computations that relate the current window to surrounding rows, like cumulative sum, rolling averages or lead/lag computations.
|
7
|
+
|
8
|
+
{help.commands.addcol-window}
|
9
|
+
|
10
|
+
With large window sizes, [:code]g'[/] (`freeze-sheet`) to calculate all cells and copy the entire sheet into a new source sheet, which will conserve CPU.
|
11
|
+
|
12
|
+
## Examples
|
13
|
+
|
14
|
+
date color price
|
15
|
+
---------- ----- -----
|
16
|
+
2024-09-01 R 30
|
17
|
+
2024-09-02 B 28
|
18
|
+
2024-09-03 R 100
|
19
|
+
2024-09-03 B 33
|
20
|
+
2024-09-03 B 99
|
21
|
+
|
22
|
+
|
23
|
+
1. [:keys]#[/] (`type-int`) on the **price** column to type as int.
|
24
|
+
2. [:keys]w[/] (`addcol-window`) on the **price** column, followed by `1 2`, to create a window consisting of 4 rows: 1 row before the current row, and 2 rows after.
|
25
|
+
3. To create a moving average of the values in the window, add a new column with a python expression: [:keys]=[/] (`addcol-expr`)
|
26
|
+
followed by `sum(price_window)/len(price_window)`
|
27
|
+
|
28
|
+
date color price price_window sum(price_window)/len(price_window)
|
29
|
+
---------- ----- ----- ------------------- -----------------------------------
|
30
|
+
2024-09-01 R 38 [4] ; 38; 28; 100 41.5
|
31
|
+
2024-09-02 B 28 [4] 38; 28; 100; 33 49.75
|
32
|
+
2024-09-03 R 100 [4] 28; 100; 33; 99 65.0
|
33
|
+
2024-09-03 B 33 [4] 100; 33; 99; 58.0
|
34
|
+
2024-09-03 B 99 [4] 33; 99; ; 33.0
|
35
|
+
|
36
|
+
|
37
|
+
## Workflows
|
38
|
+
|
39
|
+
### Create a cumulative sum
|
40
|
+
|
41
|
+
1. Set the before window size to the total number of rows in the table, and the after rows to 0. In the above example that would be `w 5 0` (`addcol-window`).
|
42
|
+
2. Add an expression ([:keys]=[/] (`addcol-expr`) of `sum(window)` where `window` is the name of the window function column.
|
43
|
+
|
44
|
+
### Compute the change between rows
|
45
|
+
|
46
|
+
1. `w 1 0` on the `foo` column to create a window function of size 1 before and 0 after.
|
47
|
+
2. Add a python expression. The window function column is 'foo_window':
|
48
|
+
`=foo_window[1] - foo_window[0] if len(foo_window) > 1 else None`
|
49
|
+
|
visidata/help.py
CHANGED
@@ -4,7 +4,24 @@ import collections
|
|
4
4
|
from visidata import VisiData, MetaSheet, ColumnAttr, Column, BaseSheet, VisiDataMetaSheet, SuspendCurses
|
5
5
|
from visidata import vd, asyncthread, ENTER, drawcache, AttrDict, TextSheet
|
6
6
|
|
7
|
-
|
7
|
+
|
8
|
+
vd.option('disp_help_flags', 'cmdpalette guides help hints inputfield inputkeys nometacols sidebar',
|
9
|
+
'''list of helper features to enable (space-separated):
|
10
|
+
- "cmdpalette": exec-longname suggestions
|
11
|
+
- "guides": guides in sidebar
|
12
|
+
- "help": help sidebar collapsed by default
|
13
|
+
- "hints": context-sensitive hints on menu line
|
14
|
+
- "inputfield": context-sensitive help for each input field
|
15
|
+
- "inputkeys": input quick reference in sidebar
|
16
|
+
- "nometacols": hide expert columns on metasheets
|
17
|
+
- "sidebar": context-sensitive sheet help in sidebar
|
18
|
+
- "all": enable all helper features''')
|
19
|
+
|
20
|
+
|
21
|
+
@VisiData.api
|
22
|
+
def wantsHelp(vd, feat):
|
23
|
+
return feat in vd.options.disp_help_flags or 'all' in vd.options.disp_help_flags
|
24
|
+
|
8
25
|
|
9
26
|
@BaseSheet.api
|
10
27
|
def hint_basichelp(sheet):
|
@@ -99,12 +116,13 @@ class HelpPane:
|
|
99
116
|
|
100
117
|
def draw(self, scr, x=None, y=None, **kwargs):
|
101
118
|
if not scr: return
|
102
|
-
# if vd.
|
119
|
+
# if not vd.wantsHelp('statushelp'):
|
103
120
|
# if self.scr:
|
104
121
|
# self.scr.erase()
|
105
122
|
# self.scr.refresh()
|
106
123
|
# self.scr = None
|
107
124
|
# return
|
125
|
+
|
108
126
|
if y is None: y=0 # show at top of screen by default
|
109
127
|
if x is None: x=0
|
110
128
|
hneeded = self.amgr.maxHeight+3
|
@@ -135,7 +153,7 @@ class HelpPane:
|
|
135
153
|
self.scr.erase()
|
136
154
|
self.scr.box()
|
137
155
|
self.amgr.draw(self.scr, y=1, x=2, **kwargs)
|
138
|
-
self.scr.
|
156
|
+
self.scr.noutrefresh()
|
139
157
|
|
140
158
|
|
141
159
|
@VisiData.api
|
@@ -176,13 +194,12 @@ BaseSheet.bindkey('gKEY_BACKSPACE', 'sysopen-help')
|
|
176
194
|
HelpSheet.addCommand(None, 'exec-command', 'quit(sheet); draw_all(); activeStack[0].execCommand(cursorRow.longname)', 'execute command on undersheet')
|
177
195
|
BaseSheet.addCommand(None, 'open-tutorial-visidata', 'launchBrowser("https://jsvine.github.io/intro-to-visidata/")', 'open https://jsvine.github.io/intro-to-visidata/')
|
178
196
|
|
179
|
-
vd.addMenuItem("Help", "VisiData tutorial", 'open-tutorial-visidata')
|
180
|
-
vd.addMenuItem("Help", 'Sheet commands', 'help-commands')
|
181
|
-
vd.addMenuItem("Help", 'All commands', 'help-commands-all')
|
182
197
|
|
183
198
|
vd.addGlobals(HelpSheet=HelpSheet)
|
184
199
|
|
185
200
|
vd.addMenuItems('''
|
201
|
+
Help > VisiData tutorial > open-tutorial-visidata
|
202
|
+
Help > All commands > help-commands-all
|
186
203
|
Help > Quick reference > sysopen-help
|
187
204
|
Help > Command list > help-commands
|
188
205
|
''')
|
visidata/indexsheet.py
CHANGED
@@ -99,7 +99,7 @@ BaseSheet.addCommand('g<', 'open-source-prev', 'vd.replace(openSource(source.nex
|
|
99
99
|
IndexSheet.addCommand('g^R', 'reload-selected', 'reloadSheets(selectedRows or rows)', 'reload all selected sheets')
|
100
100
|
|
101
101
|
# when diving into a sheet, remove the index unless it is precious
|
102
|
-
|
102
|
+
IndexSheet.addCommand('gC', 'columns-selected', 'vd.push(ColumnsSheet("all_columns", source=selectedRows))', 'open Columns Sheet with all visible columns from selected sheets')
|
103
103
|
IndexSheet.addCommand('^C', 'cancel-row', 'cancelThread(*cursorRow.currentThreads)', 'abort async thread for current sheet')
|
104
104
|
IndexSheet.addCommand('gz^C', 'cancel-rows', 'for vs in selectedRows: cancelThread(*vs.currentThreads)', 'abort async threads for selected sheets')
|
105
105
|
SheetsSheet.addCommand('Enter', 'open-row', 'dest=cursorRow; vd.sheets.remove(sheet) if not sheet.precious else None; vd.push(openRow(dest))', 'open sheet referenced in current row')
|
visidata/loaders/_pandas.py
CHANGED
@@ -28,7 +28,7 @@ def save_dta(vd, p, *sheets):
|
|
28
28
|
vs = sheets[0]
|
29
29
|
|
30
30
|
columns = [col.name for col in vs.visibleCols]
|
31
|
-
|
31
|
+
|
32
32
|
# Get data types
|
33
33
|
types = list()
|
34
34
|
dispvals = next(vs.iterdispvals(format=True))
|
@@ -154,6 +154,8 @@ class PandasSheet(Sheet):
|
|
154
154
|
readfunc = self.read_tsv
|
155
155
|
elif filetype == 'jsonl':
|
156
156
|
readfunc = partial(pd.read_json, lines=True)
|
157
|
+
elif filetype == 'hdf5':
|
158
|
+
readfunc = partial(pd.read_hdf, lines=True)
|
157
159
|
else:
|
158
160
|
readfunc = getattr(pd, 'read_'+filetype) or vd.error('no pandas.read_'+filetype)
|
159
161
|
# readfunc() handles binary and text open()
|
visidata/loaders/archive.py
CHANGED
@@ -2,21 +2,25 @@ import pathlib
|
|
2
2
|
import tarfile
|
3
3
|
import zipfile
|
4
4
|
import datetime
|
5
|
+
import os.path
|
5
6
|
from visidata.loaders import unzip_http
|
6
7
|
|
7
8
|
from visidata import vd, VisiData, asyncthread, Sheet, Progress, Menu, options
|
8
|
-
from visidata import ColumnAttr, Column, Path
|
9
|
+
from visidata import ColumnAttr, Column, Path, filesize
|
9
10
|
from visidata.type_date import date
|
10
11
|
|
11
12
|
@VisiData.api
|
12
13
|
def guess_zip(vd, p):
|
13
14
|
if not p.is_url() and zipfile.is_zipfile(p.open_bytes()):
|
14
|
-
return dict(filetype='zip')
|
15
|
+
return dict(filetype='zip', _likelihood=10)
|
15
16
|
|
16
17
|
@VisiData.api
|
17
18
|
def guess_tar(vd, p):
|
19
|
+
# an empty file will pass is_tarfile(), but can't be opened by tarfile.open()
|
20
|
+
if filesize(p) == 0:
|
21
|
+
return None
|
18
22
|
if tarfile.is_tarfile(p.open_bytes()):
|
19
|
-
return dict(filetype='tar')
|
23
|
+
return dict(filetype='tar', _likelihood=10)
|
20
24
|
|
21
25
|
@VisiData.api
|
22
26
|
def open_zip(vd, p):
|
@@ -81,13 +85,13 @@ Commands:
|
|
81
85
|
return vd.openSource(Path(fi.filename, fp=fp, filesize=fi.file_size), filetype=options.filetype)
|
82
86
|
|
83
87
|
def extract(self, *rows, path=None):
|
84
|
-
path = path or
|
88
|
+
path = path or Path('.')
|
85
89
|
|
86
90
|
files = []
|
87
91
|
for row in rows:
|
88
92
|
r, _ = row
|
89
93
|
vd.confirmOverwrite(path/r.filename) #1452
|
90
|
-
self.extract_async(row)
|
94
|
+
self.extract_async(row, path=path)
|
91
95
|
|
92
96
|
def sysopen_row(self, row):
|
93
97
|
'Extract file in row to tempdir and launch $EDITOR. Modifications will be discarded.'
|
@@ -109,6 +113,12 @@ Commands:
|
|
109
113
|
if '://' in str(self.source):
|
110
114
|
unzip_http.warning = vd.warning
|
111
115
|
self._zfp = unzip_http.RemoteZipFile(str(self.source))
|
116
|
+
elif isinstance(self.source, Path):
|
117
|
+
if self.source.has_fp(): #when opening a zip inside tar or zip
|
118
|
+
fp = self.source.open('rb')
|
119
|
+
else:
|
120
|
+
fp = self.source
|
121
|
+
self._zfp = zipfile.ZipFile(fp, 'r')
|
112
122
|
else:
|
113
123
|
self._zfp = zipfile.ZipFile(str(self.source), 'r')
|
114
124
|
|
@@ -119,18 +129,35 @@ Commands:
|
|
119
129
|
yield [zi, Path(zi.filename)]
|
120
130
|
|
121
131
|
|
132
|
+
#from https://docs.python.org/3/library/tarfile.html#tarfile.REGTYPE
|
133
|
+
tarfile_type_names = {
|
134
|
+
tarfile.REGTYPE:"file",
|
135
|
+
tarfile.AREGTYPE:"file",
|
136
|
+
tarfile.LNKTYPE:"hard link",
|
137
|
+
tarfile.SYMTYPE:"symbolic link",
|
138
|
+
tarfile.CHRTYPE:"character device",
|
139
|
+
tarfile.BLKTYPE:"block device",
|
140
|
+
tarfile.DIRTYPE:"directory",
|
141
|
+
tarfile.FIFOTYPE:"FIFO",
|
142
|
+
tarfile.CONTTYPE:"contiguous file",
|
143
|
+
tarfile.GNUTYPE_LONGNAME:"GNU tar longname",
|
144
|
+
tarfile.GNUTYPE_LONGLINK:"GNU tar longlink",
|
145
|
+
tarfile.GNUTYPE_SPARSE:"GNU tar sparse file",
|
146
|
+
}
|
122
147
|
class TarSheet(Sheet):
|
123
148
|
'Wrapper for `tarfile` library.'
|
124
149
|
rowtype = 'files' # rowdef TarInfo
|
125
150
|
columns = [
|
126
151
|
ColumnAttr('name'),
|
152
|
+
Column('ext', getter=lambda col,row: row.isdir() and '/' or os.path.splitext(row.name)[1][1:]),
|
127
153
|
ColumnAttr('size', type=int),
|
128
154
|
ColumnAttr('mtime', type=date),
|
129
|
-
|
155
|
+
Column('type', getter=lambda col, row: tarfile_type_names.get(row.type, 'unknown')),
|
130
156
|
ColumnAttr('mode', type=int),
|
131
157
|
ColumnAttr('uname'),
|
132
158
|
ColumnAttr('gname')
|
133
159
|
]
|
160
|
+
nKeys=1
|
134
161
|
|
135
162
|
def openRow(self, fi):
|
136
163
|
tfp = tarfile.open(name=str(self.source))
|
visidata/loaders/csv.py
CHANGED
@@ -12,6 +12,13 @@ vd.option('csv_lineterminator', '\r\n', 'lineterminator passed to csv.writer', r
|
|
12
12
|
vd.option('safety_first', False, 'sanitize input/output to handle edge cases, with a performance cost', replay=True)
|
13
13
|
|
14
14
|
|
15
|
+
@VisiData.api
|
16
|
+
def guess_csv_delimiter(vd, p):
|
17
|
+
'If csv_delimiter option has been modified from default, assume CSV format.'
|
18
|
+
|
19
|
+
if vd.options.csv_delimiter != vd.options.getdefault('csv_delimiter'):
|
20
|
+
return dict(filetype='csv', _likelihood=2)
|
21
|
+
|
15
22
|
@VisiData.api
|
16
23
|
def guess_csv(vd, p):
|
17
24
|
import csv
|
@@ -26,7 +33,11 @@ def guess_csv(vd, p):
|
|
26
33
|
|
27
34
|
for csvopt in dir(dialect):
|
28
35
|
if not csvopt.startswith('_'):
|
29
|
-
|
36
|
+
v = getattr(dialect, csvopt)
|
37
|
+
optname = 'csv_'+csvopt
|
38
|
+
r[optname] = v
|
39
|
+
if vd.options.get(optname) != v:
|
40
|
+
vd.warning(f'guessed option {optname}={v}')
|
30
41
|
|
31
42
|
return r
|
32
43
|
|
visidata/loaders/eml.py
CHANGED
visidata/loaders/f5log.py
CHANGED
@@ -14,7 +14,7 @@ Regex: (?:/Common/)(?P<site>[^-]+)-(?P<vstype>[^-]+)-(?P<application>[^-]+)
|
|
14
14
|
|
15
15
|
/Common/newyork-www-banking1
|
16
16
|
|
17
|
-
... | site | vstype |
|
17
|
+
... | site | vstype | application | ...
|
18
18
|
... | newyork | www | banking1 | ...
|
19
19
|
|
20
20
|
Adding to .visidatarc
|
@@ -313,7 +313,7 @@ class F5LogSheet(Sheet):
|
|
313
313
|
cmd_data = msg[cmd_data_loc + 1 :]
|
314
314
|
# split the message and the command
|
315
315
|
msg, cmd = msg[:cmd_data_loc].rsplit(" ", maxsplit=1)
|
316
|
-
# strip off the
|
316
|
+
# strip off the trailing " -" from the message
|
317
317
|
msg = msg[:-2]
|
318
318
|
object = cmd_data.split('"', maxsplit=2)
|
319
319
|
if len(object) == 3:
|
visidata/loaders/fec.py
CHANGED
@@ -106,22 +106,19 @@ class DiveSheet(Sheet):
|
|
106
106
|
self.addRow(item)
|
107
107
|
|
108
108
|
except Exception as e:
|
109
|
-
vd.warning("Can't dive on lists with
|
109
|
+
vd.warning("Can't dive on lists with heterogeneous item types.")
|
110
110
|
return False
|
111
111
|
|
112
112
|
def openRow(self, row):
|
113
113
|
if self.is_keyvalue:
|
114
114
|
cell = row["value"]
|
115
|
-
name = vd.joinSheetnames(self.name, row["key"])
|
116
|
-
|
117
115
|
if isinstance(cell, (list, dict)):
|
118
|
-
vs = self.__class__(name, source = cell)
|
116
|
+
vs = self.__class__(self.name, row["key"], source = cell)
|
119
117
|
else:
|
120
118
|
vd.warning("Nothing to dive into.")
|
121
119
|
return
|
122
120
|
else:
|
123
|
-
|
124
|
-
vs = self.__class__(name, source = self.row)
|
121
|
+
vs = self.__class__(self.name, "row", source = self.row)
|
125
122
|
|
126
123
|
success = vs.reload()
|
127
124
|
if success == False:
|
@@ -174,7 +171,7 @@ class FECScheduleSheet(Sheet):
|
|
174
171
|
|
175
172
|
for schedule_name in self.source.keys():
|
176
173
|
vs = FECItemizationSheet(
|
177
|
-
|
174
|
+
self.name, schedule_name,
|
178
175
|
schedule_name = schedule_name,
|
179
176
|
source = self.source[schedule_name],
|
180
177
|
size = len(self.source[schedule_name]),
|
@@ -225,7 +222,7 @@ class FECFiling(Sheet):
|
|
225
222
|
] else dict
|
226
223
|
|
227
224
|
vs = cls(
|
228
|
-
|
225
|
+
self.name, component_name,
|
229
226
|
component_name = component_name,
|
230
227
|
source = source_cls(),
|
231
228
|
size = 0,
|
@@ -270,7 +267,7 @@ class FECFiling(Sheet):
|
|
270
267
|
if form_type not in sheet_row.source:
|
271
268
|
sheet_row.source[form_type] = [ ]
|
272
269
|
subsheet = FECItemizationSheet(
|
273
|
-
|
270
|
+
sheet_row.name, form_type,
|
274
271
|
schedule_name = form_type,
|
275
272
|
source = [ ],
|
276
273
|
size = 0,
|
visidata/loaders/fixed_width.py
CHANGED
@@ -110,3 +110,5 @@ def save_fixed(vd, p, *vsheets):
|
|
110
110
|
for col, val in dispvals.items():
|
111
111
|
fp.write(('{0:%s%s.%s} ' % ('>' if vd.isNumeric(col) else '<', widths[col], widths[col])).format(val))
|
112
112
|
fp.write('\n')
|
113
|
+
|
114
|
+
FixedWidthColumnsSheet.options.null_value = '' # the file format cannot contain None, so use empty string instead
|
visidata/loaders/hdf5.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
from visidata import VisiData, vd, Sheet, Path, Column, ItemColumn, BaseSheet
|
1
|
+
from visidata import VisiData, vd, Sheet, Path, Column, ItemColumn, BaseSheet, anytype
|
2
|
+
from itertools import chain
|
2
3
|
|
3
4
|
@VisiData.api
|
4
5
|
def open_h5(vd, p):
|
@@ -6,8 +7,11 @@ def open_h5(vd, p):
|
|
6
7
|
|
7
8
|
VisiData.open_hdf5 = VisiData.open_h5
|
8
9
|
|
10
|
+
vd.option('hdf5_matrix_enumerate', False, 'enumerate matrix rows and columns')
|
11
|
+
|
9
12
|
class Hdf5ObjSheet(Sheet):
|
10
13
|
'Support sheets in HDF5 format.'
|
14
|
+
|
11
15
|
def iterload(self):
|
12
16
|
h5py = vd.importExternal('h5py')
|
13
17
|
source = self.source
|
@@ -26,25 +30,39 @@ class Hdf5ObjSheet(Sheet):
|
|
26
30
|
for k, v in source.items():
|
27
31
|
yield Hdf5ObjSheet(self.name, k, source=v)
|
28
32
|
elif isinstance(source, h5py.Dataset):
|
29
|
-
if len(source.shape)
|
33
|
+
if len(source.shape)==1:
|
30
34
|
if source.dtype.names:
|
31
|
-
for i, colname in enumerate(source.dtype.
|
32
|
-
|
35
|
+
for i, (colname, fmt, *_) in enumerate(source.dtype.descr):
|
36
|
+
if not colname:
|
37
|
+
colname = f"col{i}"
|
38
|
+
ctype = _guess_type(fmt)
|
39
|
+
self.addColumn(ItemColumn(colname, i, type=ctype))
|
33
40
|
yield from source # copy
|
34
41
|
else:
|
35
42
|
self.addColumn(ItemColumn(source.name, 0))
|
36
43
|
for v in source:
|
37
44
|
yield [v]
|
38
|
-
elif len(source.shape)
|
45
|
+
elif len(source.shape)==2:
|
46
|
+
matrix_enumerate = bool(self.options.hdf5_matrix_enumerate)
|
47
|
+
|
39
48
|
ncols = source.shape[1]
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
49
|
+
ctype = _guess_type(source.dtype.descr[0][1])
|
50
|
+
|
51
|
+
if matrix_enumerate:
|
52
|
+
self.addColumn(ItemColumn("row", 0, width=8, keycol=1, type=int), index=0)
|
53
|
+
for i in range(ncols):
|
54
|
+
self.addColumn(ItemColumn(f'col{i}', i+1, width=8, type=ctype), index=i+1)
|
55
|
+
self.recalc()
|
56
|
+
yield from list(list((chain((i,), row))) for i, row in enumerate(source))
|
57
|
+
else:
|
58
|
+
for i in range(ncols):
|
59
|
+
self.addColumn(ItemColumn('', i, width=8, type=ctype), index=i)
|
60
|
+
self.recalc()
|
61
|
+
yield from source # copy
|
44
62
|
else:
|
45
63
|
vd.fail('too many dimensions in shape %s' % str(source.shape))
|
46
64
|
else:
|
47
|
-
vd.fail(
|
65
|
+
vd.fail(f"too many dimensions in shape {source.shape}")
|
48
66
|
|
49
67
|
|
50
68
|
def openRow(self, row):
|
@@ -59,5 +77,11 @@ class Hdf5ObjSheet(Sheet):
|
|
59
77
|
if isinstance(row, numpy.ndarray):
|
60
78
|
return NpySheet(None, npy=row)
|
61
79
|
|
80
|
+
def _guess_type(fmt):
|
81
|
+
if 'i' in fmt or 'u' in fmt:
|
82
|
+
return int
|
83
|
+
elif 'f' in fmt:
|
84
|
+
return float
|
85
|
+
return anytype
|
62
86
|
|
63
87
|
Hdf5ObjSheet.addCommand('A', 'dive-metadata', 'vd.push(SheetDict(cursorRow.name + "_attrs", source=cursorRow.attrs))', 'open metadata sheet for object referenced in current row')
|