visidata 3.1.1__py3-none-any.whl → 3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. visidata/__init__.py +2 -2
  2. visidata/_input.py +106 -58
  3. visidata/_open.py +10 -7
  4. visidata/_types.py +2 -2
  5. visidata/aggregators.py +125 -16
  6. visidata/apps/vdsql/_ibis.py +8 -13
  7. visidata/basesheet.py +4 -3
  8. visidata/canvas.py +11 -7
  9. visidata/clipboard.py +11 -2
  10. visidata/cliptext.py +68 -23
  11. visidata/cmdlog.py +5 -1
  12. visidata/column.py +48 -33
  13. visidata/ddwplay.py +2 -2
  14. visidata/deprecated.py +96 -63
  15. visidata/errors.py +41 -5
  16. visidata/{features → experimental}/helloworld.py +1 -1
  17. visidata/experimental/liveupdate.py +1 -1
  18. visidata/expr.py +1 -0
  19. visidata/extensible.py +4 -0
  20. visidata/features/cmdpalette.py +64 -25
  21. visidata/features/describe.py +2 -2
  22. visidata/features/expand_cols.py +7 -5
  23. visidata/features/freeze.py +14 -2
  24. visidata/features/go_col.py +3 -3
  25. visidata/features/graph_zoom_y.py +47 -0
  26. visidata/features/incr.py +7 -3
  27. visidata/features/join.py +23 -12
  28. visidata/features/layout.py +8 -4
  29. visidata/features/melt.py +1 -0
  30. visidata/features/rank.py +103 -0
  31. visidata/features/reload_every.py +11 -8
  32. visidata/features/sysedit.py +14 -4
  33. visidata/features/transpose.py +1 -0
  34. visidata/features/window.py +12 -0
  35. visidata/form.py +10 -9
  36. visidata/freqtbl.py +47 -3
  37. visidata/fuzzymatch.py +11 -7
  38. visidata/graph.py +5 -3
  39. visidata/guides/AggregatorsSheet.md +84 -0
  40. visidata/guides/CommandsSheet.md +1 -0
  41. visidata/guides/MacrosSheet.md +1 -1
  42. visidata/guides/RankGuide.md +51 -0
  43. visidata/guides/TypesSheet.md +1 -1
  44. visidata/guides/WindowFunctionGuide.md +49 -0
  45. visidata/help.py +23 -6
  46. visidata/indexsheet.py +1 -1
  47. visidata/loaders/_pandas.py +3 -1
  48. visidata/loaders/archive.py +33 -6
  49. visidata/loaders/csv.py +12 -1
  50. visidata/loaders/eml.py +2 -0
  51. visidata/loaders/f5log.py +2 -2
  52. visidata/loaders/fec.py +6 -9
  53. visidata/loaders/fixed_width.py +2 -0
  54. visidata/loaders/hdf5.py +34 -10
  55. visidata/loaders/npy.py +54 -23
  56. visidata/loaders/orgmode.py +3 -2
  57. visidata/loaders/pandas_freqtbl.py +4 -0
  58. visidata/loaders/psv.py +13 -0
  59. visidata/loaders/sqlite.py +1 -1
  60. visidata/loaders/vds.py +3 -4
  61. visidata/macros.py +5 -4
  62. visidata/main.py +21 -11
  63. visidata/mainloop.py +8 -5
  64. visidata/man/parse_options.py +3 -2
  65. visidata/man/vd.1 +38 -17
  66. visidata/man/vd.txt +47 -17
  67. visidata/menu.py +10 -10
  68. visidata/metasheets.py +3 -3
  69. visidata/mouse.py +3 -0
  70. visidata/movement.py +6 -3
  71. visidata/pyobj.py +17 -9
  72. visidata/save.py +10 -2
  73. visidata/selection.py +29 -18
  74. visidata/settings.py +9 -5
  75. visidata/sheets.py +124 -48
  76. visidata/shell.py +2 -2
  77. visidata/sidebar.py +11 -8
  78. visidata/sort.py +89 -11
  79. visidata/statusbar.py +10 -9
  80. visidata/tests/test_cliptext.py +164 -0
  81. visidata/tests/test_commands.py +6 -2
  82. visidata/tests/test_menu.py +1 -1
  83. visidata/textsheet.py +34 -8
  84. visidata/themes/ascii8.py +2 -2
  85. visidata/themes/light.py +5 -0
  86. visidata/threads.py +38 -8
  87. visidata/utils.py +15 -1
  88. visidata/vendor/__init__.py +0 -0
  89. {visidata-3.1.1.data → visidata-3.3.data}/data/share/man/man1/vd.1 +38 -17
  90. {visidata-3.1.1.data → visidata-3.3.data}/data/share/man/man1/visidata.1 +38 -17
  91. {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/METADATA +62 -15
  92. {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/RECORD +98 -92
  93. {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/WHEEL +1 -1
  94. {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/entry_points.txt +1 -0
  95. visidata-3.1.1.data/scripts/vd +0 -6
  96. {visidata-3.1.1.data → visidata-3.3.data}/data/share/applications/visidata.desktop +0 -0
  97. {visidata-3.1.1.data → visidata-3.3.data}/scripts/vd2to3.vdx +0 -0
  98. {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/LICENSE.gpl3 +0 -0
  99. {visidata-3.1.1.dist-info → visidata-3.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,84 @@
1
+ ---
2
+ sheet: AggregatorSheet
3
+ ---
4
+ # Aggregations like sum, mean, and distinct
5
+
6
+ Aggregators provide summary statistics for grouped rows.
7
+
8
+ The current aggregators include:
9
+
10
+ min smallest value in the group
11
+ max largest value in the group
12
+ avg/mean average value of the group
13
+ mode most frequently appearing value in group
14
+ median median value in the group
15
+ q3/q4/q5/q10 add quantile aggregators to group (e.g. q4 adds p25, p50, p75)
16
+ sum total summation of all numbers in the group
17
+ distinct number of distinct values in the group
18
+ count number of values in the group
19
+ keymax key of the row with the largest value in the group
20
+ list gathers values in column into a list
21
+ stdev standard deviation of values
22
+
23
+ ## View a one-off aggregation of a column
24
+
25
+ - {help.commands.memo-aggregate}
26
+
27
+ ## Create an aggregator column
28
+
29
+ Aggregated columns appear in the **Frequency Table** and **Pivot Table** (grouped sheets). Aggregated values will also appear at the bottom of their columns in the source sheet.
30
+
31
+ - {help.commands.aggregate-col}
32
+
33
+ Then aggregate the sheet with one of the grouping commands:
34
+
35
+ - {help.commands.freq-col}
36
+ - {help.commands.pivot}
37
+
38
+ Aggregators can be viewed and modified on the **Columns Sheet** in the `aggregators` column.
39
+
40
+ - {help.commands.columns-sheet}
41
+
42
+ ## The Describe Sheet
43
+
44
+ To get a predefined set of summary statistics for every column in the sheet, use the **Describe Sheet* :
45
+
46
+ - {help.commands.describe-all}
47
+
48
+ ## Examples
49
+
50
+ Sample input sheet **sales**:
51
+
52
+ date color price
53
+ ---------- ----- -----
54
+ 2024-09-01 R 30
55
+ 2024-09-02 B 28
56
+ 2024-09-03 R 100
57
+ 2024-09-03 B 33
58
+ 2024-09-03 B 99
59
+
60
+ 1. Move to the `price` column
61
+ 2. Set it to currency: [:keys]$[/key]
62
+ 3. Quickly show average price
63
+ - [:keys]z+[/] (`memo-aggregate`) then enter 'avg'
64
+ 4. Add an `sum` aggregator column:
65
+ - Press [:keys]+[/] (`aggregate-column`) then enter 'sum'
66
+ 5. Move to the date column [:keys]gh[/]
67
+ 6. Generate a **Frequency Table** by `date`
68
+ - [:keys]Shift-F[/] (`freq`)
69
+
70
+
71
+ date count price_sum
72
+ ---------- ----- ---------
73
+ 2024-09-03 3 232.00
74
+ 2024-09-01 1 30.00
75
+ 2024-09-02 1 28.00
76
+
77
+ ## Creating new aggregator functions
78
+
79
+ To add a new aggregator to compute the range of the grouped values (max - min), add the following to `.visidatarc`:
80
+
81
+ [:code]vd.aggregator('range', lambda values: max(values) - min(values), 'range of values')[/]
82
+
83
+ The `values` parameter is a list of typed values from the column, with the function returning the aggregated value.
84
+ The new aggregator will now be available the next time VisiData is started.
@@ -8,6 +8,7 @@ Start typing a command longname or keyword in its helpstring.
8
8
 
9
9
  - [:code]Enter[/] to execute top command.
10
10
  - [:code]Tab[/] to highlight top command and provide a numeric jumplist.
11
+ - [:code]PgUp[/]/[:code]PgDn[/] to scroll through commands.
11
12
 
12
13
  When a command is highlighted:
13
14
 
@@ -6,7 +6,7 @@ The basic usage is:
6
6
  2. Execute a series of commands.
7
7
  3. `m` again to complete the recording, and prompt for the keystroke or longname to bind it to.
8
8
 
9
- The macro will then be executed everytime the provided keystroke or longname are used. Note: the Alt+keys and the function keys are left unbound; overriding other keys may conflict with existing bindings, now or in the future.
9
+ The macro will then be executed every time the provided keystroke or longname are used. Note: the Alt+keys and the function keys are left unbound; overriding other keys may conflict with existing bindings, now or in the future.
10
10
 
11
11
  Executing a macro will the series of commands starting on the current row and column on the current sheet.
12
12
 
@@ -0,0 +1,51 @@
1
+ # Ranking
2
+
3
+ Ranking assigns numeric ranks to rows based on column values. VisiData provides two ranking approaches: sheet-wide ranking and group-based ranking.
4
+
5
+ ## Sheet-wide ranking
6
+
7
+ [:keys]addcol-sheetrank[/] ranks all rows across the entire sheet.
8
+
9
+ Navigate to the column to rank by and execute [:keys]addcol-sheetrank[/]. A new column appears with ranks, where 1 indicates the best value.
10
+
11
+ **Example:**
12
+ ```
13
+ Name | Salary | Salary_rank
14
+ Alice | 95000 | 1
15
+ Bob | 85000 | 2
16
+ Carol | 70000 | 3
17
+ ```
18
+
19
+ ## Group-based ranking
20
+
21
+ [:keys]addcol-aggregate[/] with [:code]rank[/] aggregator ranks rows within groups defined by key columns.
22
+
23
+ 1. Set key columns with [:keys]![/] (defines groups)
24
+ 2. Navigate to the column to rank by
25
+ 3. Execute [:keys]addcol-aggregate[/]
26
+ 4. Select [:code]rank[/] aggregator
27
+
28
+ **Example with Department as key column:**
29
+ ```
30
+ Name | Department | Salary | Salary_rank
31
+ Alice | Engineering | 95000 | 1
32
+ Bob | Engineering | 85000 | 2
33
+ Carol | Sales | 70000 | 1
34
+ Dave | Sales | 65000 | 2
35
+ ```
36
+
37
+ Alice and Carol both receive rank 1 as the highest earners in their respective departments.
38
+
39
+ ## Sort direction
40
+
41
+ Ranking follows the current sort direction of the column:
42
+ - Ascending sort: lower values get better ranks
43
+ - Descending sort: higher values get better ranks
44
+
45
+ ## Usage patterns
46
+
47
+ **Global comparison:** Use [:keys]addcol-sheetrank[/] to find overall leaders across all data.
48
+
49
+ **Category comparison:** Use [:keys]addcol-aggregate[/] + [:code]rank[/] to find leaders within each group defined by key columns.
50
+
51
+ **Multiple groupings:** Set multiple key columns before group ranking for complex categorization.
@@ -17,7 +17,7 @@ VisiData pre-set defaults for formatting types:
17
17
  - `currency` removes non-numeric characters and parses the remainder as `float`.
18
18
  - `date` parses dates into date object (shown as ISO8601).
19
19
  - `vlen` formats the cell value to the length of the content
20
- - `float` uses the decimal seperator, keeping two significant digits.
20
+ - `float` uses the decimal separator, keeping two significant digits.
21
21
 
22
22
  Change float precision with:
23
23
  - {help.commands.setcol-precision-less}
@@ -0,0 +1,49 @@
1
+ ---
2
+ sheet: Sheet
3
+ ---
4
+ # Create a window over consecutive rows
5
+
6
+ Window functions enable computations that relate the current window to surrounding rows, like cumulative sum, rolling averages or lead/lag computations.
7
+
8
+ {help.commands.addcol-window}
9
+
10
+ With large window sizes, [:code]g'[/] (`freeze-sheet`) to calculate all cells and copy the entire sheet into a new source sheet, which will conserve CPU.
11
+
12
+ ## Examples
13
+
14
+ date color price
15
+ ---------- ----- -----
16
+ 2024-09-01 R 30
17
+ 2024-09-02 B 28
18
+ 2024-09-03 R 100
19
+ 2024-09-03 B 33
20
+ 2024-09-03 B 99
21
+
22
+
23
+ 1. [:keys]#[/] (`type-int`) on the **price** column to type as int.
24
+ 2. [:keys]w[/] (`addcol-window`) on the **price** column, followed by `1 2`, to create a window consisting of 4 rows: 1 row before the current row, and 2 rows after.
25
+ 3. To create a moving average of the values in the window, add a new column with a python expression: [:keys]=[/] (`addcol-expr`)
26
+ followed by `sum(price_window)/len(price_window)`
27
+
28
+ date color price price_window sum(price_window)/len(price_window)
29
+ ---------- ----- ----- ------------------- -----------------------------------
30
+ 2024-09-01 R 38 [4] ; 38; 28; 100 41.5
31
+ 2024-09-02 B 28 [4] 38; 28; 100; 33 49.75
32
+ 2024-09-03 R 100 [4] 28; 100; 33; 99 65.0
33
+ 2024-09-03 B 33 [4] 100; 33; 99; 58.0
34
+ 2024-09-03 B 99 [4] 33; 99; ; 33.0
35
+
36
+
37
+ ## Workflows
38
+
39
+ ### Create a cumulative sum
40
+
41
+ 1. Set the before window size to the total number of rows in the table, and the after rows to 0. In the above example that would be `w 5 0` (`addcol-window`).
42
+ 2. Add an expression ([:keys]=[/] (`addcol-expr`) of `sum(window)` where `window` is the name of the window function column.
43
+
44
+ ### Compute the change between rows
45
+
46
+ 1. `w 1 0` on the `foo` column to create a window function of size 1 before and 0 after.
47
+ 2. Add a python expression. The window function column is 'foo_window':
48
+ `=foo_window[1] - foo_window[0] if len(foo_window) > 1 else None`
49
+
visidata/help.py CHANGED
@@ -4,7 +4,24 @@ import collections
4
4
  from visidata import VisiData, MetaSheet, ColumnAttr, Column, BaseSheet, VisiDataMetaSheet, SuspendCurses
5
5
  from visidata import vd, asyncthread, ENTER, drawcache, AttrDict, TextSheet
6
6
 
7
- vd.option('disp_expert', 0, 'max level of options and columns to include')
7
+
8
+ vd.option('disp_help_flags', 'cmdpalette guides help hints inputfield inputkeys nometacols sidebar',
9
+ '''list of helper features to enable (space-separated):
10
+ - "cmdpalette": exec-longname suggestions
11
+ - "guides": guides in sidebar
12
+ - "help": help sidebar collapsed by default
13
+ - "hints": context-sensitive hints on menu line
14
+ - "inputfield": context-sensitive help for each input field
15
+ - "inputkeys": input quick reference in sidebar
16
+ - "nometacols": hide expert columns on metasheets
17
+ - "sidebar": context-sensitive sheet help in sidebar
18
+ - "all": enable all helper features''')
19
+
20
+
21
+ @VisiData.api
22
+ def wantsHelp(vd, feat):
23
+ return feat in vd.options.disp_help_flags or 'all' in vd.options.disp_help_flags
24
+
8
25
 
9
26
  @BaseSheet.api
10
27
  def hint_basichelp(sheet):
@@ -99,12 +116,13 @@ class HelpPane:
99
116
 
100
117
  def draw(self, scr, x=None, y=None, **kwargs):
101
118
  if not scr: return
102
- # if vd.options.disp_help <= 0:
119
+ # if not vd.wantsHelp('statushelp'):
103
120
  # if self.scr:
104
121
  # self.scr.erase()
105
122
  # self.scr.refresh()
106
123
  # self.scr = None
107
124
  # return
125
+
108
126
  if y is None: y=0 # show at top of screen by default
109
127
  if x is None: x=0
110
128
  hneeded = self.amgr.maxHeight+3
@@ -135,7 +153,7 @@ class HelpPane:
135
153
  self.scr.erase()
136
154
  self.scr.box()
137
155
  self.amgr.draw(self.scr, y=1, x=2, **kwargs)
138
- self.scr.refresh()
156
+ self.scr.noutrefresh()
139
157
 
140
158
 
141
159
  @VisiData.api
@@ -176,13 +194,12 @@ BaseSheet.bindkey('gKEY_BACKSPACE', 'sysopen-help')
176
194
  HelpSheet.addCommand(None, 'exec-command', 'quit(sheet); draw_all(); activeStack[0].execCommand(cursorRow.longname)', 'execute command on undersheet')
177
195
  BaseSheet.addCommand(None, 'open-tutorial-visidata', 'launchBrowser("https://jsvine.github.io/intro-to-visidata/")', 'open https://jsvine.github.io/intro-to-visidata/')
178
196
 
179
- vd.addMenuItem("Help", "VisiData tutorial", 'open-tutorial-visidata')
180
- vd.addMenuItem("Help", 'Sheet commands', 'help-commands')
181
- vd.addMenuItem("Help", 'All commands', 'help-commands-all')
182
197
 
183
198
  vd.addGlobals(HelpSheet=HelpSheet)
184
199
 
185
200
  vd.addMenuItems('''
201
+ Help > VisiData tutorial > open-tutorial-visidata
202
+ Help > All commands > help-commands-all
186
203
  Help > Quick reference > sysopen-help
187
204
  Help > Command list > help-commands
188
205
  ''')
visidata/indexsheet.py CHANGED
@@ -99,7 +99,7 @@ BaseSheet.addCommand('g<', 'open-source-prev', 'vd.replace(openSource(source.nex
99
99
  IndexSheet.addCommand('g^R', 'reload-selected', 'reloadSheets(selectedRows or rows)', 'reload all selected sheets')
100
100
 
101
101
  # when diving into a sheet, remove the index unless it is precious
102
- SheetsSheet.addCommand('gC', 'columns-selected', 'vd.push(ColumnsSheet("all_columns", source=selectedRows))', 'open Columns Sheet with all visible columns from selected sheets')
102
+ IndexSheet.addCommand('gC', 'columns-selected', 'vd.push(ColumnsSheet("all_columns", source=selectedRows))', 'open Columns Sheet with all visible columns from selected sheets')
103
103
  IndexSheet.addCommand('^C', 'cancel-row', 'cancelThread(*cursorRow.currentThreads)', 'abort async thread for current sheet')
104
104
  IndexSheet.addCommand('gz^C', 'cancel-rows', 'for vs in selectedRows: cancelThread(*vs.currentThreads)', 'abort async threads for selected sheets')
105
105
  SheetsSheet.addCommand('Enter', 'open-row', 'dest=cursorRow; vd.sheets.remove(sheet) if not sheet.precious else None; vd.push(openRow(dest))', 'open sheet referenced in current row')
@@ -28,7 +28,7 @@ def save_dta(vd, p, *sheets):
28
28
  vs = sheets[0]
29
29
 
30
30
  columns = [col.name for col in vs.visibleCols]
31
-
31
+
32
32
  # Get data types
33
33
  types = list()
34
34
  dispvals = next(vs.iterdispvals(format=True))
@@ -154,6 +154,8 @@ class PandasSheet(Sheet):
154
154
  readfunc = self.read_tsv
155
155
  elif filetype == 'jsonl':
156
156
  readfunc = partial(pd.read_json, lines=True)
157
+ elif filetype == 'hdf5':
158
+ readfunc = partial(pd.read_hdf, lines=True)
157
159
  else:
158
160
  readfunc = getattr(pd, 'read_'+filetype) or vd.error('no pandas.read_'+filetype)
159
161
  # readfunc() handles binary and text open()
@@ -2,21 +2,25 @@ import pathlib
2
2
  import tarfile
3
3
  import zipfile
4
4
  import datetime
5
+ import os.path
5
6
  from visidata.loaders import unzip_http
6
7
 
7
8
  from visidata import vd, VisiData, asyncthread, Sheet, Progress, Menu, options
8
- from visidata import ColumnAttr, Column, Path
9
+ from visidata import ColumnAttr, Column, Path, filesize
9
10
  from visidata.type_date import date
10
11
 
11
12
  @VisiData.api
12
13
  def guess_zip(vd, p):
13
14
  if not p.is_url() and zipfile.is_zipfile(p.open_bytes()):
14
- return dict(filetype='zip')
15
+ return dict(filetype='zip', _likelihood=10)
15
16
 
16
17
  @VisiData.api
17
18
  def guess_tar(vd, p):
19
+ # an empty file will pass is_tarfile(), but can't be opened by tarfile.open()
20
+ if filesize(p) == 0:
21
+ return None
18
22
  if tarfile.is_tarfile(p.open_bytes()):
19
- return dict(filetype='tar')
23
+ return dict(filetype='tar', _likelihood=10)
20
24
 
21
25
  @VisiData.api
22
26
  def open_zip(vd, p):
@@ -81,13 +85,13 @@ Commands:
81
85
  return vd.openSource(Path(fi.filename, fp=fp, filesize=fi.file_size), filetype=options.filetype)
82
86
 
83
87
  def extract(self, *rows, path=None):
84
- path = path or pathlib.Path('.')
88
+ path = path or Path('.')
85
89
 
86
90
  files = []
87
91
  for row in rows:
88
92
  r, _ = row
89
93
  vd.confirmOverwrite(path/r.filename) #1452
90
- self.extract_async(row)
94
+ self.extract_async(row, path=path)
91
95
 
92
96
  def sysopen_row(self, row):
93
97
  'Extract file in row to tempdir and launch $EDITOR. Modifications will be discarded.'
@@ -109,6 +113,12 @@ Commands:
109
113
  if '://' in str(self.source):
110
114
  unzip_http.warning = vd.warning
111
115
  self._zfp = unzip_http.RemoteZipFile(str(self.source))
116
+ elif isinstance(self.source, Path):
117
+ if self.source.has_fp(): #when opening a zip inside tar or zip
118
+ fp = self.source.open('rb')
119
+ else:
120
+ fp = self.source
121
+ self._zfp = zipfile.ZipFile(fp, 'r')
112
122
  else:
113
123
  self._zfp = zipfile.ZipFile(str(self.source), 'r')
114
124
 
@@ -119,18 +129,35 @@ Commands:
119
129
  yield [zi, Path(zi.filename)]
120
130
 
121
131
 
132
+ #from https://docs.python.org/3/library/tarfile.html#tarfile.REGTYPE
133
+ tarfile_type_names = {
134
+ tarfile.REGTYPE:"file",
135
+ tarfile.AREGTYPE:"file",
136
+ tarfile.LNKTYPE:"hard link",
137
+ tarfile.SYMTYPE:"symbolic link",
138
+ tarfile.CHRTYPE:"character device",
139
+ tarfile.BLKTYPE:"block device",
140
+ tarfile.DIRTYPE:"directory",
141
+ tarfile.FIFOTYPE:"FIFO",
142
+ tarfile.CONTTYPE:"contiguous file",
143
+ tarfile.GNUTYPE_LONGNAME:"GNU tar longname",
144
+ tarfile.GNUTYPE_LONGLINK:"GNU tar longlink",
145
+ tarfile.GNUTYPE_SPARSE:"GNU tar sparse file",
146
+ }
122
147
  class TarSheet(Sheet):
123
148
  'Wrapper for `tarfile` library.'
124
149
  rowtype = 'files' # rowdef TarInfo
125
150
  columns = [
126
151
  ColumnAttr('name'),
152
+ Column('ext', getter=lambda col,row: row.isdir() and '/' or os.path.splitext(row.name)[1][1:]),
127
153
  ColumnAttr('size', type=int),
128
154
  ColumnAttr('mtime', type=date),
129
- ColumnAttr('type', type=int),
155
+ Column('type', getter=lambda col, row: tarfile_type_names.get(row.type, 'unknown')),
130
156
  ColumnAttr('mode', type=int),
131
157
  ColumnAttr('uname'),
132
158
  ColumnAttr('gname')
133
159
  ]
160
+ nKeys=1
134
161
 
135
162
  def openRow(self, fi):
136
163
  tfp = tarfile.open(name=str(self.source))
visidata/loaders/csv.py CHANGED
@@ -12,6 +12,13 @@ vd.option('csv_lineterminator', '\r\n', 'lineterminator passed to csv.writer', r
12
12
  vd.option('safety_first', False, 'sanitize input/output to handle edge cases, with a performance cost', replay=True)
13
13
 
14
14
 
15
+ @VisiData.api
16
+ def guess_csv_delimiter(vd, p):
17
+ 'If csv_delimiter option has been modified from default, assume CSV format.'
18
+
19
+ if vd.options.csv_delimiter != vd.options.getdefault('csv_delimiter'):
20
+ return dict(filetype='csv', _likelihood=2)
21
+
15
22
  @VisiData.api
16
23
  def guess_csv(vd, p):
17
24
  import csv
@@ -26,7 +33,11 @@ def guess_csv(vd, p):
26
33
 
27
34
  for csvopt in dir(dialect):
28
35
  if not csvopt.startswith('_'):
29
- r['csv_'+csvopt] = getattr(dialect, csvopt)
36
+ v = getattr(dialect, csvopt)
37
+ optname = 'csv_'+csvopt
38
+ r[optname] = v
39
+ if vd.options.get(optname) != v:
40
+ vd.warning(f'guessed option {optname}={v}')
30
41
 
31
42
  return r
32
43
 
visidata/loaders/eml.py CHANGED
@@ -6,6 +6,8 @@ from visidata import VisiData, vd, Column, TableSheet, vlen
6
6
  def open_eml(vd, p):
7
7
  return EmailSheet(p.base_stem, source=p)
8
8
 
9
+ open_mhtml = open_eml
10
+
9
11
  class EmailSheet(TableSheet):
10
12
  rowtype = 'parts' # rowdef: sub-Messages
11
13
  columns = [
visidata/loaders/f5log.py CHANGED
@@ -14,7 +14,7 @@ Regex: (?:/Common/)(?P<site>[^-]+)-(?P<vstype>[^-]+)-(?P<application>[^-]+)
14
14
 
15
15
  /Common/newyork-www-banking1
16
16
 
17
- ... | site | vstype | appliction | ...
17
+ ... | site | vstype | application | ...
18
18
  ... | newyork | www | banking1 | ...
19
19
 
20
20
  Adding to .visidatarc
@@ -313,7 +313,7 @@ class F5LogSheet(Sheet):
313
313
  cmd_data = msg[cmd_data_loc + 1 :]
314
314
  # split the message and the command
315
315
  msg, cmd = msg[:cmd_data_loc].rsplit(" ", maxsplit=1)
316
- # strip off the trailling " -" from the message
316
+ # strip off the trailing " -" from the message
317
317
  msg = msg[:-2]
318
318
  object = cmd_data.split('"', maxsplit=2)
319
319
  if len(object) == 3:
visidata/loaders/fec.py CHANGED
@@ -106,22 +106,19 @@ class DiveSheet(Sheet):
106
106
  self.addRow(item)
107
107
 
108
108
  except Exception as e:
109
- vd.warning("Can't dive on lists with heterogenous item types.")
109
+ vd.warning("Can't dive on lists with heterogeneous item types.")
110
110
  return False
111
111
 
112
112
  def openRow(self, row):
113
113
  if self.is_keyvalue:
114
114
  cell = row["value"]
115
- name = vd.joinSheetnames(self.name, row["key"])
116
-
117
115
  if isinstance(cell, (list, dict)):
118
- vs = self.__class__(name, source = cell)
116
+ vs = self.__class__(self.name, row["key"], source = cell)
119
117
  else:
120
118
  vd.warning("Nothing to dive into.")
121
119
  return
122
120
  else:
123
- name = vd.joinSheetnames(self.name, "row")
124
- vs = self.__class__(name, source = self.row)
121
+ vs = self.__class__(self.name, "row", source = self.row)
125
122
 
126
123
  success = vs.reload()
127
124
  if success == False:
@@ -174,7 +171,7 @@ class FECScheduleSheet(Sheet):
174
171
 
175
172
  for schedule_name in self.source.keys():
176
173
  vs = FECItemizationSheet(
177
- vd.joinSheetnames(self.name, schedule_name),
174
+ self.name, schedule_name,
178
175
  schedule_name = schedule_name,
179
176
  source = self.source[schedule_name],
180
177
  size = len(self.source[schedule_name]),
@@ -225,7 +222,7 @@ class FECFiling(Sheet):
225
222
  ] else dict
226
223
 
227
224
  vs = cls(
228
- vd.joinSheetnames(self.name, component_name),
225
+ self.name, component_name,
229
226
  component_name = component_name,
230
227
  source = source_cls(),
231
228
  size = 0,
@@ -270,7 +267,7 @@ class FECFiling(Sheet):
270
267
  if form_type not in sheet_row.source:
271
268
  sheet_row.source[form_type] = [ ]
272
269
  subsheet = FECItemizationSheet(
273
- vd.joinSheetnames(sheet_row.name, form_type),
270
+ sheet_row.name, form_type,
274
271
  schedule_name = form_type,
275
272
  source = [ ],
276
273
  size = 0,
@@ -110,3 +110,5 @@ def save_fixed(vd, p, *vsheets):
110
110
  for col, val in dispvals.items():
111
111
  fp.write(('{0:%s%s.%s} ' % ('>' if vd.isNumeric(col) else '<', widths[col], widths[col])).format(val))
112
112
  fp.write('\n')
113
+
114
+ FixedWidthColumnsSheet.options.null_value = '' # the file format cannot contain None, so use empty string instead
visidata/loaders/hdf5.py CHANGED
@@ -1,4 +1,5 @@
1
- from visidata import VisiData, vd, Sheet, Path, Column, ItemColumn, BaseSheet
1
+ from visidata import VisiData, vd, Sheet, Path, Column, ItemColumn, BaseSheet, anytype
2
+ from itertools import chain
2
3
 
3
4
  @VisiData.api
4
5
  def open_h5(vd, p):
@@ -6,8 +7,11 @@ def open_h5(vd, p):
6
7
 
7
8
  VisiData.open_hdf5 = VisiData.open_h5
8
9
 
10
+ vd.option('hdf5_matrix_enumerate', False, 'enumerate matrix rows and columns')
11
+
9
12
  class Hdf5ObjSheet(Sheet):
10
13
  'Support sheets in HDF5 format.'
14
+
11
15
  def iterload(self):
12
16
  h5py = vd.importExternal('h5py')
13
17
  source = self.source
@@ -26,25 +30,39 @@ class Hdf5ObjSheet(Sheet):
26
30
  for k, v in source.items():
27
31
  yield Hdf5ObjSheet(self.name, k, source=v)
28
32
  elif isinstance(source, h5py.Dataset):
29
- if len(source.shape) == 1:
33
+ if len(source.shape)==1:
30
34
  if source.dtype.names:
31
- for i, colname in enumerate(source.dtype.names):
32
- self.addColumn(ItemColumn(colname, colname), index=i)
35
+ for i, (colname, fmt, *_) in enumerate(source.dtype.descr):
36
+ if not colname:
37
+ colname = f"col{i}"
38
+ ctype = _guess_type(fmt)
39
+ self.addColumn(ItemColumn(colname, i, type=ctype))
33
40
  yield from source # copy
34
41
  else:
35
42
  self.addColumn(ItemColumn(source.name, 0))
36
43
  for v in source:
37
44
  yield [v]
38
- elif len(source.shape) == 2: # matrix
45
+ elif len(source.shape)==2:
46
+ matrix_enumerate = bool(self.options.hdf5_matrix_enumerate)
47
+
39
48
  ncols = source.shape[1]
40
- for i in range(ncols):
41
- self.addColumn(ItemColumn('', i, width=8), index=i)
42
- self.recalc()
43
- yield from source # copy
49
+ ctype = _guess_type(source.dtype.descr[0][1])
50
+
51
+ if matrix_enumerate:
52
+ self.addColumn(ItemColumn("row", 0, width=8, keycol=1, type=int), index=0)
53
+ for i in range(ncols):
54
+ self.addColumn(ItemColumn(f'col{i}', i+1, width=8, type=ctype), index=i+1)
55
+ self.recalc()
56
+ yield from list(list((chain((i,), row))) for i, row in enumerate(source))
57
+ else:
58
+ for i in range(ncols):
59
+ self.addColumn(ItemColumn('', i, width=8, type=ctype), index=i)
60
+ self.recalc()
61
+ yield from source # copy
44
62
  else:
45
63
  vd.fail('too many dimensions in shape %s' % str(source.shape))
46
64
  else:
47
- vd.fail('unknown h5 object type %s' % type(source))
65
+ vd.fail(f"too many dimensions in shape {source.shape}")
48
66
 
49
67
 
50
68
  def openRow(self, row):
@@ -59,5 +77,11 @@ class Hdf5ObjSheet(Sheet):
59
77
  if isinstance(row, numpy.ndarray):
60
78
  return NpySheet(None, npy=row)
61
79
 
80
+ def _guess_type(fmt):
81
+ if 'i' in fmt or 'u' in fmt:
82
+ return int
83
+ elif 'f' in fmt:
84
+ return float
85
+ return anytype
62
86
 
63
87
  Hdf5ObjSheet.addCommand('A', 'dive-metadata', 'vd.push(SheetDict(cursorRow.name + "_attrs", source=cursorRow.attrs))', 'open metadata sheet for object referenced in current row')