visidata 2.11.1__py3-none-any.whl → 3.0.1__py3-none-any.whl

visidata/loaders/fec.py

@@ -0,0 +1,325 @@
+"""
+Filename: vdfec.py
+Last updated: 2019-04-21
+Home: https://github.com/jsvine/visidata-plugins
+Author: Jeremy Singer-Vine
+
+# Installation
+
+- Install fecfile: `pip install fecfile`
+- Add vdfec.py to your ~/.visidata directory
+- Add "import vdfec" to your ~/.visidatarc file
+
+# Usage
+
+vdfec.py enables VisiData to load .fec files from the Federal Election Commission.
+
+Once saved to your ~/.visidata directory, and imported via your ~/.visidatarc file,
+you should be able to interactively explore .fec files as you would any other 
+filetype in VisiData.
+
+From the command line:
+
+    vd path/to/my/file.fec
+
+# Thanks
+
+vdfec.py depends heavily on Evan Sonderegger's `fecfile` Python library: https://esonderegger.github.io/fecfile/
+
+... which in turn is based on Derek Willis's `Fech` Ruby library: https://github.com/dwillis/Fech
+
+... and Chris Zubak-Skees' transformation of `Fech`'s form-and-field mappings: https://github.com/PublicI/fec-parse/blob/master/lib/renderedmaps.js
+
+Thanks to all who have contributed to those projects.
+
+"""
+
+from copy import copy
+from visidata import (
+    vd,
+    Path,
+    Sheet,
+    TextSheet,
+    Column,
+    ColumnAttr,
+    ColumnItem,
+    ENTER,
+    asyncthread,
+    Progress,
+    addGlobals,
+)
+
+class DiveSheet(Sheet):
+    "A deeply-diveable, quick-diving sheet."
+
+    def reload(self):
+        mapping = self.source
+
+        self.columns = []
+        self.rows = []
+
+        self.key_type = str
+        self.size = len(mapping)
+
+        if self.size == 0:
+            return
+
+        if isinstance(mapping, list):
+            first = mapping[0]
+            if isinstance(first, dict):
+                colgetter = lambda x: x.keys()
+            elif isinstance(first, list):
+                colgetter = lambda x: list(range(len(x)))
+            else:
+                mapping = dict(enumerate(mapping))
+                self.key_type = int
+                self.size = len(mapping)
+
+        if isinstance(mapping, dict):
+            self.is_keyvalue = True
+            if self.size:
+                max_key_len = max(map(len, map(str, mapping.keys())))
+                key_width = min(50, max(max_key_len + 2, 6))
+            else:
+                key_width = None
+
+            self.addColumn(ColumnItem(
+                "key",
+                width = key_width,
+                type = self.key_type
+            ))
+            self.addColumn(ColumnItem("value"))
+            self.setKeys(self.columns[:1])
+
+            for k, v in mapping.items():
+                self.addRow({ "key": k, "value": v })
+
+        elif isinstance(mapping, list):
+            self.is_keyvalue = False
+            indices = [] 
+            for item in mapping:
+                try:
+                    cols = colgetter(item)
+                    for col in cols:
+                        if col not in indices:
+                            self.addColumn(ColumnItem(col))
+                            indices.append(col)
+
+                    self.addRow(item)
+
+                except Exception as e:
+                    vd.warning("Can't dive on lists with heterogenous item types.")
+                    return False
+
+    def dive(self):
+        if self.is_keyvalue:
+            cell = self.cursorRow["value"]
+            name = vd.joinSheetnames(self.name, self.cursorRow["key"])
+
+            if isinstance(cell, (list, dict)):
+                vs = self.__class__(name, source = cell)
+            else:
+                vd.warning("Nothing to dive into.")
+                return
+        else:
+            name = vd.joinSheetnames(self.name, "row")
+            vs = self.__class__(name, source = self.cursorRow)
+
+        success = vs.reload()
+        if success == False:
+            return
+
+        vd.push(vs)
+
+DiveSheet.addCommand(
+    ENTER,
+    'dive-row',
+    'vd.sheet.dive()'
+)
+
+class FECItemizationSheet(Sheet):
+    "A sheet to display a list of FEC itemizations from a given form/schedule."
+
+    rowtype = "itemizations"
+
+    @asyncthread
+    def reload(self):
+        self.rows = []
+        self.columns = []
+
+        if len(self.source) == 0:
+            return
+
+        for i, row in enumerate(Progress(self.source, total = len(self.source))):
+            if i == 0:
+                self.set_columns_from_row(row)
+            self.addRow(row)
+    
+    def set_columns_from_row(self, row):
+        self.columns.clear()
+        for i, name in enumerate(row.keys()):
+            self.addColumn(ColumnItem(name))
+    def dive(self):
+        vs = DiveSheet(
+            vd.joinSheetnames(self.name, "detail"),
+            source = self.cursorRow
+        )
+        vs.reload()
+        vd.push(vs)
+
+FECItemizationSheet.addCommand(
+    ENTER,
+    'dive-row',
+    'vd.sheet.dive()'
+)
+
+class FECScheduleSheet(Sheet):
+    "A sheet to display the list of itemized schedules in a filing."
+
+    rowtype = "schedules"
+
+    columns = [
+        ColumnAttr("schedule", "schedule_name", width = 14),
+        ColumnAttr("name", width = 0),
+        ColumnAttr("size", type = int),
+    ]
+
+    nKeys = 1
+
+    @asyncthread
+    def reload(self):
+        self.rows = []
+
+        for schedule_name in self.source.keys():
+            vs = FECItemizationSheet(
+                vd.joinSheetnames(self.name, schedule_name),
+                schedule_name = schedule_name,
+                source = self.source[schedule_name],
+                size = len(self.source[schedule_name]),
+            )
+            self.addRow(vs)
+
+FECScheduleSheet.addCommand(
+    ENTER,
+    'dive-row',
+    'vd.push(cursorRow)'
+)
+
+COMPONENT_SHEET_CLASSES = {
+    "header": DiveSheet,
+    "summary": DiveSheet,
+    "itemization": FECScheduleSheet,
+    "text": FECItemizationSheet,
+    "F99_text": TextSheet,
+}
+
+class FECFiling(Sheet):
+    "A sheet representing an entire .fec file."
+
+    rowtype = "components"
+    filing = None
+
+    columns = [
+        ColumnAttr("component", "component_name", width = 14),
+        ColumnAttr("name", width = 0),
+        ColumnAttr("size", type = int),
+    ]
+
+    nKeys = 1
+
+    @asyncthread
+    def reload(self):
+        from fecfile import fecparser
+        self.rows = []
+
+        row_dict = { }
+        itemization_subsheets = {}
+
+        def addSheetRow(component_name):
+            "On first encountering a component, add a row to the filing sheet"
+
+            cls = COMPONENT_SHEET_CLASSES[component_name]
+
+            source_cls = list if cls in [
+                    FECItemizationSheet,
+                    TextSheet
+                ] else dict
+
+            vs = cls(
+                vd.joinSheetnames(self.name, component_name),
+                component_name = component_name,
+                source = source_cls(),
+                size = 0,
+            )
+
+            vs.reload()
+            row_dict[component_name] = vs
+            self.addRow(vs)
+
+        src = Path(self.source.resolve())
+
+        item_iter = fecparser.iter_lines(src, { "as_strings": True })
+
+        for item in item_iter:
+            dtype = item.data_type 
+            if dtype not in row_dict.keys():
+                addSheetRow(dtype)
+
+            sheet_row = row_dict[dtype]
+
+            if dtype in [ "header", "summary" ]:
+                sheet_row.source = item.data
+                sheet_row.reload()
+
+            elif dtype == "text":
+                if len(sheet_row.source) == 0:
+                    sheet_row.set_columns_from_row(item.data)
+                sheet_row.source.append(item.data)
+                sheet_row.addRow(item.data)
+                sheet_row.size += 1
+
+            elif dtype == "F99_text":
+                sheet_row.source = item.data.split("\n")
+                sheet_row.size = len(sheet_row.source)
+
+            elif dtype == "itemization":
+                form_type = item.data["form_type"]
+
+                if form_type[0] == "S":
+                    form_type = "Schedule " + item.data["form_type"][1]
+
+                if form_type not in sheet_row.source:
+                    sheet_row.source[form_type] = [ ] 
+                    subsheet = FECItemizationSheet(
+                        vd.joinSheetnames(sheet_row.name, form_type),
+                        schedule_name = form_type,
+                        source = [ ],
+                        size = 0,
+                    )
+                    subsheet.reload()
+                    subsheet.set_columns_from_row(item.data)
+                    sheet_row.addRow(subsheet)
+                    itemization_subsheets[form_type] = subsheet
+                else:
+                    subsheet = itemization_subsheets[form_type]
+
+                subsheet.addRow(item.data)
+                subsheet.source.append(item.data)
+                subsheet.size += 1
+
+                sheet_row.source[form_type].append(item.data)
+                sheet_row.size += 1
+
+FECFiling.addCommand(
+    ENTER,
+    'dive-row',
+    'vd.push(cursorRow)'
+)
+
+def open_fec(p):
+    return FECFiling(p.base_stem, source=p)
+
+addGlobals({
+    "open_fec": open_fec,
+    "DiveSheet": DiveSheet
+})

visidata/loaders/fixed_width.py

@@ -7,7 +7,7 @@ vd.option('fixed_maxcols', 0, 'max number of fixed-width columns to create (0 is
 
 @VisiData.api
 def open_fixed(vd, p):
-    return FixedWidthColumnsSheet(p.name, source=p, headerlines=[])
+    return FixedWidthColumnsSheet(p.base_stem, source=p, headerlines=[])
 
 class FixedWidthColumn(Column):
     def __init__(self, name, i, j, **kwargs):
@@ -76,7 +76,7 @@ class FixedWidthColumnsSheet(SequenceSheet):
 
 @VisiData.api
 def save_fixed(vd, p, *vsheets):
-    with p.open_text(mode='w', encoding=vsheets[0].options.encoding) as fp:
+    with p.open(mode='w', encoding=vsheets[0].options.save_encoding) as fp:
         for sheet in vsheets:
             if len(vsheets) > 1:
                 fp.write('%s\n\n' % sheet.name)
@@ -94,5 +94,3 @@ def save_fixed(vd, p, *vsheets):
                     for col, val in dispvals.items():
                         fp.write(('{0:%s%s.%s} ' % ('>' if vd.isNumeric(col) else '<', widths[col], widths[col])).format(val))
                     fp.write('\n')
-
-            vd.status('%s save finished' % p)

visidata/loaders/frictionless.py

@@ -2,11 +2,11 @@ from visidata import VisiData, vd, Progress, IndexSheet
 
 @VisiData.api
 def open_frictionless(vd, p):
-    return FrictionlessIndexSheet(p.name, source=p)
+    return FrictionlessIndexSheet(p.base_stem, source=p)
 
 class FrictionlessIndexSheet(IndexSheet):
     def iterload(self):
-        import datapackage
-        self.dp = datapackage.Package(self.source.open_text(encoding='utf-8'))
+        datapackage = vd.importExternal('datapackage')
+        self.dp = datapackage.Package(self.source.open(encoding='utf-8'))
         for r in Progress(self.dp.resources):
             yield vd.openSource(self.source.with_name(r.descriptor['path']), filetype=r.descriptor.get('format', 'json'))

visidata/loaders/geojson.py

@@ -1,12 +1,15 @@
 from functools import reduce
+from copy import deepcopy
+
 import json
 
-from visidata import VisiData, vd, Column, asyncthread, Progress, PythonSheet, InvertedCanvas, deepcopy, date, wrapply, TypedExceptionWrapper, TypedWrapper
+from visidata import VisiData, vd, Column, asyncthread, Progress, PythonSheet, InvertedCanvas, date, wrapply, TypedExceptionWrapper, TypedWrapper
+
 
 
 @VisiData.api
 def open_geojson(vd, p):
-    return GeoJSONSheet(p.name, source=p)
+    return GeoJSONSheet(p.base_stem, source=p)
 
 class GeoJSONColumn(Column):
     def calcValue(self, row):
@@ -24,7 +27,7 @@ class GeoJSONSheet(PythonSheet):
     def iterload(self):
         self.colnames = {}
 
-        with self.source.open_text(encoding='utf-8') as fp:
+        with self.source.open(encoding='utf-8') as fp:
             ret = json.load(fp)
 
             if ret['type'] == 'FeatureCollection':
@@ -146,14 +149,14 @@ def save_geojson(vd, p, vs):
     except Exception:
         indent = vs.options.json_indent
 
-    with p.open_text(mode='w', encoding='utf-8') as fp:
+    with p.open(mode='w', encoding='utf-8') as fp:
         encoder = json.JSONEncoder(indent=indent, sort_keys=vs.options.json_sort_keys)
         for chunk in encoder.iterencode(featcoll):
             fp.write(chunk)
 
 GeoJSONSheet.addCommand('.', 'plot-row', 'vd.push(GeoJSONMap(name+"_map", sourceRows=[cursorRow], textCol=cursorCol, source=sheet))', 'plot geospatial vector in current row')
 GeoJSONSheet.addCommand('g.', 'plot-rows', 'vd.push(GeoJSONMap(name+"_map", sourceRows=rows, textCol=cursorCol, source=sheet))', 'plot all geospatial vectors in current sheet')
-GeoJSONMap.addCommand('^S', 'save-sheet', 'vd.saveSheets(inputPath("save to: ", value=getDefaultSaveName(sheet)), sheet, confirm_overwrite=options.confirm_overwrite)', 'save current sheet to filename in format determined by extension (default .geojson)')
+GeoJSONMap.addCommand('^S', 'save-sheet', 'vd.saveSheets(inputPath("save to: ", value=getDefaultSaveName(sheet)), sheet)', 'save current sheet to filename in format determined by extension (default .geojson)')
 
 vd.addGlobals({
     'GeoJSONMap': GeoJSONMap,

visidata/loaders/google.py

@@ -0,0 +1,48 @@
+
+
+from visidata import vd, VisiData, Sheet, IndexSheet, SequenceSheet, ColumnItem, Path, AttrDict, ColumnAttr, asyncthread
+
+
+def _google_creds_fn():
+
+    filename = 'google_creds.json'
+    google_creds_path = vd.pkg_resources_files('vdplus.api.google') / filename
+    import os
+    if not os.path.exists(google_creds_path):
+        vd.error(f'{filename} file does not exist in {google_creds_path.parent}\n'
+                 'Create it by following this guide: https://github.com/saulpw/visidata/blob/develop/docs/gmail.md')
+    else:
+        return str(google_creds_path)
+
+
+@VisiData.api
+def google_auth(vd, scopes=None):
+    import pickle
+    import os.path
+    import urllib.parse
+
+    SCOPES = []
+    for scope in scopes.split():
+        if not scope.startswith('https://'):
+            scope = 'https://www.googleapis.com/auth/' + scope
+        SCOPES.append(scope)
+
+    GOOGLE_TOKEN_FILE = Path(vd.options.visidata_dir)/f'google-{urllib.parse.quote_plus(str(scopes))}.pickle'
+    creds = None
+    if os.path.exists(GOOGLE_TOKEN_FILE):
+        with open(GOOGLE_TOKEN_FILE, 'rb') as fp:
+            creds = pickle.load(fp)
+
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            from google.auth.transport.requests import Request
+            creds.refresh(Request())
+        else:
+            from google_auth_oauthlib.flow import InstalledAppFlow
+            flow = InstalledAppFlow.from_client_secrets_file(_google_creds_fn(), SCOPES)
+            creds = flow.run_local_server(port=0)
+
+        with open(GOOGLE_TOKEN_FILE, 'wb') as fp:
+            pickle.dump(creds, fp)
+
+    return creds

visidata/loaders/graphviz.py

@@ -1,5 +1,5 @@
 from visidata import vd, options, TypedWrapper, asyncthread, Progress
-from visidata import wrapply, clean_to_id, VisiData
+from visidata import wrapply, VisiData
 
 vd.option('graphviz_edge_labels', True, 'whether to include edge labels on graphviz diagrams')
 
@@ -19,7 +19,7 @@ def save_dot(vd, p, vs):
 
     srccol = vs.keyCols[0]
     dstcol = vs.keyCols[1]
-    with p.open_text(mode='w', encoding='utf-8') as fp:
+    with p.open(mode='w', encoding='utf-8') as fp:
         pfp = lambda *args: print(*args, file=fp)
         pfp('graph { concentrate=true;')
         for row in Progress(vs.rows, 'saving'):
@@ -28,8 +28,8 @@ def save_dot(vd, p, vs):
             if not is_valid(src) or not is_valid(dst):
                 continue
 
-            downsrc = clean_to_id(str(src)) or src
-            downdst = clean_to_id(str(dst)) or dst
+            downsrc = vd.cleanName(str(src)) or src
+            downdst = vd.cleanName(str(dst)) or dst
             edgenotes = [c.getTypedValue(row) for c in vs.nonKeyVisibleCols if not vd.isNumeric(c)]
             edgetype = '-'.join(str(x) for x in edgenotes if is_valid(x))
             color = assignedColors.get(edgetype, None)

visidata/loaders/hdf5.py

@@ -2,14 +2,14 @@ from visidata import VisiData, vd, Sheet, Path, Column, ItemColumn, BaseSheet
 
 @VisiData.api
 def open_h5(vd, p):
-    return Hdf5ObjSheet(p.name, source=p)
+    return Hdf5ObjSheet(p.base_stem, source=p)
 
 VisiData.open_hdf5 = VisiData.open_h5
 
 class Hdf5ObjSheet(Sheet):
     'Support sheets in HDF5 format.'
     def iterload(self):
-        import h5py
+        h5py = vd.importExternal('h5py')
         source = self.source
         if isinstance(self.source, Path):
             source = h5py.File(str(self.source), 'r')
@@ -48,13 +48,13 @@ class Hdf5ObjSheet(Sheet):
 
 
     def openRow(self, row):
-        import h5py
+        h5py = vd.importExternal('h5py')
         if isinstance(row, BaseSheet):
             return row
         if isinstance(row, h5py.HLObject):
             return Hdf5ObjSheet(row)
 
-        import numpy
+        numpy = vd.importExternal('numpy')
         from .npy import NpySheet
         if isinstance(row, numpy.ndarray):
             return NpySheet(None, npy=row)

visidata/loaders/html.py

@@ -3,13 +3,26 @@ import urllib.parse
 import copy
 import itertools
 
-from visidata import VisiData, vd, Sheet, options, Column, Progress, IndexSheet, ItemColumn, InferColumnsSheet
+from visidata import VisiData, vd, Sheet, options, Column, Progress, IndexSheet, ItemColumn
 
 vd.option('html_title', '<h2>{sheet.name}</h2>', 'table header when saving to html')
 
+
+@VisiData.api
+def guess_html(vd, p):
+    with p.open() as fp:
+        r = fp.read(10240)
+        if r.strip().startswith('<'):
+            m = re.search(r, r'charset=(\S+)')
+            if m:
+                encoding = m.group(0)
+            else:
+                encoding = None
+            return dict(filetype='html', _likelihood=1, encoding=encoding)
+
 @VisiData.api
 def open_html(vd, p):
-    return HtmlTablesSheet(p.name, source=p)
+    return HtmlTablesSheet(p.base_stem, source=p)
 
 VisiData.open_htm = VisiData.open_html
 
@@ -20,12 +33,17 @@ class HtmlTablesSheet(IndexSheet):
         Column('tag', width=0, getter=lambda col,row: row.html.tag),
         Column('id', getter=lambda col,row: row.html.attrib.get('id')),
         Column('classes', getter=lambda col,row: row.html.attrib.get('class')),
+        Column('title', getter=lambda col,row: row.html.attrib.get('title')),
+        Column('aria_label', getter=lambda col,row: row.html.attrib.get('aria-label')),
+        Column('caption', getter=lambda col,row: row.html.xpath('normalize-space(./caption)') if row.html.xpath('./caption') else None, cache=True),
+        Column('summary', getter=lambda col,row: row.html.attrib.get('summary')),
+        Column('heading', getter=lambda col,row: row.html.xpath('normalize-space(./preceding-sibling::*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6][1])') or None, cache=True),
     ]
     def iterload(self):
+        lxml = vd.importExternal('lxml')
         from lxml import html
-        utf8_parser = html.HTMLParser(encoding='utf-8')
-        with self.source.open_text(encoding='utf-8') as fp:
-            doc = html.parse(fp, parser=utf8_parser, base_url=self.source.given)
+        with self.source.open(encoding='utf-8') as fp:
+            doc = html.parse(fp, parser=vd.utf8_parser, base_url=self.source.given)
         self.setKeys([self.column('name')])
         self.column('keys').hide()
         self.column('source').hide()
@@ -56,15 +74,16 @@ class HtmlLinksSheet(Sheet):
         ItemColumn('link', 2, width=40),
     ]
     def iterload(self):
+        lxml = vd.importExternal('lxml')
         from lxml.html import iterlinks
         root = self.source.getroot()
-        root.make_links_absolute(self.source.docinfo.URL)
+        root.make_links_absolute(self.source.docinfo.URL, handle_failures='ignore')
         yield from iterlinks(root)
 
     def openRow(self, row):
         return vd.openSource(row[2])
 
-class HtmlElementsSheet(InferColumnsSheet):
+class HtmlElementsSheet(Sheet):
     rowtype = 'links'  #  dict
     columns = [
         ItemColumn('__element__', width=0),
@@ -89,6 +108,7 @@ class HtmlTableSheet(Sheet):
     columns = []
 
     def iterload(self):
+        import lxml
         headers = []
 
         maxlinks = {}  # [colnum] -> nlinks:int
@@ -108,8 +128,14 @@ class HtmlTableSheet(Sheet):
             for cell in r.getchildren():
                 colspan = int(cell.attrib.get('colspan', 1))
                 rowspan = int(cell.attrib.get('rowspan', 1))
+                if isinstance(cell, lxml.etree.CommentBase):
+                    continue
                 cellval = ' '.join(x.strip() for x in cell.itertext())  # text only without markup
-                links = [urllib.parse.urljoin(self.source.base_url, x.get('href')) for x in cell.iter('a')]
+                links = [
+                    vd.callNoExceptions(urllib.parse.urljoin, self.source.base_url, x.get('href')) or x.get('href')
+                        for x in cell.iter('a')
+                ]
+
                 maxlinks[colnum] = max(maxlinks.get(colnum, 0), len(links))
 
                 if is_header(cell):
@@ -124,7 +150,7 @@ class HtmlTableSheet(Sheet):
                         cellval = ''   # use empty non-None value for subsequent rows in the rowspan
                 else:
                     while colnum >= len(row):
-                        row.append(None)
+                        row.append((None, []))
                     row[colnum] = (cellval, links)
 
                 colnum += colspan
@@ -137,8 +163,11 @@ class HtmlTableSheet(Sheet):
         if headers:
             it = itertools.zip_longest(*headers, fillvalue='')
         else:
-            it = list(list(x) for x in self.rows.pop(0))
-            it += [''] * (ncols-len(it))
+            if len(self.rows) > 0:
+                it = list(list(x) for x in self.rows.pop(0))
+                it += [''] * (ncols-len(it))
+            else:
+                it = []
 
         for colnum, names in enumerate(it):
             name = '_'.join(str(x) for x in names if x)
@@ -177,7 +206,20 @@ def save_html(vd, p, *vsheets):
                     fp.write('</tr>\n')
 
             fp.write('</table>')
-            vd.status('%s save finished' % p)
+
+
+@VisiData.lazy_property
+def utf8_parser(vd):
+    lxml = vd.importExternal('lxml')
+    return lxml.html.HTMLParser(encoding='utf-8')
+#    return lxml.etree.HTMLParser(encoding='utf-8')
+
+
+@VisiData.api
+def HTML(vd, s):
+    lxml = vd.importExternal('lxml')
+    from lxml import html
+    return html.fromstring(s, parser=vd.utf8_parser)
 
 
 VisiData.save_htm = VisiData.save_html