visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/loaders/http.py

@@ -1,11 +1,29 @@
-from visidata import Path, RepeatFile, options, vd, VisiData
+import re
 
-content_filetypes = {
-    'tab-separated-values': 'tsv'
-}
+from visidata import Path, RepeatFile, vd, VisiData
+from visidata.loaders.tsv import splitter
 
 vd.option('http_max_next', 0, 'max next.url pages to follow in http response') #848
 vd.option('http_req_headers', {}, 'http headers to send to requests')
+vd.option('http_ssl_verify', True, 'verify host and certificates for https')
+
+
+@VisiData.api
+def guessurl_mimetype(vd, path, response):
+    content_filetypes = {
+        'tab-separated-values': 'tsv'
+    }
+
+    for k in dir(vd):
+        if k.startswith('open_'):
+            ft = k[5:]
+            content_filetypes[ft] = ft
+
+    contenttype = response.getheader('content-type')
+    subtype = contenttype.split(';')[0].split('/')[-1]
+    if subtype in content_filetypes:
+        return dict(filetype=content_filetypes.get(subtype), _likelihood=10)
+
 
 
 @VisiData.api
@@ -18,54 +36,90 @@ def openurl_http(vd, path, filetype=None):
             vd.fail(f'no vd.openhttp_{sch}')
         return openfunc(Path(schemes[-1]+'://'+path.given.split('://')[1]))
 
-    import requests
+    import urllib.request
+    import urllib.error
+    import mimetypes
 
-    response = requests.get(path.given, stream=True, **vd.options.getall('http_req_'))
-    response.raise_for_status()
+    ctx = None
+    if not vd.options.http_ssl_verify:
+        import ssl
 
-    if not filetype:
-        # try auto-detect from extension
-        ext = path.suffix[1:].lower()
-        openfunc = getattr(vd, f'open_{ext}', vd.getGlobals().get(f'open_{ext}'))
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
 
-        if openfunc:
-            filetype = ext
-        else:
-            # if extension unknown, fallback to mime-type
-            contenttype = response.headers['content-type']
-            subtype = contenttype.split(';')[0].split('/')[-1]
-            filetype = content_filetypes.get(subtype, subtype)
-
-    # If no charset is provided by response headers, use the user-specified
-    # encoding option (which defaults to UTF-8) and hope for the best.  The
-    # alternative is an error because iter_lines() will produce bytes.  We're
-    # streaming so can't use response.apparent_encoding.
-    if not response.encoding:
-        response.encoding = options.encoding
+    req = urllib.request.Request(path.given, **vd.options.getall('http_req_'))
+    response = urllib.request.urlopen(req, context=ctx)
+
+    filetype = filetype or vd.guessFiletype(path, response, funcprefix='guessurl_').get('filetype')  # try guessing by url
+    filetype = filetype or vd.guessFiletype(path, funcprefix='guess_').get('filetype')  # try guessing by contents
 
     # Automatically paginate if a 'next' URL is given
-    def _iter_lines(path=path, response=response, max_next=options.http_max_next):
+    def _iter_lines(path=path, response=response, max_next=vd.options.http_max_next):
         path.responses = []
         n = 0
         while response:
             path.responses.append(response)
-            yield from response.iter_lines(decode_unicode=True)
+            with response as fp:
+                for line in splitter(response, delim=b'\n'):
+                    yield line.decode(vd.options.encoding)
+
+            linkhdr = response.getheader('Link')
+            src = None
+            if linkhdr:
+                links = parse_header_links(linkhdr)
+                link_data = {}
+                for link in links:
+                    key = link.get('rel') or link.get('url')
+                    link_data[key] = link
+                src = link_data.get('next', {}).get('url', None)
 
-            src = response.links.get('next', {}).get('url', None)
             if not src:
                 break
 
             n += 1
             if n > max_next:
-                vd.warning(f'stopping at max {max_next} pages')
+                vd.warning(f'stopping at max next pages: {max_next} pages')
                 break
 
             vd.status(f'fetching next page from {src}')
-            response = requests.get(src, stream=True, **vd.options.getall('http_req_'))
+            req = urllib.request.Request(src, **vd.options.getall('http_req_'))
+            response = urllib.request.urlopen(req)
 
     # add resettable iterator over contents as an already-open fp
     path.fptext = RepeatFile(_iter_lines())
 
     return vd.openSource(path, filetype=filetype)
 
+def parse_header_links(link_header):
+    '''Return a list of dictionaries:
+    [{'url': 'https://example.com/content?page=1', 'rel': 'prev'},
+     {'url': 'https://example.com/content?page=3', 'rel': 'next'}]
+    Takes a link header string, of the form
+    '<https://example.com/content?page=1>; rel="prev", <https://example.com/content?page=3>; rel="next"'
+    See https://datatracker.ietf.org/doc/html/rfc8288#section-3
+    '''
+
+    links = []
+    quote_space = ' \'"'
+    link_header = link_header.strip(quote_space)
+    if not link_header: return []
+    for link_value in re.split(', *<', link_header):
+        if ';' in link_value:
+            url, params = link_value.split(';', maxsplit=1)
+        else:
+            url, params = link_value, ''
+        link = {'url': url.strip('<>' + quote_space)}
+
+        for param in params.split(';'):
+            if '=' in param:
+                key, value = param.split('=')
+                key = key.strip(quote_space)
+                value = value.strip(quote_space)
+                link[key] = value
+            else:
+                break
+        links.append(link)
+    return links
+
 VisiData.openurl_https = VisiData.openurl_http

visidata/loaders/imap.py

@@ -1,11 +1,12 @@
-from visidata import vd, TableSheet, asyncthread, ColumnItem, Column, ColumnAttr, Progress
+from visidata import VisiData, vd, TableSheet, asyncthread, ColumnItem, Column, ColumnAttr, Progress
+import visidata.loaders.google
 from urllib.parse import urlparse
 
 
-def openurl_imap(p, **kwargs):
-    url = urlparse(p.given)
-    password = url.password or vd.error('no password given in url') # vd.input("imap password for %s" % user, display=False))
-    return ImapSheet(url.hostname, source=url, password=password)
+@VisiData.api
+def openurl_imap(vd, url, **kwargs):
+    url_parsed = urlparse(str(url))
+    return ImapSheet(url_parsed.hostname, source=url_parsed, password=url_parsed.password)
 
 
 class ImapSheet(TableSheet):
@@ -22,17 +23,26 @@ class ImapSheet(TableSheet):
     ]
     nKeys = 1
 
-    @asyncthread
-    def reload(self):
+    def iterload(self):
         import imaplib
         import email.parser
 
         m = imaplib.IMAP4_SSL(host=self.source.hostname)
+        # m.debug=4
         user = self.source.username
-        m.login(user, self.password)
+
+        if self.source.hostname == 'imap.gmail.com':
+            credentials=vd.google_auth(scopes='https://mail.google.com/')
+            header_template = 'user=%s\1auth=Bearer %s\1\1'
+            m.authenticate('XOAUTH2', lambda x: header_template % (user, credentials.token))
+        else:
+            if self.password is None:
+                vd.error('no password given in url') # vd.input("imap password for %s" % user, display=False))
+            m.login(user, self.source.password)
+
         typ, folders = m.list()
         for r in Progress(folders, gerund="downloading"):
-            fname = r.decode('utf-8').split()[-1][1:-1]
+            fname = r.decode('utf-8').split()[-1]
             try:
                 m.select(fname)
                 typ, data = m.search(None, 'ALL')
@@ -44,7 +54,7 @@ class ImapSheet(TableSheet):
 
                     msg = email.message_from_bytes(msgbytes[0][1])
                     msg['folder'] = fname
-                    self.addRow(msg)
+                    yield msg
 
                 m.close()
             except Exception:

visidata/loaders/jrnl.py

@@ -0,0 +1,52 @@
+'''Loader for the jrnl.sh CLI journal file format'''
+
+import re
+
+
+from visidata import VisiData, TableSheet, ItemColumn, AttrDict
+
+
+@VisiData.api
+def open_jrnl(vd, p):
+    return JrnlSheet(p.base_stem, source=p)
+
+
+class JrnlSheet(TableSheet):
+    # rowdef: AttrDict
+    columns = [
+        ItemColumn('date'),
+        ItemColumn('time'),
+        ItemColumn('title'),
+        ItemColumn('body'),
+        ItemColumn('tags'),
+    ]
+    def iterload(self):
+        re_title = re.compile(r'\[(.*?)\s(.*?)\] (.*)')
+        prevline = ''
+        for line in self.source:
+            tags = re.findall(r'(?<!\S)(@[-+*#/\w]+)', line)
+            if not prevline:
+                m = re_title.match(line)
+                if m:
+                    row = AttrDict()
+                    row.date, row.time, row.title = m.groups()
+                    row.body = ''
+                    row.tags = ' '.join(tags)
+                    yield row
+                    continue
+
+            row.body += line + '\n'
+            row.tags = ' '.join([row.tags]+tags)
+            prevline = line.strip()
+
+
+@VisiData.api
+def save_jrnl(vd, p, *vsheets):
+    with p.open(mode='w', encoding=vsheets[0].options.save_encoding) as fp:
+        for vs in vsheets:
+            for r in vs.iterrows():
+                fp.write(f'[{r.date} {r.time}] {r.title}\n')
+                body = r.body.strip()
+                if body:
+                    fp.write(body + '\n')
+                fp.write('\n')

visidata/loaders/json.py

@@ -1,30 +1,49 @@
 import json
 
-from visidata import vd, date, VisiData, PyobjSheet, deepcopy, AttrDict, stacktrace, TypedExceptionWrapper, options, visidata, ColumnItem, deduceType, wrapply, TypedWrapper, Progress, Sheet, InferColumnsSheet
+from visidata import vd, date, anytype, VisiData, PyobjSheet, AttrDict, stacktrace, TypedExceptionWrapper, AlwaysDict, ItemColumn, wrapply, TypedWrapper, Progress, Sheet
 
 vd.option('json_indent', None, 'indent to use when saving json')
 vd.option('json_sort_keys', False, 'sort object keys when saving to json')
+vd.option('json_ensure_ascii', True, 'ensure ascii encode when saving json')
 vd.option('default_colname', '', 'column name to use for non-dict rows')
 
+@VisiData.api
+def guess_json(vd, p):
+    with p.open(encoding=vd.options.encoding) as fp:
+        line = next(fp)
+
+    line = line.strip()
+
+    if line.startswith('{') and line.endswith('}'):
+        return dict(filetype='jsonl')
+
+    if line.startswith(tuple('[{')):
+        return dict(filetype='json')
+
+
 @VisiData.api
 def open_jsonobj(vd, p):
-    return JsonSheet(p.name, source=p)
+    return JsonSheet(p.base_stem, source=p)
 
 @VisiData.api
 def open_jsonl(vd, p):
-    return JsonSheet(p.name, source=p)
+    return JsonSheet(p.base_stem, source=p)
 
 VisiData.open_ndjson = VisiData.open_ldjson = VisiData.open_json = VisiData.open_jsonl
 
 
-class JsonSheet(InferColumnsSheet):
+class JsonSheet(Sheet):
+    _rowtype = AttrDict
+    def resetCols(self):
+        self._knownKeys = set()
+        super().resetCols()
+
     def iterload(self):
-        with self.source.open_text(encoding=self.options.encoding) as fp:
+        with self.open_text_source() as fp:
             for L in fp:
+                L = L.strip()
                 try:
-                    if L.startswith('#'): # skip commented lines
-                        continue
-                    elif not L.strip(): # skip blank lines
+                    if not L: # skip blank lines
                         continue
                     ret = json.loads(L, object_hook=AttrDict)
                     if isinstance(ret, list):
@@ -37,7 +56,7 @@ class JsonSheet(InferColumnsSheet):
                         e.stacktrace = stacktrace()
                         yield TypedExceptionWrapper(json.loads, L, exception=e)  # an error on one line
                     else:
-                        with self.source.open_text(encoding=self.options.encoding) as fp:
+                        with self.open_text_source() as fp:
                             ret = json.load(fp)
                             if isinstance(ret, list):
                                 yield from ret
@@ -45,22 +64,36 @@ class JsonSheet(InferColumnsSheet):
                                 yield ret
                         break
 
+    def addColumn(self, *cols, index=None):
+        for c in cols:
+            self._knownKeys.add(c.expr or c.name)
+        return super().addColumn(*cols, index=index)
+
     def addRow(self, row, index=None):
         # Wrap non-dict rows in a dummy object with a predictable key name.
         # This allows for more consistent handling of rows containing scalars
         # or lists.
         if not isinstance(row, dict):
-            v = {options.default_colname: row}
-            row = visidata.AlwaysDict(row, **v)
+            v = {self.options.default_colname: row}
+            row = AlwaysDict(row, **v)
 
-        return super().addRow(row, index=index)
+        ret = super().addRow(row, index=index)
+
+        for k in row:
+            if k not in self._knownKeys:
+                c = ItemColumn(k, type=float if isinstance(row[k], (float, int)) else anytype)
+                self.addColumn(c)
+
+        return ret
 
     def newRow(self, **fields):
-        return fields
+        return AttrDict(fields)
 
     def openRow(self, row):
         return PyobjSheet("%s[%s]" % (self.name, self.keystr(row)), source=row)
 
+JsonSheet.init('_knownKeys', set, copy=True)  # set of row keys already seen
+
 ## saving json and jsonl
 
 class _vjsonEncoder(json.JSONEncoder):
@@ -68,17 +101,23 @@ class _vjsonEncoder(json.JSONEncoder):
         return str(obj)
 
 
-def _rowdict(cols, row):
+@VisiData.api
+def get_json_value(vd, col, row):
+    o = wrapply(col.getTypedValue, row)
+    if isinstance(o, TypedExceptionWrapper):
+        o = col.sheet.options.safe_error or str(o.exception)
+    elif isinstance(o, TypedWrapper):
+        o = o.val
+    elif isinstance(o, date):
+        o = col.getDisplayValue(row)
+    return o
+
+
+def _rowdict(cols, row, keep_nulls=False):
     ret = {}
     for col in cols:
-        o = wrapply(col.getTypedValue, row)
-        if isinstance(o, TypedExceptionWrapper):
-            o = col.sheet.options.safe_error or str(o.exception)
-        elif isinstance(o, TypedWrapper):
-            o = o.val
-        elif isinstance(o, date):
-            o = col.getDisplayValue(row)
-        if o is not None:
+        o = vd.get_json_value(col, row)
+        if keep_nulls or o is not None:
             ret[col.name] = o
     return ret
 
@@ -92,13 +131,13 @@ def encode_json(vd, row, cols, enc=_vjsonEncoder(sort_keys=False)):
 @VisiData.api
 def save_json(vd, p, *vsheets):
     vs = vsheets[0]
-    with p.open_text(mode='w', encoding=vs.options.encoding) as fp:
+    with p.open(mode='w', encoding=vs.options.save_encoding) as fp:
         try:
             indent = int(vs.options.json_indent)
         except Exception:
             indent = vs.options.json_indent
 
-        jsonenc = _vjsonEncoder(indent=indent, sort_keys=vs.options.json_sort_keys)
+        jsonenc = _vjsonEncoder(indent=indent, sort_keys=vs.options.json_sort_keys, ensure_ascii=vs.options.json_ensure_ascii)
 
         if len(vsheets) == 1:
             fp.write('[\n')
@@ -107,11 +146,11 @@ def save_json(vd, p, *vsheets):
                 for i, row in enumerate(vs.iterrows()):
                     if i > 0:
                         fp.write(',\n')
-                    rd = _rowdict(vs.visibleCols, row)
+                    rd = _rowdict(vs.visibleCols, row, keep_nulls=(i==0))
                     fp.write(jsonenc.encode(rd))
             fp.write('\n]\n')
         else:
-            it = {vs.name: [_rowdict(vs.visibleCols, row) for row in vs.iterrows()] for vs in vsheets}
+            it = {vs.name: [_rowdict(vs.visibleCols, row, keep_nulls=(i==0)) for i, row in enumerate(vs.iterrows())] for vs in vsheets}
 
             with Progress(gerund='saving'):
                 for chunk in jsonenc.iterencode(it):
@@ -123,19 +162,34 @@ def write_jsonl(vs, fp):
         vcols = vs.visibleCols
         jsonenc = _vjsonEncoder()
         with Progress(gerund='saving'):
-            for row in vs.iterrows():
-                rowdict = _rowdict(vcols, row)
+            for i, row in enumerate(vs.iterrows()):
+                rowdict = _rowdict(vcols, row, keep_nulls=(i==0))
                 fp.write(jsonenc.encode(rowdict) + '\n')
 
+        if len(vs) == 0:
+            vd.warning(
+                "Output file is empty - cannot save headers without data for jsonl.\n"
+                "Use `.jsonla` filetype to save as JSONL arrays format "
+                "rather than JSONL dict format to preserve the headers."
+            )
+
 
 @VisiData.api
 def save_jsonl(vd, p, *vsheets):
-    with p.open_text(mode='w', encoding=vsheets[0].options.encoding) as fp:
+    with p.open(mode='w', encoding=vsheets[0].options.save_encoding) as fp:
         for vs in vsheets:
             vs.write_jsonl(fp)
 
 
+@VisiData.api
+def JSON(vd, s:str):
+    'Parse `s` as JSON.'
+    return json.loads(s)
+
+
 JsonSheet.options.encoding = 'utf-8'
+JsonSheet.options.regex_skip = r'^(//|#).*'
+
 VisiData.save_ndjson = VisiData.save_jsonl
 VisiData.save_ldjson = VisiData.save_jsonl
 

visidata/loaders/jsonla.py

@@ -0,0 +1,74 @@
+import json
+
+from visidata import VisiData, vd, SequenceSheet, deduceType, Progress
+
+
+@VisiData.api
+def guess_jsonla(vd, p):
+    '''A JSONLA file is a JSONL file with rows of arrays, where the first row
+    is a header array:
+
+    ["A", "B", "C"]
+    [1, "blue", true]
+    [2, "yellow", false]
+
+    The header array must be a flat array of strings
+
+    If no suitable header is found, fall back to generic JSON load.
+    '''
+
+    with p.open(encoding=vd.options.encoding) as fp:
+        first_line = next(fp)
+
+    if first_line.strip().startswith('['):
+        ret = json.loads(first_line)
+        if isinstance(ret, list) and all(isinstance(v, str) for v in ret):
+            return dict(filetype='jsonla')
+
+
+@VisiData.api
+def open_jsonla(vd, p):
+    return JsonlArraySheet(p.base_stem, source=p)
+
+
+class JsonlArraySheet(SequenceSheet):
+    rowtype = 'rows'    # rowdef: list of Python objects decoded from JSON
+    def iterload(self):
+        with self.open_text_source() as fp:
+            for L in fp:
+                yield json.loads(L)
+
+        # set column types from first row
+        for i, c in enumerate(self.columns):
+            c.type = deduceType(self.rows[0][i])
+
+
+def get_jsonla_rows(sheet, cols):
+    for row in Progress(sheet.rows):
+        yield [vd.get_json_value(col, row) for col in cols]
+
+
+class _vjsonEncoder(json.JSONEncoder):
+    def default(self, obj):
+        return str(obj)
+
+
+def write_jsonla(vs, fp):
+        vcols = vs.visibleCols
+        jsonenc = _vjsonEncoder()
+        with Progress(gerund='saving'):
+            header = [col.name for col in vcols]
+            fp.write(jsonenc.encode(header) + '\n')
+            rows = get_jsonla_rows(vs, vcols)
+            for row in rows:
+                fp.write(jsonenc.encode(row) + '\n')
+
+
+@VisiData.api
+def save_jsonla(vd, p, *vsheets):
+    with p.open(mode='w', encoding=vsheets[0].options.save_encoding) as fp:
+        for vs in vsheets:
+            write_jsonla(vs, fp)
+
+
+JsonlArraySheet.options.regex_skip = r'^(//|#).*'

visidata/loaders/lsv.py

@@ -1,17 +1,19 @@
 import collections
 
+#1179  Line Separated Values for e.g. awk
+
 from visidata import VisiData, Sheet, ItemColumn
 
 
 @VisiData.api
 def open_lsv(vd, p):
-    return LsvSheet(p.name, source=p)
+    return LsvSheet(p.base_stem, source=p)
 
 
 @VisiData.api
 def save_lsv(vd, p, *vsheets):
     vs = vsheets[0]
-    with p.open_text(mode='w', encoding=vs.options.encoding) as fp:
+    with p.open(mode='w', encoding=vs.options.save_encoding) as fp:
         for row in vs.iterrows():
             for col in vs.visibleCols:
                 fp.write('%s: %s\n' % (col.name, col.getDisplayValue(row)))
@@ -33,17 +35,19 @@ class LsvSheet(Sheet):
         self._knownCols = set()
         row = collections.defaultdict(str)
         k = ''
-        for line in self.source.open_text():
-            line = line.strip()
-            if not line:
-                yield row
-                row = collections.defaultdict(str)
 
-            if ':' in line:
-                k, line = line.split(':', maxsplit=1)
-            # else append to previous k
+        with self.open_text_source() as fp:
+            for line in fp:
+                line = line.strip()
+                if not line:
+                    yield row
+                    row = collections.defaultdict(str)
+
+                if ':' in line:
+                    k, line = line.split(':', maxsplit=1)
+                # else append to previous k
 
-            row[k.strip()] += line.strip()
+                row[k.strip()] += line.strip()
 
         if row:
             yield row

visidata/loaders/mailbox.py

@@ -0,0 +1,40 @@
+from visidata import VisiData, Sheet, ItemColumn, date, Column
+
+
+@VisiData.api
+def open_mbox(vd, p):
+    return MboxSheet(p.base_stem, source=p, format='mbox')
+
+@VisiData.api
+def open_maildir(vd, p):
+    return MboxSheet(p.base_stem, source=p, format='Maildir')
+
+@VisiData.api
+def open_mmdf(vd, p):
+    return MboxSheet(p.base_stem, source=p, format='MMDF')
+
+@VisiData.api
+def open_babyl(vd, p):
+    return MboxSheet(p.base_stem, source=p, format='Babyl')
+
+@VisiData.api
+def open_mh(vd, p):
+    return MboxSheet(p.base_stem, source=p, format='MH')
+
+
+class MboxSheet(Sheet):
+    columns = [
+        ItemColumn('Date', type=date),
+        ItemColumn('From'),
+        ItemColumn('To'),
+        ItemColumn('Cc'),
+        ItemColumn('Subject'),
+        Column('Payload', getter=lambda c,r: r.get_payload(decode=True),
+                          setter=lambda c,r,v: r.set_payload(v)),
+    ]
+    def iterload(self):
+        import mailbox
+        cls = getattr(mailbox, self.format)
+        self.mailbox = cls(str(self.source), create=False)
+        for r in self.mailbox.itervalues():
+            yield r

visidata/loaders/markdown.py

@@ -26,7 +26,7 @@ def write_md(p, *vsheets, md_style='orgmode'):
     else:
         delim = '|'
 
-    with p.open_text(mode='w', encoding=vsheets[0].options.encoding) as fp:
+    with p.open(mode='w', encoding=vsheets[0].options.save_encoding) as fp:
         for vs in vsheets:
             if len(vsheets) > 1:
                 fp.write('# %s\n\n' % vs.name)
@@ -44,8 +44,6 @@ def write_md(p, *vsheets, md_style='orgmode'):
                     fp.write(s)
             fp.write('\n')
 
-    vd.status('%s save finished' % p)
-
 
 @VisiData.api
 def save_md(vd, p, *sheets):