visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/loaders/api_zulip.py

@@ -0,0 +1,249 @@
+import time
+
+from visidata import vd, VisiData, BaseSheet, Sheet, TextSheet, PyobjSheet
+from visidata import ItemColumn, Column, vlen, date, asyncsingle, ENTER, AttrDict
+
+vd.option('zulip_batch_size', -100, 'number of messages to fetch per call (<0 to fetch before anchor)')
+vd.option('zulip_anchor', 1000000000, 'message id to start fetching from')
+vd.option('zulip_delay_s', 0.00001, 'seconds to wait between calls (0 to stop after first)')
+vd.option('zulip_api_key', '', 'Zulip API key')
+vd.option('zulip_email', '', 'Email for use with Zulip API key')
+
+
+@VisiData.api
+def open_zulip(vd, p):
+    vd.importExternal('zulip')
+    import zulip
+
+    if not vd.options.zulip_api_key:
+        vd.warning('zulip_api_key must be set first')
+        vd.status('Enter your login email and Zulip API key (see _https://zulip.com/api/api-keys_).')
+        email = vd.input(f'Login email for {p.given}: ', record=False)
+        api_key = vd.input(f'Zulip API key: ', record=False)
+
+        vd.setPersistentOptions(zulip_email=email, zulip_api_key=api_key)
+
+    vd.z_client = zulip.Client(site=p.given, api_key=vd.options.zulip_api_key, email=vd.options.zulip_email)
+
+    return vd.subscribedStreams
+
+
+VisiData.openhttp_zulip = VisiData.open_zulip
+
+
+@VisiData.api
+def z_rpc(vd, r, result_field_name=None):
+    if r['result'] != 'success':
+        return PyobjSheet(result_field_name+'_error', source=r)
+    elif result_field_name:
+        return PyobjSheet(result_field_name, source=r[result_field_name])
+
+
+@VisiData.lazy_property
+def allStreams(vd):
+    return ZulipStreamsSheet("all_streams", zulip_func='get_streams', zulip_result_key="streams", zulip_kwargs=dict(include_public=True, include_subscribed=True))
+
+
+@VisiData.lazy_property
+def subscribedStreams(vd):
+    return ZulipStreamsSheet("subscriptions", zulip_func='get_subscriptions', zulip_result_key="subscriptions")
+
+
+@VisiData.lazy_property
+def allMessages(vd):
+    return ZulipMessagesSheet("all_messages")
+
+
+@VisiData.api
+def parseColumns(vd, fieldlist):
+    for cname in fieldlist:
+        kwargs = {}
+        while not cname[0].isalpha():
+            if cname[0] == '#': kwargs['type'] = int
+            elif cname[0] == '@': kwargs['type'] = date
+            elif cname[0] == '-': kwargs['width'] = 0
+            else: break
+            cname = cname[1:]
+        yield ItemColumn(cname, **kwargs)
+
+
+class ZulipAPISheet(Sheet):
+    zulip_func = None
+    zulip_result_key = ''
+    zulip_args = []
+    zulip_kwargs = {}
+    fields = ''
+
+    def iterload(self):
+        self.columns = []
+        for c in vd.parseColumns(self.fields.split()):
+            self.addColumn(c)
+
+        zulip_func = self.zulip_func
+        if isinstance(zulip_func, str): # allow later binding for startup perf
+            zulip_func = getattr(vd.z_client, zulip_func)
+        r = zulip_func(*self.zulip_args, **self.zulip_kwargs)
+        if r['result'] != 'success':
+            vd.push(PyobjSheet(self.zulip_result_key+'_error', source=r))
+            return
+        yield from r[self.zulip_result_key]
+
+    def addRow(self, r, **kwargs):
+        return super().addRow(AttrDict(r), **kwargs)
+
+
+class ZulipStreamsSheet(ZulipAPISheet):
+    guide = '''# Zulip Streams
+
+- `Enter` to open recent messages from the stream
+- `z Enter` to open list of topics from the stream
+'''
+    rowtype = 'streams'  # rowdef: dict of stream from server
+    fields = '-#stream_id name @date_created description -rendered_description -invite_only -is_web_public -stream_post_policy -history_public_to_subscribers -#first_message_id -#message_retention_days -is_announcement_only'
+
+    def openRow(self, r):
+        return ZulipMessagesSheet(r.name, filters=dict(stream=r.name))
+
+    def openCell(self, c, r):
+        return ZulipTopicsSheet(r.name+'_topics',
+                             zulip_func=vd.z_client.get_stream_topics,
+                             zulip_args=[r.stream_id],
+                             zulip_result_key='topics')
+
+
+class ZulipTopicsSheet(ZulipAPISheet):
+    rowtype = 'topics'  # rowdef: dict of topic from server
+    fields='name #max_id'
+    def openRow(self, r):
+        return ZulipMessagesSheet(f'{r.name}:{r.subject}', filters=dict(stream=r.name, topic=r.subject))
+
+
+class ZulipMembersSheet(ZulipAPISheet):
+    guide = '''# Zulip Members
+- `Enter` to open list of messages from this member
+'''
+    rowtype = 'members'  # rowdef: dict of member from server
+    fields = '''-#user_id full_name email timezone @date_joined -#avatar_version -is_admin -is_owner -is_guest -is_bot -#role -is_active -avatar_url -bot_type -#bot_owner_id'''
+    def openRow(self, r):
+        return ZulipMessagesSheet(r.display_recipient, filters=dict(stream=r.display_recipient))
+
+
+class ZulipMessagesSheet(Sheet):
+    guide = '''# Zulip Messages Sheet
+Loads continuously starting with most recent, until all messages have been read.
+
+- `Ctrl+C` to cancel loading.
+- `Enter` to open message in word-wrapped text sheet
+'''
+    rowtype = 'messages'  # rowdef: dict of message from server
+#    fields = ''
+    columns = [
+        ItemColumn('timestamp', type=date, fmtstr='%Y-%m-%d %H:%M'),
+        ItemColumn('sender', 'sender_full_name'),
+        ItemColumn('sender_email', width=0),
+        ItemColumn('recipient', 'display_recipient'),
+        ItemColumn('subject'),
+        ItemColumn('content'),
+        ItemColumn('client', width=0),
+        ItemColumn('reactions', type=vlen),
+        ItemColumn('submessages', type=vlen),
+        ItemColumn('flags', width=0),
+    ]
+    filters={}
+
+    @asyncsingle # kill previous thread
+    def reload(self):
+        self.rows = []
+        narrow = list(self.filters.items())
+        n = self.options.zulip_batch_size
+        req = AttrDict(
+            num_before = -n if n < 0 else 0,
+            num_after = n if n > 0 else 0,
+            anchor = self.options.zulip_anchor,
+            apply_markdown = False,
+            narrow = narrow)
+
+        while True:
+            r = vd.z_client.call_endpoint(url='messages', method='GET', request=req)
+            if r['result'] == 'success':
+                if not r['messages']: break
+                for i, msg in enumerate(r['messages']):
+                    self.addRow(msg, index=i)
+                req['anchor'] = min(msg['id'] for msg in r['messages'])-1
+                s = self.options.zulip_delay_s
+                if s <= 0:
+                    break
+                time.sleep(s)
+
+#        vd.status('finished loading, waiting for new messages')
+#        vd.z_client.call_on_each_event(self.received_event, ['message'], narrow=narrow)
+
+    def get_channel_name(self, r):
+        recp = r['display_recipient']
+        if isinstance(recp, list):  # private message
+            return '[%s]' % recp[0]['full_name']
+        else:
+            return '%s:%s' % (recp, r['subject'])
+
+    def update_message(self, msgid, content):
+        req = {
+            "message_id": msgid,
+            "content": content
+        }
+        vd.z_rpc(vd.z_client.update_message(req))
+
+    def openRow(self, r):
+        vs = TextSheet(self.get_channel_name(r), source=[r["content"]])
+        vs.options.wrap = True
+        return vs
+
+    def received_event(self, event):
+        if event['type'] == 'message':
+            self.addRow(event['message'])
+
+    def reply_message(self, msg, row):
+        recp = row['display_recipient']
+        if isinstance(recp, list):
+            for dest in recp:
+                self.send_message(msg, row['subject'], dest['email'], 'private')
+        else:
+            self.send_message(msg, row['subject'], dest, 'stream')
+
+    def send_message(self, msg, subject, dest, msgtype='stream'):
+        req = {
+            'type': msgtype,
+            'content': msg,
+            'subject': subject,
+            'to': dest,
+        }
+        r = vd.z_client.send_message(req)
+
+        if r['result'] != 'success':
+            vd.push(PyobjSheet('send_message_result', source=r))
+
+
+vd.addGlobals({
+    'ZulipMembersSheet': ZulipMembersSheet,
+    'ZulipStreamsSheet': ZulipStreamsSheet,
+    'ZulipAPISheet': ZulipAPISheet,
+    'ZulipMessagesSheet': ZulipMessagesSheet,
+})
+
+ZulipAPISheet.addCommand('', 'open-zulip-profile', 'vd.push(PyobjSheet("profile", source=z_client.get_profile()))', 'open connected user\'s profile')
+ZulipAPISheet.addCommand('', 'open-zulip-members', 'vd.push(ZulipMembersSheet("members", zulip_func=z_client.get_users, zulip_result_key="members"))', 'open list of all members')
+ZulipAPISheet.addCommand('', 'open-zulip-streams', 'vd.push(vd.allStreams)', 'open list of all streams')
+ZulipAPISheet.addCommand('', 'open-zulip-subs', 'vd.push(vd.subscribedStreams)', 'open list of subscribed streams')
+ZulipAPISheet.addCommand('', 'open-zulip-msgs', 'vd.push(vd.allMessages)', 'open list of all messages')
+
+ZulipMessagesSheet.addCommand('', 'reply-zulip-msg', 'reply_message(input(cursorRow["display_recipient"][1]["short_name"]+"> ", "message"), cursorRow)', 'reply to current topic')
+ZulipMessagesSheet.addCommand('', 'edit-zulip-msg', 'update_message(cursorRow["id"], editCell(3, cursorRowIndex))', 'edit message content')
+
+vd.addMenuItems('''
+File > Zulip > profile > open-zulip-profile
+File > Zulip > member list > open-zulip-members
+File > Zulip > streams > open-zulip-streams
+File > Zulip > subscriptions > open-zulip-subs
+File > Zulip > messages > open-zulip-subs
+File > Zulip > reply > reply-zulip-msg
+File > Zulip > edit message > edit-zulip-msg
+''')

visidata/loaders/archive.py

@@ -8,13 +8,23 @@ from visidata import vd, VisiData, asyncthread, Sheet, Progress, Menu, options
 from visidata import ColumnAttr, Column, Path
 from visidata.type_date import date
 
+@VisiData.api
+def guess_zip(vd, p):
+    if not p.is_url() and zipfile.is_zipfile(p.open_bytes()):
+        return dict(filetype='zip')
+
+@VisiData.api
+def guess_tar(vd, p):
+    if tarfile.is_tarfile(p.fp):
+        return dict(filetype='tar')
+
 @VisiData.api
 def open_zip(vd, p):
-    return vd.ZipSheet(p.name, source=p)
+    return vd.ZipSheet(p.base_stem, source=p)
 
 @VisiData.api
 def open_tar(vd, p):
-    return TarSheet(p.name, source=p)
+    return TarSheet(p.base_stem, source=p)
 
 VisiData.open_tgz = VisiData.open_tar
 VisiData.open_txz = VisiData.open_tar
@@ -37,16 +47,33 @@ class ZipSheet(Sheet):
                getter=lambda col, row: datetime.datetime(*row[0].date_time)),
     ]
     nKeys = 2
+    guide = '''# Zip Sheet
+This is a list of files contained in the zipfile {sheet.displaySource}.
+
+Once extracted, files can be loaded with `ENTER`.
+
+Commands:
+
+- `x` to extract current file to current directory
+- `gx` to extract selected files to current directory
+- `zx` to extract current file to a given pathname
+- `gzx`  to extract selected files to given directory
+
+'''
 
     def openZipFile(self, fp, *args, **kwargs):
         '''Use VisiData input to handle password-protected zip files.'''
+        if isinstance(fp, zipfile.ZipFile):
+            zip_open =  fp.open
+        elif isinstance(fp, unzip_http.RemoteZipFile):
+            zip_open = fp._open
         try:
-            return fp.open(*args, **kwargs)
+            return zip_open(*args, **kwargs)
         except RuntimeError as err:
             if 'password required' in err.args[0]:
                 pwd = vd.input(f'{args[0].filename} is encrypted, enter password: ', display=False)
-                return fp.open(*args, **kwargs, pwd=pwd.encode('utf-8'))
-            vd.error(err)
+                return zip_open(*args, **kwargs, pwd=pwd.encode('utf-8'))
+            vd.exceptionCaught(err)
 
     def openRow(self, row):
             fi, zpath = row
@@ -59,10 +86,16 @@ class ZipSheet(Sheet):
         files = []
         for row in rows:
             r, _ = row
-            if (path/r.filename).exists():
-                vd.confirm(f'{r.filename} exists, overwrite? ')  #1452
+            vd.confirmOverwrite(path/r.filename)  #1452
             self.extract_async(row)
 
+    def sysopen_row(self, row):
+        'Extract file in row to tempdir and launch $EDITOR.  Modifications will be discarded.'
+        import tempfile
+        with tempfile.TemporaryDirectory() as tempdir:
+            self.zfp.extract(member=row[0], path=tempdir)
+            vd.launchExternalEditorPath(Path(tempdir)/row[0].filename)
+
     @asyncthread
     def extract_async(self, *rows, path=None):
         'Extract rows to *path*, without confirmation.'
@@ -115,6 +148,7 @@ ZipSheet.addCommand('x', 'extract-file', 'extract(cursorRow)', 'extract current
 ZipSheet.addCommand('gx', 'extract-selected', 'extract(*onlySelectedRows)', 'extract selected files to current directory')
 ZipSheet.addCommand('zx', 'extract-file-to', 'extract(cursorRow, path=inputPath("extract to: "))', 'extract current file to given pathname')
 ZipSheet.addCommand('gzx', 'extract-selected-to', 'extract(*onlySelectedRows, path=inputPath("extract %d files to: " % nSelectedRows))', 'extract selected files to given directory')
+ZipSheet.addCommand('Ctrl+O', 'sysopen-row', 'sysopen_row(cursorRow)', 'open $EDITOR with current file (modifications will be discarded)')
 
 vd.addMenu(Menu('File', Menu('Extract',
         Menu('current file', 'extract-file'),

visidata/loaders/arrow.py

@@ -7,17 +7,17 @@ from visidata import Sheet, VisiData, TypedWrapper, anytype, date, vlen, Column,
 @VisiData.api
 def open_arrow(vd, p):
     'Apache Arrow IPC file format'
-    return ArrowSheet(p.name, source=p)
+    return ArrowSheet(p.base_stem, source=p)
 
 
 @VisiData.api
 def open_arrows(vd, p):
     'Apache Arrow IPC streaming format'
-    return ArrowSheet(p.name, source=p)
+    return ArrowSheet(p.base_stem, source=p)
 
 
 def arrow_to_vdtype(t):
-    import pyarrow as pa
+    pa = vd.importExternal('pyarrow')
 
     arrow_to_vd_typemap = {
         pa.lib.Type_BOOL: bool,
@@ -44,7 +44,7 @@ def arrow_to_vdtype(t):
         pa.lib.Type_LARGE_BINARY: vlen,
 #            pa.lib.Type_FIXED_SIZE_BINARY: bytes,
 #            pa.lib.Type_STRING: str,
-        pa.lib.Type_LARGE_STRING: vlen,
+#            pa.lib.Type_LARGE_STRING: vlen,  #2003
 #            pa.lib.Type_LIST: list,
 #            pa.lib.Type_LARGE_LIST: list,
 #            pa.lib.Type_FIXED_SIZE_LIST: list,
@@ -58,7 +58,7 @@ def arrow_to_vdtype(t):
 
 class ArrowSheet(Sheet):
     def iterload(self):
-        import pyarrow as pa
+        pa = vd.importExternal('pyarrow')
 
         try:
             with pa.OSFile(str(self.source), 'rb') as fp:
@@ -81,8 +81,8 @@ class ArrowSheet(Sheet):
 
 @VisiData.api
 def save_arrow(vd, p, sheet, streaming=False):
-    import pyarrow as pa
-    import numpy as np
+    pa = vd.importExternal('pyarrow')
+    np = vd.importExternal('numpy')
 
     typemap = {
         anytype: pa.string(),

visidata/loaders/conll.py

@@ -0,0 +1,49 @@
+__author__ = "Paul McCann <polm@dampfkraft.com>"
+
+from visidata import vd, VisiData, TableSheet, ItemColumn
+
+@VisiData.api
+def open_conll(vd, p):
+    return ConllSheet(p.base_stem, source=p)
+
+
+@VisiData.api
+def open_conllu(vd, p):
+    return ConllSheet(p.base_stem, source=p)
+
+
+class ConllSheet(TableSheet):
+    rowtype='tokens'
+    # see here for reference:
+    # https://universaldependencies.org/format.html
+    columns=[
+        # Usually an integer, but can be prefixed like "dev-s1"
+        ItemColumn('sent_id', 0, type=str),
+        # token ID is almost always an integer, but can be technically be a decimal between 0 and 1.
+        # starts from 1 for each sentence.
+        ItemColumn('token_id', 1, type=int),
+        # form from the raw input, aka surface
+        ItemColumn('form', 2, type=str),
+        ItemColumn('lemma', 3, type=str),
+        ItemColumn('upos', 4, type=str),
+        ItemColumn('xpos', 5, type=str),
+        ItemColumn('feats', 6, type=dict),
+        ItemColumn('head', 7, type=int),
+        ItemColumn('deprel', 8, type=str),
+        # possibly list of pairs, but often? unused
+        ItemColumn('deps', 9),
+        # empty or a dictionary
+        ItemColumn('misc', 10, type=dict),
+    ]
+    def iterload(self):
+        pyconll = vd.importExternal('pyconll')
+
+        # sent_id + token_id will be unique
+        self.setKeys([self.columns[0], self.columns[1]])
+
+        with self.source.open(encoding='utf-8') as fp:
+            for sent in pyconll.load.iter_sentences(fp):
+                sent_id = sent.id
+                for token in sent:
+                    yield [sent_id, token.id, token._form, token.lemma, token.upos,
+                            token.xpos, token.feats, token.head, token.deprel, token.deps, token.misc]

visidata/loaders/csv.py

@@ -1,5 +1,3 @@
-import csv
-
 from visidata import vd, VisiData, SequenceSheet, options, stacktrace
 from visidata import TypedExceptionWrapper, Progress
 
@@ -11,13 +9,25 @@ vd.option('csv_escapechar', None, 'escapechar passed to csv.reader', replay=True
 vd.option('csv_lineterminator', '\r\n', 'lineterminator passed to csv.writer', replay=True)
 vd.option('safety_first', False, 'sanitize input/output to handle edge cases, with a performance cost', replay=True)
 
-csv.field_size_limit(2**31-1) # Windows has max 32-bit
 
-options_num_first_rows = 10
+@VisiData.api
+def guess_csv(vd, p):
+    import csv
+    csv.field_size_limit(2**31-1)  #288 Windows has max 32-bit
+    line = next(p.open())
+    if ',' in line:
+        dialect = csv.Sniffer().sniff(line)
+        r = dict(filetype='csv', _likelihood=0)
+
+        for csvopt in dir(dialect):
+            if not csvopt.startswith('_'):
+                r['csv_'+csvopt] = getattr(dialect, csvopt)
+
+        return r
 
 @VisiData.api
 def open_csv(vd, p):
-    return CsvSheet(p.name, source=p)
+    return CsvSheet(p.base_stem, source=p)
 
 def removeNulls(fp):
     for line in fp:
@@ -28,7 +38,10 @@ class CsvSheet(SequenceSheet):
 
     def iterload(self):
         'Convert from CSV, first handling header row specially.'
-        with self.source.open_text(encoding=self.options.encoding) as fp:
+        import csv
+        csv.field_size_limit(2**31-1)  #288 Windows has max 32-bit
+
+        with self.open_text_source() as fp:
             if options.safety_first:
                 rdr = csv.reader(removeNulls(fp), **options.getall('csv_'))
             else:
@@ -47,7 +60,10 @@ class CsvSheet(SequenceSheet):
 @VisiData.api
 def save_csv(vd, p, sheet):
     'Save as single CSV file, handling column names as first line.'
-    with p.open_text(mode='w', encoding=sheet.options.encoding, newline='') as fp:
+    import csv
+    csv.field_size_limit(2**31-1)  #288 Windows has max 32-bit
+
+    with p.open(mode='w', encoding=sheet.options.save_encoding, newline='') as fp:
         cw = csv.writer(fp, **options.getall('csv_'))
         colnames = [col.name for col in sheet.visibleCols]
         if ''.join(colnames):
@@ -57,6 +73,8 @@ def save_csv(vd, p, sheet):
             for dispvals in sheet.iterdispvals(format=True):
                 cw.writerow(dispvals.values())
 
+CsvSheet.options.regex_skip = '^#.*'
+
 vd.addGlobals({
     'CsvSheet': CsvSheet
 })

visidata/loaders/eml.py

@@ -4,7 +4,7 @@ from visidata import VisiData, vd, Column, TableSheet, vlen
 
 @VisiData.api
 def open_eml(vd, p):
-    return EmailSheet(p.name, source=p)
+    return EmailSheet(p.base_stem, source=p)
 
 class EmailSheet(TableSheet):
     rowtype = 'parts'  # rowdef: sub-Messages
@@ -16,7 +16,7 @@ class EmailSheet(TableSheet):
     def iterload(self):
         import email
         parser = email.parser.Parser()
-        with self.source.open_text(encoding='utf-8') as fp:
+        with self.source.open(encoding='utf-8') as fp:
             yield from parser.parse(fp).walk()
 
 @EmailSheet.api
@@ -27,8 +27,7 @@ def extract_part(sheet, givenpath, part):
 @EmailSheet.api
 def extract_parts(sheet, givenpath, *parts):
     'Save all *parts* to Path *givenpath*.'
-    if givenpath.exists() and sheet.options.confirm_overwrite:
-        vd.confirm("%s already exists. overwrite? " % givenpath.given)
+    vd.confirmOverwrite(givenpath, f'{givenpath} already exists, extract anyway?')
 
     vd.status('saving %s parts to %s' % (len(parts), givenpath.given))