visidata 2.11.1__py3-none-any.whl → 3.0.1__py3-none-any.whl

visidata/loaders/arrow.py

@@ -7,17 +7,17 @@ from visidata import Sheet, VisiData, TypedWrapper, anytype, date, vlen, Column,
 @VisiData.api
 def open_arrow(vd, p):
     'Apache Arrow IPC file format'
-    return ArrowSheet(p.name, source=p)
+    return ArrowSheet(p.base_stem, source=p)
 
 
 @VisiData.api
 def open_arrows(vd, p):
     'Apache Arrow IPC streaming format'
-    return ArrowSheet(p.name, source=p)
+    return ArrowSheet(p.base_stem, source=p)
 
 
 def arrow_to_vdtype(t):
-    import pyarrow as pa
+    pa = vd.importExternal('pyarrow')
 
     arrow_to_vd_typemap = {
         pa.lib.Type_BOOL: bool,
@@ -44,7 +44,7 @@ def arrow_to_vdtype(t):
         pa.lib.Type_LARGE_BINARY: vlen,
 #            pa.lib.Type_FIXED_SIZE_BINARY: bytes,
 #            pa.lib.Type_STRING: str,
-        pa.lib.Type_LARGE_STRING: vlen,
+#            pa.lib.Type_LARGE_STRING: vlen,  #2003
 #            pa.lib.Type_LIST: list,
 #            pa.lib.Type_LARGE_LIST: list,
 #            pa.lib.Type_FIXED_SIZE_LIST: list,
@@ -58,7 +58,7 @@ def arrow_to_vdtype(t):
 
 class ArrowSheet(Sheet):
     def iterload(self):
-        import pyarrow as pa
+        pa = vd.importExternal('pyarrow')
 
         try:
             with pa.OSFile(str(self.source), 'rb') as fp:
@@ -81,8 +81,8 @@ class ArrowSheet(Sheet):
 
 @VisiData.api
 def save_arrow(vd, p, sheet, streaming=False):
-    import pyarrow as pa
-    import numpy as np
+    pa = vd.importExternal('pyarrow')
+    np = vd.importExternal('numpy')
 
     typemap = {
         anytype: pa.string(),

visidata/loaders/conll.py

@@ -0,0 +1,49 @@
+__author__ = "Paul McCann <polm@dampfkraft.com>"
+
+from visidata import vd, VisiData, TableSheet, ItemColumn
+
+@VisiData.api
+def open_conll(vd, p):
+    return ConllSheet(p.base_stem, source=p)
+
+
+@VisiData.api
+def open_conllu(vd, p):
+    return ConllSheet(p.base_stem, source=p)
+
+
+class ConllSheet(TableSheet):
+    rowtype='tokens'
+    # see here for reference:
+    # https://universaldependencies.org/format.html
+    columns=[
+        # Usually an integer, but can be prefixed like "dev-s1"
+        ItemColumn('sent_id', 0, type=str),
+        # token ID is almost always an integer, but can be technically be a decimal between 0 and 1.
+        # starts from 1 for each sentence.
+        ItemColumn('token_id', 1, type=int),
+        # form from the raw input, aka surface
+        ItemColumn('form', 2, type=str),
+        ItemColumn('lemma', 3, type=str),
+        ItemColumn('upos', 4, type=str),
+        ItemColumn('xpos', 5, type=str),
+        ItemColumn('feats', 6, type=dict),
+        ItemColumn('head', 7, type=int),
+        ItemColumn('deprel', 8, type=str),
+        # possibly list of pairs, but often? unused
+        ItemColumn('deps', 9),
+        # empty or a dictionary
+        ItemColumn('misc', 10, type=dict),
+    ]
+    def iterload(self):
+        pyconll = vd.importExternal('pyconll')
+
+        # sent_id + token_id will be unique
+        self.setKeys([self.columns[0], self.columns[1]])
+
+        with self.source.open(encoding='utf-8') as fp:
+            for sent in pyconll.load.iter_sentences(fp):
+                sent_id = sent.id
+                for token in sent:
+                    yield [sent_id, token.id, token._form, token.lemma, token.upos,
+                            token.xpos, token.feats, token.head, token.deprel, token.deps, token.misc]

visidata/loaders/csv.py

@@ -1,5 +1,3 @@
-import csv
-
 from visidata import vd, VisiData, SequenceSheet, options, stacktrace
 from visidata import TypedExceptionWrapper, Progress
 
@@ -11,13 +9,25 @@ vd.option('csv_escapechar', None, 'escapechar passed to csv.reader', replay=True
 vd.option('csv_lineterminator', '\r\n', 'lineterminator passed to csv.writer', replay=True)
 vd.option('safety_first', False, 'sanitize input/output to handle edge cases, with a performance cost', replay=True)
 
-csv.field_size_limit(2**31-1) # Windows has max 32-bit
 
-options_num_first_rows = 10
+@VisiData.api
+def guess_csv(vd, p):
+    import csv
+    csv.field_size_limit(2**31-1)  #288 Windows has max 32-bit
+    line = next(p.open())
+    if ',' in line:
+        dialect = csv.Sniffer().sniff(line)
+        r = dict(filetype='csv', _likelihood=0)
+
+        for csvopt in dir(dialect):
+            if not csvopt.startswith('_'):
+                r['csv_'+csvopt] = getattr(dialect, csvopt)
+
+        return r
 
 @VisiData.api
 def open_csv(vd, p):
-    return CsvSheet(p.name, source=p)
+    return CsvSheet(p.base_stem, source=p)
 
 def removeNulls(fp):
     for line in fp:
@@ -28,7 +38,10 @@ class CsvSheet(SequenceSheet):
 
     def iterload(self):
         'Convert from CSV, first handling header row specially.'
-        with self.source.open_text(encoding=self.options.encoding) as fp:
+        import csv
+        csv.field_size_limit(2**31-1)  #288 Windows has max 32-bit
+
+        with self.open_text_source() as fp:
             if options.safety_first:
                 rdr = csv.reader(removeNulls(fp), **options.getall('csv_'))
             else:
@@ -47,7 +60,10 @@ class CsvSheet(SequenceSheet):
 @VisiData.api
 def save_csv(vd, p, sheet):
     'Save as single CSV file, handling column names as first line.'
-    with p.open_text(mode='w', encoding=sheet.options.encoding, newline='') as fp:
+    import csv
+    csv.field_size_limit(2**31-1)  #288 Windows has max 32-bit
+
+    with p.open(mode='w', encoding=sheet.options.save_encoding, newline='') as fp:
         cw = csv.writer(fp, **options.getall('csv_'))
         colnames = [col.name for col in sheet.visibleCols]
         if ''.join(colnames):
@@ -57,6 +73,8 @@ def save_csv(vd, p, sheet):
             for dispvals in sheet.iterdispvals(format=True):
                 cw.writerow(dispvals.values())
 
+CsvSheet.options.regex_skip = '^#.*'
+
 vd.addGlobals({
     'CsvSheet': CsvSheet
 })

visidata/loaders/eml.py

@@ -4,7 +4,7 @@ from visidata import VisiData, vd, Column, TableSheet, vlen
 
 @VisiData.api
 def open_eml(vd, p):
-    return EmailSheet(p.name, source=p)
+    return EmailSheet(p.base_stem, source=p)
 
 class EmailSheet(TableSheet):
     rowtype = 'parts'  # rowdef: sub-Messages
@@ -16,7 +16,7 @@ class EmailSheet(TableSheet):
     def iterload(self):
         import email
         parser = email.parser.Parser()
-        with self.source.open_text(encoding='utf-8') as fp:
+        with self.source.open(encoding='utf-8') as fp:
             yield from parser.parse(fp).walk()
 
 @EmailSheet.api
@@ -27,8 +27,7 @@ def extract_part(sheet, givenpath, part):
 @EmailSheet.api
 def extract_parts(sheet, givenpath, *parts):
     'Save all *parts* to Path *givenpath*.'
-    if givenpath.exists() and sheet.options.confirm_overwrite:
-        vd.confirm("%s already exists. overwrite? " % givenpath.given)
+    vd.confirmOverwrite(givenpath, f'{givenpath} already exists, extract anyway?')
 
     vd.status('saving %s parts to %s' % (len(parts), givenpath.given))