visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/loaders/scrape.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+
+__all__=[ 'SelectorColumn', 'soupstr' ]
+
+import os.path
+from urllib.parse import urljoin
+
+import concurrent.futures
+import functools
+
+from visidata import vd, VisiData, TableSheet, vdtype, Column, AttrColumn, Progress, date
+
+
+@VisiData.api
+def soup(vd, s):
+    bs4 = vd.importExternal('bs4', 'beautifulsoup4')
+    from bs4 import BeautifulSoup
+    return BeautifulSoup(s, 'html.parser')
+
+
+@VisiData.api
+def open_scrape(vd, p):
+    bs4 = vd.importExternal('bs4', 'beautifulsoup4')
+
+    vd.enable_requests_cache()
+    if p.is_url():
+        return HtmlDocsSheet(p.base_stem, source=p, urls=[p.given])
+    else:
+        return HtmlElementsSheet(p.base_stem, source=p, elements=None)
+
+VisiData.openhttp_scrape = VisiData.open_scrape
+
+def node_name(node):
+    me = node.name
+    class_ = node.attrs.get("class")
+    if class_:
+        me += '.' + class_[0]
+    id_ = node.attrs.get("id")
+    if id_:
+        me += '#' + id_
+    return me
+
+@functools.lru_cache(maxsize=None)
+def calc_selector(node):
+    if not node.parent:
+        return ''
+
+    psel = calc_selector(node.parent)
+    oursel = node_name(node)
+    if not psel:
+        return oursel
+
+    root = list(node.parents)[-1]
+
+    combinedsel = psel+' '+oursel
+    if len(root.select(combinedsel)) == len(root.select(oursel)):
+        return oursel
+
+    return combinedsel
+
+
+class HtmlAttrColumn(Column):
+    def calcValue(self, row):
+        return row.attrs.get(self.expr)
+
+
+def prev_header(r):
+    hdrtags = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+    try:
+        i = hdrtags.index(r.name)
+        return r.find_previous(hdrtags[:i-1])
+    except Exception:
+        return r.find_previous(hdrtags)
+
+
+# one row per element
+class HtmlElementsSheet(TableSheet):
+    guide = '''# HTMLElements
+
+This is a list of HTML elements from _{sheet.source}_ as parsed by `beautifulsoup4`.
+
+Standard VisiData exploration techniques can be used to find relevant data, which will help determine the proper selector to use.
+
+- `Enter` to dive into children of cursor element (or children of all selected rows with `g Enter`)
+- `go` to batch open links in selected rows on new RequestsSheet, which will fetch each page
+- `~` to use the `soupstr` type to join all the text elements
+'''
+    # source=[element, ...]
+    rowtype='dom nodes'  # rowdef soup.element
+    columns = [
+        Column('name', getter=lambda c,r: node_name(r)),
+        Column('selector', getter=lambda c,r: calc_selector(r), cache='async', width=0),
+        AttrColumn('string'),
+        Column('depth', cache=True, getter=lambda c,r: list(c.sheet.html_parents(r))),
+        Column('prev_header', getter=lambda c,r: prev_header(r), cache=True),
+        HtmlAttrColumn('href', expr='href'),
+    ]
+    def iterload(self):
+        for el in self.elements or [vd.soup(self.source.read_text())]:
+            for x in el.find_all():
+                if x.string:
+                    yield x
+
+    def html_parents(self, row):
+        while row.parent and row.parent is not row:
+            yield row.parent
+            row = row.parent
+
+    @property
+    def rootSource(self):
+        return self.rootSheet.source
+
+    def openRows(self, rows):
+        realurls = [urljoin(self.rootSource.given, r.attrs.get('href')) for r in rows]
+        yield HtmlDocsSheet(self.name, 'scrape', source=self, urls=realurls)
+
+    def openRow(self, row):
+        'opening a single row'
+        return HtmlElementsSheet('', source=self, elements=[row])
+
+
+class DocsSelectorColumn(Column):
+    def calcValue(self, row):
+        return [x for x in row.soup.select(self.expr)]
+
+class SelectorColumn(Column):
+    def calcValue(self, row):
+        return [x for x in row.select(self.expr)]
+
+
+# urls=list of urls to scrape
+class HtmlDocsSheet(TableSheet):
+    help='''# HtmlDocsSheet
+
+- `Enter` to open the current request as list of HTMLElements
+- `;` to add column of elements matching given css selector
+  - this is how to cross-tabulate data from multiple pages
+'''
+    rowtype='requests'  # rowdef: requests.Response
+    columns = [
+        AttrColumn('url'),
+        AttrColumn('status_code', type=int),
+        AttrColumn('from_cache'),
+        AttrColumn('fetched_at', 'created_at', type=date, width=0),
+        AttrColumn('expires', type=date),
+        AttrColumn('reason'),
+        AttrColumn('soup.title.string'),
+    ]
+    def iterload(self):
+        requests = vd.importExternal('requests')
+        self.colnames = {}
+#        with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+#            yield from executor.map(requests.get, Progress(self.urls))
+        for url in Progress(self.urls):
+            yield requests.get(url)
+
+    def addRow(self, row, index=None):
+        super().addRow(row, index=index)
+        row.soup = vd.callNoExceptions(vd.soup, row.text)
+
+    def openRow(self, row):
+        return HtmlElementsSheet(row.url, source=self, elements=[row.soup])
+
+def soupstr(coll):
+    return ' '.join(v.string for v in coll)
+
+vdtype(soupstr, 's')
+
+@TableSheet.api
+def scrape_urls(sheet, col, rows):
+    return HtmlDocsSheet(sheet.name, "selected_urls", urls=[col.getTypedValue(r) for r in rows])
+
+HtmlElementsSheet.addCommand('~', 'type-soupstr', 'cursorCol.type=soupstr', 'set type of current column to list of html elements')
+HtmlElementsSheet.addCommand('go', 'open-rows', 'for vs in openRows(selectedRows): vd.push(vs)', 'open sheet for each selected element')
+TableSheet.addCommand('gzo', 'scrape-cells', 'vd.push(scrape_urls(cursorCol, selectedRows))', 'open HTML Documents sheet from selected URLs')
+HtmlDocsSheet.addCommand(';', 'addcol-selector', 'sel=input("css selector: ", type="selector"); addColumn(DocsSelectorColumn(sel, expr=sel, cache="async"))', 'add column derived from css selector of current column')
+HtmlElementsSheet.addCommand(';', 'addcol-selector', 'sel=input("css selector: ", type="selector"); addColumn(SelectorColumn(sel, expr=sel, cache="async"))',  'add column derived from css selector of current column')
+
+vd.addGlobals({
+    'HtmlDocsSheet':SelectorColumn,
+    'SelectorColumn':SelectorColumn,
+    'DocsSelectorColumn':DocsSelectorColumn,
+    'soupstr':soupstr
+})
+
+vd.addMenuItem('Data', '+Scrape', 'selected cells', 'scrape-cells')

visidata/loaders/shp.py

@@ -1,13 +1,14 @@
 import json
+from copy import copy
 
-from visidata import VisiData, vd, Sheet, Column, Progress, date, copy, InvertedCanvas, asyncthread
+from visidata import VisiData, vd, Sheet, Column, Progress, date, InvertedCanvas, asyncthread
 
 # requires pyshp
 
 
 @VisiData.api
 def open_shp(vd, p):
-    return ShapeSheet(p.name, source=p)
+    return ShapeSheet(p.base_stem, source=p)
 
 VisiData.open_dbf = VisiData.open_shp
 
@@ -33,7 +34,7 @@ class ShapeSheet(Sheet):
         Column('shapeType', width=0, getter=lambda col,row: row.shape.shapeType)
     ]
     def iterload(self):
-        import shapefile
+        shapefile = vd.importExternal('shapefile', 'pyshp')
         self.sf = shapefile.Reader(str(self.source))
         self.reloadCols()
         for shaperec in Progress(self.sf.iterShapeRecords(), total=self.sf.numRecords):
@@ -97,10 +98,10 @@ def save_geojson(vd, p, vs):
         'type': 'FeatureCollection',
         'features': features,
     }
-    with p.open_text(mode='w', encoding=vs.options.encoding) as fp:
+    with p.open(mode='w', encoding=vs.options.save_encoding) as fp:
         for chunk in json.JSONEncoder().iterencode(featcoll):
             fp.write(chunk)
 
 ShapeSheet.addCommand('.', 'plot-row', 'vd.push(ShapeMap(name+"_map", source=sheet, sourceRows=[cursorRow], textCol=cursorCol))', 'plot geospatial vector in current row')
 ShapeSheet.addCommand('g.', 'plot-rows', 'vd.push(ShapeMap(name+"_map", source=sheet, sourceRows=rows, textCol=cursorCol))', 'plot all geospatial vectors in current sheet')
-ShapeMap.addCommand('^S', 'save-sheet', 'vd.saveSheets(inputPath("save to: ", value=getDefaultSaveName(sheet)), sheet, confirm_overwrite=options.confirm_overwrite)', 'save current sheet to filename in format determined by extension (default .geojson)')
+ShapeMap.addCommand('^S', 'save-sheet', 'vd.saveSheets(inputPath("save to: ", value=getDefaultSaveName(sheet)), sheet)', 'save current sheet to filename in format determined by extension (default .geojson)')

visidata/loaders/spss.py

@@ -1,22 +1,21 @@
-from visidata import VisiData, Sheet, Progress, asyncthread, ColumnItem
+from visidata import VisiData, Sheet, Progress, asyncthread, ItemColumn, vd
 
 
 @VisiData.api
 def open_spss(vd, p):
-    return SpssSheet(p.name, source=p)
+    return SpssSheet(p.base_stem, source=p)
 VisiData.open_sav = VisiData.open_spss
 
 
 class SpssSheet(Sheet):
-    @asyncthread
-    def reload(self):
-        import savReaderWriter
+    def loader(self):
+        savReaderWriter = vd.importExternal('savReaderWriter')
         self.rdr = savReaderWriter.SavReader(str(self.source))
         with self.rdr as reader:
             self.columns = []
             for i, vname in enumerate(reader.varNames):
                 vtype = float if reader.varTypes[vname] == 0 else str
-                self.addColumn(ColumnItem(vname.decode('utf-8'), i, type=vtype))
+                self.addColumn(ItemColumn(vname.decode('utf-8'), i, type=vtype))
 
             self.rows = []
             for r in Progress(reader, total=reader.shape.nrows):

visidata/loaders/sqlite.py

@@ -1,6 +1,7 @@
+from copy import copy
 import re
 
-from visidata import VisiData, vd, Sheet, options, Column, Progress, anytype, ColumnItem, asyncthread, TypedExceptionWrapper, TypedWrapper, IndexSheet, copy, clean_to_id, vlen
+from visidata import VisiData, vd, Sheet, options, Column, Progress, anytype, ColumnItem, asyncthread, TypedExceptionWrapper, TypedWrapper, IndexSheet, vlen
 from visidata.type_date import date
 
 vd.option('sqlite_onconnect', '', 'sqlite statement to execute after opening a connection')
@@ -16,13 +17,21 @@ def requery(url, **kwargs):
     return urlunparse(url_parts)
 
 
+@VisiData.api
+def guess_sqlite(vd, p):
+    if p.open_bytes().read(16).startswith(b'SQLite format'):
+        return dict(filetype='sqlite')
+
+
 @VisiData.api
 def open_sqlite(vd, p):
-    return SqliteIndexSheet(p.name, source=p)
+    if not p.is_local():
+        vd.fail('sqlite requires an uncompressed, local file')
+    return SqliteIndexSheet(p.base_stem, source=p)
 
 @VisiData.api
 def openurl_sqlite(vd, p, filetype=None):
-    return SqliteIndexSheet(p.name, source=p)
+    return SqliteIndexSheet(p.base_stem, source=p)
 
 VisiData.open_sqlite3 = VisiData.open_sqlite
 VisiData.open_db = VisiData.open_sqlite
@@ -32,15 +41,14 @@ class SqliteSheet(Sheet):
     'Provide functionality for importing SQLite databases.'
     savesToSource = True
     defer = True
-
-    def resolve(self):
-        'Resolve all the way back to the original source Path.'
-        return self.source.resolve()
+    query = ''
+    tableName = ''
 
     def conn(self):
         import sqlite3
-        pathname = str(self.resolve())
-        url = pathname if '://' in pathname else f'file:{pathname}'
+        localpath = self.rootSheet().source
+
+        url = localpath if localpath.is_url() else f'file:{localpath.resolve()}'
         url = requery(url, **self.options.getall('sqlite_param_'))
 
         con = sqlite3.connect(url, uri=True, **self.options.getall('sqlite_connect_'))
@@ -49,13 +57,25 @@ class SqliteSheet(Sheet):
             con.execute(self.options.sqlite_onconnect)
         return con
 
+    def rawSql(self, q:str) -> 'SqliteSheet':
+        return SqliteSheet('query', source=self.source, query=q)
+
+    @property
+    def sidebar(self):
+        if self.query:
+            return '# SQL\n' + self.query
+        else:
+            return super().sidebar
+
     def execute(self, conn, sql, parms=None):
         parms = parms or []
         vd.debug(sql)
         return conn.execute(sql, parms)
 
-    def iterload(self):
-        import sqlite3
+    def iterload_table(self, tblname:str):
+        '''Generate all rows from `tblname` in database at self.source,
+        including type information from table_xinfo(), and getting each rowid
+        if available (for simpler updates).'''
 
         def parse_sqlite_type(t):
             m = re.match(r'(\w+)(\((\d+)(,(\d+))?\))?', t.upper())
@@ -70,7 +90,6 @@ class SqliteSheet(Sheet):
 
         self.rowidColumn = None
         with self.conn() as conn:
-            tblname = self.tableName
             if not isinstance(self, SqliteIndexSheet):
                 self.columns = []
                 for r in self.execute(conn, 'PRAGMA TABLE_XINFO("%s")' % tblname):
@@ -92,6 +111,32 @@ class SqliteSheet(Sheet):
                 r = self.execute(conn, 'SELECT NULL, * FROM "%s"' % tblname)
             yield from Progress(r, total=r.rowcount-1)
 
+    def iterload_query(self, query:str):
+        '''Generate rows from `query` to database at self.source,
+        including type information from table_xinfo(), and getting each rowid
+        if available (for simpler updates).'''
+
+        with self.conn() as conn:
+            self.columns = []
+            for c in type(self).columns:
+                self.addColumn(copy(c))
+
+            self.result = self.execute(conn, query, parms=getattr(self, 'parms', []))
+
+            for i, desc in enumerate(self.result.description):
+                self.addColumn(ColumnItem(desc[0], i))
+
+            for row in self.result:
+                yield row
+
+    def iterload(self):
+        if self.tableName:
+            yield from self.iterload_table(self.tableName)
+        elif self.query:
+            yield from self.iterload_query(self.query)
+        else:
+            vd.fail('no query or tablename to load')
+
     @asyncthread
     def putChanges(self):
         adds, mods, dels = self.getDeferredChanges()
@@ -192,24 +237,14 @@ class SqliteIndexSheet(SqliteSheet, IndexSheet):
         self.preloadHook()
         self.reload()
 
-class SqliteQuerySheet(SqliteSheet):
-    def iterload(self):
-        with self.conn() as conn:
-            self.columns = []
-            for c in type(self).columns:
-                self.addColumn(copy(c))
-            self.result = self.execute(conn, self.query, parms=getattr(self, 'parms', []))
-            for i, desc in enumerate(self.result.description):
-                self.addColumn(ColumnItem(desc[0], i))
-
-            for row in self.result:
-                yield row
-
 
 
 @VisiData.api
 def save_sqlite(vd, p, *vsheets):
     import sqlite3
+    import json
+    jsonenc = json.JSONEncoder()  #1589: list/dict values as json
+
     conn = sqlite3.connect(str(p))
     conn.text_factory = lambda s, enc=vsheets[0].options.encoding: s.decode(enc)
     conn.row_factory = sqlite3.Row
@@ -231,7 +266,7 @@ def save_sqlite(vd, p, *vsheets):
     vd.sync()
 
     for vs in vsheets:
-        tblname = clean_to_id(vs.name)
+        tblname = vd.cleanName(vs.name)
         sqlcols = []
         for col in vs.visibleCols:
             sqlcols.append('"%s" %s' % (col.name, sqltypes.get(col.type, 'TEXT')))
@@ -247,6 +282,8 @@ def save_sqlite(vd, p, *vsheets):
                         v = options.safe_error
                     else:
                         v = None
+                elif isinstance(v, (list, tuple, dict)):
+                    v = jsonenc.encode(v)
                 elif not isinstance(v, (int, float, str)):
                     v = col.getDisplayValue(r)
                 sqlvals.append(v)
@@ -255,16 +292,19 @@ def save_sqlite(vd, p, *vsheets):
 
     conn.commit()
 
-    vd.status("%s save finished" % p)
 
+SqliteSheet.addCommand('', 'exec-sql', 'vd.push(rawSql(input("execute SQL: ", type="sql")))', 'execute raw SQL statement')
 
 SqliteIndexSheet.addCommand('a', 'add-table', 'fail("create a new table by saving a sheet to this database file")', 'stub; add table by saving a sheet to the db file instead')
 SqliteIndexSheet.bindkey('ga', 'add-table')
 SqliteSheet.options.header = 0
 VisiData.save_db = VisiData.save_sqlite
 
+vd.addMenuItems('''
+    Data > execute SQL query > exec-sql
+''')
+
 vd.addGlobals({
     'SqliteIndexSheet': SqliteIndexSheet,
     'SqliteSheet': SqliteSheet,
-    'SqliteQuerySheet': SqliteQuerySheet
 })

visidata/loaders/texttables.py

@@ -7,7 +7,7 @@ try:
         def save_table(path, *sheets, fmt=fmt):
             import tabulate
 
-            with path.open_text(mode='w', encoding=sheets[0].options.encoding) as fp:
+            with path.open(mode='w', encoding=sheets[0].options.save_encoding) as fp:
                 for vs in sheets:
                     fp.write(tabulate.tabulate(
                         vs.itervals(*vs.visibleCols, format=True),

visidata/loaders/toml.py

@@ -0,0 +1,60 @@
+from visidata import (
+    ColumnItem,
+    PythonSheet,
+    VisiData,
+    asyncthread,
+    deduceType,
+    vd,
+)
+
+
+@VisiData.api
+def open_toml(vd, p):
+    return TomlSheet(p.base_stem, source=p)
+
+
+class TomlSheet(PythonSheet):
+    """A Sheet representing the top level of a loaded TOML file.
+
+    This is an intentionally minimal loader with cues taken from
+    VisiData built-in JSON and Python object sheet types.
+    """
+    guide = '''# Toml Sheet
+This sheet represents the top level of {sheet.source.name}.{sheet.source.ext}.
+
+Each cell within this sheet can contain dictionaries (representing TOML key:value pairs), lists (representing TOML arrays), or scalars.
+
+Some helpful commands when working with cells of lists and dictionaries:
+
+- `(` (`expand-col`) on a column with lists or dictionaries will "expand" the structures in the cells into new columns within the current sheet.
+- `zEnter` on a cell with lists or dictionaries will "dive" into the current cell, expanding its structures into rows and columns in a separate sheet.
+'''
+
+    rowtype = "values"  # rowdef: dict values, possibly nested
+
+    def loader(self):
+        """Loading a TOML file produces a single dict. Use
+        its keys as column headings, and populate a single
+        row.
+        """
+        self.columns = []
+        self.rows = []
+
+        try:
+            # Python 3.11+
+            import tomllib
+        except ModuleNotFoundError:
+            # Python 3.10 and below
+            tomllib = vd.importExternal("tomli")
+
+        data = tomllib.loads(self.source.read_text())
+        for k, v in data.items():
+            self.addColumn(ColumnItem(k, type=deduceType(v)))
+        self.addRow(data)
+
+
+vd.addGlobals(
+    {
+        "TomlSheet": TomlSheet,
+    }
+)

visidata/loaders/tsv.py

@@ -2,8 +2,10 @@ import os
 import contextlib
 import itertools
 import collections
+import math
+import time
 
-from visidata import vd, asyncthread, options, Progress, ColumnItem, SequenceSheet, Sheet, FileExistsError, getType, VisiData
+from visidata import vd, asyncthread, options, Progress, ColumnItem, SequenceSheet, Sheet, VisiData
 from visidata import namedlist, filesize
 
 vd.option('delimiter', '\t', 'field delimiter to use for tsv/usv filetype', replay=True)
@@ -14,23 +16,55 @@ vd.option('tsv_safe_tab', '\u001f', 'replacement for tab character when saving t
 
 @VisiData.api
 def open_tsv(vd, p):
-    return TsvSheet(p.name, source=p)
+    return TsvSheet(p.base_stem, source=p)
+
+
+def adaptive_bufferer(fp, max_buffer_size=65536):
+    """Loading e.g. tsv files goes faster with a large buffer. But when the input stream
+    is slow (e.g. 1 byte/second) and the buffer size is large, it can take a long time until
+    the buffer is filled. Only when the buffer is filled (or the input stream is finished)
+    you can see the data visiualized in visidata. That's why we use an adaptive buffer.
+    For fast input streams, the buffer becomes large, for slow input streams, the buffer stays
+    small"""
+    buffer_size = 8
+    processed_buffer_size = 0
+    previous_start_time = time.time()
+    while True:
+        next_chunk = fp.read(max(buffer_size, 1))
+        if not next_chunk:
+            break
 
+        yield next_chunk
 
-def splitter(fp, delim='\n'):
-    'Generates one line/row/record at a time from fp, separated by delim'
+        processed_buffer_size += len(next_chunk)
 
-    buf = ''
-    while True:
-        nextbuf = fp.read(65536)
-        if not nextbuf:
-            break
-        buf += nextbuf
+        current_time = time.time()
+        current_delta = current_time - previous_start_time
+
+        if current_delta < 1:
+            # if it takes longer than one second to fill the buffer, double the size of the buffer
+            buffer_size = min(buffer_size * 2, max_buffer_size)
+        else:
+            # if it takes less than one second, increase the buffer size so it takes about
+            # 1 second to fill it
+            previous_start_time = current_time
+            buffer_size = math.ceil(min(processed_buffer_size / current_delta, max_buffer_size))
+            processed_buffer_size = 0
+
+def splitter(stream, delim='\n'):
+    'Generates one line/row/record at a time from stream, separated by delim'
+
+    buf = type(delim)()
+
+    for chunk in stream:
+        buf += chunk
 
         *rows, buf = buf.split(delim)
         yield from rows
 
-    yield from buf.rstrip(delim).split(delim)
+    buf = buf.rstrip(delim)  # trim empty trailing lines
+    if buf:
+        yield from buf.rstrip(delim).split(delim)
 
 
 # rowdef: list
@@ -42,8 +76,8 @@ class TsvSheet(SequenceSheet):
         delim = self.delimiter or self.options.delimiter
         rowdelim = self.row_delimiter or self.options.row_delimiter
 
-        with self.source.open_text(encoding=self.options.encoding) as fp:
-                for line in splitter(fp, rowdelim):
+        with self.open_text_source() as fp:
+                for line in splitter(adaptive_bufferer(fp), rowdelim):
                     if not line:
                         continue
 
@@ -63,7 +97,7 @@ def save_tsv(vd, p, vs, delimiter='', row_delimiter=''):
     rowsep = row_delimiter or vs.options.row_delimiter
     trdict = vs.safe_trdict()
 
-    with p.open_text(mode='w', encoding=vs.options.encoding) as fp:
+    with p.open(mode='w', encoding=vs.options.save_encoding) as fp:
         colhdr = unitsep.join(col.name.translate(trdict) for col in vs.visibleCols) + rowsep
         fp.write(colhdr)
 
@@ -71,8 +105,6 @@ def save_tsv(vd, p, vs, delimiter='', row_delimiter=''):
             fp.write(unitsep.join(dispvals.values()))
             fp.write(rowsep)
 
-    vd.status('%s save finished' % p)
-
 
 @Sheet.api
 def append_tsv_row(vs, row):
@@ -87,14 +119,24 @@ def append_tsv_row(vs, row):
         trdict = vs.safe_trdict()
         unitsep = options.delimiter
 
-        with vs.source.open_text(mode='w', encoding=vs.options.encoding) as fp:
+        with vs.source.open(mode='w') as fp:
             colhdr = unitsep.join(col.name.translate(trdict) for col in vs.visibleCols) + vs.options.row_delimiter
             if colhdr.strip():  # is anything but whitespace
                 fp.write(colhdr)
 
-    with vs.source.open_text(mode='a', encoding=vs.options.encoding) as fp:
-        fp.write('\t'.join(col.getDisplayValue(row) for col in vs.visibleCols) + '\n')
+    newrow = ''
+
+    contents = vs.source.open(mode='r').read()
+    if not contents.endswith('\n'):  #1569
+        newrow += '\n'
+
+    newrow += '\t'.join(col.getDisplayValue(row) for col in vs.visibleCols) + '\n'
+
+    with vs.source.open(mode='a') as fp:
+        fp.write(newrow)
+
 
+TsvSheet.options.regex_skip = '^#.*'
 
 vd.addGlobals({
     'TsvSheet': TsvSheet,