visidata 2.11.1__py3-none-any.whl → 3.0.1__py3-none-any.whl

visidata/apps/vdsql/bigquery.py

@@ -0,0 +1,61 @@
+'''
+Specify the billing_project_id as the netloc, and the actual dataset_id as the path:
+
+    vdsql bigquery://<billing_project>/<dataset_id>''
+'''
+
+
+from visidata import vd, VisiData, Sheet, AttrColumn
+from . import IbisTableSheet, IbisTableIndexSheet, IbisConnectionPool
+
+import ibis
+import ibis.expr.operations as ops
+
+@VisiData.api
+def openurl_bigquery(vd, p, filetype=None):
+    vd.configure_ibis()
+    vd.configure_bigquery()
+    return BigqueryDatabaseIndexSheet(p.base_stem, source=p, ibis_con=None)
+
+vd.openurl_bq = vd.openurl_bigquery
+
+
+@VisiData.api
+def configure_bigquery(vd):
+    @ibis.bigquery.add_operation(ops.TimestampDiff)
+    def bq_timestamp_diff(t, expr):
+        op = expr.op()
+        left = t.translate(op.left)
+        right = t.translate(op.right)
+        return f"TIMESTAMP_DIFF({left}, {right}, SECOND)"
+
+
+class BigqueryDatabaseIndexSheet(Sheet):
+    rowtype = 'databases'  # rowdef: DatasetListItem
+    columns = [
+#        AttrColumn('project', width=0),
+        AttrColumn('dataset_id'),
+        AttrColumn('friendly_name'),
+        AttrColumn('full_dataset_id', width=0),
+        AttrColumn('labels'),
+    ]
+    nKeys = 1
+
+    @property
+    def con(self):
+        if not self.ibis_con:
+            import ibis
+            self.ibis_con = ibis.connect(self.source)
+        return self.ibis_con
+
+    def iterload(self):
+        yield from self.con.client.list_datasets(project=self.source.name)
+
+    def openRow(self, row):
+        return IbisTableIndexSheet(row.dataset_id,
+                                   database_name=self.source.name+'.'+row.dataset_id,
+                                   ibis_con=self.con,
+                                   ibis_conpool=IbisConnectionPool(f"{self.source}/{row.dataset_id}"),
+                                   source=row,
+                                   filetype=None,
+                                   sheet_type=IbisTableSheet)

visidata/apps/vdsql/clickhouse.py

@@ -0,0 +1,53 @@
+import time
+
+from visidata import BaseException, vd, VisiData, Progress
+
+from ._ibis import IbisTableSheet, IbisTableIndexSheet, IbisConnectionPool
+
+
+@VisiData.api
+def openurl_clickhouse(vd, p, filetype=None):
+    vd.configure_ibis()
+    return IbisTableIndexSheet(p.base_stem, source=p, filetype=None, database_name=None,
+                               ibis_conpool=IbisConnectionPool(p), sheet_type=ClickhouseSheet)
+
+vd.openurl_clickhouses = vd.openurl_clickhouse
+
+
+class ClickhouseSheet(IbisTableSheet):
+    @property
+    def countRows(self):
+        if self.total_rows is not None:
+            return self.total_rows
+        return super().countRows
+
+    def iterload(self):
+        with self.con as con:
+            qid = None
+            try:
+                if self.query is None:
+                    self.query = self.baseQuery(con)
+
+                self.reloadColumns(self.query, start=0)  # columns based on query without metadata
+                sqlstr = con.compile(self.query.limit(self.options.ibis_limit or None))
+
+                with Progress(gerund='clickhousing', sheet=self) as prog:
+                    settings = {'max_block_size': 10000}
+                    with con.con.query_rows_stream(sqlstr, settings) as stream:
+                        prog.total = int(stream.source.summary['total_rows_to_read'])
+                        prog.made = 0
+                        for row in stream:
+                            prog.made += 1
+                            yield row
+                        self.total_rows = prog.total
+
+            except Exception as e:
+                raise
+            except BaseException:
+                if qid:
+                    con.con.cancel(qid)
+
+
+ClickhouseSheet.init('total_rows', lambda: None)
+
+#ClickhouseSheet.class_options.sql_always_count = True

visidata/apps/vdsql/setup.py

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: MIT
+
+from setuptools import setup, find_packages
+from pathlib import Path
+
+
+exec(Path('__about__.py').read_text())
+
+
+def readme():
+    return Path("README.md").read_text()
+
+
+def requirements():
+    return Path("requirements.txt").read_text().splitlines()
+
+def requirements_extra():
+    return Path("requirements-extra.txt").read_text().splitlines()
+
+
+setup(
+        name="vdsql",
+        version=__version__,
+        description=__description__,
+        long_description=readme(),
+        long_description_content_type="text/markdown",
+        classifiers=[
+            "Development Status :: 4 - Beta",
+            "Programming Language :: Python :: 3",
+        ],
+        keywords="visidata sql rdbms ibis substrait",
+        author="Saul Pwanson",
+        url="https://github.com/visidata/vdsql",
+        python_requires=">=3.9",
+        packages=find_packages(exclude=["tests"]),
+        entry_points={'visidata.plugins': 'vdsql=visidata.apps.vdsql'},
+        scripts=['vdsql'],
+        install_requires=requirements(),
+        extra_requires=requirements_extra(),
+)

visidata/apps/vdsql/snowflake.py

@@ -0,0 +1,67 @@
+import time
+
+from visidata import vd, BaseException, VisiData
+
+from ._ibis import IbisTableSheet, IbisConnectionPool, IbisTableIndexSheet
+
+
+@VisiData.api
+def openurl_snowflake(vd, p, filetype=None):
+    return IbisTableIndexSheet(p.base_stem, source=p, filetype=None, database_name=None,
+                               ibis_conpool=IbisConnectionPool(p),
+                               sheet_type=SnowflakeSheet)
+
+
+class SnowflakeSheet(IbisTableSheet):
+    @property
+    def countRows(self):
+        r = super().countRows
+        if r is None and self.cursor is None:
+            return None  # no cursor yet
+        return r
+
+    def executeSql(self, sql):
+        assert self.cursor is None
+
+        with self.con as con:
+            con = con.con
+
+            if self.warehouse:
+                con.execute(f'USE WAREHOUSE {self.warehouse}')
+
+            with con.begin() as c:
+                snowflake_conn = c.connection.dbapi_connection
+                cursor = self.cursor = snowflake_conn.cursor()
+                cursor.execute_async(sql)
+                while snowflake_conn.is_still_running(snowflake_conn.get_query_status(cursor.sfqid)):
+                    time.sleep(.1)
+
+        cursor.get_results_from_sfqid(cursor.sfqid)
+        yield from cursor.fetchall()
+
+        self.cursor = None
+
+    def iterload(self):
+        try:
+            with self.con as con:
+                if self.query is None:
+                    self.query = self.baseQuery(con)
+                yield from self.executeSql(self.ibis_to_sql(self.withRowcount(self.baseQuery(con))))
+        except BaseException:
+            if self.cursor:
+                self.cancelQuery(self.cursor.sfqid)
+            raise
+
+        self.reloadColumns(self.query)  # columns based on query without metadata
+
+    def cancelQuery(self, qid):
+        vd.status(f'canceling "{qid}"')
+        with self.con as con:
+            with con.begin() as con:
+                cursor = con.connection.dbapi_connection.cursor()
+                cursor.execute(f"SELECT SYSTEM$CANCEL_QUERY('{qid}')")
+                vd.status(cursor.fetchall())
+
+
+SnowflakeSheet.init('cursor', lambda: None)
+SnowflakeSheet.init('warehouse', str)

visidata/apps/vgit/__init__.py

@@ -0,0 +1,13 @@
+from . import abort, statusbar
+from . import (
+    grep,
+    config,
+    branch,
+    remote,
+    blame,
+    status,
+    log,
+    diff,
+    stash,
+    repos,
+)

{vgit → visidata/apps/vgit}/blame.py

@@ -4,7 +4,7 @@ from .gitsheet import GitSheet
 
 
 @VisiData.api
-def git_blame(vd, args, **kwargs):
+def git_blame(vd, gitpath, args, **kwargs):
     if args and not args[-1].startswith('-'):
         fn = args[-1]
         return GitBlame('blame', fn, source=Path(fn), **kwargs)
@@ -20,6 +20,9 @@ class FormatColumn(Column):
 # source = GitSheet; .gitfile=GitFile
 class GitBlame(GitSheet):
     rowtype = 'lines'
+    guide = '''
+        # git blame
+    '''
     columns = [
         ItemColumn('sha', width=0),
         ItemColumn('orig_linenum', width=0, type=int),
@@ -70,4 +73,4 @@ class GitBlame(GitSheet):
 
 #GitBlame.addCommand(ENTER, 'diff-line', 'openDiff(str(gitfile), cursorRow[0]["sha"]+"^", cursorRow[0]["sha"])', 'open diff of the commit when this line changed')
 
-#GitStatus.addCommand(None, 'git-blame', 'vd.push(GitBlame(cursorRow, source=sheet))', 'push blame for this file'),
+#GitStatus.addCommand(None, 'git-blame', 'vd.push(GitBlame(cursorRow, source=sheet))', 'push blame for this file')

{vgit → visidata/apps/vgit}/branch.py

@@ -4,17 +4,17 @@ from visidata import vd, Column, VisiData, ItemColumn, AttrColumn, Path, AttrDic
 
 from .gitsheet import GitSheet
 
-vd.option('color_git_current_branch', 'underline', 'color of current branch on branches sheet')
-vd.option('color_git_remote_branch', 'cyan', 'color of remote branches on branches sheet')
+vd.theme_option('color_git_current_branch', 'underline', 'color of current branch on branches sheet')
+vd.theme_option('color_git_remote_branch', 'cyan', 'color of remote branches on branches sheet')
 
 
 @VisiData.api
-def git_branch(vd, args):
+def git_branch(vd, p, args):
     nonListArgs = '--track --no-track --set-upstream-to -u --unset-upstream -m -M -c -C -d -D --edit-description'.split()
     if any(x in args for x in nonListArgs):
         return
 
-    return GitBranch('git-branch-list', source=Path('.'), git_args=args)
+    return GitBranch('git-branch-list', source=p, git_args=args)
 
 
 def _remove_prefix(text, prefix):
@@ -32,7 +32,7 @@ class GitBranchNameColumn(Column):
 
 
 class GitBranch(GitSheet):
-    help = '''
+    guide = '''
         # git branch
         List of all branches, including relevant metadata.
 
@@ -61,16 +61,25 @@ class GitBranch(GitSheet):
     nKeys = 1
 
     def iterload(self):
-        branches_lines = self.git_lines('branch',
-                                        '--list',
-                                        '-vv',
-                                        '--no-color',
-                                        *self.git_args)
-        for line in branches_lines:
-            if '->' in line:
-                continue
+        branches_lines = self.git_lines(
+            'branch',
+            '--list',
+            '--format',
+            ' '.join((
+                '%(if)%(symref)%(then)yes%(else)no%(end)',
+                '%(HEAD) %(refname:short) %(objectname:short)',
+                '%(if)%(upstream)%(then)[%(upstream:short)',
+                '%(if)%(upstream:track)%(then): %(upstream:track,nobracket)%(end)]',
+                '%(end)',
+                '%(contents:subject)'
+            )),
+            '-vv',
+            '--no-color',
+            *self.git_args)
 
-            m = re.match(r'''(?P<current>\*?)\s+
+        for line in branches_lines:
+            m = re.match(r'''(?P<is_symref>(yes|no)?)\s+
+                             (?P<current>\*?)\s+
                              (?P<localbranch>\S+)\s+
                              (?P<refid>\w+)\s+
                              (?:\[
@@ -78,8 +87,12 @@ class GitBranch(GitSheet):
                                \s*(?P<extra>.*?)
                              \])?
                              \s*(?P<msg>.*)''', line, re.VERBOSE)
-            if m:
-                yield AttrDict(m.groupdict())
+            if not m:
+                continue
+            branch_details = AttrDict(m.groupdict())
+            if branch_details.is_symref == 'yes':
+                continue
+            yield branch_details
 
         branch_stats = self.gitRootSheet.gitBranchStatuses
         for row in Progress(self.rows):
@@ -126,6 +139,7 @@ def gitBranchStatuses(sheet):
     return ret
 
 
+GitSheet.addCommand('', 'git-open-branches', 'vd.push(git_branch(source, []))', 'push branches sheet')
 GitSheet.addCommand('', 'git-branch-create', 'git("branch", input("create branch: ", type="branch"))', 'create a new branch off the current checkout')
 GitBranch.addCommand('', 'git-branch-checkout', 'git("checkout", cursorRow.localbranch)', 'checkout this branch')
 
@@ -135,4 +149,5 @@ vd.addMenuItems('''
     Git > Branch > delete > git-branch-delete
     Git > Branch > rename > git-branch-rename
     Git > Branch > checkout > git-branch-checkout
+    Git > Open > branches > git-open-branches
 ''')

{vgit → visidata/apps/vgit}/config.py

@@ -8,9 +8,9 @@ from .gitsheet import GitSheet
 CONFIG_CONTEXTS = ('local', 'global', 'system')
 
 @VisiData.api
-def git_config(vd, args):
+def git_config(vd, p, args):
     if not args or '-l' in args:
-        return GitConfig('git-config', source=Path('.'))
+        return GitConfig('git-config', source=p)
 
 vd.git_options = vd.git_config
 
@@ -37,7 +37,7 @@ class GitConfigColumn(Column):
 
 
 class GitConfig(GitSheet):
-    help = '''
+    guide = '''
         # git config
         Add, edit, or delete git config options.
 

visidata/apps/vgit/diff.py

@@ -0,0 +1,169 @@
+from visidata import vd, VisiData, ItemColumn, RowColorizer, AttrDict, Column
+
+from .gitsheet import GitSheet
+
+vd.option('git_diff_algo', 'minimal', 'algorithm to use for git diff')
+vd.theme_option('color_git_hunk_add', 'green', 'color for added hunk lines')
+vd.theme_option('color_git_hunk_del', 'red', 'color for deleted hunk lines')
+vd.theme_option('color_git_hunk_diff', 'yellow', 'color for hunk diffs')
+
+@VisiData.api
+def git_diff(vd, p, args):
+    return GitDiffSheet('git-diff', source=p, gitargs=args)
+
+def _parseStartCount(s):
+    sc = s.split(',')
+    if len(sc) == 2:
+        return sc
+    if len(sc) == 1:
+        return sc[0], 1
+
+
+class GitDiffSheet(GitSheet):
+    columns = [
+        ItemColumn('a_fn', width=0),
+        ItemColumn('fn', 'b_fn', width=30, hoffset=-28),
+        ItemColumn('a_lineno', type=int, width=0),
+        ItemColumn('lineno', 'b_lineno', type=int, width=8),
+        Column('count', width=10, getter=lambda c,r: c.sheet.hunkCount(r)),
+        ItemColumn('context'),
+        ItemColumn('lines', type=''.join),
+    ]
+
+    guide = '''# {sheet.cursorRow.a_fn}
+{sheet.cursorLines}'''
+
+    def hunkCount(self, row):
+        return f'-{row.a_count}/+{row.b_count}'
+
+    @property
+    def cursorLines(self):
+        r = ''
+        for line in self.cursorRow.lines[2:]:
+            if line.startswith('-'):
+                line = '[:git_hunk_del]' + line + '[/]'
+            elif line.startswith('+'):
+                line = '[:git_hunk_add]' + line + '[/]'
+
+            r += line + '\n'
+        r = r[4:]
+        return r
+
+    def iterload(self):
+        current_hunk = None
+
+        for line in self.git_lines('diff --patch --inter-hunk-context=2 --find-renames --no-color --no-prefix', *self.gitargs):
+            if line.startswith('diff'):
+                diff_started = True
+                continue
+            if not diff_started:
+                continue
+
+            if line.startswith('---'):
+                hunk_lines = [line]  # new file
+                leftfn = line[4:]
+            elif line.startswith('+++'):
+                hunk_lines.append(line)
+                rightfn = line[4:]
+            elif line.startswith('@@'):
+                hunk_lines.append(line)
+                _, linenums, context = line.split('@@')
+                leftlinenums, rightlinenums = linenums.split()
+                leftstart, leftcount = _parseStartCount(leftlinenums[1:])
+                rightstart, rightcount = _parseStartCount(rightlinenums[1:])
+                current_hunk = AttrDict(
+                    a_fn=leftfn,
+                    b_fn=rightfn,
+                    context=context,
+                    a_lineno=int(leftstart),
+                    a_count=0,
+                    b_lineno=int(rightstart),
+                    b_count=0,
+                    lines=hunk_lines
+                )
+                yield current_hunk
+                hunk_lines = hunk_lines[:2]  # keep file context only
+
+            elif line[0] in ' +-':
+                current_hunk.lines.append(line)
+                if line[0] == '+':
+                    current_hunk.a_count += 1
+                elif line[0] == '-':
+                    current_hunk.b_count += 1
+
+    def openRow(self, row):
+        return HunkViewer(f'{row.a_fn}:{row.a_lineno}', source=self.source, hunks=[row])
+
+
+class HunkViewer(GitSheet):
+    colorizers = [
+        RowColorizer(4, 'color_git_hunk_add', lambda s,c,r,v: r and r.old != r.new and r.old is None),
+        RowColorizer(4, 'color_git_hunk_del', lambda s,c,r,v: r and r.old != r.new and r.new is None),
+        RowColorizer(5, 'color_git_hunk_diff', lambda s,c,r,v: r and r.old != r.new and r.new is not None and r.old is not None),
+    ]
+    columns = [
+        ItemColumn('1', 'old', width=40),
+        ItemColumn('2', 'new', width=40),
+    ]
+
+    def draw(self, scr):
+        self.column('1').width=self.windowWidth//2-1
+        self.column('2').width=self.windowWidth//2-1
+        super().draw(scr)
+
+    def iterload(self):
+        nextDelIdx = None
+        for hunk in self.hunks:
+          for line in hunk.lines[3:]:  # diff without the patch headers
+            typech = line[0]
+            line = line[1:]
+            if typech == '-':  # deleted
+                yield AttrDict(hunk=hunk, type=typech, old=line)
+                if nextDelIdx is None:
+                    nextDelIdx = len(self.rows)-1
+            elif typech == '+':  # added
+                if nextDelIdx is not None:
+                    if nextDelIdx < len(self.rows):
+                        self.rows[nextDelIdx].new = line
+                        nextDelIdx += 1
+                        continue
+
+                yield AttrDict(hunk=hunk, type=typech, new=line)
+                nextDelIdx = None
+            elif typech == ' ':  # unchanged
+                yield AttrDict(hunk=hunk, type=typech, old=line, new=line)
+                nextDelIdx = None
+            else:
+                continue  # header
+
+
+HunkViewer.addCommand('2', 'git-apply-hunk', 'source.git_apply(cursorRow.hunk, "--cached"); reload()', 'apply this hunk to the index and move to the next hunk')
+HunkViewer.addCommand('1', 'git-remove-hunk', 'source.git_apply(cursorRow.hunk, "--reverse"); reload()', 'remove this hunk from staging')
+HunkViewer.addCommand('Enter', 'git-skip-hunk', 'hunks.pop(0); reload()', 'move to the next hunk without applying this hunk')
+HunkViewer.addCommand('d', 'delete-line', 'source[7].pop(cursorRow[3]); reload()', 'delete a line from the patch')
+
+#HunksSheet.addCommand('g^J', 'git-diff-selected', 'vd.push(HunkViewer(selectedRows or rows, source=sheet))', 'view the diffs for the selected hunks (or all hunks)')
+
+@GitDiffSheet.api
+def git_apply(sheet, row, *args):
+    sheet.git('apply -p0 -', *args, _in='\n'.join(row.lines)+'\n')
+
+    c = sheet.hunkCount(row)
+    vd.status(f'applied hunk ({c})')
+    sheet.reload()
+
+
+#DiffSheet.addCommand('[', '', 'cursorRowIndex = findDiffRow(cursorCol.refnum, cursorRowIndex, -1)', 'go to previous diff')
+#DiffSheet.addCommand(']', '', 'cursorRowIndex = findDiffRow(cursorCol.refnum, cursorRowIndex, +1)', 'go to next diff')
+
+GitDiffSheet.addCommand('a', 'git-add-hunk', 'git_apply(cursorRow, "--cached")', 'apply this hunk to the index')
+
+vd.addMenuItems('''
+    Git > Stage > current hunk > git-add-hunk
+    Git > Stage > current hunk > git-add-hunk
+''')
+
+vd.addGlobals(
+    GitDiffSheet=GitDiffSheet,
+    HunkViewer=HunkViewer
+)