visidata 2.11.dev0__py3-none-any.whl → 3.0__py3-none-any.whl

visidata/apps/vdsql/bigquery.py

@@ -0,0 +1,61 @@
+'''
+Specify the billing_project_id as the netloc, and the actual dataset_id as the path:
+
+    vdsql bigquery://<billing_project>/<dataset_id>''
+'''
+
+
+from visidata import vd, VisiData, Sheet, AttrColumn
+from . import IbisTableSheet, IbisTableIndexSheet, IbisConnectionPool
+
+import ibis
+import ibis.expr.operations as ops
+
+@VisiData.api
+def openurl_bigquery(vd, p, filetype=None):
+    vd.configure_ibis()
+    vd.configure_bigquery()
+    return BigqueryDatabaseIndexSheet(p.base_stem, source=p, ibis_con=None)
+
+vd.openurl_bq = vd.openurl_bigquery
+
+
+@VisiData.api
+def configure_bigquery(vd):
+    @ibis.bigquery.add_operation(ops.TimestampDiff)
+    def bq_timestamp_diff(t, expr):
+        op = expr.op()
+        left = t.translate(op.left)
+        right = t.translate(op.right)
+        return f"TIMESTAMP_DIFF({left}, {right}, SECOND)"
+
+
+class BigqueryDatabaseIndexSheet(Sheet):
+    rowtype = 'databases'  # rowdef: DatasetListItem
+    columns = [
+#        AttrColumn('project', width=0),
+        AttrColumn('dataset_id'),
+        AttrColumn('friendly_name'),
+        AttrColumn('full_dataset_id', width=0),
+        AttrColumn('labels'),
+    ]
+    nKeys = 1
+
+    @property
+    def con(self):
+        if not self.ibis_con:
+            import ibis
+            self.ibis_con = ibis.connect(self.source)
+        return self.ibis_con
+
+    def iterload(self):
+        yield from self.con.client.list_datasets(project=self.source.name)
+
+    def openRow(self, row):
+        return IbisTableIndexSheet(row.dataset_id,
+                                   database_name=self.source.name+'.'+row.dataset_id,
+                                   ibis_con=self.con,
+                                   ibis_conpool=IbisConnectionPool(f"{self.source}/{row.dataset_id}"),
+                                   source=row,
+                                   filetype=None,
+                                   sheet_type=IbisTableSheet)

visidata/apps/vdsql/clickhouse.py

@@ -0,0 +1,53 @@
+import time
+
+from visidata import BaseException, vd, VisiData, Progress
+
+from ._ibis import IbisTableSheet, IbisTableIndexSheet, IbisConnectionPool
+
+
+@VisiData.api
+def openurl_clickhouse(vd, p, filetype=None):
+    vd.configure_ibis()
+    return IbisTableIndexSheet(p.base_stem, source=p, filetype=None, database_name=None,
+                               ibis_conpool=IbisConnectionPool(p), sheet_type=ClickhouseSheet)
+
+vd.openurl_clickhouses = vd.openurl_clickhouse
+
+
+class ClickhouseSheet(IbisTableSheet):
+    @property
+    def countRows(self):
+        if self.total_rows is not None:
+            return self.total_rows
+        return super().countRows
+
+    def iterload(self):
+        with self.con as con:
+            qid = None
+            try:
+                if self.query is None:
+                    self.query = self.baseQuery(con)
+
+                self.reloadColumns(self.query, start=0)  # columns based on query without metadata
+                sqlstr = con.compile(self.query.limit(self.options.ibis_limit or None))
+
+                with Progress(gerund='clickhousing', sheet=self) as prog:
+                    settings = {'max_block_size': 10000}
+                    with con.con.query_rows_stream(sqlstr, settings) as stream:
+                        prog.total = int(stream.source.summary['total_rows_to_read'])
+                        prog.made = 0
+                        for row in stream:
+                            prog.made += 1
+                            yield row
+                        self.total_rows = prog.total
+
+            except Exception as e:
+                raise
+            except BaseException:
+                if qid:
+                    con.con.cancel(qid)
+
+
+ClickhouseSheet.init('total_rows', lambda: None)
+
+#ClickhouseSheet.class_options.sql_always_count = True

visidata/apps/vdsql/setup.py

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: MIT
+
+from setuptools import setup, find_packages
+from pathlib import Path
+
+
+exec(Path('__about__.py').read_text())
+
+
+def readme():
+    return Path("README.md").read_text()
+
+
+def requirements():
+    return Path("requirements.txt").read_text().splitlines()
+
+def requirements_extra():
+    return Path("requirements-extra.txt").read_text().splitlines()
+
+
+setup(
+        name="vdsql",
+        version=__version__,
+        description=__description__,
+        long_description=readme(),
+        long_description_content_type="text/markdown",
+        classifiers=[
+            "Development Status :: 4 - Beta",
+            "Programming Language :: Python :: 3",
+        ],
+        keywords="visidata sql rdbms ibis substrait",
+        author="Saul Pwanson",
+        url="https://github.com/visidata/vdsql",
+        python_requires=">=3.9",
+        packages=find_packages(exclude=["tests"]),
+        entry_points={'visidata.plugins': 'vdsql=visidata.apps.vdsql'},
+        scripts=['vdsql'],
+        install_requires=requirements(),
+        extra_requires=requirements_extra(),
+)

visidata/apps/vdsql/snowflake.py

@@ -0,0 +1,67 @@
+import time
+
+from visidata import vd, BaseException, VisiData
+
+from ._ibis import IbisTableSheet, IbisConnectionPool, IbisTableIndexSheet
+
+
+@VisiData.api
+def openurl_snowflake(vd, p, filetype=None):
+    return IbisTableIndexSheet(p.base_stem, source=p, filetype=None, database_name=None,
+                               ibis_conpool=IbisConnectionPool(p),
+                               sheet_type=SnowflakeSheet)
+
+
+class SnowflakeSheet(IbisTableSheet):
+    @property
+    def countRows(self):
+        r = super().countRows
+        if r is None and self.cursor is None:
+            return None  # no cursor yet
+        return r
+
+    def executeSql(self, sql):
+        assert self.cursor is None
+
+        with self.con as con:
+            con = con.con
+
+            if self.warehouse:
+                con.execute(f'USE WAREHOUSE {self.warehouse}')
+
+            with con.begin() as c:
+                snowflake_conn = c.connection.dbapi_connection
+                cursor = self.cursor = snowflake_conn.cursor()
+                cursor.execute_async(sql)
+                while snowflake_conn.is_still_running(snowflake_conn.get_query_status(cursor.sfqid)):
+                    time.sleep(.1)
+
+        cursor.get_results_from_sfqid(cursor.sfqid)
+        yield from cursor.fetchall()
+
+        self.cursor = None
+
+    def iterload(self):
+        try:
+            with self.con as con:
+                if self.query is None:
+                    self.query = self.baseQuery(con)
+                yield from self.executeSql(self.ibis_to_sql(self.withRowcount(self.baseQuery(con))))
+        except BaseException:
+            if self.cursor:
+                self.cancelQuery(self.cursor.sfqid)
+            raise
+
+        self.reloadColumns(self.query)  # columns based on query without metadata
+
+    def cancelQuery(self, qid):
+        vd.status(f'canceling "{qid}"')
+        with self.con as con:
+            with con.begin() as con:
+                cursor = con.connection.dbapi_connection.cursor()
+                cursor.execute(f"SELECT SYSTEM$CANCEL_QUERY('{qid}')")
+                vd.status(cursor.fetchall())
+
+
+SnowflakeSheet.init('cursor', lambda: None)
+SnowflakeSheet.init('warehouse', str)

visidata/apps/vgit/__init__.py

@@ -0,0 +1,13 @@
+from . import abort, statusbar
+from . import (
+    grep,
+    config,
+    branch,
+    remote,
+    blame,
+    status,
+    log,
+    diff,
+    stash,
+    repos,
+)

visidata/apps/vgit/__main__.py

@@ -0,0 +1,3 @@
+from .main import vgit_cli
+
+vgit_cli()

visidata/apps/vgit/abort.py

@@ -0,0 +1,23 @@
+from visidata import vd, Menu
+from .gitsheet import GitSheet
+
+
+@GitSheet.api
+def abortWhatever(sheet):
+    inp = sheet.gitInProgress()
+    if inp.startswith('cherry-pick'):
+        sheet.modifyGit('cherry-pick', '--abort')
+    elif inp.startswith('merg'):
+        sheet.modifyGit('merge', '--abort')
+    elif inp.startswith('bisect'):
+        sheet.modifyGit('bisect', 'reset')
+    elif inp.startswith('rebas') or inp.startswith('apply'):
+        sheet.modifyGit('rebase', '--abort')  # or --quit?
+    else:
+        vd.status('nothing to abort')
+
+
+GitSheet.addCommand('^A', 'git-abort', 'abortWhatever()', 'abort the current in-progress action')
+
+
+vd.addMenuItems('Git > Abort > git-abort')

visidata/apps/vgit/blame.py

@@ -0,0 +1,76 @@
+from visidata import vd, Column, VisiData, ItemColumn, Path, AttrDict, date
+
+from .gitsheet import GitSheet
+
+
+@VisiData.api
+def git_blame(vd, gitpath, args, **kwargs):
+    if args and not args[-1].startswith('-'):
+        fn = args[-1]
+        return GitBlame('blame', fn, source=Path(fn), **kwargs)
+
+
+class FormatColumn(Column):
+    def calcValue(self, row):
+        return self.expr.format(**row)
+
+
+# rowdef: (hdr, orig_linenum, linenum, line)
+#   hdr = { 'sha': .., 'orig_linenum': .., 'final_linenum': .. }
+# source = GitSheet; .gitfile=GitFile
+class GitBlame(GitSheet):
+    rowtype = 'lines'
+    guide = '''
+        # git blame
+    '''
+    columns = [
+        ItemColumn('sha', width=0),
+        ItemColumn('orig_linenum', width=0, type=int),
+        ItemColumn('final_linenum', width=0, type=int),
+        ItemColumn('author', width=15),
+        ItemColumn('author_time', width=13, type=date),
+        FormatColumn('committer', width=0, expr='{committer} {committer_mail}'),
+        ItemColumn('committer_time', width=0, type=date),
+        ItemColumn('linenum', width=6, type=int),
+        ItemColumn('line', width=72),
+    ]
+
+    def iterload(self):
+        lines = list(self.git_lines('blame', '--porcelain', str(self.source)))
+        i = 0
+        headers = {}  # [sha1] -> hdr
+        while i < len(lines):
+            # header
+            parts = lines[i].split()
+            sha, orig, final = parts[:3]
+            if len(parts) > 3:
+                nlines_this_group = parts[3]
+
+            if sha not in headers:
+                hdr = AttrDict(sha=sha, orig_linenum=orig, final_linenum=final)
+                headers[sha] = hdr
+            else:
+                hdr = headers[sha]
+
+            while lines[i][0] != '\t':
+                try:
+                    k, v = lines[i].split(maxsplit=1)
+                    k = k.replace('-', '_')
+                    if '_time' in k:
+                        v = int(v)
+                    hdr[k] = v
+                except Exception:
+                    vd.status(lines[i])
+                i += 1
+
+            yield AttrDict(
+                linenum=final,
+                line=lines[i][1:],
+                **hdr
+            )
+            i += 1
+
+
+#GitBlame.addCommand(ENTER, 'diff-line', 'openDiff(str(gitfile), cursorRow[0]["sha"]+"^", cursorRow[0]["sha"])', 'open diff of the commit when this line changed')
+
+#GitStatus.addCommand(None, 'git-blame', 'vd.push(GitBlame(cursorRow, source=sheet))', 'push blame for this file')

visidata/apps/vgit/branch.py

@@ -0,0 +1,153 @@
+import re
+
+from visidata import vd, Column, VisiData, ItemColumn, AttrColumn, Path, AttrDict, RowColorizer, date, Progress
+
+from .gitsheet import GitSheet
+
+vd.theme_option('color_git_current_branch', 'underline', 'color of current branch on branches sheet')
+vd.theme_option('color_git_remote_branch', 'cyan', 'color of remote branches on branches sheet')
+
+
+@VisiData.api
+def git_branch(vd, p, args):
+    nonListArgs = '--track --no-track --set-upstream-to -u --unset-upstream -m -M -c -C -d -D --edit-description'.split()
+    if any(x in args for x in nonListArgs):
+        return
+
+    return GitBranch('git-branch-list', source=p, git_args=args)
+
+
+def _remove_prefix(text, prefix):
+    if text.startswith(prefix):
+        return text[len(prefix):]
+    return text
+
+
+class GitBranchNameColumn(Column):
+    def calcValue(self, row):
+        return _remove_prefix(row.localbranch, 'remotes/')
+
+    def putValue(self, row, val):
+        self.sheet.loggit('branch', '-v', '--move', row.localbranch, val)
+
+
+class GitBranch(GitSheet):
+    guide = '''
+        # git branch
+        List of all branches, including relevant metadata.
+
+        - `d` to mark a branch for deletion
+        - `e` on the _branch_ column to rename the branch
+        - `z Ctrl+S` to commit changes
+    '''
+    defer = True
+    rowtype = 'branches'  # rowdef: AttrDict from regex (in reload below)
+    columns = [
+        GitBranchNameColumn('branch', width=20),
+#        Column('remote', getter=lambda c,r: r['localbranch'].startswith('remotes/') and '*' or '', width=3),
+        ItemColumn('head_commitid', 'refid', width=0),
+        ItemColumn('tracking', 'remotebranch'),
+        ItemColumn('upstream'),
+        ItemColumn('merge_base', 'merge_name', width=20),
+        ItemColumn('extra', width=0),
+        ItemColumn('head_commitmsg', 'msg', width=50),
+        ItemColumn('last_commit', type=date),
+        ItemColumn('last_author'),
+    ]
+    colorizers = [
+        RowColorizer(10, 'color_git_current_branch', lambda s,c,r,v: r and r['current']),
+        RowColorizer(10, 'color_git_remote_branch', lambda s,c,r,v: r and r['localbranch'].startswith('remotes/')),
+    ]
+    nKeys = 1
+
+    def iterload(self):
+        branches_lines = self.git_lines(
+            'branch',
+            '--list',
+            '--format',
+            ' '.join((
+                '%(if)%(symref)%(then)yes%(else)no%(end)',
+                '%(HEAD) %(refname:short) %(objectname:short)',
+                '%(if)%(upstream)%(then)[%(upstream:short)',
+                '%(if)%(upstream:track)%(then): %(upstream:track,nobracket)%(end)]',
+                '%(end)',
+                '%(contents:subject)'
+            )),
+            '-vv',
+            '--no-color',
+            *self.git_args)
+
+        for line in branches_lines:
+            m = re.match(r'''(?P<is_symref>(yes|no)?)\s+
+                             (?P<current>\*?)\s+
+                             (?P<localbranch>\S+)\s+
+                             (?P<refid>\w+)\s+
+                             (?:\[
+                               (?P<remotebranch>[^\s\]:]+):?
+                               \s*(?P<extra>.*?)
+                             \])?
+                             \s*(?P<msg>.*)''', line, re.VERBOSE)
+            if not m:
+                continue
+            branch_details = AttrDict(m.groupdict())
+            if branch_details.is_symref == 'yes':
+                continue
+            yield branch_details
+
+        branch_stats = self.gitRootSheet.gitBranchStatuses
+        for row in Progress(self.rows):
+            merge_base = self.git_all("show-branch", "--merge-base", row.localbranch, self.gitRootSheet.branch, _ok_code=[0,1]).strip()
+            row.update(dict(
+                merge_name = self.git_all("name-rev", "--name-only", merge_base).strip() if merge_base else '',
+                upstream = branch_stats.get(row.localbranch),
+                last_commit = self.git_all("show", "--no-patch", '--pretty=%ai', row.localbranch).strip(),
+                last_author = self.git_all("show", "--no-patch", '--pretty=%an', row.localbranch).strip()
+            ))
+
+    def commitAddRow(self, row):
+        self.loggit('branch', row.localbranch)
+
+    def commitDeleteRow(self, row):
+        self.loggit('branch', '--delete', _remove_prefix(row.localbranch, 'remotes/'))
+
+
+@GitSheet.lazy_property
+def gitBranchStatuses(sheet):
+    ret = {}  # localbranchname -> "+5/-2"
+    for branch_status in sheet.git_lines('for-each-ref', '--format=%(refname:short) %(upstream:short) %(upstream:track)', 'refs/heads'):
+        m = re.search(r'''(\S+)\s*
+                          (\S+)?\s*
+                          (\[
+                          (ahead.(\d+)),?\s*
+                          (behind.(\d+))?
+                          \])?''', branch_status, re.VERBOSE)
+        if not m:
+            vd.status('unmatched branch status: ' + branch_status)
+            continue
+
+        localb, remoteb, _, _, nahead, _, nbehind = m.groups()
+        if nahead:
+            r = '+%s' % nahead
+        else:
+            r = ''
+        if nbehind:
+            if r:
+                r += '/'
+            r += '-%s' % nbehind
+        ret[localb] = r
+
+    return ret
+
+
+GitSheet.addCommand('', 'git-open-branches', 'vd.push(git_branch(source, []))', 'push branches sheet')
+GitSheet.addCommand('', 'git-branch-create', 'git("branch", input("create branch: ", type="branch"))', 'create a new branch off the current checkout')
+GitBranch.addCommand('', 'git-branch-checkout', 'git("checkout", cursorRow.localbranch)', 'checkout this branch')
+
+
+vd.addMenuItems('''
+    Git > Branch > add > git-branch-create
+    Git > Branch > delete > git-branch-delete
+    Git > Branch > rename > git-branch-rename
+    Git > Branch > checkout > git-branch-checkout
+    Git > Open > branches > git-open-branches
+''')

visidata/apps/vgit/config.py

@@ -0,0 +1,95 @@
+from itertools import islice
+
+from visidata import vd, Column, VisiData, ItemColumn, AttrColumn, Path, AttrDict
+
+from .gitsheet import GitSheet
+
+
+CONFIG_CONTEXTS = ('local', 'global', 'system')
+
+@VisiData.api
+def git_config(vd, p, args):
+    if not args or '-l' in args:
+        return GitConfig('git-config', source=p)
+
+vd.git_options = vd.git_config
+
+def batched(iterable, n=1):
+    "Batch data into tuples of length n. The last batch may be shorter."
+    # batched('ABCDEFG', 3) --> ABC DEF G
+    assert n >= 1, 'n must be at least one'
+
+    while (batch := tuple(islice(iter(iterable), n))):
+        yield batch
+
+
+class GitConfigColumn(Column):
+    def calcValue(self, row):
+        return row.get(self.expr)
+
+    def putValue(self, row, val):
+        if val is None:
+            self.sheet.loggit('config', '--unset', '--'+self.expr, row['option'])
+        else:
+            self.sheet.loggit('config', '--'+self.expr, row['option'], val)
+
+        row[self.expr] = val
+
+
+class GitConfig(GitSheet):
+    guide = '''
+        # git config
+        Add, edit, or delete git config options.
+
+        - Make changes using standard commands like `a`dd and `e`dit
+        - `z Ctrl+S` to commit changes to git config file.
+    '''
+    rowtype = 'git options'  # rowdef: [scope, origin, opt, val]
+    defer = True
+    columns = [
+        ItemColumn('option', width=20),
+    ]
+    nKeys = 1
+
+    def iterload(self):
+        cmd = self.git_iter('config', '--list', '--show-scope', '--show-origin', '-z')
+        self.gitopts = {}
+        scopes = {c.name:c for c in self.columns}
+        for row in batched(cmd, 3):
+            if len(row) < 3:
+                break
+
+            scope, origin, optval = row
+            opt, val = optval.split('\n', 1)
+
+            if opt in self.gitopts:
+                r = self.gitopts[opt]
+            else:
+                r = AttrDict(option=opt)
+                self.gitopts[opt] = r
+                yield r
+
+            r[scope] = val
+
+            if scope not in scopes:
+                c = GitConfigColumn(scope, expr=scope)
+                self.addColumn(c)
+                scopes[scope] = c
+
+    def commitDeleteRow(self, row):
+        for k in CONFIG_CONTEXTS:
+            if row.get(k):
+                self.loggit('config', '--unset', '--'+k, row['option'])
+
+    def commitAddRow(self, row):
+        for k in CONFIG_CONTEXTS:
+            if row.get(k):
+                self.loggit('config', '--add', '--'+k, row['option'], row.get(k))
+
+
+GitSheet.addCommand('gO', 'git-config', 'vd.push(GitConfig("git_config", source=Path(".")))', 'push sheet of git options')
+
+
+vd.addMenuItems('''
+    Git > Config > git-config
+''')