tabpull 0.1.0__py3-none-any.whl

tabpull/app.py

@@ -0,0 +1,385 @@
+"""Export Tableau crosstab CSVs for the dashboard sheets you name.
+
+Commands:
+  setup  save a site's personal access token and SSO session
+  add    record a job (prompts, or flags for the site, view, sheets, and filters)
+  remove delete saved jobs from the jobs file
+  run    export jobs from the jobs file into the current folder
+  login  refresh a site's SSO session
+
+Run tabpull with no command at a terminal to open the interactive screen.
+Piped, it prints configured sites and saved jobs.
+"""
+
+import argparse
+import shlex
+import sys
+import tomllib
+from collections.abc import Sequence
+from pathlib import Path
+from typing import NoReturn
+
+import questionary
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import sync_playwright
+from rich.rule import Rule
+
+from tabpull import ui, wizard
+from tabpull.add import add_job, add_job_from_flags
+from tabpull.cli import VERSION_FLAGS, version
+from tabpull.filters import (
+    filters_for_run,
+    params_for_run,
+    parse_filter_spec,
+    parse_param_spec,
+)
+from tabpull.home import file_flags, show_home
+from tabpull.jobs import JobError, check_known, saved_jobs_or_exit
+from tabpull.remove import remove_jobs
+from tabpull.run import open_report, run_jobs
+from tabpull.tableau import (
+    MissingSettingsError,
+    Settings,
+    UnknownSiteError,
+    check_site_name,
+    jobs_path,
+    list_sites,
+    load_site,
+    sso_login,
+)
+
+
+def _require_flag_shape(args: argparse.Namespace) -> None:
+    if not args.view:
+        msg = (
+            'Pass --view Workbook/View, and --sheet at least once, '
+            'to add a job without prompts.'
+        )
+        raise SystemExit(msg)
+    if not args.sheets:
+        msg = 'Pass --sheet at least once to add a job without prompts.'
+        raise SystemExit(msg)
+
+
+def _configured_name(explicit: str | None) -> str:
+    if explicit:
+        try:
+            return check_site_name(explicit)
+        except ValueError as e:
+            raise SystemExit(str(e)) from e
+    names = list_sites()
+    if len(names) == 1:
+        return names[0]
+    if not names:
+        msg = 'No Tableau site configured. Run: tabpull setup'
+    else:
+        msg = f'Pass --site. Configured sites: {", ".join(names)}'
+    raise SystemExit(msg)
+
+
+def _pick_site(explicit: str | None) -> Settings:
+    if explicit or not ui.interactive():
+        return load_site(_configured_name(explicit))
+    names = list_sites()
+    if len(names) == 1:
+        return load_site(names[0])
+    if not names:
+        print('No Tableau site configured. Starting setup.')
+        return load_site(wizard.main(None))
+    choices = []
+    for name in names:
+        settings = load_site(name)
+        label = f'{name}  ({settings.server}, site {settings.site or "(default)"})'
+        choices.append(questionary.Choice(label, value=name))
+    return load_site(
+        ui.ask(questionary.select('Site', choices=choices, style=ui.STYLE))
+    )
+
+
+def _cmd_login(site: str | None) -> None:
+    try:
+        settings = _pick_site(site)
+    except (UnknownSiteError, MissingSettingsError, ValueError) as e:
+        raise SystemExit(str(e)) from e
+    print(
+        f'Signing in to {settings.server}, site {settings.site or "(default)"} '
+        f'(local name {settings.name}).'
+    )
+    with sync_playwright() as pw:
+        sso_login(pw, settings)
+
+
+def _cmd_add(args: argparse.Namespace, jobs_file: Path) -> None:
+    flag_mode = args.view or args.sheets or args.filter_specs or args.params
+    try:
+        if flag_mode or not ui.interactive():
+            _require_flag_shape(args)
+            add_job_from_flags(load_site(_configured_name(args.site)), args, jobs_file)
+            return
+        ui.console.print(Rule('[bold]tabpull add', align='left', style='blue'))
+        add_job(_pick_site(args.site), jobs_file, args.name)
+    except (
+        JobError,
+        UnknownSiteError,
+        MissingSettingsError,
+        PlaywrightError,
+        tomllib.TOMLDecodeError,
+        OSError,
+        ValueError,
+    ) as e:
+        message = str(e).partition('\n')[0] or repr(e)
+        raise SystemExit(message) from e
+
+
+def _cmd_run(args: argparse.Namespace, jobs_file: Path, out_dir: Path) -> int:
+    jobs = saved_jobs_or_exit(jobs_file)
+    selected = [job for job in jobs if not args.names or job.name in args.names]
+    try:
+        check_known(args.names, jobs)
+        overrides = [parse_filter_spec(spec) for spec in args.filter_specs or []]
+        param_overrides = [parse_param_spec(spec) for spec in args.params or []]
+        selected = [
+            params_for_run(filters_for_run(job, overrides), param_overrides)
+            for job in selected
+        ]
+    except JobError as e:
+        raise SystemExit(str(e)) from e
+    report = open_report(selected, out_dir)
+    try:
+        failed = run_jobs(selected, out_dir, report, parallel=args.parallel)
+    except JobError as e:
+        raise SystemExit(str(e)) from e
+    filter_flags = [
+        flag for spec in args.filter_specs or [] for flag in ('--filter', spec)
+    ]
+    param_flags = [flag for spec in args.params or [] for flag in ('--param', spec)]
+    parallel_flags = ['--parallel', str(args.parallel)] if args.parallel != 1 else []
+    rerun = shlex.join(
+        [
+            'tabpull',
+            'run',
+            *file_flags(args),
+            *parallel_flags,
+            *filter_flags,
+            *param_flags,
+            '--',
+            *failed,
+        ]
+    )
+    report.summary(
+        len(selected) - len(failed), len(selected), rerun if failed else None
+    )
+    return 1 if failed else 0
+
+
+class _Parser(argparse.ArgumentParser):
+    def error(self, message: str) -> NoReturn:
+        print(f'error: {message}')
+        print(self.format_usage().rstrip())
+        print(f'help: run `{self.prog} --help` for flags and examples')
+        sys.exit(2)
+
+
+def _add_file_flags(
+    parser: argparse.ArgumentParser, *, out: bool, default: object
+) -> None:
+    parser.add_argument(
+        '--jobs',
+        type=Path,
+        default=default,
+        help='jobs file for this run (default: the config directory jobs.toml)',
+    )
+    if out:
+        parser.add_argument(
+            '--out',
+            type=Path,
+            default=default,
+            help='output folder for this run (default: the current folder)',
+        )
+
+
+def _at_least_one(text: str) -> int:
+    try:
+        value = int(text)
+    except ValueError as e:
+        msg = f'parallel must be an integer, not {text!r}'
+        raise argparse.ArgumentTypeError(msg) from e
+    if value < 1:
+        msg = 'parallel must be at least 1'
+        raise argparse.ArgumentTypeError(msg)
+    return value
+
+
+def _parser() -> tuple[_Parser, dict[str, _Parser]]:
+    parser = _Parser(
+        prog='tabpull',
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(*VERSION_FLAGS, action='version', version=version())
+    _add_file_flags(parser, out=True, default=None)
+    commands = parser.add_subparsers(dest='command')
+    setup = commands.add_parser('setup', help='save a site token and browser session')
+    setup.add_argument('--site', help='local name for this site; each job refers to it')
+    add = commands.add_parser(
+        'add',
+        help='save a job, prompting unless view and sheets are flags',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""examples:
+  tabpull add
+  tabpull add --view SalesWorkbook/Overview --sheet "Order Detail" --name daily
+  tabpull add --view SalesWorkbook/Overview --sheet Totals \\
+    --filter "Region=West|Central" --filter "Order Date=2026-09-01..2026-09-25"
+""",
+    )
+    add.add_argument('--site', help='local site name from tabpull setup')
+    add.add_argument('--view', help='Workbook/View, or a view URL')
+    add.add_argument('--name', help='job name')
+    add.add_argument(
+        '--sheet',
+        action='append',
+        dest='sheets',
+        metavar='SHEET',
+        help='worksheet to crosstab (repeat for several)',
+    )
+    add.add_argument(
+        '--filter',
+        action='append',
+        dest='filter_specs',
+        metavar='SPEC',
+        help='Field=a|b or Field=min..max (min.. or ..max leaves that side open); optional " @Sheet" (repeatable)',
+    )
+    add.add_argument(
+        '--param',
+        action='append',
+        dest='params',
+        metavar='NAME=VALUE',
+        help='parameter Name=value (repeatable)',
+    )
+    _add_file_flags(add, out=False, default=argparse.SUPPRESS)
+    remove = commands.add_parser(
+        'remove',
+        help='delete saved jobs from the jobs file',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""examples:
+  tabpull remove daily-west
+  tabpull remove daily-west weekly-east
+  tabpull remove
+
+With no names, a terminal asks which jobs to remove, then confirms.
+Otherwise pass the names. The jobs file is rewritten from the jobs that
+remain. Removing every job leaves that file empty.
+""",
+    )
+    remove.add_argument(
+        'names',
+        nargs='*',
+        help='jobs to delete (prompt when omitted on a terminal)',
+    )
+    _add_file_flags(remove, out=False, default=argparse.SUPPRESS)
+    run = commands.add_parser(
+        'run',
+        help='export jobs into the current folder',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""writes <out>/<job>/<sheet>.csv, spaces in names become _
+
+--filter uses the same Field=a|b or Field=min..max syntax as add, including
+an open side (min.. or ..max). It applies to every job in this run and
+does not rewrite the jobs file. With no @Sheet it replaces the job's saved
+filter on that field on every sheet; with @Sheet, only that sheet's. A field
+the job does not filter is added, on @Sheet or else the first sheet.
+
+--param uses the same Name=value syntax as add. It applies to every job in
+this run and does not rewrite the jobs file. A saved param of that name is
+replaced. A name the job does not have is added for this run.
+
+examples:
+  tabpull run
+  tabpull run daily-west
+  tabpull run daily-west --filter "Order Date=2026-09-01.." --out ~/reports
+  tabpull run daily-west weekly-east --filter "Order Date=2026-09-01..2026-09-07"
+  tabpull run daily-west --param "Top N=10"
+  tabpull run --parallel 4
+""",
+    )
+    run.add_argument('names', nargs='*', help='only these jobs (default: all)')
+    run.add_argument(
+        '--filter',
+        action='append',
+        dest='filter_specs',
+        metavar='SPEC',
+        help=(
+            'override Field=a|b or Field=min..max for this run only '
+            '(repeatable; applied to every selected job; jobs file unchanged)'
+        ),
+    )
+    run.add_argument(
+        '--param',
+        action='append',
+        dest='params',
+        metavar='NAME=VALUE',
+        help=(
+            'override Name=value for this run only '
+            '(repeatable; applied to every selected job; jobs file unchanged)'
+        ),
+    )
+    run.add_argument(
+        '--parallel',
+        type=_at_least_one,
+        default=1,
+        help='how many jobs to export at once (default: 1)',
+    )
+    _add_file_flags(run, out=True, default=argparse.SUPPRESS)
+    login = commands.add_parser('login', help='refresh a site SSO browser session')
+    login.add_argument(
+        '--site', help='local site name (default: the only configured site)'
+    )
+    return parser, commands.choices
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser, commands = _parser()
+    args, extra = parser.parse_known_args(argv)
+    if extra:
+        (commands.get(args.command) or parser).error(
+            f'unrecognized arguments: {" ".join(extra)}'
+        )
+    jobs_file = args.jobs or jobs_path()
+    out_dir = args.out or Path()
+    match args.command:
+        case 'setup':
+            wizard.main(args.site)
+        case 'login':
+            _cmd_login(args.site)
+        case 'add':
+            _cmd_add(args, jobs_file)
+        case 'remove':
+            remove_jobs(jobs_file, args.names)
+        case 'run':
+            return _cmd_run(args, jobs_file, out_dir)
+        case _ if ui.interactive():
+            from tabpull.tui import run_tui
+
+            run_tui(jobs_file, out_dir)
+        case _:
+            show_home(args, jobs_file, out_dir)
+    return 0
+
+
+def cli() -> None:
+    try:
+        code = main()
+    except KeyboardInterrupt:
+        print(file=sys.stderr)
+        code = 130
+    except SystemExit as e:
+        if not isinstance(e.code, str):
+            raise
+        print(f'error: {e.code}')
+        code = 1
+    sys.exit(code)
+
+
+if __name__ == '__main__':
+    print('Run: tabpull setup | add | remove | run | login', file=sys.stderr)
+    sys.exit(2)

tabpull/cli.py

@@ -0,0 +1,17 @@
+import sys
+from importlib.metadata import version as installed_version
+
+VERSION_FLAGS = ('-v', '-V', '--version')
+
+
+def version() -> str:
+    return installed_version('tabpull')
+
+
+def main() -> None:
+    if any(sys.argv[1:] == [flag] for flag in VERSION_FLAGS):
+        print(version())
+        return
+    from tabpull.app import cli
+
+    cli()

tabpull/embed.py

@@ -0,0 +1,248 @@
+import codecs
+import csv
+import io
+import json
+import string
+from collections.abc import Callable
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any, TypedDict
+
+from playwright.sync_api import BrowserContext, Page
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
+from tabpull import ui
+from tabpull.filters import normalize_range_bound, resolved_filters
+from tabpull.jobs import Job, JobError, RangeFilter, ValuesFilter, slug
+from tabpull.tableau import Settings
+
+VIZ_LOAD_TIMEOUT_MS = 180_000
+DOWNLOAD_TIMEOUT_MS = 300_000
+# A fake page on the Tableau origin: the embedded viz then loads first-party, so the SSO
+# cookies apply and "restrict embedding to these domains" site settings can't block it.
+HOST_PATH = '/__crosstab_host__'
+
+HOST_PAGE = string.Template("""<!doctype html>
+<meta charset="utf-8">
+<body>
+<script type="module">
+try {
+  const api = await import($script_url);
+  window.tableauEnums = { CrosstabFileFormat: api.CrosstabFileFormat, FilterDomainType: api.FilterDomainType, FilterUpdateType: api.FilterUpdateType };
+  const viz = new api.TableauViz();
+  viz.src = $view_url;
+  viz.toolbar = 'hidden';
+  viz.hideTabs = true;
+  viz.addEventListener(api.TableauEventType.FirstInteractive, () => { window.vizState = 'ready'; });
+  viz.addEventListener(api.TableauEventType.VizLoadError, (e) => {
+    window.vizState = 'error: ' + (e.detail?.message ?? JSON.stringify(e.detail));
+  });
+  window.viz = viz;
+  document.body.appendChild(viz);
+} catch (e) {
+  window.vizState = 'error: could not load the Tableau Embedding API: ' + e;
+}
+</script>
+""")
+
+STORY_REFUSAL = 'Stories are not supported; use the dashboard inside it.'
+_STORY_GUARD_JS = (
+    "if (window.viz.workbook.activeSheet.sheetType === 'story') "
+    'throw new Error(' + json.dumps(STORY_REFUSAL) + ');'
+)
+
+STORY_JS = '() => {\n  ' + _STORY_GUARD_JS + '\n}'
+
+INSPECT_JS = (
+    'async () => {\n  '
+    + _STORY_GUARD_JS
+    + """
+  const active = window.viz.workbook.activeSheet;
+  const worksheets = active.sheetType === 'worksheet' ? [active] : active.worksheets;
+  const current = (f) => {
+    if (f.filterType === 'categorical') return f.isAllSelected ? '(All)' : f.appliedValues.map((v) => v.formattedValue).join(' | ');
+    if (f.filterType === 'range') return (f.minValue?.formattedValue ?? '') + ' .. ' + (f.maxValue?.formattedValue ?? '');
+    return '';
+  };
+  const sheets = [];
+  for (const ws of worksheets) {
+    const filters = await ws.getFiltersAsync();
+    sheets.push({ name: ws.name, filters: filters.map((f) => ({ field: f.fieldName, type: f.filterType, current: current(f) })) });
+  }
+  const params = await window.viz.workbook.getParametersAsync();
+  return { sheets, params: params.map((p) => ({ name: p.name, current: p.currentValue.formattedValue })) };
+}"""
+)
+
+APPLY_JS = """async ({ filters, params }) => {
+  const viz = window.viz;
+  const active = viz.workbook.activeSheet;
+  const worksheets = active.sheetType === 'worksheet' ? [active] : active.worksheets;
+  const toValue = (v) => {
+    if (typeof v !== 'string') return v;
+    if (v.trim() === '') return v;
+    // Tableau reads range-filter Dates as UTC calendar days, whatever the browser timezone.
+    // An impossible ISO date must not be returned: `new Date` shifts it (2024-02-31 -> March 2).
+    const iso = /^(\\d{4})-(\\d{2})-(\\d{2})$/.exec(v);
+    if (iso) {
+      const year = Number(iso[1]);
+      const month = Number(iso[2]);
+      const day = Number(iso[3]);
+      const parsed = new Date(v + 'T00:00:00Z');
+      if (
+        Number.isNaN(parsed.getTime()) ||
+        parsed.getUTCFullYear() !== year ||
+        parsed.getUTCMonth() !== month - 1 ||
+        parsed.getUTCDate() !== day
+      ) throw new Error(v + ' is not a real date; use YYYY-MM-DD or M/D/YYYY');
+      return parsed;
+    }
+    if (/^-?\\d+(\\.\\d+)?$/.test(v)) return Number(v);
+    throw new Error(v + ' is a relative date or a date computed at run time; use YYYY-MM-DD or M/D/YYYY');
+  };
+  // applyRangeFilterAsync refuses a null bound, so an open side takes the filter's own endpoint.
+  const domainOf = async (ws, f) => {
+    const found = (await ws.getFiltersAsync()).find((x) => x.fieldName === f.field && x.filterType === 'range');
+    if (!found) throw new Error('No range filter on ' + f.field + ' in ' + f.sheet + ' to take an open end from; give both min and max');
+    return found.getDomainAsync(window.tableauEnums.FilterDomainType.Database);
+  };
+  for (const [name, value] of Object.entries(params)) await viz.workbook.changeParameterValueAsync(name, value);
+  for (const f of filters) {
+    const ws = worksheets.find((w) => w.name === f.sheet);
+    if (!ws) throw new Error('Sheet not in this view: ' + f.sheet);
+    if (f.values) {
+      await ws.applyFilterAsync(f.field, f.values, window.tableauEnums.FilterUpdateType.Replace);
+      continue;
+    }
+    const domain = f.min == null || f.max == null ? await domainOf(ws, f) : null;
+    await ws.applyRangeFilterAsync(f.field, {
+      min: f.min == null ? domain.min.value : toValue(f.min),
+      max: f.max == null ? domain.max.value : toValue(f.max),
+    });
+  }
+}"""
+
+EXPORT_JS = '(sheet) => window.viz.exportCrosstabAsync(sheet, window.tableauEnums.CrosstabFileFormat.CSV)'
+
+
+class SheetInfo(TypedDict):
+    name: str
+    filters: list[dict[str, str]]
+
+
+class ViewInfo(TypedDict):
+    sheets: list[SheetInfo]
+    params: list[dict[str, str]]
+
+
+def filter_payload(item: ValuesFilter | RangeFilter) -> dict[str, Any]:
+    raw = asdict(item)
+    if isinstance(item, RangeFilter):
+        raw['min'] = normalize_range_bound(item.min)
+        raw['max'] = normalize_range_bound(item.max)
+    return raw
+
+
+def normalize_csv(raw: bytes) -> str:
+    """Tableau's crosstab "CSV" is UTF-16 and tab-separated."""
+    if not raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
+        return raw.decode('utf-8-sig')
+    out = io.StringIO()
+    csv.writer(out, lineterminator='\n').writerows(
+        csv.reader(io.StringIO(raw.decode('utf-16')), delimiter='\t')
+    )
+    return out.getvalue()
+
+
+def _refuse_story(page: Page) -> None:
+    try:
+        page.evaluate(STORY_JS)
+    except PlaywrightError as e:
+        if STORY_REFUSAL not in str(e):
+            raise
+        msg = STORY_REFUSAL
+        raise JobError(msg) from e
+
+
+def _publish_csv(path: Path, text: str) -> None:
+    if ui.stopped():
+        msg = 'Cancelled.'
+        raise JobError(msg)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    partial = path.with_name(f'.{path.name}.partial')
+    try:
+        partial.write_text(text, encoding='utf-8-sig')
+        if ui.stopped():
+            msg = 'Cancelled.'
+            raise JobError(msg)
+        partial.replace(path)
+    finally:
+        partial.unlink(missing_ok=True)
+
+
+def open_view(context: BrowserContext, settings: Settings, view: str) -> Page:
+    page = context.new_page()
+    host = settings.server + HOST_PATH
+    html = HOST_PAGE.substitute(
+        script_url=json.dumps(
+            f'{settings.server}/javascripts/api/tableau.embedding.3.latest.min.js'
+        ),
+        view_url=json.dumps(settings.view_url(view)),
+    )
+    page.route(host, lambda route: route.fulfill(content_type='text/html', body=html))
+    page.goto(host)
+    try:
+        page.wait_for_function('() => window.vizState', timeout=VIZ_LOAD_TIMEOUT_MS)
+    except PlaywrightTimeoutError as e:
+        msg = f'{view!r} did not load within {VIZ_LOAD_TIMEOUT_MS // 1000}s. Check the path and that you can open it in a browser.'
+        raise JobError(msg) from e
+    state = page.evaluate('() => window.vizState')
+    if state != 'ready':
+        msg = f'{view!r}: {state}'
+        raise JobError(msg)
+    try:
+        _refuse_story(page)
+    except (JobError, PlaywrightError):
+        page.close()
+        raise
+    return page
+
+
+def export_embed(
+    context: BrowserContext,
+    settings: Settings,
+    job: Job,
+    out_dir: Path,
+    *,
+    on_sheet: Callable[[int, str], None] | None = None,
+) -> list[Path]:
+    filters = resolved_filters(job)
+    payloads = [filter_payload(item) for item in filters]
+    page = open_view(context, settings, job.view)
+    try:
+        page.evaluate(
+            APPLY_JS,
+            {
+                'filters': payloads,
+                'params': job.params,
+            },
+        )
+        written = []
+        for sheet in job.sheets:
+            if ui.stopped():
+                msg = 'Cancelled.'
+                raise JobError(msg)
+            if on_sheet is not None:
+                on_sheet(len(written), sheet)
+            with page.expect_download(timeout=DOWNLOAD_TIMEOUT_MS) as download:
+                page.evaluate(EXPORT_JS, sheet)
+            if ui.stopped():
+                msg = 'Cancelled.'
+                raise JobError(msg)
+            path = out_dir / slug(job.name) / f'{slug(sheet)}.csv'
+            _publish_csv(path, normalize_csv(Path(download.value.path()).read_bytes()))
+            written.append(path)
+        return written
+    finally:
+        page.close()