bqlens 0.1.0__py3-none-any.whl

bqlens/__init__.py

@@ -0,0 +1,3 @@
+"""bqlens — find wasted BigQuery spend in one command."""
+
+__version__ = "0.1.0"

bqlens/cli.py

@@ -0,0 +1,135 @@
+"""Command line interface for bqlens."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+from .models import DEFAULT_PRICE_PER_TIB, ScanResult
+from .report import render_html, render_json, render_terminal
+from .rules import run_all
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="bqlens",
+        description="Find wasted BigQuery spend. Read-only; never touches your table data.",
+    )
+    parser.add_argument("--version", action="version", version=f"bqlens {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    scan = sub.add_parser("scan", help="Scan query history for recoverable spend")
+    scan.add_argument("--project", help="GCP project ID to scan")
+    scan.add_argument(
+        "--region",
+        default="region-us",
+        help="BigQuery region of the JOBS view (default: region-us)",
+    )
+    scan.add_argument("--days", type=int, default=30, help="Days of history (default: 30)")
+    scan.add_argument(
+        "--price-per-tib",
+        type=float,
+        default=DEFAULT_PRICE_PER_TIB,
+        help=f"On-demand price per TiB scanned (default: {DEFAULT_PRICE_PER_TIB})",
+    )
+    scan.add_argument(
+        "--min-gib",
+        type=float,
+        default=1.0,
+        help="Ignore queries smaller than this many GiB (default: 1.0)",
+    )
+    scan.add_argument(
+        "--min-repeats",
+        type=int,
+        default=10,
+        help="Repeats before a query shape is flagged for materialisation (default: 10)",
+    )
+    scan.add_argument(
+        "--format",
+        choices=["terminal", "json", "html"],
+        default="terminal",
+        help="Output format (default: terminal)",
+    )
+    scan.add_argument("--out", help="Write output to this file instead of stdout")
+    scan.add_argument("--no-color", action="store_true", help="Disable ANSI colour")
+    scan.add_argument(
+        "--demo",
+        action="store_true",
+        help="Run against built-in synthetic history. No credentials needed.",
+    )
+    scan.add_argument(
+        "--fail-over",
+        type=float,
+        default=None,
+        metavar="USD",
+        help="Exit non-zero if monthly recoverable spend exceeds this. For CI.",
+    )
+    return parser
+
+
+def cmd_scan(args: argparse.Namespace) -> int:
+    if args.demo:
+        from .demo import generate
+
+        jobs = generate()
+        project = "demo-project"
+    else:
+        if not args.project:
+            print("error: --project is required (or use --demo)", file=sys.stderr)
+            return 2
+        from .fetch import fetch_jobs
+
+        jobs = fetch_jobs(project=args.project, region=args.region, days=args.days)
+        project = args.project
+
+    if not jobs:
+        print("No query jobs found in that window.", file=sys.stderr)
+        return 0
+
+    min_bytes = int(args.min_gib * 1024**3)
+    findings = run_all(
+        jobs,
+        price_per_tib=args.price_per_tib,
+        min_bytes=min_bytes,
+        min_repeats=args.min_repeats,
+    )
+
+    result = ScanResult(
+        findings=findings,
+        total_jobs=len(jobs),
+        total_spend_usd=sum(j.cost(args.price_per_tib) for j in jobs),
+        days=args.days,
+        project=project,
+        price_per_tib=args.price_per_tib,
+    )
+
+    if args.format == "json":
+        output = render_json(result)
+    elif args.format == "html":
+        output = render_html(result)
+    else:
+        colour = sys.stdout.isatty() and not args.no_color
+        output = render_terminal(result, colour=colour)
+
+    if args.out:
+        Path(args.out).write_text(output, encoding="utf-8")
+        print(f"Wrote {args.out}", file=sys.stderr)
+    else:
+        print(output)
+
+    if args.fail_over is not None and result.monthly_recoverable_usd > args.fail_over:
+        return 1
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command == "scan":
+        return cmd_scan(args)
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

bqlens/demo.py

@@ -0,0 +1,188 @@
+"""Synthetic job history so the scanner can be tried without credentials.
+
+`bqlens scan --demo` runs the full pipeline against this. It is also what the
+test suite asserts against, so the numbers below are fixtures, not decoration.
+"""
+
+from __future__ import annotations
+
+import random
+from datetime import datetime, timedelta
+
+from .models import Job
+
+GIB = 1024**3
+
+_USERS = [
+    "analytics@example.com",
+    "dbt-runner@example.iam.gserviceaccount.com",
+    "looker@example.iam.gserviceaccount.com",
+    "priya@example.com",
+]
+
+
+def _job(
+    idx: int,
+    query: str,
+    gib: float,
+    user: str,
+    *,
+    cache_hit: bool = False,
+    error: str | None = None,
+    when: datetime | None = None,
+) -> Job:
+    billed = int(gib * GIB)
+    return Job(
+        job_id=f"demo_job_{idx:05d}",
+        user_email=user,
+        creation_time=when or (datetime.utcnow() - timedelta(hours=idx % 720)),
+        query=query,
+        total_bytes_billed=0 if cache_hit else billed,
+        total_bytes_processed=billed,
+        cache_hit=cache_hit,
+        statement_type="SELECT",
+        referenced_tables=["demo-project.warehouse.events"],
+        total_slot_ms=int(gib * 1200),
+        error_result=error,
+    )
+
+
+def generate(seed: int = 7) -> list[Job]:
+    """A plausible 30 days of query history for a mid-sized data team."""
+    rng = random.Random(seed)
+    jobs: list[Job] = []
+    idx = 0
+
+    # 1. A Looker dashboard hammering the same unaggregated query all month.
+    dashboard_sql = """
+        SELECT user_id, event_name, event_timestamp, platform, country,
+               session_id, revenue_usd
+        FROM `demo-project.warehouse.events`
+        WHERE event_date >= '2026-08-01'
+    """
+    for _ in range(280):
+        idx += 1
+        jobs.append(_job(idx, dashboard_sql, rng.uniform(7.5, 9.0), "looker@example.iam.gserviceaccount.com"))
+
+    # 2. GA4-style wildcard scans with no _TABLE_SUFFIX filter. The classic.
+    wildcard_sql = """
+        SELECT event_name, COUNT(*) AS n
+        FROM `demo-project.analytics_311.events_*`
+        GROUP BY event_name
+    """
+    for _ in range(22):
+        idx += 1
+        jobs.append(_job(idx, wildcard_sql, rng.uniform(140, 190), "analytics@example.com"))
+
+    # 3. Analysts exploring with SELECT * LIMIT, believing LIMIT is cheap.
+    for _ in range(46):
+        idx += 1
+        jobs.append(
+            _job(
+                idx,
+                "SELECT * FROM `demo-project.warehouse.events` LIMIT 100",
+                rng.uniform(58, 72),
+                "priya@example.com",
+            )
+        )
+
+    # 4. SELECT * feeding a transform that uses four columns.
+    for _ in range(35):
+        idx += 1
+        jobs.append(
+            _job(
+                idx,
+                "SELECT * FROM `demo-project.warehouse.transactions` "
+                "WHERE txn_date BETWEEN '2026-08-01' AND '2026-08-31'",
+                rng.uniform(18, 26),
+                "dbt-runner@example.iam.gserviceaccount.com",
+            )
+        )
+
+    # 5. A broken scheduled query retrying all month and billing every time.
+    for _ in range(64):
+        idx += 1
+        jobs.append(
+            _job(
+                idx,
+                "SELECT customer_id, SUM(amount) FROM `demo-project.warehouse.ledger` "
+                "WHERE posted_date >= '2026-08-01' GROUP BY 1",
+                rng.uniform(11, 15),
+                "dbt-runner@example.iam.gserviceaccount.com",
+                error="Not found: Table demo-project:warehouse.ledger_v2",
+            )
+        )
+
+    # 6. An unfiltered full scan somebody schedules nightly.
+    for _ in range(30):
+        idx += 1
+        jobs.append(
+            _job(
+                idx,
+                "SELECT customer_id, lifetime_value, segment "
+                "FROM `demo-project.warehouse.customer_360`",
+                rng.uniform(34, 44),
+                "analytics@example.com",
+            )
+        )
+
+    # 7. A join missing its ON clause.
+    for _ in range(6):
+        idx += 1
+        jobs.append(
+            _job(
+                idx,
+                "SELECT a.user_id, b.campaign_id "
+                "FROM `demo-project.warehouse.users` a "
+                "JOIN `demo-project.warehouse.campaigns` b "
+                "WHERE a.country = 'IN'",
+                rng.uniform(25, 33),
+                "priya@example.com",
+            )
+        )
+
+    # 8. Plenty of ordinary, well-written, cheap queries — the healthy majority.
+    for _ in range(900):
+        idx += 1
+        col = rng.choice(["event_name", "platform", "country"])
+        jobs.append(
+            _job(
+                idx,
+                f"SELECT {col}, COUNT(*) FROM `demo-project.warehouse.events` "
+                f"WHERE event_date = '2026-09-{rng.randint(1, 17):02d}' GROUP BY 1",
+                rng.uniform(0.05, 0.9),
+                rng.choice(_USERS),
+            )
+        )
+
+    # 9. Legitimate heavy analytical work: large, well-written, distinct
+    #    queries. This is the bulk of a healthy bill and nothing should flag
+    #    it. Without this block the demo would claim an absurd savings rate.
+    _dims = ["country", "platform", "channel", "device_type", "cohort", "tier", "region"]
+    _metrics = ["revenue_usd", "sessions", "orders", "refunds", "margin_usd", "units"]
+    _tables = ["transactions", "sessions_daily", "orders", "subscriptions", "inventory_daily"]
+    for i in range(300):
+        idx += 1
+        dim = _dims[i % len(_dims)]
+        metric = _metrics[(i // 7) % len(_metrics)]
+        table = _tables[(i // 3) % len(_tables)]
+        jobs.append(
+            _job(
+                idx,
+                f"SELECT {dim}, DATE_TRUNC(txn_date, MONTH) AS m, SUM({metric}) AS total_{i} "
+                f"FROM `demo-project.warehouse.{table}` t "
+                f"JOIN `demo-project.warehouse.dim_customer` c ON t.customer_id = c.customer_id "
+                f"WHERE txn_date >= '2026-0{rng.randint(1, 8)}-01' "
+                f"GROUP BY 1, 2",
+                rng.uniform(40, 90),
+                rng.choice(_USERS),
+            )
+        )
+
+    # 10. Cache hits — free, and correctly ignored by every rule.
+    for _ in range(150):
+        idx += 1
+        jobs.append(_job(idx, dashboard_sql, 8.0, "looker@example.iam.gserviceaccount.com", cache_hit=True))
+
+    rng.shuffle(jobs)
+    return jobs

bqlens/fetch.py

@@ -0,0 +1,65 @@
+"""Read query history from INFORMATION_SCHEMA.JOBS.
+
+Read-only. bqlens never reads a row of your actual table data — only job
+metadata and the SQL text of the queries themselves.
+"""
+
+from __future__ import annotations
+
+from .models import Job
+
+# The JOBS view is region-qualified: region-us, region-eu, region-asia-south1, ...
+JOBS_SQL_REGIONAL = """
+SELECT
+  job_id,
+  user_email,
+  creation_time,
+  query,
+  total_bytes_billed,
+  total_bytes_processed,
+  cache_hit,
+  statement_type,
+  total_slot_ms,
+  referenced_tables,
+  -- error_result is a STRUCT<reason, location, debug_info, message>, so it
+  -- cannot be cast to STRING directly. Serialise it, and keep NULL as NULL
+  -- so that `is_failed` stays false for jobs that succeeded.
+  IF(error_result.reason IS NULL, NULL, TO_JSON_STRING(error_result)) AS error_result
+FROM `{project}`.`{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT
+WHERE creation_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL @days DAY)
+  AND job_type = 'QUERY'
+  AND query IS NOT NULL
+ORDER BY total_bytes_billed DESC
+LIMIT @max_jobs
+"""
+
+
+def fetch_jobs(
+    project: str,
+    region: str = "region-us",
+    days: int = 30,
+    max_jobs: int = 20000,
+) -> list[Job]:
+    """Pull query jobs from BigQuery. Requires google-cloud-bigquery."""
+    try:
+        from google.cloud import bigquery
+    except ImportError as exc:  # pragma: no cover - depends on optional extra
+        raise SystemExit(
+            "google-cloud-bigquery is not installed.\n"
+            "Install it with:  pip install 'bqlens[bigquery]'\n"
+            "Or try the scanner first with:  bqlens scan --demo"
+        ) from exc
+
+    client = bigquery.Client(project=project)
+    sql = JOBS_SQL_REGIONAL.format(project=project, region=region)
+    config = bigquery.QueryJobConfig(
+        query_parameters=[
+            bigquery.ScalarQueryParameter("days", "INT64", days),
+            bigquery.ScalarQueryParameter("max_jobs", "INT64", max_jobs),
+        ],
+        # Reading job metadata is itself billable. Cap it so the scanner can
+        # never be the expensive thing in someone's bill.
+        maximum_bytes_billed=10 * 1024**3,
+    )
+    rows = client.query(sql, job_config=config).result()
+    return [Job.from_row(dict(row)) for row in rows]

bqlens/models.py

@@ -0,0 +1,161 @@
+"""Core data structures for bqlens."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+# BigQuery on-demand analysis pricing. Default is the US multi-region list
+# price per TiB scanned. Override with --price-per-tib for your region or
+# negotiated rate.
+DEFAULT_PRICE_PER_TIB = 6.25
+
+BYTES_PER_TIB = 1024**4
+
+
+def bytes_to_usd(num_bytes: float, price_per_tib: float = DEFAULT_PRICE_PER_TIB) -> float:
+    """Convert bytes billed into on-demand dollars."""
+    if num_bytes <= 0:
+        return 0.0
+    return (num_bytes / BYTES_PER_TIB) * price_per_tib
+
+
+def fmt_usd(amount: float) -> str:
+    """Format dollars without rendering real money as a meaningless $0.00.
+
+    A finding worth a third of a cent is still a real finding — on a bill
+    100x larger it is $0.33 — so we say "<$0.01" rather than "$0.00", which
+    reads as a bug.
+    """
+    if amount >= 0.01:
+        return f"${amount:,.2f}"
+    if amount > 0:
+        return "<$0.01"
+    return "$0.00"
+
+
+def human_bytes(num_bytes: float) -> str:
+    """Render a byte count in the largest sensible binary unit."""
+    step = 1024.0
+    value = float(num_bytes)
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if abs(value) < step or unit == "TiB":
+            if unit == "B":
+                return f"{value:.0f} B"
+            return f"{value:.2f} {unit}"
+        value /= step
+    return f"{value:.2f} TiB"
+
+
+@dataclass
+class Job:
+    """One BigQuery query job, normalised from INFORMATION_SCHEMA.JOBS."""
+
+    job_id: str
+    user_email: str
+    creation_time: datetime
+    query: str
+    total_bytes_billed: int
+    total_bytes_processed: int
+    cache_hit: bool = False
+    statement_type: str = "SELECT"
+    referenced_tables: list[str] = field(default_factory=list)
+    total_slot_ms: int = 0
+    destination_table: str | None = None
+    error_result: str | None = None
+
+    @property
+    def is_failed(self) -> bool:
+        return bool(self.error_result)
+
+    def cost(self, price_per_tib: float = DEFAULT_PRICE_PER_TIB) -> float:
+        return bytes_to_usd(self.total_bytes_billed, price_per_tib)
+
+    @classmethod
+    def from_row(cls, row: Any) -> "Job":
+        """Build a Job from a BigQuery result row (or any mapping-like object)."""
+        get = row.get if hasattr(row, "get") else lambda k, d=None: getattr(row, k, d)
+
+        refs = get("referenced_tables") or []
+        normalised_refs = []
+        for ref in refs:
+            if isinstance(ref, str):
+                normalised_refs.append(ref)
+            else:
+                # BigQuery returns STRUCT<project_id, dataset_id, table_id>
+                project = ref.get("project_id") if hasattr(ref, "get") else getattr(ref, "project_id", None)
+                dataset = ref.get("dataset_id") if hasattr(ref, "get") else getattr(ref, "dataset_id", None)
+                table = ref.get("table_id") if hasattr(ref, "get") else getattr(ref, "table_id", None)
+                normalised_refs.append(f"{project}.{dataset}.{table}")
+
+        return cls(
+            job_id=get("job_id") or "",
+            user_email=get("user_email") or "unknown",
+            creation_time=get("creation_time"),
+            query=get("query") or "",
+            total_bytes_billed=int(get("total_bytes_billed") or 0),
+            total_bytes_processed=int(get("total_bytes_processed") or 0),
+            cache_hit=bool(get("cache_hit")),
+            statement_type=get("statement_type") or "SELECT",
+            referenced_tables=normalised_refs,
+            total_slot_ms=int(get("total_slot_ms") or 0),
+            destination_table=get("destination_table"),
+            error_result=get("error_result"),
+        )
+
+
+@dataclass
+class Finding:
+    """One thing costing money that could stop costing money."""
+
+    rule_id: str
+    title: str
+    # Dollars per period that this finding accounts for.
+    wasted_usd: float
+    # Dollars per period we believe are *recoverable* if the fix is applied.
+    recoverable_usd: float
+    detail: str
+    fix: str
+    severity: str = "medium"  # low | medium | high
+    job_count: int = 0
+    sample_job_ids: list[str] = field(default_factory=list)
+    owners: list[str] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if self.recoverable_usd > self.wasted_usd:
+            self.recoverable_usd = self.wasted_usd
+
+
+@dataclass
+class ScanResult:
+    """Everything one scan produced."""
+
+    findings: list[Finding]
+    total_jobs: int
+    total_spend_usd: float
+    days: int
+    project: str
+    price_per_tib: float = DEFAULT_PRICE_PER_TIB
+
+    @property
+    def recoverable_usd(self) -> float:
+        return sum(f.recoverable_usd for f in self.findings)
+
+    @property
+    def recoverable_pct(self) -> float:
+        if self.total_spend_usd <= 0:
+            return 0.0
+        return 100.0 * self.recoverable_usd / self.total_spend_usd
+
+    @property
+    def monthly_spend_usd(self) -> float:
+        if self.days <= 0:
+            return 0.0
+        return self.total_spend_usd * (30.0 / self.days)
+
+    @property
+    def monthly_recoverable_usd(self) -> float:
+        if self.days <= 0:
+            return 0.0
+        return self.recoverable_usd * (30.0 / self.days)