ripple-sql 0.1.0__py3-none-any.whl

ripple/usage/ingest.py

@@ -0,0 +1,414 @@
+"""Turn a query-history export into per-model counts, refusing what it can't know.
+
+Accepts JSONL (one object per line) or a JSON array, so the native output of
+`snow sql --format json` and `bq query --format=json` ingests as-is. Field
+names follow Snowflake's QUERY_HISTORY columns with common aliases, matched
+case-insensitively.
+
+Attribution refuses instead of guessing: an unqualified table name in a
+statement with no database/schema context could live anywhere, so it lands in
+a visible "no context" bucket even when it happens to match a model's name.
+That collision (same table name, different database) is exactly how usage
+tools end up confidently wrong.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections import Counter
+from datetime import datetime, timezone
+from pathlib import Path
+
+TEXT_KEYS = ("query_text", "text", "sql", "statement")
+ID_KEYS = ("query_id", "id", "statement_id", "job_id")
+START_KEYS = ("start_time", "started_at", "start", "creation_time")
+DATABASE_KEYS = ("database_name", "database", "catalog")
+SCHEMA_KEYS = ("schema_name", "schema")
+PRINCIPAL_KEYS = ("user_name", "user", "principal")
+
+# Snowflake's INFORMATION_SCHEMA.QUERY_HISTORY hard cap. Exactly this many
+# rows almost always means the export was cut short, not that the week ended.
+SNOWFLAKE_ROW_CAP = 10_000
+
+OUTSIDE_TABLE_LIMIT = 20
+
+
+def store_path(root: str | Path) -> Path:
+    return Path(root) / ".ripple" / "usage.json"
+
+
+def models_fingerprint(model_names: list[str]) -> str:
+    """Identity of the model set a store was computed against, so a summary
+    read after models were added or renamed can say 'stale' instead of
+    silently reporting the old project's inventory."""
+    return hashlib.sha256("|".join(sorted({n.lower() for n in model_names})).encode()).hexdigest()
+
+
+def _get(record: dict, keys: tuple[str, ...]):
+    lowered = {str(k).lower(): v for k, v in record.items()}
+    for key in keys:
+        value = lowered.get(key)
+        if value not in (None, ""):
+            return value
+    return None
+
+
+def _parse_time(value) -> datetime | None:
+    if value is None:
+        return None
+    if isinstance(value, (int, float)):
+        try:
+            seconds = value / 1000.0 if value > 1e12 else float(value)
+            return datetime.fromtimestamp(seconds, tz=timezone.utc)
+        except (OverflowError, OSError, ValueError):
+            return None
+    text = str(value).strip().replace("Z", "+00:00")
+    try:
+        parsed = datetime.fromisoformat(text)
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        # a valid export can mix offset-less and offset-bearing rows; naive
+        # values read as UTC, or min()/max() over the window raises TypeError
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed
+
+
+def _load_records(raw: bytes) -> tuple[list[dict], int]:
+    """Records plus the count of lines that were not usable."""
+    text = raw.decode("utf-8", errors="replace").strip()
+    if text.startswith("["):
+        try:
+            data = json.loads(text)
+        except json.JSONDecodeError:
+            return [], 1
+        records = [r for r in data if isinstance(r, dict)]
+        return records, len(data) - len(records)
+    records, invalid = [], 0
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError:
+            invalid += 1
+            continue
+        if isinstance(record, dict):
+            records.append(record)
+        else:
+            invalid += 1
+    return records, invalid
+
+
+def _statement_refs(statement) -> list[tuple[str, str, bool, bool]]:
+    """(name, display, qualified, is_write) per table reference, CTEs excluded.
+
+    Only statements that read or write data count. GRANT, SHOW, DESCRIBE,
+    ALTER and the rest are administrative: counting them made a model look
+    alive because someone touched its permissions, and SHOW TABLES IN a
+    schema fabricated the schema itself as an outside table.
+    """
+    from sqlglot import exp
+
+    activity_kinds = (
+        exp.Query,
+        exp.Insert,
+        exp.Create,
+        exp.Merge,
+        exp.Update,
+        exp.Delete,
+        exp.Drop,
+        exp.Copy,
+    )
+    if not isinstance(statement, activity_kinds):
+        return []
+    ctes = {c.alias_or_name.lower() for c in statement.find_all(exp.CTE) if c.alias_or_name}
+    write_ids: set[int] = set()
+    if isinstance(
+        statement, (exp.Insert, exp.Create, exp.Merge, exp.Update, exp.Delete, exp.Drop, exp.Copy)
+    ):
+        target = statement.this
+        if target is not None:
+            nodes = [target] if isinstance(target, exp.Table) else list(target.find_all(exp.Table))
+            write_ids.update(id(node) for node in nodes)
+
+    refs = []
+    for node in statement.find_all(exp.Table):
+        name = (node.name or "").lower()
+        if not name:
+            continue
+        if name in ctes and not (node.db or node.catalog):
+            # only an UNQUALIFIED reference can mean the CTE; a qualified
+            # analytics.orders inside a CTE named orders is the real table,
+            # and dropping it erased the statement's only genuine read
+            continue
+        parts = [p for p in (node.catalog, node.db, node.name) if p]
+        refs.append(
+            (name, ".".join(parts).lower(), bool(node.db or node.catalog), id(node) in write_ids)
+        )
+    return refs
+
+
+def _identity_index(model_aliases: dict[str, set[str]] | None) -> tuple[dict, dict]:
+    """(dotted alias -> model, model -> dotted alias set), lowercased.
+
+    Only uniquely owned aliases resolve; a dotted spelling claimed by two
+    models resolves to neither. Bare aliases carry no identity and are
+    ignored here.
+    """
+    if not model_aliases:
+        return {}, {}
+    owners: dict[str, set[str]] = {}
+    dotted: dict[str, set[str]] = {}
+    for model, aliases in model_aliases.items():
+        lowered = model.lower()
+        for alias in aliases or ():
+            a = str(alias).lower()
+            if "." not in a:
+                continue
+            owners.setdefault(a, set()).add(lowered)
+            dotted.setdefault(lowered, set()).add(a)
+    index = {a: next(iter(ms)) for a, ms in owners.items() if len(ms) == 1}
+    return index, dotted
+
+
+def _suffix_compatible(display: str, aliases: set[str]) -> bool:
+    return any(
+        display == a or a.endswith("." + display) or display.endswith("." + a) for a in aliases
+    )
+
+
+def _ref_prefix(display: str, session_db: str | None) -> str | None:
+    """The database-ish evidence a leaf match rests on, for conflict
+    detection. A three-part reference names its database outright;
+    otherwise the session database is the evidence; a two-part reference
+    outside any session contributes its first token."""
+    parts = display.split(".")
+    if len(parts) >= 3:
+        return parts[0]
+    if session_db:
+        return session_db
+    if len(parts) == 2:
+        return parts[0]
+    return None
+
+
+def ingest_file(
+    path: str | Path,
+    model_names: list[str],
+    dialect: str | None,
+    model_aliases: dict[str, set[str]] | None = None,
+) -> dict:
+    """Aggregate one export into the store dict. Raises ValueError when the
+    file holds no usable records, with the expected shape spelled out.
+
+    Attribution runs in two phases so identity conflicts are seen before
+    any counting: phase one parses every statement and gathers which
+    database each leaf-name match would rest on; phase two attributes.
+    A model with dbt-manifest identity (dotted aliases) rejects foreign
+    databases outright; a model without identity that is referenced from
+    several databases in one export is refused into the ambiguous bucket,
+    because merging them is exactly how usage counts end up confidently
+    wrong.
+    """
+    import sqlglot
+
+    path = Path(path)
+    raw = path.read_bytes()
+    records, invalid = _load_records(raw)
+    if not records:
+        raise ValueError(
+            f"no usable records in {path.name}. Expected JSONL (one object per line) "
+            "or a JSON array, each record with a query_text field "
+            "(Snowflake QUERY_HISTORY column names work as-is)."
+        )
+
+    occurrences = Counter(name.lower() for name in model_names)
+    unique_models = {n for n, c in occurrences.items() if c == 1}
+    ambiguous_models = {n for n, c in occurrences.items() if c > 1}
+    alias_index, dotted_aliases = _identity_index(model_aliases)
+
+    stats = dict.fromkeys(
+        (
+            "total",
+            "touched_tables",
+            "matched",
+            "no_context",
+            "outside_only",
+            "no_tables",
+            "unparsed",
+        ),
+        0,
+    )
+    principals: set[str] = set()
+    seen_ids: set[str] = set()
+    duplicates = 0
+    times: list[datetime] = []
+    parsed_records: list[tuple] = []
+    leaf_prefixes: dict[str, set[str]] = {}
+
+    for record in records:
+        text = _get(record, TEXT_KEYS)
+        if not text or not isinstance(text, str):
+            invalid += 1
+            continue
+        qid = _get(record, ID_KEYS)
+        if qid is not None:
+            qid = str(qid)
+            if qid in seen_ids:
+                duplicates += 1
+                continue
+            seen_ids.add(qid)
+
+        stats["total"] += 1
+        principal = _get(record, PRINCIPAL_KEYS)
+        if principal:
+            principals.add(str(principal))
+        started = _parse_time(_get(record, START_KEYS))
+        if started:
+            times.append(started)
+        session_db = _get(record, DATABASE_KEYS)
+        session_db = str(session_db).lower() if session_db else None
+        has_context = bool(session_db or _get(record, SCHEMA_KEYS))
+
+        try:
+            statements = sqlglot.parse(text, read=dialect)
+        except Exception:
+            stats["unparsed"] += 1
+            continue
+
+        refs = {}
+        for statement in statements:
+            if statement is None:
+                continue
+            for name, display, qualified, is_write in _statement_refs(statement):
+                # keyed by the qualified spelling: a join of a.foo and b.foo
+                # is two tables, not one
+                key = (display, is_write)
+                prior = refs.get(key)
+                refs[key] = (name, (prior[1] if prior else False) or qualified)
+        if not refs:
+            stats["no_tables"] += 1
+            continue
+        stats["touched_tables"] += 1
+        parsed_records.append((started, has_context, refs))
+
+        for (display, _is_write), (name, qualified) in refs.items():
+            if (
+                name in unique_models
+                and name not in dotted_aliases
+                and display not in alias_index
+                and (qualified or has_context)
+            ):
+                prefix = _ref_prefix(display, session_db)
+                if prefix:
+                    leaf_prefixes.setdefault(name, set()).add(prefix)
+
+    conflicted = {name for name, prefixes in leaf_prefixes.items() if len(prefixes) > 1}
+
+    models: dict[str, dict] = {}
+    outside: dict[str, int] = {}
+    ambiguous_hit: set[str] = set()
+
+    def count(name: str, is_write: bool, started) -> None:
+        entry = models.setdefault(name, {"reads": 0, "writes": 0, "last_seen": None})
+        entry["writes" if is_write else "reads"] += 1
+        if started and (entry["last_seen"] is None or started.isoformat() > entry["last_seen"]):
+            entry["last_seen"] = started.isoformat()
+
+    for started, has_context, refs in parsed_records:
+        matched = refused = external = 0
+        for (display, is_write), (name, qualified) in refs.items():
+            resolved = alias_index.get(display)
+            if resolved is not None:
+                matched += 1
+                count(resolved, is_write, started)
+                continue
+            if not qualified and not has_context:
+                # could be any database's table with this name; refusing here
+                # is the whole difference between a count and a guess
+                refused += 1
+                continue
+            if name in unique_models:
+                known = dotted_aliases.get(name)
+                if known and qualified and not _suffix_compatible(display, known):
+                    # identity is known and this reference names somewhere
+                    # else: another database's table, not this model
+                    external += 1
+                    outside[display] = outside.get(display, 0) + 1
+                    continue
+                if name in conflicted:
+                    ambiguous_hit.add(name)
+                    continue
+                matched += 1
+                count(name, is_write, started)
+            elif name in ambiguous_models:
+                ambiguous_hit.add(name)
+            else:
+                external += 1
+                outside[display] = outside.get(display, 0) + 1
+
+        if matched:
+            stats["matched"] += 1
+        elif refused:
+            stats["no_context"] += 1
+        elif external:
+            stats["outside_only"] += 1
+
+    window_start = min(times).isoformat() if times else None
+    window_end = max(times).isoformat() if times else None
+    days = ((max(times) - min(times)).total_seconds() / 86400.0) if len(times) > 1 else None
+
+    truncated = "unknown (hand-carried export)"
+    if len(records) == SNOWFLAKE_ROW_CAP:
+        cap_name = (
+            "Snowflake's INFORMATION_SCHEMA cap"
+            if dialect == "snowflake"
+            else "the export's row limit"
+        )
+        truncated = (
+            f"likely: exactly {SNOWFLAKE_ROW_CAP:,} rows is {cap_name}, "
+            "so the window was probably cut short"
+        )
+
+    top_outside = dict(sorted(outside.items(), key=lambda kv: -kv[1])[:OUTSIDE_TABLE_LIMIT])
+    seen = set(models)
+    not_seen = sorted(n for n in unique_models if n not in seen)
+
+    return {
+        "version": 1,
+        "models_fingerprint": models_fingerprint(model_names),
+        "manifest": {
+            "source": path.name,
+            "sha256": hashlib.sha256(raw).hexdigest(),
+            "ingested_at": datetime.now(tz=timezone.utc).isoformat(),
+            "invalid_lines": invalid,
+            "duplicate_ids": duplicates,
+            "window": {"start": window_start, "end": window_end, "days": days},
+            "principals": len(principals),
+            "truncated": truncated,
+        },
+        "project_models": len(unique_models) + len(ambiguous_models),
+        "statements": stats,
+        "models": models,
+        "not_seen": not_seen,
+        "ambiguous_names": sorted(ambiguous_hit),
+        "outside": top_outside,
+    }
+
+
+def write_store(root: str | Path, store: dict) -> Path:
+    path = store_path(root)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(store, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return path
+
+
+def read_store(root: str | Path) -> dict | None:
+    path = store_path(root)
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None

ripple/usage/report.py

@@ -0,0 +1,131 @@
+"""Render the usage store for a human. Plain text, honesty built into the copy.
+
+The banned word is banned here above all: nothing this module prints may call
+a model "unused". The window is always printed next to any absence claim,
+because the absence is a fact about the window, not about the model.
+"""
+
+from __future__ import annotations
+
+SEEN_LIMIT = 10
+NOT_SEEN_LIMIT = 8
+OUTSIDE_LIMIT = 5
+
+
+def _day(iso: str | None) -> str:
+    return iso[:10] if iso else "?"
+
+
+def _window_phrase(manifest: dict) -> str:
+    window = manifest["window"]
+    if not window["start"]:
+        return "no timestamps in this export"
+    phrase = f"{_day(window['start'])} to {_day(window['end'])}"
+    if window["days"] is not None:
+        phrase += f" ({window['days']:.1f} days)"
+    return phrase
+
+
+def render(store: dict) -> str:
+    manifest = store["manifest"]
+    stats = store["statements"]
+    users = f"{manifest['principals']} user" + ("s" if manifest["principals"] != 1 else "")
+    collected = manifest.get("collected")
+    # a collected store's "source" is a temp spool filename the user never
+    # made; say where the rows actually came from instead
+    source = (
+        f"{collected['platform']} ({collected['connection']})" if collected else manifest["source"]
+    )
+    lines = [f"{stats['total']:,} statements from {source} · {_window_phrase(manifest)} · {users}"]
+    if not str(manifest["truncated"]).startswith("unknown"):
+        lines.append(f"  export truncated? {manifest['truncated']}")
+
+    touched = stats["touched_tables"]
+    if touched:
+        pct = 100.0 * stats["matched"] / touched
+        lines.append(
+            f"\n{stats['matched']:,} of {touched:,} table-touching statements "
+            f"matched this project's models ({pct:.0f}%)"
+        )
+    parts = []
+    if stats["no_context"]:
+        parts.append(f"{stats['no_context']:,} skipped rather than guessed (no database context)")
+    if stats["outside_only"]:
+        parts.append(f"{stats['outside_only']:,} only touch tables outside this project")
+    if stats["unparsed"]:
+        parts.append(f"{stats['unparsed']:,} would not parse")
+    if stats["no_tables"]:
+        parts.append(f"{stats['no_tables']:,} touch no tables (SHOW, USE, ...)")
+    for part in parts:
+        lines.append(f"  {part}")
+
+    models = store["models"]
+    if models:
+        lines.append(f"\nSeen running: {len(models)} of {store['project_models']} models")
+        ranked = sorted(models.items(), key=lambda kv: -(kv[1]["reads"] + kv[1]["writes"]))
+        width = max(len(name) for name, _ in ranked[:SEEN_LIMIT])
+        for name, entry in ranked[:SEEN_LIMIT]:
+            lines.append(
+                f"  {name.ljust(width)}  {entry['reads']} reads · {entry['writes']} writes · "
+                f"last seen {_day(entry['last_seen'])}"
+            )
+        if len(ranked) > SEEN_LIMIT:
+            lines.append(f"  ...and {len(ranked) - SEEN_LIMIT} more")
+
+    not_seen = store["not_seen"]
+    if not_seen:
+        preview = ", ".join(not_seen[:NOT_SEEN_LIMIT])
+        more = (
+            f", and {len(not_seen) - NOT_SEEN_LIMIT} more" if len(not_seen) > NOT_SEEN_LIMIT else ""
+        )
+        lines.append(f"\nNot seen in this window: {len(not_seen)} models")
+        lines.append(f"  {preview}{more}")
+        lines.append("  This window can't call a model dead: a quarterly job looks identical to a")
+        lines.append("  dead one. Not seen here means exactly that, and nothing more.")
+
+    if store["ambiguous_names"]:
+        lines.append(
+            f"\n{len(store['ambiguous_names'])} name(s) claimed by several models or several "
+            f"databases were left uncounted rather than guessed: "
+            f"{', '.join(store['ambiguous_names'])}"
+        )
+
+    outside = store["outside"]
+    if outside:
+        lines.append("\nAlso read by these queries, but defined outside this project:")
+        ranked_outside = sorted(outside.items(), key=lambda kv: -kv[1])
+        for table, count in ranked_outside[:OUTSIDE_LIMIT]:
+            lines.append(f"  {table} ({count})")
+        if len(ranked_outside) > OUTSIDE_LIMIT:
+            lines.append(f"  ...and {len(ranked_outside) - OUTSIDE_LIMIT} more")
+        lines.append("  That's activity your SQL files alone can't see: dashboards, scripts,")
+        lines.append("  other teams. Their columns can be added with ripple ingest-schema.")
+
+    return "\n".join(lines)
+
+
+def render_empty() -> str:
+    return """No usage data yet. The short way, if snow, bq, or databricks is set up here:
+
+  ripple collect-usage
+
+runs your own warehouse CLI with the login you already have, ingests the
+result, and deletes the raw rows. Or export by hand and hand the file over:
+
+  ripple ingest-usage history.json
+
+Snowflake, your own queries, last 7 days, no permission needed:
+
+  snow sql -q "SELECT query_id, query_text, database_name, schema_name,
+    user_name, start_time
+    FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY(RESULT_LIMIT => 10000))" \\
+    --format json > history.json
+
+Getting exactly 10,000 rows back means you hit Snowflake's cap and the window
+was cut short; Ripple will say so. For everyone's queries over 365 days, the
+same columns come from SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY, which needs one
+admin grant (IMPORTED PRIVILEGES on the SNOWFLAKE database).
+
+Any JSONL or JSON-array file with a query_text field works, whichever
+warehouse it came from. The file stays on this machine, and Ripple stores
+per-table counts only, never the query text itself."""