ripple-sql 0.1.0__py3-none-any.whl

ripple/render_shims.py

@@ -0,0 +1,209 @@
+"""Builtin shims for macros whose bodies a repo clone never contains.
+
+dbt-core's cross-database builtins (dbt.date_trunc, dbt.concat, ...) ship
+with dbt itself; dbt_utils/fivetran_utils bodies live in dbt_packages/,
+which a raw clone lacks. The generic unknown-macro stub renders NULL, which
+parses but erases every value argument — holdout round 6 lost five dbt_jira
+cases through dbt.date_trunc alone, and dbt_utils.date_spine's NULL parsed
+as a relation literally named null.
+
+Only lineage-faithful shapes belong here: each shim either reproduces the
+macro's documented SQL closely enough that the value arguments survive, or
+degrades to a NULL literal (never an empty string — 'select  as x' parses
+as a bare column and fabricates an edge). Every entry is pinned by a test
+in tests/test_dbt_builtin_macros.py naming the real-repo miss behind it.
+"""
+
+from __future__ import annotations
+
+import re
+
+
+def _sql(value) -> str:
+    """Arguments arrive as jinja strings, numbers, or undefined stubs."""
+    text = str(value).strip()
+    return text if text else "null"
+
+
+def _part(datepart) -> str:
+    """Dateparts arrive quoted or bare; emit one canonical quoted spelling."""
+    return str(datepart).strip().strip("'\"").lower() or "day"
+
+
+def fallback_group_by(n=0, **kwargs):
+    # dbt_utils.group_by(n) emits the whole clause; ordinals reference
+    # already-listed select items, so lineage is unchanged
+    try:
+        count = int(kwargs.get("n", n))
+    except (TypeError, ValueError):
+        return ""
+    if count <= 0:
+        return ""
+    return "group by " + ", ".join(str(i) for i in range(1, count + 1))
+
+
+def fallback_empty(*args, **kwargs) -> str:
+    return ""
+
+
+def fallback_listagg(
+    measure="", delimiter_text="','", order_by_clause="", limit_num=None, **kwargs
+):
+    # the measure expression carries the lineage; the ordering clause is
+    # structural (same rule as a window ORDER BY). Cost holdout round 3
+    # two dbt_hubspot cases.
+    if not measure:
+        return "null"
+    return f"listagg({measure}, {delimiter_text})"
+
+
+def fallback_surrogate_key(field_list=None, **kwargs):
+    # the fields carry the lineage, the hash contributes nothing beyond
+    # them. Cost holdout round 4 all ten danish_democracy_data misses.
+    if isinstance(field_list, str):
+        field_list = [field_list]
+    parts = [
+        f"coalesce(cast({field} as varchar), '_')"
+        for field in (field_list or [])
+        if isinstance(field, str) and field.strip()
+    ]
+    if not parts:
+        return "null"
+    return "md5(" + " || '-' || ".join(parts) + ")"
+
+
+def fallback_date_trunc(datepart="day", date="", **kwargs):
+    # dbt_jira lost updated_at_week / open_until through the NULL stub
+    # (holdout round 6, five cases)
+    return f"date_trunc('{_part(datepart)}', {_sql(date)})"
+
+
+def fallback_dateadd(datepart="day", interval=0, from_date_or_timestamp="", **kwargs):
+    # the datepart is quoted: bare `day` parses as a COLUMN in
+    # bigquery/postgres/duckdb and fabricated t.day sources (review
+    # of cycle 7)
+    return f"dateadd('{_part(datepart)}', {_sql(interval)}, {_sql(from_date_or_timestamp)})"
+
+
+def fallback_datediff(first_date="", second_date="", datepart="day", **kwargs):
+    return f"datediff('{_part(datepart)}', {_sql(first_date)}, {_sql(second_date)})"
+
+
+def fallback_current_timestamp(**kwargs) -> str:
+    return "current_timestamp"
+
+
+def fallback_concat(fields=None, **kwargs):
+    if isinstance(fields, str):
+        fields = [fields]
+    parts = [f for f in (fields or []) if isinstance(f, str) and f.strip()]
+    return " || ".join(parts) if parts else "null"
+
+
+def fallback_safe_cast(field="", type="varchar", **kwargs):  # noqa: A002 - dbt's arg name
+    return f"cast({_sql(field)} as {_sql(type)})"
+
+
+def fallback_split_part(string_text="", delimiter_text="','", part_number=1, **kwargs):
+    return f"split_part({_sql(string_text)}, {_sql(delimiter_text)}, {_sql(part_number)})"
+
+
+def fallback_hash(field="", **kwargs):
+    return f"md5(cast({_sql(field)} as varchar))"
+
+
+def fallback_string_agg(field_to_agg="", delimiter="','", **kwargs):
+    # fivetran_utils.string_agg's NULL stub severed dbt_jira's whole
+    # multiselect field_value chain (holdout round 6)
+    if not field_to_agg:
+        return "null"
+    return f"string_agg({_sql(field_to_agg)}, {_sql(delimiter)})"
+
+
+def fallback_pass_through(
+    pass_through_variable=None, identifier=None, transform="", _var=None, **kwargs
+) -> str:
+    """fivetran_utils persist/fill_pass_through_columns: renders the
+    project-configured pass-through columns as ', ident.col as alias'
+    entries, nothing when unconfigured. Entries are strings or
+    {name, alias, transform_sql} dicts per the fivetran contract; a
+    transform_sql's own expression is opaque, so the column itself is the
+    honest value source."""
+    entries = _var(pass_through_variable, []) if (_var and pass_through_variable) else []
+    parts: list[str] = []
+    for entry in entries if isinstance(entries, list) else []:
+        if isinstance(entry, str):
+            name, alias = entry, None
+        elif isinstance(entry, dict):
+            name, alias = entry.get("name"), entry.get("alias")
+        else:
+            continue
+        if not name:
+            continue
+        expr = f"{identifier}.{name}" if identifier else str(name)
+        parts.append(f", {expr} as {alias or name}")
+    return "".join(parts)
+
+
+fallback_pass_through.wants_var = True
+
+
+def fallback_slugify(string="", **kwargs) -> str:
+    # runs at jinja level (feeds alias names, not SQL); the NULL stub made
+    # dbt_jira's pivot emit a column literally named null
+    slug = re.sub(r"[^a-z0-9_]+", "_", str(string).lower())
+    return f"_{slug}" if slug[:1].isdigit() else slug
+
+
+def fallback_date_spine(datepart="day", start_date="", end_date="", **kwargs) -> str:
+    # the spine is generated data: its column has no upstream, and the
+    # bounds are literals or clock functions. Rendering NULL inside
+    # from (...) parsed as a relation named null (holdout round 6,
+    # dbt_jira issue_day_id). The real macro names the column
+    # date_{datepart} (the cycle-7 review: 'hour' spines exist).
+    part = _part(datepart)
+    kind = "date" if part in ("day", "week", "month", "quarter", "year") else "timestamp"
+    return f"select cast(null as {kind}) as date_{part}"
+
+
+def _type(name: str):
+    return lambda **kwargs: name
+
+
+# Only lineage-safe shapes belong here; the NULL stub covers everything else.
+BUILTIN_PACKAGE_MACROS = {
+    ("dbt_utils", "group_by"): fallback_group_by,
+    # emits ", source_relation" only in multi-source unions; an empty
+    # partition arg keeps OVER (PARTITION BY x) parseable and feeds no column
+    ("fivetran_utils", "partition_by_source_relation"): fallback_empty,
+    # emits ", col, ..." only when the pass-through var is configured;
+    # unconfigured (the offline default) it emits nothing. The NULL stub
+    # rendered comma-less NULLs after the last select item and killed
+    # whole dbt_salesforce models (holdout round 7). When the project DOES
+    # configure the var, the columns render (the cycle-8 review:
+    # returning empty erased configured lineage)
+    ("fivetran_utils", "persist_pass_through_columns"): fallback_pass_through,
+    ("fivetran_utils", "fill_pass_through_columns"): fallback_pass_through,
+    ("dbt", "listagg"): fallback_listagg,
+    ("dbt_utils", "generate_surrogate_key"): fallback_surrogate_key,
+    # the pre-1.0 spelling of the same macro
+    ("dbt_utils", "surrogate_key"): fallback_surrogate_key,
+    ("dbt", "date_trunc"): fallback_date_trunc,
+    ("dbt", "dateadd"): fallback_dateadd,
+    ("dbt", "datediff"): fallback_datediff,
+    ("dbt", "current_timestamp"): fallback_current_timestamp,
+    ("dbt", "concat"): fallback_concat,
+    ("dbt", "safe_cast"): fallback_safe_cast,
+    ("dbt", "split_part"): fallback_split_part,
+    ("dbt", "hash"): fallback_hash,
+    ("dbt", "type_string"): _type("varchar"),
+    ("dbt", "type_timestamp"): _type("timestamp"),
+    ("dbt", "type_int"): _type("integer"),
+    ("dbt", "type_bigint"): _type("bigint"),
+    ("dbt", "type_float"): _type("float"),
+    ("dbt", "type_numeric"): _type("numeric(28,6)"),
+    ("dbt", "type_boolean"): _type("boolean"),
+    ("fivetran_utils", "string_agg"): fallback_string_agg,
+    ("dbt_utils", "slugify"): fallback_slugify,
+    ("dbt_utils", "date_spine"): fallback_date_spine,
+}

ripple/schemas.py

@@ -0,0 +1,155 @@
+"""Agent-ingested warehouse schemas.
+
+Ripple never connects to a warehouse. When a project reads tables it does
+not define, the user's own agent (or the user, via the CLI) fetches those
+tables' columns from information_schema and hands them over. They persist
+in .ripple/schemas.json at the project root: committable, sorted keys,
+stable order, so the file diffs cleanly and can be shared with a team.
+
+Format:
+    {"tables": {"<name>": {"columns": ["a", "b"], "origin": "ingested"}}}
+
+Names may be bare or schema/database qualified; matching is case
+insensitive. project.py exposes loaded tables as known external relations.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+from pathlib import Path
+
+
+def schemas_path(root: str | Path) -> Path:
+    return Path(root) / ".ripple" / "schemas.json"
+
+
+def load_schemas(root: str | Path) -> dict[str, list[str]]:
+    """Table name (lowercased) -> column list (lowercased). {} when the file
+    is absent or unreadable; a broken file must never break the project load."""
+    path = schemas_path(root)
+    if not path.is_file():
+        return {}
+    try:
+        doc = json.loads(path.read_text(errors="replace", encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return {}
+    try:
+        tables = normalize_tables(doc)
+    except ValueError:
+        return {}
+    return {name.lower(): [c.lower() for c in cols] for name, cols in tables.items()}
+
+
+def normalize_tables(payload: dict) -> dict[str, list[str]]:
+    """Accept the shapes agents and files actually produce:
+
+    - {"tables": {name: {"columns": [...]}}}   the canonical file format
+    - {"tables": {name: [...]}}                shorthand
+    - {name: [...]} or {name: {"columns": [...]}}  bare mapping
+    - {"rows": [{"table": ..., "column": ...}]}    tabular query results
+    """
+    if not isinstance(payload, dict):
+        raise ValueError("expected a JSON object")
+    if isinstance(payload.get("rows"), list):
+        tables: dict[str, list[str]] = {}
+        for row in payload["rows"]:
+            if not isinstance(row, dict):
+                continue
+            table = row.get("table") or row.get("table_name")
+            column = row.get("column") or row.get("column_name")
+            if not table or not column:
+                continue
+            bucket = tables.setdefault(str(table), [])
+            if str(column) not in bucket:
+                bucket.append(str(column))
+        if not tables:
+            raise ValueError("no usable rows: each row needs 'table' and 'column'")
+        return tables
+    mapping = payload.get("tables") if isinstance(payload.get("tables"), dict) else payload
+    tables = {}
+    for name, entry in mapping.items():
+        columns = entry.get("columns") if isinstance(entry, dict) else entry
+        if not isinstance(columns, list) or not all(isinstance(c, str) for c in columns):
+            raise ValueError(f"table '{name}': expected a list of column names")
+        deduped: list[str] = []
+        for column in columns:
+            if column not in deduped:
+                deduped.append(column)
+        tables[str(name)] = deduped
+    if not tables:
+        raise ValueError("no tables found in payload")
+    return tables
+
+
+def parse_csv(text: str) -> dict[str, list[str]]:
+    """CSV with a table,column header (information_schema exports)."""
+    reader = csv.DictReader(io.StringIO(text))
+    fields = {(f or "").strip().lower(): f for f in (reader.fieldnames or [])}
+    table_key = fields.get("table") or fields.get("table_name")
+    column_key = fields.get("column") or fields.get("column_name")
+    if not table_key or not column_key:
+        raise ValueError("CSV needs 'table' and 'column' header fields")
+    tables: dict[str, list[str]] = {}
+    for row in reader:
+        table = (row.get(table_key) or "").strip()
+        column = (row.get(column_key) or "").strip()
+        if not table or not column:
+            continue
+        bucket = tables.setdefault(table, [])
+        if column not in bucket:
+            bucket.append(column)
+    return tables
+
+
+def merge_schemas(root: str | Path, tables: dict[str, list[str]]) -> dict:
+    """Merge new tables into .ripple/schemas.json and report what changed.
+
+    Matching is case insensitive; the file keeps lowercase names so the same
+    table ingested twice with different casing stays one entry. New columns
+    append after existing ones, so a re-ingest diffs as pure additions.
+    """
+    path = schemas_path(root)
+    existing: dict[str, dict] = {}
+    if path.is_file():
+        try:
+            doc = json.loads(path.read_text(errors="replace", encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            doc = {}
+        raw = doc.get("tables") if isinstance(doc, dict) else None
+        if isinstance(raw, dict):
+            for name, entry in raw.items():
+                columns = entry.get("columns") if isinstance(entry, dict) else entry
+                if isinstance(columns, list):
+                    existing[name.lower()] = {
+                        "columns": [str(c).lower() for c in columns],
+                        "origin": (entry.get("origin") if isinstance(entry, dict) else None)
+                        or "ingested",
+                    }
+
+    added: list[str] = []
+    updated: list[str] = []
+    columns_added = 0
+    for name, columns in tables.items():
+        key = name.lower()
+        entry = existing.get(key)
+        if entry is None:
+            entry = {"columns": [], "origin": "ingested"}
+            existing[key] = entry
+            added.append(key)
+        new = [c.lower() for c in columns if c.lower() not in entry["columns"]]
+        if new and key not in added:
+            updated.append(key)
+        entry["columns"].extend(new)
+        columns_added += len(new)
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    ordered = {name: existing[name] for name in sorted(existing)}
+    path.write_text(json.dumps({"tables": ordered}, indent=2) + "\n", encoding="utf-8")
+    return {
+        "tables_added": sorted(added),
+        "tables_updated": sorted(updated),
+        "columns_added": columns_added,
+        "path": str(path),
+    }

ripple/semantic.py

@@ -0,0 +1,232 @@
+"""Downstream reach into the dbt semantic layer, metrics, and exposures.
+
+The BI layer, where it is declared as code, is just more edges. A dbt metric is a
+node that derives from a measure, which is a column of a model Ripple already
+mapped. A dbt exposure is a node-level leaf hanging off the models it reads. So a
+column rename can be traced to the metric and the dashboard it breaks, offline,
+with no warehouse connection.
+
+This reads the dbt schema YAML directly (the same files a raw dbt project ships),
+so it works without a compiled manifest. Nothing here fabricates a mapping: a
+metric whose measure we cannot resolve to a real column is dropped, not guessed.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from ripple.project import IGNORE_DIRS
+
+_REF = re.compile(r"ref\(\s*['\"]([^'\"]+)['\"]\s*(?:,\s*['\"]([^'\"]+)['\"])?\s*\)")
+
+
+@dataclass(frozen=True)
+class DownstreamEdge:
+    """A model column (or the model itself) feeding a BI node declared in code."""
+
+    src_model: str
+    src_column: str  # "*" means node-level (the whole model)
+    dst_model: str  # "metric:x" | "exposure:y" | "semantic:orders.customer"
+    dst_column: str
+    trust: str
+    reason: str = ""
+
+
+def _ref_model(value) -> str | None:
+    """The model name a `model: ref('orders')` points at."""
+    if not isinstance(value, str):
+        return None
+    m = _REF.search(value)
+    if m:
+        return m.group(2) or m.group(1)
+    return value.strip() or None
+
+
+def _iter_schema_docs(root: Path):
+    for path in sorted(root.rglob("*.yml")) + sorted(root.rglob("*.yaml")):
+        parts = path.relative_to(root).parts[:-1]
+        if any(p in IGNORE_DIRS or p.startswith(".") for p in parts):
+            continue
+        try:
+            doc = yaml.safe_load(path.read_text(errors="replace", encoding="utf-8"))
+        except (yaml.YAMLError, OSError):
+            continue
+        if not isinstance(doc, dict):
+            continue
+        if "semantic_models" in doc or "metrics" in doc or "exposures" in doc:
+            yield doc
+        elif any(
+            isinstance(m, dict) and ("semantic_model" in m or "metrics" in m)
+            for m in (doc.get("models") or [])
+            if m
+        ):
+            # dbt's newer inline syntax: semantic_model nested under a model
+            yield doc
+
+
+def _measure_name(spec) -> str | None:
+    if isinstance(spec, str):
+        return spec
+    if isinstance(spec, dict):
+        return spec.get("name")
+    return None
+
+
+def load_downstream(root: Path, known_models: set[str]) -> list[DownstreamEdge]:
+    """Every BI edge we can resolve from the project's declared-as-code layer."""
+    known = {m.lower() for m in known_models}
+    # global measure name -> (model, column). Measure names are unique per project.
+    measures: dict[str, tuple[str, str]] = {}
+    edges: list[DownstreamEdge] = []
+
+    docs = list(_iter_schema_docs(root))
+
+    # pass 0: dbt's inline syntax (jaffle-shop migrated to it): the semantic
+    # model is declared under the model entry itself, dimensions/entities on
+    # its columns, metrics on the model with the column as name or expr
+    for doc in docs:
+        for entry in doc.get("models") or []:
+            if not isinstance(entry, dict):
+                continue
+            sm = entry.get("semantic_model")
+            has_inline = isinstance(sm, dict) or entry.get("metrics")
+            if not has_inline or (isinstance(sm, dict) and sm.get("enabled") is False):
+                continue
+            model = entry.get("name")
+            if not model or model.lower() not in known:
+                continue
+            for col in entry.get("columns") or []:
+                if not (isinstance(col, dict) and col.get("name")):
+                    continue
+                cname = col["name"]
+                for kind in ("dimension", "entity"):
+                    spec = col.get(kind)
+                    if not isinstance(spec, dict):
+                        continue
+                    dname = spec.get("name") or cname
+                    edges.append(
+                        DownstreamEdge(
+                            src_model=model,
+                            src_column=cname,
+                            dst_model=f"semantic:{model}.{dname}",
+                            dst_column=dname,
+                            trust="verified",
+                        )
+                    )
+                measure = col.get("measure")
+                if isinstance(measure, dict) and measure.get("name"):
+                    measures[measure["name"]] = (model, cname)
+            for metric in entry.get("metrics") or []:
+                if not (isinstance(metric, dict) and metric.get("name")):
+                    continue
+                expr = metric.get("expr", metric["name"])
+                if isinstance(expr, str) and expr.isidentifier():
+                    edges.append(
+                        DownstreamEdge(
+                            src_model=model,
+                            src_column=expr,
+                            dst_model=f"metric:{metric['name']}",
+                            dst_column="value",
+                            trust="verified",
+                        )
+                    )
+                # a constant or computed expr has no single source column:
+                # dropped, never guessed
+
+    # pass 1: semantic models give measures/dimensions/entities their columns
+    for doc in docs:
+        for sm in doc.get("semantic_models") or []:
+            if not isinstance(sm, dict):
+                continue
+            model = _ref_model(sm.get("model"))
+            if not model or model.lower() not in known:
+                continue
+            sm_name = sm.get("name", model)
+            for m in sm.get("measures") or []:
+                if isinstance(m, dict) and m.get("name"):
+                    col = m.get("expr") or m["name"]
+                    if isinstance(col, str) and col.isidentifier():
+                        measures[m["name"]] = (model, col)
+            for kind in ("dimensions", "entities"):
+                for d in sm.get(kind) or []:
+                    if not (isinstance(d, dict) and d.get("name")):
+                        continue
+                    col = d.get("expr") or d["name"]
+                    if not (isinstance(col, str) and col.isidentifier()):
+                        continue
+                    edges.append(
+                        DownstreamEdge(
+                            src_model=model,
+                            src_column=col,
+                            dst_model=f"semantic:{sm_name}.{d['name']}",
+                            dst_column=d["name"],
+                            trust="verified",
+                        )
+                    )
+
+    # pass 2: metrics resolve through their measure(s) to the underlying column
+    for doc in docs:
+        for metric in doc.get("metrics") or []:
+            if not (isinstance(metric, dict) and metric.get("name")):
+                continue
+            params = metric.get("type_params") or {}
+            names: list[str] = []
+            for key in ("measure", "numerator", "denominator"):
+                n = _measure_name(params.get(key))
+                if n:
+                    names.append(n)
+            for m in params.get("measures") or []:
+                n = _measure_name(m)
+                if n:
+                    names.append(n)
+            for n in names:
+                loc = measures.get(n)
+                if not loc:
+                    continue  # unresolved measure: drop, never fake
+                edges.append(
+                    DownstreamEdge(
+                        src_model=loc[0],
+                        src_column=loc[1],
+                        dst_model=f"metric:{metric['name']}",
+                        dst_column="value",
+                        trust="verified",
+                    )
+                )
+
+    # pass 3: exposures are node-level (a dashboard reads the whole model)
+    for doc in docs:
+        for exp in doc.get("exposures") or []:
+            if not (isinstance(exp, dict) and exp.get("name")):
+                continue
+            deps = exp.get("depends_on") or exp.get("refs") or []
+            if isinstance(deps, str):
+                deps = [deps]
+            for dep in deps:
+                model = (
+                    _ref_model(dep)
+                    if isinstance(dep, str)
+                    else _ref_model((dep or {}).get("name") if isinstance(dep, dict) else None)
+                )
+                if model and model.lower() in known:
+                    edges.append(
+                        DownstreamEdge(
+                            src_model=model,
+                            src_column="*",
+                            dst_model=f"exposure:{exp['name']}",
+                            dst_column="*",
+                            trust="review_required",
+                            reason=f"declared {exp.get('type', 'exposure')}, node-level",
+                        )
+                    )
+
+    # dedupe
+    seen, out = set(), []
+    for e in edges:
+        if e not in seen:
+            seen.add(e)
+            out.append(e)
+    return out