ripple-sql 0.1.0__py3-none-any.whl

ripple/project.py

@@ -0,0 +1,167 @@
+"""Project discovery and loading.
+
+Finds what kind of SQL project we're in and yields models with parseable SQL.
+Three modes, best evidence first:
+
+1. dbt with a compiled manifest (target/manifest.json): exact compiled SQL,
+   explicit dependency edges, adapter-declared dialect.
+2. dbt raw (no manifest): models/**/*.sql with lightweight Jinja conversion.
+   ref('x') becomes x, source('a','b') becomes a__b.
+3. Plain SQL directory: every .sql file is a model named by its stem.
+
+Models from mode 2 and 3 carry lower baseline trust than mode 1; the graph
+layer folds that into per-edge confidence.
+
+This module is the public API and the mode dispatcher; the loaders package
+does the work.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ripple.loaders.dbt import _load_one_dbt, resolve_dbt_dialect
+from ripple.loaders.dbt_config import _dbt_owned_dirs
+from ripple.loaders.identity import _apply_ingested_schemas, _qualify_collisions
+from ripple.loaders.sqldir import _load_sql_dir
+from ripple.loaders.types import (
+    ADAPTER_TO_DIALECT,
+    DEFAULT_DIALECT,
+    IGNORE_DIRS,
+    Model,
+    Project,
+    _find_dbt_projects,
+    find_project_root,
+    relative_posix,
+)
+
+__all__ = [
+    "ADAPTER_TO_DIALECT",
+    "DEFAULT_DIALECT",
+    "IGNORE_DIRS",
+    "Model",
+    "Project",
+    "find_project_root",
+    "load_project",
+    "resolve_dbt_dialect",
+]
+
+
+def load_project(path: str | Path = ".", dialect: str | None = None) -> Project:
+    start = Path(path).resolve()
+    root = find_project_root(start)
+    # If we're standing above several dbt projects (a monorepo), gather them
+    # all. find_project_root only walks UP to the nearest one, so scan DOWN.
+    scan_root = root if (root / "dbt_project.yml").exists() else start
+    dbt_dirs = _find_dbt_projects(scan_root)
+
+    if len(dbt_dirs) > 1:
+        project = _load_monorepo(scan_root, dbt_dirs, dialect)
+    elif len(dbt_dirs) == 1:
+        project = _load_one_dbt(dbt_dirs[0], dialect)
+    elif (root / "dbt_project.yml").exists():
+        project = _load_one_dbt(root, dialect)
+    else:
+        project = _load_sql_dir(start, dialect)
+    if not project.models and dbt_dirs:
+        project = _sql_beside_dbt(start, dbt_dirs, dialect) or project
+    elif project.models and dbt_dirs:
+        # a repo can hold raw build SQL beside its dbt project(s), even
+        # inside a project dir, beside its models/ (nyc cbbr, holdout
+        # round 4). Only the directories dbt actually owns are pruned;
+        # pruning whole project dirs made cbbr/sql invisible.
+        owned = [d for pdir in dbt_dirs for d in _dbt_owned_dirs(pdir)]
+        # the dbt project's declared warehouse is the best evidence for the
+        # SQL beside it; sniffing those files alone reported "Assuming
+        # postgres" inside Snowflake's own tasty_bytes demo
+        inherited = None if project.dialect_assumed else project.dialect
+        stray = _load_sql_dir(scan_root, dialect or inherited, exclude=owned)
+        if stray.models:
+            project.models.extend(stray.models)
+            project.warnings.extend(w for w in stray.warnings if not w.startswith("No .sql files"))
+            project.warnings.append(
+                f"{len(stray.models)} plain-SQL models outside the dbt "
+                f"project{'s' if len(dbt_dirs) != 1 else ''} were loaded as well."
+            )
+    _qualify_collisions(project)
+    _apply_ingested_schemas(project, extra_roots=(scan_root, start))
+    from ripple.lookml import load_lookml_downstream
+    from ripple.semantic import load_downstream
+
+    downstream = []
+    if project.mode != "sql-dir":
+        downstream += load_downstream(project.root, {m.name for m in project.models})
+    downstream += load_lookml_downstream(project.root)  # Looker, where the extra is installed
+    project.downstream = downstream
+    return project
+
+
+def _sql_beside_dbt(start: Path, dbt_dirs: list[Path], dialect: str | None) -> Project | None:
+    """The repo's other SQL, when the dbt projects define no models.
+
+    dagster-open-platform ships a dbt project of macros, tests and snapshots
+    with no models/ directory, and fifteen plain .sql files beside it. Reporting
+    zero there was true of the dbt project and useless about the repo.
+
+    Deliberately narrow. It runs only when the dbt side found nothing, so a repo
+    that already resolves cannot start counting its migrations as models. The
+    dbt directories are excluded, since their macros are Jinja fragments rather
+    than models.
+    """
+    project = _load_sql_dir(start, dialect, exclude=dbt_dirs)
+    if not project.models:
+        return None
+    project.warnings.insert(
+        0,
+        "The dbt project here defines no models, so this is the rest of the repo's SQL instead.",
+    )
+    return project
+
+
+def _load_monorepo(root: Path, project_dirs: list[Path], dialect: str | None) -> Project:
+    """Union of several dbt projects, each parsed in its own dialect.
+
+    Cross-project references stay unresolved and degrade to review_required in
+    the graph layer, which is honest: one warehouse rarely reads another's
+    tables directly, and where it does we don't pretend to know the columns.
+    """
+    from collections import Counter
+
+    all_models: list[Model] = []
+    all_sources: list[Model] = []
+    macro_sources: list = []
+    inner_warnings: list[str] = []
+    island_dialects: dict[str, str] = {}
+    merged_vars: dict = {}
+
+    for pdir in project_dirs:
+        sub = _load_one_dbt(pdir, dialect)
+        for model in sub.models:
+            model.dialect = sub.dialect
+            model.jinja_vars = sub.jinja_vars
+        all_models.extend(sub.models)
+        all_sources.extend(sub.sources)
+        macro_sources.extend(sub.macro_sources)
+        inner_warnings.extend(sub.warnings)
+        merged_vars.update(sub.jinja_vars)
+        label = relative_posix(pdir, root) or pdir.name
+        island_dialects[label] = sub.dialect
+
+    dialect_counts = Counter(m.dialect for m in all_models if m.dialect)
+    primary = dialect or (dialect_counts.most_common(1)[0][0] if dialect_counts else "snowflake")
+    listing = ", ".join(f"{p} ({d})" for p, d in sorted(island_dialects.items()))
+    warnings = [
+        f"Found {len(project_dirs)} dbt projects, each parsed in its own dialect: {listing}."
+    ] + inner_warnings
+
+    return Project(
+        root=root,
+        dialect=primary,
+        mode="dbt-monorepo",
+        models=all_models,
+        sources=all_sources,
+        warnings=warnings,
+        macro_sources=macro_sources,
+        package_name=next((pkg for pkg, _ in macro_sources if pkg), None),
+        jinja_vars=merged_vars,
+    )

ripple/render.py

@@ -0,0 +1,426 @@
+"""Tier-b Jinja: really render dbt models instead of regex-stripping them.
+
+Three tiers of evidence for a model's SQL (best first):
+  a. compiled SQL from target/manifest.json (exact, handled in project.py)
+  b. jinja2 render with a real macro registry (this module): control flow
+     executes, the project's own macros expand, package-namespaced calls
+     like {{ ga4.unnest_key(...) }} resolve, and adapter.dispatch finds
+     default__/<dialect>__ implementations
+  c. the regex converter in engine/jinja.py (last resort)
+
+A render that fails falls through to tier c. A model that still won't parse
+is reported honestly, never guessed at.
+
+The registry design mirrors sqlmesh's dbt loader: macros are registered as
+late-bound wrappers before anything compiles, so compile order never
+matters; unknown macros degrade to NULL (parseable, no fake lineage).
+"""
+
+from __future__ import annotations
+
+import datetime as _datetime
+import itertools as _itertools
+import re
+import re as _re_mod
+
+from jinja2 import BaseLoader, ChainableUndefined, Environment
+from jinja2.utils import missing as _missing
+
+from ripple.render_shims import BUILTIN_PACKAGE_MACROS as _BUILTIN_PACKAGE_MACROS
+
+_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
+
+
+class _Anything(ChainableUndefined):
+    """Undefined that survives calls, attribute access, and iteration.
+
+    Unknown macros render as NULL: '{{ unknown_macro(x) }} as col' becomes
+    'NULL as col', which parses, keeps the column, and carries no fake
+    lineage. The engine's honesty labels take it from there.
+
+    A pure undefined name/attribute chain that was never called
+    ({{params.dataset_name_raw}}, Airflow-templated dbt) is different: it
+    names something outside the render's knowledge, and the round-trip
+    convention cites it as written. Such a chain stringifies back to its
+    own {{...}} spelling; the relation-position rewrite in preprocess
+    carries it into the emitted table name. Calls, and attribute misses on
+    DEFINED objects, still render NULL.
+    """
+
+    __slots__ = ("_chain",)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        name = self._undefined_name
+        self._chain = (
+            name
+            if self._undefined_obj is _missing and isinstance(name, str) and _IDENT.match(name)
+            else None
+        )
+
+    def __getattr__(self, name):
+        if name.startswith("_"):
+            raise AttributeError(name)
+        if self._chain and _IDENT.match(name):
+            out = _Anything(name=name)
+            out._chain = f"{self._chain}.{name}"
+            return out
+        return self
+
+    def __getitem__(self, key):
+        return self.__getattr__(key) if isinstance(key, str) else self
+
+    def __call__(self, *args, **kwargs):
+        called = _Anything(name=self._undefined_name)
+        called._chain = None
+        return called
+
+    def __iter__(self):
+        return iter(())
+
+    def __str__(self):
+        if self._chain:
+            return "{{" + self._chain + "}}"
+        return "NULL"
+
+
+def _ref(*args, **kwargs) -> str:
+    # ref('model') or ref('package', 'model')
+    return str(args[-1]) if args else "unknown_ref"
+
+
+def _source(source_name: str, table_name: str, **kwargs) -> str:
+    return f"{source_name}__{table_name}"
+
+
+class _VarValue(str):
+    """Missing-var stub: renders as the quoted var name, survives range()/int()."""
+
+    __slots__ = ()
+
+    def __index__(self):
+        return 0
+
+    def __int__(self):
+        return 0
+
+    def __getitem__(self, item):
+        return _VarValue(self.strip("'"))
+
+
+_MISSING = object()
+
+
+def _make_var(declared: dict | None = None):
+    """dbt precedence minus CLI --vars: project vars, call default, name stub."""
+    declared = declared or {}
+
+    def _var(name: str, default=_MISSING):
+        if name in declared:
+            return declared[name]
+        # var('x', none) must return None (dbt semantics): no-default is a sentinel
+        return default if default is not _MISSING else _VarValue(f"'{name}'")
+
+    return _var
+
+
+def _config(*args, **kwargs) -> str:
+    return ""
+
+
+class _MacroReturn(Exception):
+    """dbt's return(): control flow out of a macro, caught at invocation."""
+
+    def __init__(self, value=None):
+        super().__init__()
+        self.value = value
+
+
+def _return(value=None):
+    raise _MacroReturn(value)
+
+
+def _returnable(macro):
+    """A macro call evaluates to its return() value; body output otherwise."""
+
+    def call(*args, **kwargs):
+        try:
+            return macro(*args, **kwargs)
+        except _MacroReturn as ret:
+            return ret.value
+
+    return call
+
+
+class _MacroRegistry:
+    """(package, name) -> compiled jinja macro. Shared and mutable: late binding."""
+
+    def __init__(self, root_package: str | None, dialect: str):
+        self.root_package = root_package
+        self.dialect = dialect
+        self.macros: dict[tuple[str | None, str], object] = {}
+        self.package_order: list[str | None] = []
+
+    def register(self, package: str | None, name: str, macro) -> None:
+        self.macros[(package, name)] = macro
+        if package not in self.package_order:
+            self.package_order.append(package)
+
+    def lookup(self, name: str, package: str | None = None):
+        """dbt search order: explicit package, else root project, then others."""
+        if package is not None:
+            return self.macros.get((package, name))
+        for pkg in [self.root_package] + [p for p in self.package_order if p != self.root_package]:
+            hit = self.macros.get((pkg, name))
+            if hit is not None:
+                return hit
+        return None
+
+    def dispatch(self, macro_name: str, macro_namespace: str | None = None):
+        for candidate in (f"{self.dialect}__{macro_name}", f"default__{macro_name}"):
+            hit = self.lookup(candidate, None)
+            if hit is None and macro_namespace:
+                hit = self.lookup(candidate, macro_namespace)
+            if hit is not None:
+                return hit
+        return _Anything(name=macro_name)
+
+
+class _LateBound:
+    """Callable placeholder registered before compile; resolves at call time."""
+
+    __slots__ = ("_registry", "_package", "_name")
+
+    def __init__(self, registry: _MacroRegistry, package: str | None, name: str):
+        self._registry = registry
+        self._package = package
+        self._name = name
+
+    def __call__(self, *args, **kwargs):
+        macro = self._registry.lookup(self._name, self._package)
+        if macro is None:
+            return _Anything(name=self._name)
+        return macro(*args, **kwargs)
+
+
+class _MacroNamespace:
+    """{{ pkg.macro(...) }} resolves through the registry; unknown attrs -> NULL."""
+
+    def __init__(self, registry: _MacroRegistry | None, package: str, var_fn=None):
+        self._registry = registry
+        self._package = package
+        self._var_fn = var_fn
+
+    def __getattr__(self, name: str):
+        if name.startswith("__"):
+            raise AttributeError(name)
+        if self._registry is not None and self._registry.lookup(name, self._package) is not None:
+            return _LateBound(self._registry, self._package, name)
+        builtin = _BUILTIN_PACKAGE_MACROS.get((self._package, name))
+        if builtin is not None:
+            if getattr(builtin, "wants_var", False) and self._var_fn is not None:
+                import functools
+
+                return functools.partial(builtin, _var=self._var_fn)
+            return builtin
+        return _Anything(name=f"{self._package}.{name}")
+
+
+class _AdapterNoOp:
+    """Unmodeled adapter surface: calls return '', attribute chains keep
+    no-opping, and str() is empty so expression position never leaks a
+    function repr into SQL."""
+
+    __slots__ = ()
+
+    def __call__(self, *args, **kwargs):
+        return ""
+
+    def __str__(self):
+        return ""
+
+    def __getattr__(self, name):
+        if name.startswith("__"):
+            raise AttributeError(name)
+        return self
+
+
+_ADAPTER_NOOP = _AdapterNoOp()
+
+
+class _Adapter:
+    """adapter.dispatch resolves against the registry; everything else no-ops."""
+
+    def __init__(self, registry: _MacroRegistry | None = None):
+        self._registry = registry
+
+    def dispatch(self, macro_name, macro_namespace=None):
+        if self._registry is not None:
+            return self._registry.dispatch(macro_name, macro_namespace)
+        return _Anything(name=str(macro_name))
+
+    def quote(self, name):
+        # identity, not '""': ground truth cites unquoted spellings, and a
+        # no-op '' poisons comparisons like elementary's edr_quote_column
+        # (cycle-11 review, F5)
+        return name
+
+    def __getattr__(self, name):
+        return _ADAPTER_NOOP
+
+
+class _Modules:
+    datetime = _datetime
+    re = _re_mod
+    itertools = _itertools
+
+
+_SET_BLOCK_RE = re.compile(r"{%-?\s*(call|materialization|test)\b.*?{%-?\s*end\1\s*-?%}", re.S)
+_MACRO_DEF_RE = re.compile(r"{%-?\s*macro\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(")
+_MACRO_DEF_SUB_RE = re.compile(r"({%-?\s*macro\s+)([A-Za-z_][A-Za-z0-9_]*)(\s*\()")
+# Definition names are mangled before compile so no call site can reach the
+# raw jinja macro: a same-file sibling (or recursive) call would bypass the
+# registry wrapper and its _MacroReturn would unwind through the CALLER's
+# remainder into the caller's wrapper, returning the inner macro's value from
+# the outer macro (cycle-11 review, F1: shopify max_columns came out 24, not
+# 9966). With the rename every call resolves late through the registry, so
+# each invocation catches only its own return(). Must not start with "_":
+# jinja does not export underscore names from template modules.
+_MACRO_DEF_PREFIX = "ripple_macro_def__"
+
+
+def build_environment(
+    model_name: str,
+    registry: _MacroRegistry | None = None,
+    project_vars: dict | None = None,
+) -> Environment:
+    env = Environment(
+        loader=BaseLoader(),
+        undefined=_Anything,
+        extensions=["jinja2.ext.do", "jinja2.ext.loopcontrols"],
+    )
+    dialect = registry.dialect if registry else "snowflake"
+    env.globals.update(
+        {
+            "ref": _ref,
+            "source": _source,
+            "var": _make_var(project_vars),
+            "config": _config,
+            "this": model_name,
+            "target": {"name": "prod", "schema": "analytics", "database": "prod", "type": dialect},
+            # True keeps the incremental WHERE, which proves columns that a
+            # SELECT * chain from an undeclared source would otherwise hide
+            "is_incremental": lambda: True,
+            "should_full_refresh": lambda: False,
+            "execute": False,
+            "adapter": _Adapter(registry),
+            "return": _return,
+            "run_query": lambda *a, **k: [],
+            "log": lambda *a, **k: "",
+            "exceptions": _Anything(name="exceptions"),
+            "modules": _Modules(),
+        }
+    )
+    # vendored macros still win: the namespace checks the registry first
+    for pkg in {p for p, _ in _BUILTIN_PACKAGE_MACROS}:
+        env.globals.setdefault(pkg, _MacroNamespace(registry, pkg, var_fn=_make_var(project_vars)))
+    return env
+
+
+def load_project_macros(env: Environment, macro_pairs, registry: _MacroRegistry) -> int:
+    """Register a dbt project's macros, package-attributed and late-bound."""
+    # pass 0: learn every macro name; wire wrappers + namespaces before compiling
+    names_by_pkg: dict[str | None, set[str]] = {}
+    for pkg, source in macro_pairs:
+        for name in _MACRO_DEF_RE.findall(source):
+            names_by_pkg.setdefault(pkg, set()).add(name)
+    for pkg, names in names_by_pkg.items():
+        if pkg is not None:
+            env.globals.setdefault(pkg, _MacroNamespace(registry, pkg))
+        for name in names:
+            env.globals.setdefault(name, _LateBound(registry, None, name))
+
+    # pass 1: compile once; real macros land in the shared registry.
+    # Sibling names resolve through module vars bound to the OWN package
+    # first (dbt's search order), then the bare-name globals from pass 0.
+    loaded = 0
+    for pkg, source in macro_pairs:
+        if "{% macro" not in source and "{%- macro" not in source:
+            continue
+        mangled = _MACRO_DEF_SUB_RE.sub(rf"\g<1>{_MACRO_DEF_PREFIX}\g<2>\g<3>", source)
+        try:
+            template = env.from_string(_SET_BLOCK_RE.sub("", mangled))
+            module = template.make_module(
+                vars={name: _LateBound(registry, pkg, name) for name in names_by_pkg.get(pkg, ())}
+            )
+        except Exception:
+            continue
+        for name in dir(module):
+            if name.startswith(_MACRO_DEF_PREFIX):
+                real_name = name[len(_MACRO_DEF_PREFIX) :]
+            elif name.startswith("_"):
+                continue
+            else:
+                real_name = name
+            attr = getattr(module, name)
+            if callable(attr):
+                registry.register(pkg, real_name, _returnable(attr))
+                loaded += 1
+    return loaded
+
+
+def build_project_environment(
+    macro_sources=None,
+    root_package: str | None = None,
+    dialect: str = "snowflake",
+    project_vars: dict | None = None,
+) -> Environment:
+    """Environment with the project's macros loaded.
+
+    macro_sources: list of (package_name | None, source_text). Bare strings
+    are accepted and treated as root-package.
+    """
+    registry = _MacroRegistry(root_package, dialect)
+    env = build_environment(model_name="this_model", registry=registry, project_vars=project_vars)
+    pairs = []
+    for item in macro_sources or []:
+        if isinstance(item, str):
+            pairs.append((root_package, item))
+        else:
+            pairs.append(item)
+    if pairs:
+        load_project_macros(env, pairs, registry)
+    return env
+
+
+def render_dbt_sql(
+    sql: str,
+    model_name: str,
+    env: Environment | None = None,
+    jinja_vars: dict | None = None,
+) -> str:
+    """Render a dbt model's Jinja to plain SQL. Raises on failure."""
+    trimmed = _SET_BLOCK_RE.sub("", sql)
+    if env is None:
+        env = build_environment(model_name, project_vars=jinja_vars)
+    context = {"this": model_name}
+    if jinja_vars:
+        # monorepo shares one env; each island's models still see their own vars
+        context["var"] = _make_var(jinja_vars)
+    rendered = env.from_string(trimmed).render(**context)
+    # a var value can itself be jinja ("{{ ref('sessions') }}" in
+    # dbt_project.yml); re-render bounded, then give up honestly
+    for _ in range(3):
+        if "{{" not in rendered and "{%" not in rendered:
+            break
+        try:
+            again = env.from_string(rendered).render(**context)
+        except Exception:
+            break
+        if again == rendered:
+            break
+        rendered = again
+    # a render that eats the whole model is a failure, not a success
+    if not re.search(r"\bselect\b", rendered, re.I):
+        raise ValueError("render produced no SELECT")
+    return rendered