duckrun 0.3.17.dev2__tar.gz → 0.3.17.dev3__tar.gz

{duckrun-0.3.17.dev2/duckrun.egg-info → duckrun-0.3.17.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.3.17.dev2
+Version: 0.3.17.dev3
 Summary: A dbt adapter that runs SQL in DuckDB and materializes to Delta Lake (delta_rs).
 Author: mim
 License: MIT

duckrun-0.3.17.dev3/dbt/adapters/duckrun/__version__.py

@@ -0,0 +1 @@
+version = "0.3.17.dev3"

duckrun-0.3.17.dev3/dbt/adapters/duckrun/delta_dml.py

@@ -0,0 +1,297 @@
+"""Route raw SQL DML against duckrun-managed (Delta-backed) relations to delta_rs.
+
+duckrun intercepts writes at the dbt *materialization* layer (a model/seed/snapshot goes through
+the materialization macros -> store_relation -> delta_rs). But a duckrun relation is surfaced as a
+read-only ``delta_scan`` view, so *raw* DML sent straight to the connection — ``delete from``,
+``update``, ``insert into ... select``, ``alter table ... add column``, ``create table ... as
+select`` — lands on a view and fails ("Can only delete from base table"), or would create a native
+DuckDB table that bypasses Delta entirely.
+
+This module intercepts those statements (at the cursor, see environment.DuckrunCursorWrapper) and
+applies them to the Delta table **via delta_rs only**, then refreshes the ``delta_scan`` view — so
+nothing relies on a native, mutable DuckDB table, and every op works on local AND abfss/OneLake
+stores (delta_rs carries ``storage_options``). ``create table ... as`` writes a new Delta table;
+the mutate forms (delete/update/insert/alter) apply only when a Delta table already exists at the
+target (otherwise the statement passes through — e.g. the test's native ``fact``/``seed``).
+
+``drop table`` unregisters the ``delta_scan`` view AND overwrites the table (via delta_rs) to a
+one-column ``TOMBSTONE_COLUMN`` marker, which discovery recognizes and hides. It does NOT delete
+data: delta_rs has no drop, and removing the Delta files would be a filesystem hack that fails on
+object stores. The directory persists until a human purges it; a later ``create table ... as``
+overwrites the tombstone with real data and the table is live again.
+
+The seed loader's own SQL (``create table <t> (<col defs>)``, ``insert ... values``, ``COPY``) lands
+on a native DuckDB table, not a Delta table: ``create table (<col defs>)`` doesn't match the
+``... as select`` form, and while ``insert ... values`` now *does* match a form here, the mutate
+guard only applies it when a Delta table already exists at the target — the seed's native table has
+none, so it falls through untouched. duckrun's own materializations emit ``create ... view`` (not
+``table``), so they pass through too.
+"""
+import re
+from typing import List, Optional, Tuple
+
+from . import engine
+
+# `drop table` tombstone: a dropped relation is overwritten (via delta_rs) to a table whose ONLY
+# column is this marker, so (a) discovery recognizes it as dropped and hides it, and (b) anyone who
+# opens the files sees an obviously-not-a-real-table schema rather than a plausible empty table. No
+# data is deleted — the directory stays until a human purges it; a later `create table ... as`
+# overwrites the marker schema with real data and the table is live again.
+TOMBSTONE_COLUMN = "__duckrun_deleted__"
+
+
+def _columns_are_tombstone(colnames) -> bool:
+    return [str(c).lower() for c in colnames] == [TOMBSTONE_COLUMN]
+
+
+def is_dropped(con, location: str, storage_options=None) -> bool:
+    """True if the Delta table at ``location`` is a duckrun drop-tombstone (single marker column).
+
+    Used by discovery (dbt + connection API) to hide dropped tables. Best-effort: anything that
+    can't be opened/scanned is treated as 'not a tombstone' (let normal handling deal with it).
+    """
+    loc_sql = str(location).replace("'", "''")
+    try:
+        rel = con.execute(f"select * from delta_scan('{loc_sql}') limit 0")
+        return _columns_are_tombstone([d[0] for d in rel.description])
+    except Exception:
+        return False
+
+# --- statement matchers (leading-anchored, DOTALL so multi-line bodies match) ----------------
+_CREATE_AS = re.compile(
+    r"\s*create\s+table\s+(?:if\s+not\s+exists\s+)?(?P<rel>.+?)\s+as\s+(?P<body>select\b.*)",
+    re.I | re.S,
+)
+_INSERT_SELECT = re.compile(
+    r"\s*insert\s+into\s+(?P<rel>.+?)\s+(?P<body>select\b.*)", re.I | re.S
+)
+_INSERT_VALUES = re.compile(
+    r"\s*insert\s+into\s+(?P<rel>.+?)\s*(?:\((?P<cols>[^)]*)\))?\s*values\s+(?P<body>\(.+)",
+    re.I | re.S,
+)
+_DELETE = re.compile(
+    r"\s*delete\s+from\s+(?P<rel>.+?)(?:\s+where\s+(?P<where>.+))?\s*;?\s*", re.I | re.S
+)
+_UPDATE = re.compile(
+    r"\s*update\s+(?P<rel>.+?)\s+set\s+(?P<set>.+?)(?:\s+where\s+(?P<where>.+?))?\s*;?\s*",
+    re.I | re.S,
+)
+_ALTER_ADD = re.compile(
+    r"\s*alter\s+table\s+(?P<rel>.+?)\s+add\s+column\s+(?P<col>\S+)\s+(?P<def>.+?)\s*;?\s*",
+    re.I | re.S,
+)
+_DROP = re.compile(
+    r"\s*drop\s+table\s+(?:if\s+exists\s+)?(?P<rel>[^\s;]+)\s*;?\s*", re.I | re.S
+)
+
+
+def _fullmatch(pattern, sql):
+    return pattern.fullmatch(sql.strip())
+
+
+def _split_relation(rel: str) -> Tuple[Optional[str], Optional[str]]:
+    """`"db"."schema"."tbl"` / `schema.tbl` / `tbl` -> (schema, identifier), quotes stripped."""
+    parts = [p.strip().strip('"') for p in rel.strip().split(".")]
+    if not parts or not parts[-1]:
+        return None, None
+    identifier = parts[-1]
+    schema = parts[-2] if len(parts) >= 2 else None
+    return schema, identifier
+
+
+def _split_top_level_commas(s: str) -> List[str]:
+    """Split on commas that aren't inside parentheses or quotes (so ``left(email, 3)`` stays whole)."""
+    out, depth, start, quote = [], 0, 0, None
+    for i, ch in enumerate(s):
+        if quote:
+            if ch == quote:
+                quote = None
+        elif ch in ("'", '"'):
+            quote = ch
+        elif ch in "([":
+            depth += 1
+        elif ch in ")]":
+            depth -= 1
+        elif ch == "," and depth == 0:
+            out.append(s[start:i])
+            start = i + 1
+    out.append(s[start:])
+    return [p.strip() for p in out if p.strip()]
+
+
+class _DeltaDML:
+    """One attempt to handle a statement; ``run()`` returns True if it was applied to Delta."""
+
+    def __init__(self, cursor, root_path: str, storage_options, default_schema=None):
+        self.cursor = cursor
+        self.root_path = root_path.rstrip("/")
+        self.so = storage_options
+        self.default_schema = default_schema
+
+    def _loc(self, schema: str, identifier: str) -> str:
+        return f"{self.root_path}/{schema}/{identifier}"
+
+    def _resolve(self, rel: str):
+        """(schema, identifier, location) for ``rel``, falling back to default_schema for an
+        unqualified name (the connection API relies on a current database). (None, None, None) when
+        no schema can be determined."""
+        schema, identifier = _split_relation(rel)
+        schema = schema or self.default_schema
+        if not schema or not identifier:
+            return None, None, None
+        return schema, identifier, self._loc(schema, identifier)
+
+    def _exists(self, loc: str) -> bool:
+        return engine.table_exists(loc, self.so)
+
+    def _refresh_view(self, rel: str, schema: str, loc: str) -> None:
+        loc_sql = loc.replace("'", "''")
+        self.cursor.execute(f'create schema if not exists "{schema}"')
+        self.cursor.execute(
+            f"create or replace view {rel} as select * from delta_scan('{loc_sql}')"
+        )
+
+    def try_handle(self, sql: str) -> bool:
+        m = _fullmatch(_CREATE_AS, sql)
+        if m and "__duckrun" not in m.group("rel"):
+            return self._create_as(m)
+        m = _fullmatch(_INSERT_SELECT, sql)
+        if m:
+            return self._mutate(m, self._insert_select)
+        m = _fullmatch(_INSERT_VALUES, sql)
+        if m:
+            return self._mutate(m, self._insert_values)
+        m = _fullmatch(_DELETE, sql)
+        if m:
+            return self._mutate(m, self._delete)
+        m = _fullmatch(_UPDATE, sql)
+        if m:
+            return self._mutate(m, self._update)
+        m = _fullmatch(_ALTER_ADD, sql)
+        if m:
+            return self._mutate(m, self._alter_add)
+        m = _fullmatch(_DROP, sql)
+        if m:
+            return self._drop(m)
+        return False
+
+    # -- create table <rel> as <select>: always materialize as a duckrun Delta table -----------
+    def _create_as(self, m) -> bool:
+        rel = m.group("rel").strip()
+        schema, identifier, loc = self._resolve(rel)
+        if not loc:
+            return False
+        data = self.cursor.sql(m.group("body"))
+        # overwrite_schema so this replaces a prior table (or a drop-tombstone) wholesale — a live
+        # table is recreated with the real schema, clearing any tombstone marker.
+        engine.write_delta(loc, data, "overwrite", overwrite_schema=True, storage_options=self.so)
+        self._refresh_view(rel, schema, loc)
+        return True
+
+    # -- forms that only apply when a Delta table already exists at the target ------------------
+    def _mutate(self, m, op) -> bool:
+        rel = m.group("rel").strip()
+        schema, identifier, loc = self._resolve(rel)
+        if not loc or not self._exists(loc):
+            return False  # native relation (e.g. the test's `fact`/`seed`) -> let DuckDB handle it
+        op(m, rel, schema, loc)
+        self._refresh_view(rel, schema, loc)
+        return True
+
+    def _delete(self, m, rel, schema, loc) -> None:
+        where = m.group("where")
+        engine._delta_table(loc, self.so).delete(predicate=where.strip() if where else None)
+
+    def _update(self, m, rel, schema, loc) -> None:
+        updates = {}
+        for assign in _split_top_level_commas(m.group("set")):
+            col, _, expr = assign.partition("=")
+            updates[col.strip().strip('"')] = expr.strip()
+        where = m.group("where")
+        engine._delta_table(loc, self.so).update(
+            updates=updates, predicate=where.strip() if where else None
+        )
+
+    def _insert_select(self, m, rel, schema, loc) -> None:
+        data = self.cursor.sql(m.group("body"))
+        engine.write_delta(loc, data, "append", storage_options=self.so)
+
+    def _insert_values(self, m, rel, schema, loc) -> None:
+        # `insert into <rel> [(<cols>)] values (...)`: evaluate the VALUES tuples through DuckDB and
+        # project them onto the FULL target Delta schema (so append schemas match) — supplied columns
+        # come from the literals, any unsupplied target column is filled with a typed NULL.
+        loc_sql = loc.replace("'", "''")
+        template = self.cursor.sql(f"select * from delta_scan('{loc_sql}') limit 0")
+        target_cols = list(template.columns)
+        target_types = [str(t) for t in template.types]
+        by_lower = {c.lower(): c for c in target_cols}
+
+        cols = m.group("cols")
+        if cols:  # explicit column list → canonicalize to the target's casing
+            provided = [by_lower.get(c.strip().strip('"').lower(), c.strip().strip('"'))
+                        for c in cols.split(",")]
+        else:  # positional → the literals supply every target column, in order
+            provided = target_cols
+        provided_set = {c for c in provided}
+
+        quoted = ", ".join('"' + c + '"' for c in provided)
+        inner = f"(values {m.group('body')}) v({quoted})"
+        # Cast every projected column to the TARGET column's type — both supplied values and the
+        # typed NULLs — so the appended Arrow schema matches the table exactly. This is also what a
+        # plain SQL INSERT does (a literal is coerced to the column type), and it stops a literal
+        # whose inferred type is wider than the column (e.g. a ::timestamp into a DATE column) from
+        # forcing delta_rs to add a new writer feature on append (TimestampWithoutTimezone).
+        exprs = [
+            f'cast(v."{col}" as {typ}) as "{col}"' if col in provided_set
+            else f'cast(null as {typ}) as "{col}"'
+            for col, typ in zip(target_cols, target_types)
+        ]
+        data = self.cursor.sql(f"select {', '.join(exprs)} from {inner}")
+        engine.write_delta(loc, data, "append", storage_options=self.so)
+
+    def _alter_add(self, m, rel, schema, loc) -> None:
+        col = m.group("col").strip().strip('"')
+        # Keep only the column type (drop any DEFAULT/NULL clause); add it as an all-null column by
+        # rewriting the table with overwrite_schema so delta_rs accepts the widened schema.
+        coltype = re.split(r"\s+default\b|\s+null\b", m.group("def"), flags=re.I)[0].strip() or "VARCHAR"
+        loc_sql = loc.replace("'", "''")
+        data = self.cursor.sql(
+            f'select *, cast(null as {coltype}) as "{col}" from delta_scan(\'{loc_sql}\')'
+        )
+        engine.write_delta(loc, data, "overwrite", overwrite_schema=True, storage_options=self.so)
+
+    def _drop(self, m) -> bool:
+        # `drop table` on a duckrun relation: unregister the delta_scan view AND, via delta_rs,
+        # overwrite the table to a one-column tombstone (TOMBSTONE_COLUMN) so a later glob discovery
+        # hides it. NO data is deleted — delta_rs has no drop, and removing the Delta files would be
+        # a filesystem hack that fails on object stores. The directory persists until a human purges
+        # it; a later `create table ... as` overwrites the tombstone with real data. If the relation
+        # isn't a duckrun-managed Delta table, fall through and let DuckDB drop the native table.
+        rel = m.group("rel").strip()
+        schema, identifier, loc = self._resolve(rel)
+        if not loc or not self._exists(loc):
+            return False
+        tombstone = self.cursor.sql(f"select true as {TOMBSTONE_COLUMN}")
+        engine.write_delta(loc, tombstone, "overwrite", overwrite_schema=True, storage_options=self.so)
+        self.cursor.execute(f"drop view if exists {rel}")
+        return True
+
+
+def handle(cursor, root_path, storage_options, sql: str, default_schema=None) -> bool:
+    """Apply ``sql`` to Delta if it's a DML form targeting a duckrun-managed relation, using
+    ``cursor`` to evaluate any SELECT body and to (re)create the ``delta_scan`` view.
+
+    Every handled form goes through delta_rs (``engine.write_delta`` / ``DeltaTable.delete`` /
+    ``.update``), which carries ``storage_options`` and so works on local AND abfss/OneLake stores.
+    ``default_schema`` resolves an unqualified table name (the connection API has a current
+    database; the dbt path always renders fully-qualified names so passes None).
+    Returns True if handled (the caller must NOT also run it on DuckDB), False to pass through —
+    anything unrecognized, or (for the mutate forms) a target that isn't a Delta table.
+    """
+    if not root_path:
+        return False
+    # Cheap pre-filter: only the candidate DML verbs.
+    head = sql.lstrip()[:7].lower()
+    if not head.startswith(("delete", "update", "insert", "create", "alter", "drop")):
+        return False
+    return _DeltaDML(cursor, root_path, storage_options, default_schema).try_handle(sql)

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3}/dbt/adapters/duckrun/delta_plugin.py

@@ -6,6 +6,7 @@ connection (``configure_connection``), and on ``store()`` hands the model relati
 straight to delta_rs. DuckDB relations expose the Arrow C-stream interface, which
 deltalake 1.x consumes directly, so there is no pyarrow dependency.
 """
+import re
 from typing import Any, Optional
 
 from dbt.adapters.duckdb.plugins import BasePlugin
@@ -155,11 +156,22 @@ class Plugin(BasePlugin):
         # Table-like (non-incremental) models always overwrite. Incremental models
         # overwrite on first run / full-refresh, then apply the incremental strategy.
         if not incremental or full_refresh or not exists:
+            # This branch is a CREATE OR REPLACE: a table model, a --full-refresh, or a first run.
+            # When we are REPLACING an existing table (exists), allow delta_rs to replace the schema
+            # wholesale (schema_mode="overwrite") — the model SQL defines the new schema, exactly as
+            # `CREATE OR REPLACE TABLE` does on every other warehouse. Without it, delta_rs's strict
+            # overwrite keeps the OLD schema/protocol and so can't change a column's type or write a
+            # column needing a new writer feature the old table lacks (e.g. retyping to ::timestamp /
+            # timestampNtz). This is scoped to the full-rebuild replace ONLY — NOT append, safeappend,
+            # merge, or microbatch, which must keep their strict, schema-stable writes. A fresh create
+            # (not exists) doesn't need it. A user's explicit merge_schema still wins.
+            overwrite_schema = exists and not merge_schema
             with engine.mem_profile("overwrite", con=cur):
                 engine.write_delta(
                     path, data, "overwrite",
                     partition_by=partition_by,
                     merge_schema=merge_schema,
+                    overwrite_schema=overwrite_schema,
                     storage_options=storage_options,
                     compaction_threshold=self._compaction_threshold,
                 )
@@ -201,7 +213,7 @@ class Plugin(BasePlugin):
                     insert_only=(strategy == "insert"),
                     update_columns=cfg.get("merge_update_columns"),
                     exclude_columns=cfg.get("merge_exclude_columns"),
-                    predicates=self._merge_predicates(cfg),
+                    predicates=self._merge_predicates(cfg, data.columns),
                     update_condition=self._rewrite_merge_aliases(cfg.get("merge_update_condition")),
                     insert_condition=self._rewrite_merge_aliases(cfg.get("merge_insert_condition")),
                     merge_schema=evolve_schema,
@@ -470,16 +482,37 @@ class Plugin(BasePlugin):
             return None
         return str(expr).replace("DBT_INTERNAL_DEST", "target").replace("DBT_INTERNAL_SOURCE", "source")
 
+    @staticmethod
+    def _qualify_predicate(expr, columns):
+        """Prefix bare references to known target columns with ``target.``.
+
+        duckrun folds ``incremental_predicates`` into the merge condition
+        (``target.k = source.k AND <predicate>``). A bare column there (e.g. ``id != 2``) exists
+        on BOTH the source and target, so delta_rs rejects it as an ambiguous reference. dbt's
+        ``incremental_predicates`` constrain the existing/target rows (the delete+insert delete, the
+        merge ON), so we qualify bare column tokens to ``target.``. Only exact column-name tokens
+        that aren't already qualified (preceded by ``.``) or quoted/literal are rewritten — literals
+        and functions (e.g. ``current_date``, which is not a column) are left untouched."""
+        if not expr or not columns:
+            return expr
+        # Longest names first so a column that's a prefix of another isn't partially matched.
+        for col in sorted({str(c) for c in columns}, key=len, reverse=True):
+            # whole-word col, not preceded by '.', a word char, or a quote (already qualified/quoted).
+            pattern = re.compile(r'(?<![.\w"\'])' + re.escape(col) + r'\b', re.I)
+            expr = pattern.sub(lambda m: "target." + m.group(0), expr)
+        return expr
+
     @classmethod
-    def _merge_predicates(cls, cfg: dict):
+    def _merge_predicates(cls, cfg: dict, columns=None):
         """dbt ``incremental_predicates`` (or ``predicates``), with dbt's standard merge
-        aliases rewritten to the ones delta_rs uses here."""
+        aliases rewritten to the ones delta_rs uses here and bare column refs qualified to
+        ``target.`` (see ``_qualify_predicate``)."""
         preds = cfg.get("incremental_predicates") or cfg.get("predicates")
         if not preds:
             return None
         if isinstance(preds, str):
             preds = [preds]
-        return [cls._rewrite_merge_aliases(p) for p in preds]
+        return [cls._qualify_predicate(cls._rewrite_merge_aliases(p), columns) for p in preds]
 
     @staticmethod
     def _resolve_schema_change(on_schema_change, path, data, storage_options) -> bool:

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3}/dbt/adapters/duckrun/engine.py

@@ -536,6 +536,37 @@ def table_exists(path: str, storage_options: Optional[Dict[str, str]] = None) ->
         return False
 
 
+def delta_stats(cur, path: str, storage_options: Optional[Dict[str, str]] = None):
+    """Cheap table statistics for ``dbt docs generate``, read from the Delta **log** (no data scan).
+
+    ``DeltaTable.get_add_actions()`` carries per-file ``num_records`` / ``size_bytes`` /
+    ``modification_time``; summing rows+bytes and taking the latest mtime gives the whole table's
+    stats without opening any data file. Aggregation goes through the DuckDB cursor (``cur``) via a
+    replacement scan over the arro3 table — no pyarrow dependency.
+
+    Returns ``{"num_rows", "bytes", "last_modified"}`` (last_modified = epoch milliseconds), or
+    ``None`` on ANY failure (a drop-tombstone, a missing table, an unreachable/credential-less remote
+    store). Best-effort by design: a statless catalog is fine, but a docs build must never break.
+    """
+    try:
+        add_actions = _delta_table(path, storage_options).get_add_actions()  # noqa: F841 (replacement scan)
+        row = cur.sql(
+            "select coalesce(sum(num_records), 0)::bigint, "
+            "coalesce(sum(size_bytes), 0)::bigint, "
+            "max(modification_time)::bigint from add_actions"
+        ).fetchone()
+    except Exception as exc:  # best-effort: docs stats must never fail catalog generation
+        logger.debug(f"duckrun: no Delta stats for {path!r}: {exc}")
+        return None
+    if row is None:
+        return None
+    return {
+        "num_rows": int(row[0]),
+        "bytes": int(row[1]),
+        "last_modified": int(row[2]) if row[2] is not None else None,
+    }
+
+
 # Delta column-metadata key under which we stash a dbt column description, and the dollar-quote
 # label used to embed arbitrary comment text (newlines, quotes, dollar signs) in COMMENT ON SQL.
 _DELTA_COMMENT_KEY = "comment"

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3}/dbt/adapters/duckrun/environment.py

@@ -12,10 +12,52 @@ process. We do the same for plugin sources here: instead of registering a Python
 ``CREATE OR REPLACE VIEW <source> AS <scan sql>``. No pyarrow, no copying the source into a table,
 and no dependence on dbt-duckdb's per-cursor relation re-registration.
 """
-from dbt.adapters.duckdb.environments.local import LocalEnvironment
+from dbt.adapters.duckdb.environments.local import (
+    DuckDBConnectionWrapper,
+    DuckDBCursorWrapper,
+    LocalEnvironment,
+)
+
+from . import delta_dml
+
+
+class DuckrunCursorWrapper(DuckDBCursorWrapper):
+    """Cursor wrapper that routes raw DML against duckrun-managed (Delta-backed) relations to
+    delta_rs instead of running it on the read-only ``delta_scan`` view.
+
+    Every SQL statement — whether issued by dbt's connection manager or by the adapter-test
+    harness (which goes straight to ``conn.handle.cursor().execute``) — funnels through here, so
+    this is the single production interception point. Non-matching statements, parameterized
+    statements (the seed loader's ``insert ... values (?)``), and DML against native relations all
+    fall through to DuckDB unchanged. See delta_dml.handle.
+    """
+
+    def __init__(self, cursor, credentials):
+        super().__init__(cursor)
+        self._duckrun_creds = credentials
+
+    def execute(self, sql, bindings=None):
+        if bindings is None:
+            creds = self._duckrun_creds
+            if delta_dml.handle(
+                self._cursor,
+                getattr(creds, "root_path", None),
+                getattr(creds, "storage_options", None),
+                sql,
+            ):
+                return self._cursor  # applied to Delta; nothing to run on DuckDB
+        return super().execute(sql, bindings)
 
 
 class DuckrunEnvironment(LocalEnvironment):
+    def handle(self):
+        # Swap dbt-duckdb's cursor wrapper for ours so raw DML on Delta relations is intercepted
+        # on every cursor (connection-manager AND test-harness paths) — see DuckrunCursorWrapper.
+        h = super().handle()
+        if isinstance(h, DuckDBConnectionWrapper):
+            h._cursor = DuckrunCursorWrapper(h._cursor._cursor, self.creds)
+        return h
+
     def load_source(self, plugin_name: str, source_config):
         plugin = self._plugins.get(plugin_name)
         # Only special-case the duckrun plugin (it knows how to turn a source into scan SQL).

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3}/dbt/adapters/duckrun/impl.py

@@ -13,6 +13,7 @@ from dbt.adapters.events.logging import AdapterLogger
 from dbt.adapters.duckdb.connections import DuckDBConnectionManager
 from dbt.adapters.duckdb.impl import DuckDBAdapter
 
+from dbt.adapters.duckrun import delta_dml
 from dbt.adapters.duckrun import remote
 from dbt.adapters.duckrun import secret
 from dbt.adapters.duckrun.credentials import DuckrunCredentials
@@ -248,6 +249,22 @@ class DuckrunAdapter(DuckDBAdapter):
         if not discovered:
             return in_memory
 
+        # Hide drop-tombstones: a `drop table` overwrites the table to a one-column marker (no data
+        # deleted). Such a table must not surface as a relation. Check before registering.
+        root_path = getattr(self.config.credentials, "root_path", "") or ""
+        so = getattr(self.config.credentials, "storage_options", None)
+        cur = self._cursor()
+        live = []
+        for rel in discovered:
+            loc = (root_path.rstrip("/") + "/" + str(rel.schema).strip('"')
+                   + "/" + str(rel.identifier).strip('"'))
+            if delta_dml.is_dropped(cur, loc, so):
+                continue
+            live.append(rel)
+        discovered = live
+        if not discovered:
+            return in_memory
+
         # Physically register each discovered Delta table as a delta_scan view so read-only
         # commands (dbt test/show/docs) can query models without a prior in-process run.
         for rel in discovered:
@@ -268,3 +285,71 @@ class DuckrunAdapter(DuckDBAdapter):
         ]
         merged.extend(discovered)
         return merged
+
+    # --- dbt docs: table stats from the Delta log -------------------------------------------------
+    # The stock catalog query (duckrun__get_catalog) emits only column metadata, so dbt-docs shows an
+    # empty Stats panel (issue #3). dbt assembles the panel from columns named
+    # stats:<key>:{label,value,description,include}; we enrich the catalog agate table with those,
+    # sourced from each relation's Delta log (engine.delta_stats — no data scan). Done in Python here
+    # rather than in SQL because byte size / last-modified live in the Delta log, not DuckDB metadata.
+    _STATS_SPEC = (
+        ("num_rows", "Row Count", "Number of rows in the table"),
+        ("bytes", "Approximate Size", "Approximate size of the table on disk (bytes)"),
+        ("last_modified", "Last Modified", "Time of the most recent Delta commit (UTC)"),
+    )
+
+    def get_catalog(self, *args, **kwargs):
+        table, exceptions = super().get_catalog(*args, **kwargs)
+        return self._with_delta_stats(table), exceptions
+
+    def get_catalog_by_relations(self, *args, **kwargs):
+        table, exceptions = super().get_catalog_by_relations(*args, **kwargs)
+        return self._with_delta_stats(table), exceptions
+
+    def _with_delta_stats(self, table):
+        """Return ``table`` with stats:* columns appended, sourced per-relation from the Delta log.
+
+        A relation with no Delta table at ``root_path/schema/name`` (a native ``view``, a
+        drop-tombstone) gets ``include=False`` stats, so dbt leaves it statless. Best-effort: if
+        anything goes wrong the original table is returned unchanged — docs must never break.
+        """
+        from datetime import datetime, timezone
+        from dbt_common.clients.agate_helper import table_from_data_flat
+        from . import engine
+
+        root_path = getattr(self.config.credentials, "root_path", "") or ""
+        if not root_path or len(table.rows) == 0:
+            return table
+        so = getattr(self.config.credentials, "storage_options", None)
+        cur = self._cursor()
+
+        cache = {}
+
+        def stats_for(schema, name):
+            key = (schema, name)
+            if key not in cache:
+                loc = (root_path.rstrip("/") + "/" + str(schema).strip('"')
+                       + "/" + str(name).strip('"'))
+                cache[key] = (None if delta_dml.is_dropped(cur, loc, so)
+                              else engine.delta_stats(cur, loc, so))
+            return cache[key]
+
+        cols = list(table.column_names)
+        stat_cols = [f"stats:{k}:{p}" for k, _, _ in self._STATS_SPEC
+                     for p in ("label", "value", "description", "include")]
+        rows = []
+        for r in table.rows:
+            d = dict(zip(cols, r))
+            st = stats_for(d.get("table_schema"), d.get("table_name"))
+            for k, label, desc in self._STATS_SPEC:
+                present = st is not None and st.get(k) is not None
+                if k == "last_modified" and present:
+                    val = datetime.fromtimestamp(st[k] / 1000, tz=timezone.utc).isoformat()
+                else:
+                    val = st.get(k) if present else None
+                d[f"stats:{k}:label"] = label
+                d[f"stats:{k}:value"] = val
+                d[f"stats:{k}:description"] = desc
+                d[f"stats:{k}:include"] = bool(present)
+            rows.append(d)
+        return table_from_data_flat(rows, cols + stat_cols)

duckrun-0.3.17.dev3/dbt/include/duckrun/macros/catalog.sql

@@ -0,0 +1,122 @@
+{#-- duckrun catalog: report Delta-backed relations as BASE TABLE.
+
+  Every duckrun model is physically a DuckDB *view* over `delta_scan('<location>')`, so the stock
+  dbt-duckdb catalog (duckdb_views() -> 'VIEW') reports them as views — which is dishonest: they are
+  Delta *tables*, and `dbt docs generate` / is_incremental() treat them as tables. We override the
+  catalog so a view whose definition reads from `delta_scan(...)` is reported as `BASE TABLE`, while
+  genuine `view`-materialized models (no delta_scan) stay `VIEW`. Comments (table + column) come
+  through unchanged from WS4's COMMENT ON, which get_catalog already reads.
+
+  Stats are intentionally not synthesized here: the duckrun conformance catalog fixtures use
+  `no_stats()`, so row/byte counts from the Delta log aren't required to pass — and computing them
+  per relation would re-open every table during docs generate. (If stats are wanted later, pull
+  num_records / size_bytes from DeltaTable.get_add_actions and cache per build.)
+#}
+{% macro duckrun__get_catalog(information_schema, schemas) -%}
+  {%- call statement('catalog', fetch_result=True) -%}
+    with relations AS (
+      select
+        t.table_name
+        , t.database_name
+        , t.schema_name
+        , 'BASE TABLE' as table_type
+        , t.comment as table_comment
+      from duckdb_tables() t
+      WHERE t.database_name = '{{ database }}'
+      UNION ALL
+      SELECT v.view_name as table_name
+      , v.database_name
+      , v.schema_name
+      -- A delta_scan view is a Delta table surfaced for reads; report it as a table, not a view.
+      , case when v.sql ilike '%delta_scan(%' then 'BASE TABLE' else 'VIEW' end as table_type
+      , v.comment as table_comment
+      from duckdb_views() v
+      WHERE v.database_name = '{{ database }}'
+    )
+    select
+        '{{ database }}' as table_database,
+        r.schema_name as table_schema,
+        r.table_name,
+        r.table_type,
+        r.table_comment,
+        c.column_name,
+        c.column_index as column_index,
+        c.data_type as column_type,
+        c.comment as column_comment,
+        NULL as table_owner
+    -- join on database too: an attached DB can hold a same-named table in the same schema, and
+    -- without this the columns of that shadow relation would bleed into the result.
+    FROM relations r JOIN duckdb_columns() c ON r.database_name = c.database_name AND r.schema_name = c.schema_name AND r.table_name = c.table_name
+    WHERE (
+        {%- for schema in schemas -%}
+          upper(r.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
+        {%- endfor -%}
+    )
+    ORDER BY
+        r.schema_name,
+        r.table_name,
+        c.column_index
+  {%- endcall -%}
+  {{ return(load_result('catalog').table) }}
+{%- endmacro %}
+
+
+{#-- duckrun catalog, scoped to a specific set of relations (the `get_catalog_relations`
+  capability dbt uses when it can ask for just the relations it cares about — see
+  BaseAdapter._get_one_catalog_by_relations). Same delta_scan-view → BASE TABLE rule as
+  duckrun__get_catalog above; the only difference is the WHERE clause filters to the passed
+  relations (by schema, and by identifier when one is given) instead of to whole schemas.
+  Without this, dbt falls back to default__get_catalog_relations, which just raises
+  "not implemented for duckrun". #}
+{% macro duckrun__get_catalog_relations(information_schema, relations) -%}
+  {%- set database = information_schema.database -%}
+  {%- call statement('catalog', fetch_result=True) -%}
+    with relations AS (
+      select
+        t.table_name
+        , t.database_name
+        , t.schema_name
+        , 'BASE TABLE' as table_type
+        , t.comment as table_comment
+      from duckdb_tables() t
+      WHERE t.database_name = '{{ database }}'
+      UNION ALL
+      SELECT v.view_name as table_name
+      , v.database_name
+      , v.schema_name
+      -- A delta_scan view is a Delta table surfaced for reads; report it as a table, not a view.
+      , case when v.sql ilike '%delta_scan(%' then 'BASE TABLE' else 'VIEW' end as table_type
+      , v.comment as table_comment
+      from duckdb_views() v
+      WHERE v.database_name = '{{ database }}'
+    )
+    select
+        '{{ database }}' as table_database,
+        r.schema_name as table_schema,
+        r.table_name,
+        r.table_type,
+        r.table_comment,
+        c.column_name,
+        c.column_index as column_index,
+        c.data_type as column_type,
+        c.comment as column_comment,
+        NULL as table_owner
+    -- join on database too: an attached DB can hold a same-named table in the same schema, and
+    -- without this the columns of that shadow relation would bleed into the result.
+    FROM relations r JOIN duckdb_columns() c ON r.database_name = c.database_name AND r.schema_name = c.schema_name AND r.table_name = c.table_name
+    WHERE (
+        {%- for relation in relations -%}
+          (
+            upper(r.schema_name) = upper('{{ relation.schema }}')
+            {%- if relation.identifier %} and upper(r.table_name) = upper('{{ relation.identifier }}'){%- endif -%}
+          )
+          {%- if not loop.last %} or {% endif -%}
+        {%- endfor -%}
+    )
+    ORDER BY
+        r.schema_name,
+        r.table_name,
+        c.column_index
+  {%- endcall -%}
+  {{ return(load_result('catalog').table) }}
+{%- endmacro %}

duckrun-0.3.17.dev3/dbt/include/duckrun/macros/materializations/snapshot.sql

@@ -0,0 +1,144 @@
+{#
+  dbt `snapshot` materialization, backed by Delta Lake.
+
+  Why duckrun needs its own: dbt's default snapshot materialization runs `create table` /
+  `merge` as SQL DDL/DML against the warehouse. On duckrun the warehouse is Delta + an
+  in-memory DuckDB session, so the default snapshot lands only in the in-memory catalog and is
+  LOST across processes — a snapshot appears to work within one `dbt` invocation but never
+  persists (real SCD2 is cross-invocation). This materialization persists the snapshot to Delta
+  via the same store path the table/incremental materializations use.
+
+  How it maps onto duckrun's supported merge (no SQL MERGE, no merge_clauses):
+
+    * First run / --full-refresh: `build_snapshot_table` SELECT -> overwrite the Delta table.
+    * Subsequent runs: `snapshot_staging_table` (dbt's own SCD2 change detection) produces the
+      'insert' rows (new versions, fresh dbt_scd_id) and the 'update'/'delete' rows (close the
+      open version, carry the new dbt_valid_to under the *existing* dbt_scd_id). We project away
+      the dbt_change_type / dbt_unique_key helper columns and MERGE on dbt_scd_id with
+      merge_update_columns=[dbt_valid_to]:
+        - close rows  -> matched     -> update dbt_valid_to    (only that column)
+        - insert rows -> not matched -> insert the new version
+      The change_type partition is implicit: closes share the open version's scd_id (so they
+      match), inserts get a brand-new scd_id (so they don't). This reproduces dbt's
+      default__snapshot_merge_sql exactly with the merge controls delta_rs can express.
+#}
+{% materialization snapshot, adapter='duckrun' %}
+
+  {%- set strategy_name = config.get('strategy') -%}
+  {%- set unique_key = config.get('unique_key') -%}
+
+  {%- set p = duckrun__delta_paths() -%}
+  {%- set target_relation = p['target'] -%}
+  {%- set tmp_relation = p['tmp'] -%}
+  {%- set location = p['location'] -%}
+  {%- set _loc_sql = location | replace("'", "''") -%}
+
+  {#-- Version/existence of the Delta table, captured before the model reads anything, so the
+       merge can pin OCC to it (single snapshot for the staging read and the merge commit). --#}
+  {%- set read_version = adapter.delta_version(location) -%}
+  {%- set exists = adapter.delta_table_exists(location) -%}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+  {%- do adapter.create_schema(target_relation) -%}
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  {% set strategy_macro = strategy_dispatch(strategy_name) %}
+  {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", model['config'], exists) %}
+
+  {% if not exists or should_full_refresh() %}
+
+    {#-- First run (or full refresh): build the initial snapshot and overwrite the Delta table. --#}
+    {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}
+    {{ check_time_data_types(build_sql) }}
+
+    {% call statement('stage_model') -%}
+      create or replace view {{ tmp_relation }} as {{ build_sql }}
+    {%- endcall %}
+    {{ adapter.commit() }}
+    {%- set columns = adapter.get_columns_in_relation(tmp_relation) -%}
+    {% do adapter.store_relation('duckrun', tmp_relation, columns, location, 'delta', {
+        'incremental': false,
+        'full_refresh': true,
+        'invocation_id': invocation_id,
+    }) %}
+    {% call statement('drop_stage') -%}
+      drop view if exists {{ tmp_relation }}
+    {%- endcall %}
+
+  {% else %}
+
+    {% set snapshot_cols = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %}
+    {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, snapshot_cols, strategy) }}
+
+    {#-- Pin the existing snapshot read to read_version so the staging change-detection sees one
+         consistent version (matches the merge's OCC pin below). --#}
+    {% call statement('register_this') -%}
+      create or replace view {{ target_relation }} as
+        select * from delta_scan('{{ _loc_sql }}'{% if read_version is not none %}, version => {{ read_version }}{% endif %})
+    {%- endcall %}
+
+    {% set staging_sql = snapshot_staging_table(strategy, sql, target_relation) %}
+    {{ check_time_data_types(staging_sql) }}
+    {% call statement('stage_model') -%}
+      create or replace view {{ tmp_relation }} as {{ staging_sql }}
+    {%- endcall %}
+    {{ adapter.commit() }}
+
+    {#-- Drop dbt's staging-only helper columns; the merge source must match the snapshot table. --#}
+    {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}
+    {% if unique_key | is_list %}
+      {% for key in strategy.unique_key %}
+        {% do remove_columns.append('dbt_unique_key_' + loop.index|string) %}
+        {% do remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) %}
+      {% endfor %}
+    {% endif %}
+    {% set source_columns = adapter.get_columns_in_relation(tmp_relation)
+                                 | rejectattr('name', 'in', remove_columns) | list %}
+    {% set col_csv = source_columns | map(attribute='name') | join(', ') %}
+
+    {%- set merge_src = api.Relation.create(
+          database=target_relation.database,
+          schema=target_relation.schema,
+          identifier=target_relation.identifier ~ '__duckrun_snap_src',
+          type='view') -%}
+    {% call statement('stage_merge_src') -%}
+      create or replace view {{ merge_src }} as select {{ col_csv }} from {{ tmp_relation }}
+    {%- endcall %}
+    {{ adapter.commit() }}
+    {%- set columns = adapter.get_columns_in_relation(merge_src) -%}
+
+    {#-- Merge on dbt_scd_id; matched (closes) update only dbt_valid_to, unmatched (new versions)
+         insert. read_version pins OCC to the version the staging read. --#}
+    {% do adapter.store_relation('duckrun', merge_src, columns, location, 'delta', {
+        'incremental': true,
+        'incremental_strategy': 'merge',
+        'unique_key': snapshot_cols.dbt_scd_id,
+        'merge_update_columns': [snapshot_cols.dbt_valid_to],
+        'read_version': read_version,
+        'dbt_believes_exists': true,
+        'full_refresh': false,
+        'on_schema_change': 'ignore',
+        'invocation_id': invocation_id,
+    }) %}
+
+    {% call statement('drop_stage') -%}
+      drop view if exists {{ merge_src }}; drop view if exists {{ tmp_relation }}
+    {%- endcall %}
+
+  {% endif %}
+
+  {#-- Surface the snapshot as a delta_scan view over the freshly written HEAD. --#}
+  {%- do adapter.create_schema(target_relation) -%}
+  {% call statement('main') -%}
+    create or replace view {{ target_relation }} as select * from delta_scan('{{ _loc_sql }}')
+  {%- endcall %}
+
+  {% do persist_docs(target_relation, model) %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+  {{ adapter.commit() }}
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{% endmaterialization %}

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3}/duckrun/session.py

@@ -14,7 +14,7 @@ from typing import Dict, List, Optional
 
 import duckdb
 
-from dbt.adapters.duckrun import engine, remote, secret
+from dbt.adapters.duckrun import delta_dml, engine, remote, secret
 from . import auth
 
 
@@ -164,6 +164,10 @@ class DuckSession:
                 continue
             self.con.execute(f"CREATE SCHEMA IF NOT EXISTS {_qid(schema)}")
             for table in tables:
+                # Hide drop-tombstones (a `drop table` overwrites the table to a one-column marker;
+                # no data is deleted, the files persist, but the table must not surface).
+                if delta_dml.is_dropped(self.con, self.table_path(schema, table), self.storage_options):
+                    continue
                 self._register_view(schema, table)
                 registered.append(f"{schema}.{table}")
 
@@ -204,20 +208,29 @@ class DuckSession:
     # ---- Spark-shaped surface --------------------------------------------------------------
 
     def sql(self, query: str) -> "DataFrame":
-        """Run a **read** query and return a :class:`DataFrame`. ``conn.sql()`` is read-only: the
-        tables are registered as read-only ``delta_scan`` views, so it passes straight through to
-        DuckDB. Time-travel works for free — ``conn.sql("from delta_scan('path', version => 0)")``.
-
-        Writes go through the Spark-shaped surface, not SQL: ``df.write.saveAsTable`` (create /
-        append) and the ``conn.delta_table(name)`` handle —
-        ``.merge(...)`` / ``.delete()`` / ``.update()`` / ``.replaceWhere()``. A Delta-write
-        statement is rejected up front (not executed) — a bare DuckDB ``CREATE TABLE … AS`` would
-        otherwise silently make an ephemeral DuckDB-local table that never reaches Delta.
-        ``CREATE TEMP/VIEW`` and other DuckDB-local scratch DDL still pass through.
+        """Run a query and return a :class:`DataFrame`.
+
+        Reads pass straight through to DuckDB over the ``delta_scan`` views (time-travel works for
+        free — ``conn.sql("from delta_scan('path', version => 0)")``).
+
+        Delta **DML** is applied to the Delta table via delta_rs (works local AND on OneLake):
+        ``create table … as select`` (overwrite), ``insert into … select``/``insert into … values``
+        (append), ``delete``/``update`` (delta_rs delete/update), ``alter table … add column``, and
+        ``drop table`` (tombstone — marks the table dropped without deleting data; a human purges
+        the files). After a DML statement the catalog is refreshed.
+
+        ``merge`` isn't expressible via delta_rs DML here — use the Spark write surface instead:
+        ``df.write.saveAsTable(...)`` or
+        ``conn.delta_table(name).merge(...)/.delete()/.update()/.replaceWhere()``.
+        ``CREATE TEMP/VIEW`` and other DuckDB-local scratch DDL pass through to DuckDB.
         """
+        if delta_dml.handle(self.con, self.root_path, self.storage_options, query,
+                            default_schema=self._current_database):
+            self.refresh(quiet=True)
+            return DataFrame(self.con.sql("SELECT 'ok' AS status"), self)
         if _is_delta_write(query):
             raise ValueError(
-                "conn.sql() is read-only (Delta tables are registered as read-only views). "
+                "conn.sql() can't run a SQL MERGE via delta_rs. "
                 "Use the Spark write API: df.write.saveAsTable(...) to create/append, or "
                 "conn.delta_table(name).merge(...)/.delete()/.update()/.replaceWhere()."
             )

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3/duckrun.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.3.17.dev2
+Version: 0.3.17.dev3
 Summary: A dbt adapter that runs SQL in DuckDB and materializes to Delta Lake (delta_rs).
 Author: mim
 License: MIT

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3}/duckrun.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ pyproject.toml
 dbt/adapters/duckrun/__init__.py
 dbt/adapters/duckrun/__version__.py
 dbt/adapters/duckrun/credentials.py
+dbt/adapters/duckrun/delta_dml.py
 dbt/adapters/duckrun/delta_plugin.py
 dbt/adapters/duckrun/engine.py
 dbt/adapters/duckrun/environment.py
@@ -17,6 +18,7 @@ dbt/include/duckrun/macros/catalog.sql
 dbt/include/duckrun/macros/materializations/_delta_core.sql
 dbt/include/duckrun/macros/materializations/delta.sql
 dbt/include/duckrun/macros/materializations/incremental.sql
+dbt/include/duckrun/macros/materializations/snapshot.sql
 dbt/include/duckrun/macros/materializations/table.sql
 duckrun/__init__.py
 duckrun/auth.py

{duckrun-0.3.17.dev2 → duckrun-0.3.17.dev3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "duckrun"
-version = "0.3.17.dev2"
+version = "0.3.17.dev3"
 description = "A dbt adapter that runs SQL in DuckDB and materializes to Delta Lake (delta_rs)."
 readme = "README.md"
 license = {text = "MIT"}

duckrun-0.3.17.dev2/dbt/adapters/duckrun/__version__.py

@@ -1 +0,0 @@
-version = "0.3.17.dev2"

duckrun-0.3.17.dev2/dbt/include/duckrun/macros/catalog.sql

@@ -1,59 +0,0 @@
-{#-- duckrun catalog: report Delta-backed relations as BASE TABLE.
-
-  Every duckrun model is physically a DuckDB *view* over `delta_scan('<location>')`, so the stock
-  dbt-duckdb catalog (duckdb_views() -> 'VIEW') reports them as views — which is dishonest: they are
-  Delta *tables*, and `dbt docs generate` / is_incremental() treat them as tables. We override the
-  catalog so a view whose definition reads from `delta_scan(...)` is reported as `BASE TABLE`, while
-  genuine `view`-materialized models (no delta_scan) stay `VIEW`. Comments (table + column) come
-  through unchanged from WS4's COMMENT ON, which get_catalog already reads.
-
-  Stats are intentionally not synthesized here: the duckrun conformance catalog fixtures use
-  `no_stats()`, so row/byte counts from the Delta log aren't required to pass — and computing them
-  per relation would re-open every table during docs generate. (If stats are wanted later, pull
-  num_records / size_bytes from DeltaTable.get_add_actions and cache per build.)
-#}
-{% macro duckrun__get_catalog(information_schema, schemas) -%}
-  {%- call statement('catalog', fetch_result=True) -%}
-    with relations AS (
-      select
-        t.table_name
-        , t.database_name
-        , t.schema_name
-        , 'BASE TABLE' as table_type
-        , t.comment as table_comment
-      from duckdb_tables() t
-      WHERE t.database_name = '{{ database }}'
-      UNION ALL
-      SELECT v.view_name as table_name
-      , v.database_name
-      , v.schema_name
-      -- A delta_scan view is a Delta table surfaced for reads; report it as a table, not a view.
-      , case when v.sql ilike '%delta_scan(%' then 'BASE TABLE' else 'VIEW' end as table_type
-      , v.comment as table_comment
-      from duckdb_views() v
-      WHERE v.database_name = '{{ database }}'
-    )
-    select
-        '{{ database }}' as table_database,
-        r.schema_name as table_schema,
-        r.table_name,
-        r.table_type,
-        r.table_comment,
-        c.column_name,
-        c.column_index as column_index,
-        c.data_type as column_type,
-        c.comment as column_comment,
-        NULL as table_owner
-    FROM relations r JOIN duckdb_columns() c ON r.schema_name = c.schema_name AND r.table_name = c.table_name
-    WHERE (
-        {%- for schema in schemas -%}
-          upper(r.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-        {%- endfor -%}
-    )
-    ORDER BY
-        r.schema_name,
-        r.table_name,
-        c.column_index
-  {%- endcall -%}
-  {{ return(load_result('catalog').table) }}
-{%- endmacro %}