duckrun 0.3.17.dev1__tar.gz → 0.3.17.dev3__tar.gz

{duckrun-0.3.17.dev1/duckrun.egg-info → duckrun-0.3.17.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckrun
-Version: 0.3.17.dev1
+Version: 0.3.17.dev3
 Summary: A dbt adapter that runs SQL in DuckDB and materializes to Delta Lake (delta_rs).
 Author: mim
 License: MIT
@@ -11,6 +11,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: dbt-duckdb>=1.8
+Requires-Dist: dbt-core<2.0,>=1.8
 Requires-Dist: duckdb==1.5.4.dev18
 Requires-Dist: deltalake<1.5.1,>=1.5.0
 Requires-Dist: requests

duckrun-0.3.17.dev3/dbt/adapters/duckrun/__version__.py

@@ -0,0 +1 @@
+version = "0.3.17.dev3"

duckrun-0.3.17.dev3/dbt/adapters/duckrun/delta_dml.py

@@ -0,0 +1,297 @@
+"""Route raw SQL DML against duckrun-managed (Delta-backed) relations to delta_rs.
+
+duckrun intercepts writes at the dbt *materialization* layer (a model/seed/snapshot goes through
+the materialization macros -> store_relation -> delta_rs). But a duckrun relation is surfaced as a
+read-only ``delta_scan`` view, so *raw* DML sent straight to the connection — ``delete from``,
+``update``, ``insert into ... select``, ``alter table ... add column``, ``create table ... as
+select`` — lands on a view and fails ("Can only delete from base table"), or would create a native
+DuckDB table that bypasses Delta entirely.
+
+This module intercepts those statements (at the cursor, see environment.DuckrunCursorWrapper) and
+applies them to the Delta table **via delta_rs only**, then refreshes the ``delta_scan`` view — so
+nothing relies on a native, mutable DuckDB table, and every op works on local AND abfss/OneLake
+stores (delta_rs carries ``storage_options``). ``create table ... as`` writes a new Delta table;
+the mutate forms (delete/update/insert/alter) apply only when a Delta table already exists at the
+target (otherwise the statement passes through — e.g. the test's native ``fact``/``seed``).
+
+``drop table`` unregisters the ``delta_scan`` view AND overwrites the table (via delta_rs) to a
+one-column ``TOMBSTONE_COLUMN`` marker, which discovery recognizes and hides. It does NOT delete
+data: delta_rs has no drop, and removing the Delta files would be a filesystem hack that fails on
+object stores. The directory persists until a human purges it; a later ``create table ... as``
+overwrites the tombstone with real data and the table is live again.
+
+The seed loader's own SQL (``create table <t> (<col defs>)``, ``insert ... values``, ``COPY``) lands
+on a native DuckDB table, not a Delta table: ``create table (<col defs>)`` doesn't match the
+``... as select`` form, and while ``insert ... values`` now *does* match a form here, the mutate
+guard only applies it when a Delta table already exists at the target — the seed's native table has
+none, so it falls through untouched. duckrun's own materializations emit ``create ... view`` (not
+``table``), so they pass through too.
+"""
+import re
+from typing import List, Optional, Tuple
+
+from . import engine
+
+# `drop table` tombstone: a dropped relation is overwritten (via delta_rs) to a table whose ONLY
+# column is this marker, so (a) discovery recognizes it as dropped and hides it, and (b) anyone who
+# opens the files sees an obviously-not-a-real-table schema rather than a plausible empty table. No
+# data is deleted — the directory stays until a human purges it; a later `create table ... as`
+# overwrites the marker schema with real data and the table is live again.
+TOMBSTONE_COLUMN = "__duckrun_deleted__"
+
+
+def _columns_are_tombstone(colnames) -> bool:
+    return [str(c).lower() for c in colnames] == [TOMBSTONE_COLUMN]
+
+
+def is_dropped(con, location: str, storage_options=None) -> bool:
+    """True if the Delta table at ``location`` is a duckrun drop-tombstone (single marker column).
+
+    Used by discovery (dbt + connection API) to hide dropped tables. Best-effort: anything that
+    can't be opened/scanned is treated as 'not a tombstone' (let normal handling deal with it).
+    """
+    loc_sql = str(location).replace("'", "''")
+    try:
+        rel = con.execute(f"select * from delta_scan('{loc_sql}') limit 0")
+        return _columns_are_tombstone([d[0] for d in rel.description])
+    except Exception:
+        return False
+
+# --- statement matchers (leading-anchored, DOTALL so multi-line bodies match) ----------------
+_CREATE_AS = re.compile(
+    r"\s*create\s+table\s+(?:if\s+not\s+exists\s+)?(?P<rel>.+?)\s+as\s+(?P<body>select\b.*)",
+    re.I | re.S,
+)
+_INSERT_SELECT = re.compile(
+    r"\s*insert\s+into\s+(?P<rel>.+?)\s+(?P<body>select\b.*)", re.I | re.S
+)
+_INSERT_VALUES = re.compile(
+    r"\s*insert\s+into\s+(?P<rel>.+?)\s*(?:\((?P<cols>[^)]*)\))?\s*values\s+(?P<body>\(.+)",
+    re.I | re.S,
+)
+_DELETE = re.compile(
+    r"\s*delete\s+from\s+(?P<rel>.+?)(?:\s+where\s+(?P<where>.+))?\s*;?\s*", re.I | re.S
+)
+_UPDATE = re.compile(
+    r"\s*update\s+(?P<rel>.+?)\s+set\s+(?P<set>.+?)(?:\s+where\s+(?P<where>.+?))?\s*;?\s*",
+    re.I | re.S,
+)
+_ALTER_ADD = re.compile(
+    r"\s*alter\s+table\s+(?P<rel>.+?)\s+add\s+column\s+(?P<col>\S+)\s+(?P<def>.+?)\s*;?\s*",
+    re.I | re.S,
+)
+_DROP = re.compile(
+    r"\s*drop\s+table\s+(?:if\s+exists\s+)?(?P<rel>[^\s;]+)\s*;?\s*", re.I | re.S
+)
+
+
+def _fullmatch(pattern, sql):
+    return pattern.fullmatch(sql.strip())
+
+
+def _split_relation(rel: str) -> Tuple[Optional[str], Optional[str]]:
+    """`"db"."schema"."tbl"` / `schema.tbl` / `tbl` -> (schema, identifier), quotes stripped."""
+    parts = [p.strip().strip('"') for p in rel.strip().split(".")]
+    if not parts or not parts[-1]:
+        return None, None
+    identifier = parts[-1]
+    schema = parts[-2] if len(parts) >= 2 else None
+    return schema, identifier
+
+
+def _split_top_level_commas(s: str) -> List[str]:
+    """Split on commas that aren't inside parentheses or quotes (so ``left(email, 3)`` stays whole)."""
+    out, depth, start, quote = [], 0, 0, None
+    for i, ch in enumerate(s):
+        if quote:
+            if ch == quote:
+                quote = None
+        elif ch in ("'", '"'):
+            quote = ch
+        elif ch in "([":
+            depth += 1
+        elif ch in ")]":
+            depth -= 1
+        elif ch == "," and depth == 0:
+            out.append(s[start:i])
+            start = i + 1
+    out.append(s[start:])
+    return [p.strip() for p in out if p.strip()]
+
+
+class _DeltaDML:
+    """One attempt to handle a statement; ``run()`` returns True if it was applied to Delta."""
+
+    def __init__(self, cursor, root_path: str, storage_options, default_schema=None):
+        self.cursor = cursor
+        self.root_path = root_path.rstrip("/")
+        self.so = storage_options
+        self.default_schema = default_schema
+
+    def _loc(self, schema: str, identifier: str) -> str:
+        return f"{self.root_path}/{schema}/{identifier}"
+
+    def _resolve(self, rel: str):
+        """(schema, identifier, location) for ``rel``, falling back to default_schema for an
+        unqualified name (the connection API relies on a current database). (None, None, None) when
+        no schema can be determined."""
+        schema, identifier = _split_relation(rel)
+        schema = schema or self.default_schema
+        if not schema or not identifier:
+            return None, None, None
+        return schema, identifier, self._loc(schema, identifier)
+
+    def _exists(self, loc: str) -> bool:
+        return engine.table_exists(loc, self.so)
+
+    def _refresh_view(self, rel: str, schema: str, loc: str) -> None:
+        loc_sql = loc.replace("'", "''")
+        self.cursor.execute(f'create schema if not exists "{schema}"')
+        self.cursor.execute(
+            f"create or replace view {rel} as select * from delta_scan('{loc_sql}')"
+        )
+
+    def try_handle(self, sql: str) -> bool:
+        m = _fullmatch(_CREATE_AS, sql)
+        if m and "__duckrun" not in m.group("rel"):
+            return self._create_as(m)
+        m = _fullmatch(_INSERT_SELECT, sql)
+        if m:
+            return self._mutate(m, self._insert_select)
+        m = _fullmatch(_INSERT_VALUES, sql)
+        if m:
+            return self._mutate(m, self._insert_values)
+        m = _fullmatch(_DELETE, sql)
+        if m:
+            return self._mutate(m, self._delete)
+        m = _fullmatch(_UPDATE, sql)
+        if m:
+            return self._mutate(m, self._update)
+        m = _fullmatch(_ALTER_ADD, sql)
+        if m:
+            return self._mutate(m, self._alter_add)
+        m = _fullmatch(_DROP, sql)
+        if m:
+            return self._drop(m)
+        return False
+
+    # -- create table <rel> as <select>: always materialize as a duckrun Delta table -----------
+    def _create_as(self, m) -> bool:
+        rel = m.group("rel").strip()
+        schema, identifier, loc = self._resolve(rel)
+        if not loc:
+            return False
+        data = self.cursor.sql(m.group("body"))
+        # overwrite_schema so this replaces a prior table (or a drop-tombstone) wholesale — a live
+        # table is recreated with the real schema, clearing any tombstone marker.
+        engine.write_delta(loc, data, "overwrite", overwrite_schema=True, storage_options=self.so)
+        self._refresh_view(rel, schema, loc)
+        return True
+
+    # -- forms that only apply when a Delta table already exists at the target ------------------
+    def _mutate(self, m, op) -> bool:
+        rel = m.group("rel").strip()
+        schema, identifier, loc = self._resolve(rel)
+        if not loc or not self._exists(loc):
+            return False  # native relation (e.g. the test's `fact`/`seed`) -> let DuckDB handle it
+        op(m, rel, schema, loc)
+        self._refresh_view(rel, schema, loc)
+        return True
+
+    def _delete(self, m, rel, schema, loc) -> None:
+        where = m.group("where")
+        engine._delta_table(loc, self.so).delete(predicate=where.strip() if where else None)
+
+    def _update(self, m, rel, schema, loc) -> None:
+        updates = {}
+        for assign in _split_top_level_commas(m.group("set")):
+            col, _, expr = assign.partition("=")
+            updates[col.strip().strip('"')] = expr.strip()
+        where = m.group("where")
+        engine._delta_table(loc, self.so).update(
+            updates=updates, predicate=where.strip() if where else None
+        )
+
+    def _insert_select(self, m, rel, schema, loc) -> None:
+        data = self.cursor.sql(m.group("body"))
+        engine.write_delta(loc, data, "append", storage_options=self.so)
+
+    def _insert_values(self, m, rel, schema, loc) -> None:
+        # `insert into <rel> [(<cols>)] values (...)`: evaluate the VALUES tuples through DuckDB and
+        # project them onto the FULL target Delta schema (so append schemas match) — supplied columns
+        # come from the literals, any unsupplied target column is filled with a typed NULL.
+        loc_sql = loc.replace("'", "''")
+        template = self.cursor.sql(f"select * from delta_scan('{loc_sql}') limit 0")
+        target_cols = list(template.columns)
+        target_types = [str(t) for t in template.types]
+        by_lower = {c.lower(): c for c in target_cols}
+
+        cols = m.group("cols")
+        if cols:  # explicit column list → canonicalize to the target's casing
+            provided = [by_lower.get(c.strip().strip('"').lower(), c.strip().strip('"'))
+                        for c in cols.split(",")]
+        else:  # positional → the literals supply every target column, in order
+            provided = target_cols
+        provided_set = {c for c in provided}
+
+        quoted = ", ".join('"' + c + '"' for c in provided)
+        inner = f"(values {m.group('body')}) v({quoted})"
+        # Cast every projected column to the TARGET column's type — both supplied values and the
+        # typed NULLs — so the appended Arrow schema matches the table exactly. This is also what a
+        # plain SQL INSERT does (a literal is coerced to the column type), and it stops a literal
+        # whose inferred type is wider than the column (e.g. a ::timestamp into a DATE column) from
+        # forcing delta_rs to add a new writer feature on append (TimestampWithoutTimezone).
+        exprs = [
+            f'cast(v."{col}" as {typ}) as "{col}"' if col in provided_set
+            else f'cast(null as {typ}) as "{col}"'
+            for col, typ in zip(target_cols, target_types)
+        ]
+        data = self.cursor.sql(f"select {', '.join(exprs)} from {inner}")
+        engine.write_delta(loc, data, "append", storage_options=self.so)
+
+    def _alter_add(self, m, rel, schema, loc) -> None:
+        col = m.group("col").strip().strip('"')
+        # Keep only the column type (drop any DEFAULT/NULL clause); add it as an all-null column by
+        # rewriting the table with overwrite_schema so delta_rs accepts the widened schema.
+        coltype = re.split(r"\s+default\b|\s+null\b", m.group("def"), flags=re.I)[0].strip() or "VARCHAR"
+        loc_sql = loc.replace("'", "''")
+        data = self.cursor.sql(
+            f'select *, cast(null as {coltype}) as "{col}" from delta_scan(\'{loc_sql}\')'
+        )
+        engine.write_delta(loc, data, "overwrite", overwrite_schema=True, storage_options=self.so)
+
+    def _drop(self, m) -> bool:
+        # `drop table` on a duckrun relation: unregister the delta_scan view AND, via delta_rs,
+        # overwrite the table to a one-column tombstone (TOMBSTONE_COLUMN) so a later glob discovery
+        # hides it. NO data is deleted — delta_rs has no drop, and removing the Delta files would be
+        # a filesystem hack that fails on object stores. The directory persists until a human purges
+        # it; a later `create table ... as` overwrites the tombstone with real data. If the relation
+        # isn't a duckrun-managed Delta table, fall through and let DuckDB drop the native table.
+        rel = m.group("rel").strip()
+        schema, identifier, loc = self._resolve(rel)
+        if not loc or not self._exists(loc):
+            return False
+        tombstone = self.cursor.sql(f"select true as {TOMBSTONE_COLUMN}")
+        engine.write_delta(loc, tombstone, "overwrite", overwrite_schema=True, storage_options=self.so)
+        self.cursor.execute(f"drop view if exists {rel}")
+        return True
+
+
+def handle(cursor, root_path, storage_options, sql: str, default_schema=None) -> bool:
+    """Apply ``sql`` to Delta if it's a DML form targeting a duckrun-managed relation, using
+    ``cursor`` to evaluate any SELECT body and to (re)create the ``delta_scan`` view.
+
+    Every handled form goes through delta_rs (``engine.write_delta`` / ``DeltaTable.delete`` /
+    ``.update``), which carries ``storage_options`` and so works on local AND abfss/OneLake stores.
+    ``default_schema`` resolves an unqualified table name (the connection API has a current
+    database; the dbt path always renders fully-qualified names so passes None).
+    Returns True if handled (the caller must NOT also run it on DuckDB), False to pass through —
+    anything unrecognized, or (for the mutate forms) a target that isn't a Delta table.
+    """
+    if not root_path:
+        return False
+    # Cheap pre-filter: only the candidate DML verbs.
+    head = sql.lstrip()[:7].lower()
+    if not head.startswith(("delete", "update", "insert", "create", "alter", "drop")):
+        return False
+    return _DeltaDML(cursor, root_path, storage_options, default_schema).try_handle(sql)

{duckrun-0.3.17.dev1 → duckrun-0.3.17.dev3}/dbt/adapters/duckrun/delta_plugin.py

@@ -6,6 +6,7 @@ connection (``configure_connection``), and on ``store()`` hands the model relati
 straight to delta_rs. DuckDB relations expose the Arrow C-stream interface, which
 deltalake 1.x consumes directly, so there is no pyarrow dependency.
 """
+import re
 from typing import Any, Optional
 
 from dbt.adapters.duckdb.plugins import BasePlugin
@@ -155,13 +156,25 @@ class Plugin(BasePlugin):
         # Table-like (non-incremental) models always overwrite. Incremental models
         # overwrite on first run / full-refresh, then apply the incremental strategy.
         if not incremental or full_refresh or not exists:
-            engine.write_delta(
-                path, data, "overwrite",
-                partition_by=partition_by,
-                merge_schema=merge_schema,
-                storage_options=storage_options,
-                compaction_threshold=self._compaction_threshold,
-            )
+            # This branch is a CREATE OR REPLACE: a table model, a --full-refresh, or a first run.
+            # When we are REPLACING an existing table (exists), allow delta_rs to replace the schema
+            # wholesale (schema_mode="overwrite") — the model SQL defines the new schema, exactly as
+            # `CREATE OR REPLACE TABLE` does on every other warehouse. Without it, delta_rs's strict
+            # overwrite keeps the OLD schema/protocol and so can't change a column's type or write a
+            # column needing a new writer feature the old table lacks (e.g. retyping to ::timestamp /
+            # timestampNtz). This is scoped to the full-rebuild replace ONLY — NOT append, safeappend,
+            # merge, or microbatch, which must keep their strict, schema-stable writes. A fresh create
+            # (not exists) doesn't need it. A user's explicit merge_schema still wins.
+            overwrite_schema = exists and not merge_schema
+            with engine.mem_profile("overwrite", con=cur):
+                engine.write_delta(
+                    path, data, "overwrite",
+                    partition_by=partition_by,
+                    merge_schema=merge_schema,
+                    overwrite_schema=overwrite_schema,
+                    storage_options=storage_options,
+                    compaction_threshold=self._compaction_threshold,
+                )
             return
 
         # Resolve the incremental strategy: default to merge when a unique_key is
@@ -194,31 +207,33 @@ class Plugin(BasePlugin):
             # prune the target (right for small incremental deltas into a large table). A model
             # whose source is itself huge can set merge_streamed_exec=true to stream it instead.
             sx = cfg.get("merge_streamed_exec")
-            engine.merge_delta(
-                path, data, unique_key,
-                insert_only=(strategy == "insert"),
-                update_columns=cfg.get("merge_update_columns"),
-                exclude_columns=cfg.get("merge_exclude_columns"),
-                predicates=self._merge_predicates(cfg),
-                update_condition=self._rewrite_merge_aliases(cfg.get("merge_update_condition")),
-                insert_condition=self._rewrite_merge_aliases(cfg.get("merge_insert_condition")),
-                merge_schema=evolve_schema,
-                max_spill_size=cfg.get("merge_max_spill_size"),
-                streamed_exec=(False if sx is None else bool(sx)),
-                # Pin the merge target to the version the model read (vB, captured before it read
-                # {{ this }}), so OCC validates (vB, HEAD] — the read and the commit are one snapshot.
-                read_version=cfg.get("read_version"),
-                storage_options=storage_options,
-                compaction_threshold=self._compaction_threshold,
-            )
+            with engine.mem_profile("merge", con=cur):
+                engine.merge_delta(
+                    path, data, unique_key,
+                    insert_only=(strategy == "insert"),
+                    update_columns=cfg.get("merge_update_columns"),
+                    exclude_columns=cfg.get("merge_exclude_columns"),
+                    predicates=self._merge_predicates(cfg, data.columns),
+                    update_condition=self._rewrite_merge_aliases(cfg.get("merge_update_condition")),
+                    insert_condition=self._rewrite_merge_aliases(cfg.get("merge_insert_condition")),
+                    merge_schema=evolve_schema,
+                    max_spill_size=cfg.get("merge_max_spill_size"),
+                    streamed_exec=(False if sx is None else bool(sx)),
+                    # Pin the merge target to the version the model read (vB, captured before it read
+                    # {{ this }}), so OCC validates (vB, HEAD] — read and commit are one snapshot.
+                    read_version=cfg.get("read_version"),
+                    storage_options=storage_options,
+                    compaction_threshold=self._compaction_threshold,
+                )
         elif strategy == "append":
-            engine.write_delta(
-                path, data, "append",
-                partition_by=partition_by,
-                merge_schema=merge_schema,
-                storage_options=storage_options,
-                compaction_threshold=self._compaction_threshold,
-            )
+            with engine.mem_profile("append", con=cur):
+                engine.write_delta(
+                    path, data, "append",
+                    partition_by=partition_by,
+                    merge_schema=merge_schema,
+                    storage_options=storage_options,
+                    compaction_threshold=self._compaction_threshold,
+                )
         elif strategy == "safeappend":
             # Optimistic append: commit only if the table version has not moved since the model
             # *started* (read_version, captured before it read {{ this }}), else fail so dbt errors
@@ -226,14 +241,15 @@ class Plugin(BasePlugin):
             # is what closes the read→write gap: a writer that commits any time during the build
             # makes this fail instead of appending a duplicate. No dedup — that's the SQL's job.
             # Compare-and-swap via delta_rs max_commit_retries=0 (see engine).
-            engine.append_if_unchanged(
-                path, data,
-                read_version=cfg.get("read_version"),
-                partition_by=partition_by,
-                merge_schema=merge_schema,
-                storage_options=storage_options,
-                compaction_threshold=self._compaction_threshold,
-            )
+            with engine.mem_profile("safeappend", con=cur):
+                engine.append_if_unchanged(
+                    path, data,
+                    read_version=cfg.get("read_version"),
+                    partition_by=partition_by,
+                    merge_schema=merge_schema,
+                    storage_options=storage_options,
+                    compaction_threshold=self._compaction_threshold,
+                )
         else:
             raise ValueError(
                 f"Unknown incremental_strategy '{strategy}'. "
@@ -466,16 +482,37 @@ class Plugin(BasePlugin):
             return None
         return str(expr).replace("DBT_INTERNAL_DEST", "target").replace("DBT_INTERNAL_SOURCE", "source")
 
+    @staticmethod
+    def _qualify_predicate(expr, columns):
+        """Prefix bare references to known target columns with ``target.``.
+
+        duckrun folds ``incremental_predicates`` into the merge condition
+        (``target.k = source.k AND <predicate>``). A bare column there (e.g. ``id != 2``) exists
+        on BOTH the source and target, so delta_rs rejects it as an ambiguous reference. dbt's
+        ``incremental_predicates`` constrain the existing/target rows (the delete+insert delete, the
+        merge ON), so we qualify bare column tokens to ``target.``. Only exact column-name tokens
+        that aren't already qualified (preceded by ``.``) or quoted/literal are rewritten — literals
+        and functions (e.g. ``current_date``, which is not a column) are left untouched."""
+        if not expr or not columns:
+            return expr
+        # Longest names first so a column that's a prefix of another isn't partially matched.
+        for col in sorted({str(c) for c in columns}, key=len, reverse=True):
+            # whole-word col, not preceded by '.', a word char, or a quote (already qualified/quoted).
+            pattern = re.compile(r'(?<![.\w"\'])' + re.escape(col) + r'\b', re.I)
+            expr = pattern.sub(lambda m: "target." + m.group(0), expr)
+        return expr
+
     @classmethod
-    def _merge_predicates(cls, cfg: dict):
+    def _merge_predicates(cls, cfg: dict, columns=None):
         """dbt ``incremental_predicates`` (or ``predicates``), with dbt's standard merge
-        aliases rewritten to the ones delta_rs uses here."""
+        aliases rewritten to the ones delta_rs uses here and bare column refs qualified to
+        ``target.`` (see ``_qualify_predicate``)."""
         preds = cfg.get("incremental_predicates") or cfg.get("predicates")
         if not preds:
             return None
         if isinstance(preds, str):
             preds = [preds]
-        return [cls._rewrite_merge_aliases(p) for p in preds]
+        return [cls._qualify_predicate(cls._rewrite_merge_aliases(p), columns) for p in preds]
 
     @staticmethod
     def _resolve_schema_change(on_schema_change, path, data, storage_options) -> bool:

{duckrun-0.3.17.dev1 → duckrun-0.3.17.dev3}/dbt/adapters/duckrun/engine.py

@@ -196,6 +196,133 @@ def _effective_mem_limit_source() -> str:
     return "physical RAM"
 
 
+# --------------------------------------------------------------- memory profiling (opt-in)
+# A merge that OOMs has three suspects sharing one process: DuckDB (producing the source), the
+# Arrow buffers delta_rs collects when streamed_exec=False, and delta_rs's own merge pool. RSS
+# alone can't tell them apart. With DUCKRUN_MEM_PROFILE set, mem_profile() samples this process's
+# RSS *and* DuckDB's own allocation through a write/merge and logs the split, so "who's the slob"
+# is measured, not inferred. Off by default: no thread, no samples, no overhead in production.
+
+def _proc_rss_bytes() -> Optional[int]:
+    """Resident set size of THIS process in bytes — the number the OOM-killer actually watches;
+    None if it can't be read. Linux: VmRSS from /proc/self/status. Windows: WorkingSetSize."""
+    try:
+        with open("/proc/self/status") as fh:
+            for line in fh:
+                if line.startswith("VmRSS:"):
+                    return int(line.split()[1]) * 1024  # value is in kB
+    except (OSError, ValueError, IndexError):
+        pass
+    try:  # Windows: GetProcessMemoryInfo -> WorkingSetSize
+        from ctypes import wintypes
+
+        class _PMC(ctypes.Structure):
+            _fields_ = [("cb", ctypes.c_ulong), ("PageFaultCount", ctypes.c_ulong)] + [
+                (n, ctypes.c_size_t) for n in (
+                    "PeakWorkingSetSize", "WorkingSetSize", "QuotaPeakPagedPoolUsage",
+                    "QuotaPagedPoolUsage", "QuotaPeakNonPagedPoolUsage", "QuotaNonPagedPoolUsage",
+                    "PagefileUsage", "PeakPagefileUsage")
+            ]
+        # argtypes are required: GetCurrentProcess returns the pseudo-handle (-1), which overflows
+        # ctypes' default int marshalling unless the parameter is typed as a HANDLE.
+        k32 = ctypes.windll.kernel32
+        k32.GetCurrentProcess.restype = wintypes.HANDLE
+        psapi = ctypes.windll.psapi
+        psapi.GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(_PMC), ctypes.c_ulong]
+        psapi.GetProcessMemoryInfo.restype = wintypes.BOOL
+        p = _PMC()
+        p.cb = ctypes.sizeof(_PMC)
+        if psapi.GetProcessMemoryInfo(k32.GetCurrentProcess(), ctypes.byref(p), p.cb):
+            return int(p.WorkingSetSize)
+    except Exception:
+        pass
+    return None
+
+
+def _duckdb_mem_bytes(con):
+    """(allocated_bytes, temp_spill_bytes) DuckDB currently holds, via duckdb_memory(); None on any
+    error. Runs on a *separate* cursor so it's safe to call while another query streams on `con` —
+    and this is a diagnostic-only path, so it must never raise into the real write/merge."""
+    if con is None:
+        return None
+    try:
+        cur = con.cursor()  # duckdb's cursor() is a new connection on the same instance
+        row = cur.execute(
+            "SELECT coalesce(sum(memory_usage_bytes), 0), "
+            "coalesce(sum(temporary_storage_bytes), 0) FROM duckdb_memory()"
+        ).fetchone()
+        return (int(row[0]), int(row[1]))
+    except Exception:
+        return None
+
+
+class _MemSampler:
+    """Background RSS / DuckDB-memory sampler for one write or merge. See mem_profile()."""
+
+    def __init__(self, label: str, con=None, interval: float = 0.1):
+        self.label = label
+        self.con = con
+        self.interval = interval
+        self._thread = None
+        self._stop = None
+        self.samples = 0
+        self.peak_rss = 0
+        self.duckdb_at_rss_peak = None        # DuckDB alloc at the instant RSS peaked
+        self.duckdb_spill_at_rss_peak = None
+        self.peak_duckdb = 0                  # DuckDB's own high-water, independently
+
+    def __enter__(self):
+        if not os.environ.get("DUCKRUN_MEM_PROFILE"):
+            return self  # disabled: no thread, no overhead
+        import threading
+        self._stop = threading.Event()
+        self._thread = threading.Thread(
+            target=self._run, name=f"duckrun-mem-{self.label}", daemon=True)
+        self._thread.start()
+        return self
+
+    def _run(self):
+        while not self._stop.is_set():
+            rss = _proc_rss_bytes()
+            dd = _duckdb_mem_bytes(self.con)
+            self.samples += 1
+            if dd is not None and dd[0] > self.peak_duckdb:
+                self.peak_duckdb = dd[0]
+            if rss is not None and rss > self.peak_rss:
+                self.peak_rss = rss
+                if dd is not None:
+                    self.duckdb_at_rss_peak, self.duckdb_spill_at_rss_peak = dd
+            self._stop.wait(self.interval)
+
+    def __exit__(self, *exc):
+        if self._thread is None:
+            return False
+        self._stop.set()
+        self._thread.join(timeout=2.0)
+
+        def mb(n):
+            return "n/a" if n is None else f"{n / 2 ** 20:,.0f} MB"
+
+        non_duck = None
+        if self.peak_rss and self.duckdb_at_rss_peak is not None:
+            non_duck = max(0, self.peak_rss - self.duckdb_at_rss_peak)
+        logger.info(
+            f"mem[{self.label}]: peak RSS={mb(self.peak_rss)} | "
+            f"DuckDB peak={mb(self.peak_duckdb)} "
+            f"(at RSS-peak {mb(self.duckdb_at_rss_peak)}, spill {mb(self.duckdb_spill_at_rss_peak)}) | "
+            f"non-DuckDB~={mb(non_duck)} (delta_rs + Arrow) | samples={self.samples}"
+        )
+        return False
+
+
+def mem_profile(label: str, con=None, interval: float = 0.1):
+    """Context manager that profiles a write/merge's memory when DUCKRUN_MEM_PROFILE is set, else a
+    no-op. Wraps an engine call so RSS, DuckDB's allocation, and the delta_rs/Arrow remainder are
+    measured for that phase and logged once on exit. `con` (the DuckDB connection) enables the
+    DuckDB-vs-delta_rs split; omit it to log RSS only. Diagnostic only — never affects the write."""
+    return _MemSampler(label, con=con, interval=interval)
+
+
 # How the effective memory limit is split between the two big consumers that can peak at the
 # same time during a merge — DuckDB (producing the source relation) and delta_rs (the merge
 # pool). They share one cap, so the shares must sum *under* 1.0 or we've just moved the OOM; each
@@ -409,6 +536,37 @@ def table_exists(path: str, storage_options: Optional[Dict[str, str]] = None) ->
         return False
 
 
+def delta_stats(cur, path: str, storage_options: Optional[Dict[str, str]] = None):
+    """Cheap table statistics for ``dbt docs generate``, read from the Delta **log** (no data scan).
+
+    ``DeltaTable.get_add_actions()`` carries per-file ``num_records`` / ``size_bytes`` /
+    ``modification_time``; summing rows+bytes and taking the latest mtime gives the whole table's
+    stats without opening any data file. Aggregation goes through the DuckDB cursor (``cur``) via a
+    replacement scan over the arro3 table — no pyarrow dependency.
+
+    Returns ``{"num_rows", "bytes", "last_modified"}`` (last_modified = epoch milliseconds), or
+    ``None`` on ANY failure (a drop-tombstone, a missing table, an unreachable/credential-less remote
+    store). Best-effort by design: a statless catalog is fine, but a docs build must never break.
+    """
+    try:
+        add_actions = _delta_table(path, storage_options).get_add_actions()  # noqa: F841 (replacement scan)
+        row = cur.sql(
+            "select coalesce(sum(num_records), 0)::bigint, "
+            "coalesce(sum(size_bytes), 0)::bigint, "
+            "max(modification_time)::bigint from add_actions"
+        ).fetchone()
+    except Exception as exc:  # best-effort: docs stats must never fail catalog generation
+        logger.debug(f"duckrun: no Delta stats for {path!r}: {exc}")
+        return None
+    if row is None:
+        return None
+    return {
+        "num_rows": int(row[0]),
+        "bytes": int(row[1]),
+        "last_modified": int(row[2]) if row[2] is not None else None,
+    }
+
+
 # Delta column-metadata key under which we stash a dbt column description, and the dollar-quote
 # label used to embed arbitrary comment text (newlines, quotes, dollar signs) in COMMENT ON SQL.
 _DELTA_COMMENT_KEY = "comment"