run-cache 2.3.2__tar.gz → 2.4.0__tar.gz

{run_cache-2.3.2 → run_cache-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: run-cache
-Version: 2.3.2
+Version: 2.4.0
 Summary: dbt plugin that skips redundant model executions by caching results from previous runs
 Project-URL: Homepage, https://runcache.com
 License: Copyright (c) 2026 Fivetran, Inc.

run_cache-2.4.0/src/dbt_run_cache/_version.py

@@ -0,0 +1 @@
+__version__ = "2.4.0"

{run_cache-2.3.2 → run_cache-2.4.0}/src/dbt_run_cache/adapters/__init__.py

@@ -8,6 +8,7 @@ from dbt_run_cache.adapters.bigquery import BigQueryAdapterExtension
 from dbt_run_cache.adapters.postgres import PostgresAdapterExtension
 from dbt_run_cache.adapters.snowflake import SnowflakeAdapterExtension
 from dbt_run_cache.adapters.databricks import DatabricksAdapterExtension
+from dbt_run_cache.adapters.redshift import RedshiftAdapterExtension
 
 
 ADAPTER_EXTENSION_MAPPING = {
@@ -15,6 +16,7 @@ ADAPTER_EXTENSION_MAPPING = {
     "bigquery": BigQueryAdapterExtension,
     "snowflake": SnowflakeAdapterExtension,
     "databricks": DatabricksAdapterExtension,
+    "redshift": RedshiftAdapterExtension,
 }
 
 

{run_cache-2.3.2 → run_cache-2.4.0}/src/dbt_run_cache/adapters/base.py

@@ -52,6 +52,9 @@ class BaseAdapterExtension(abc.ABC):
     """Catalogs that should not have their last modified / view definition tracked"""
     SYSTEM_METADATA_SCHEMAS: t.List[str] = ["information_schema"]
     """Schemas that should not have their last modified / view definition tracked"""
+    IMPLEMENTS_CUSTOM_CLONE: bool = False
+    """When True, the adapter extension handles clone execution via clone() rather than
+    the caller executing the server-provided clone_sqls directly."""
 
     CLONE_CHAIN_DEPTH_LIMIT: t.Optional[int] = None
     """How many clones of clones can be created before the database throws a "Cannot have more than N chained clones"-style error.
@@ -105,6 +108,26 @@ class BaseAdapterExtension(abc.ABC):
         _, agate_table = self.adapter.execute(sql, fetch=fetch)
         return agate_table
 
+    def clone(
+        self,
+        clone_sqls: t.Iterable[str],
+        clone_source: str,
+        clone_target: str,
+    ) -> None:
+        """Execute a dialect-specific clone of clone_source into clone_target.
+
+        Only called when IMPLEMENTS_CUSTOM_CLONE is True. Subclasses that set
+        IMPLEMENTS_CUSTOM_CLONE = True must override this method.
+
+        Args:
+            clone_sqls: Server-provided fallback SQL (may be used if custom logic fails).
+            clone_source: Fully-qualified quoted source table name.
+            clone_target: Fully-qualified quoted target table name.
+        """
+        raise NotImplementedError(
+            f"{type(self).__name__} sets IMPLEMENTS_CUSTOM_CLONE=True but does not implement clone()"
+        )
+
     def current_timestamp_utc(self) -> datetime:
         """Get the current UTC time from the database."""
         raise NotImplementedError(

run_cache-2.4.0/src/dbt_run_cache/adapters/redshift.py

@@ -0,0 +1,438 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from concurrent.futures import Future
+from dataclasses import replace
+from datetime import datetime, timezone
+from multiprocessing import get_context
+import typing as t
+from dbt.adapters.sql import SQLAdapter
+from sqlglot import TokenType, tokenize
+
+
+from sqlglot import exp
+
+from dbt_run_cache import events
+from dbt_run_cache.adapters.base import BaseAdapterExtension
+from dbt_run_cache.adapters.common import (
+    build_information_schema_filter,
+    ViewDefinition,
+    group_tables_by_catalog,
+)
+from dbt_run_cache.utils import set_invocation_context
+
+
+class RedshiftAdapterExtension(BaseAdapterExtension):
+    DEFAULT_SCHEMA_NAME = "public"
+    SHOULD_RELEASE_CONNECTION: bool = True
+    SYSTEM_METADATA_SCHEMAS: t.List[str] = ["information_schema", "pg_catalog"]
+    IMPLEMENTS_CUSTOM_CLONE: bool = True
+
+    _SYS_QUERY_DETAIL_LOOKBACK_MINUTES = 30
+
+    def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._catalog_adapters: t.Dict[str, SQLAdapter] = {}
+
+    @property
+    def supports_view_last_modified(self) -> bool:
+        return False
+
+    @property
+    def supports_current_timestamp_utc(self) -> bool:
+        return True
+
+    def current_timestamp_utc(self) -> datetime:
+        return (
+            self.execute(
+                "SELECT (SYSDATE AT TIME ZONE CURRENT_SETTING('timezone')) AT TIME ZONE 'UTC'",
+                fetch=True,
+            )
+            .rows[0][0]
+            .replace(tzinfo=timezone.utc)
+        )
+
+    def rollback(self) -> None:
+        """Roll back the current transaction for Redshift.
+
+        Redshift uses implicit transactions, so we need to rollback directly
+        on the connection handle rather than checking transaction_open flag.
+        """
+        conn = self.adapter.connections.get_if_exists()
+        if conn is not None and conn.handle:
+            try:
+                conn.handle.rollback()
+            except Exception as e:
+                events.fire_debug_event("Failed to rollback Redshift connection: {}", str(e))
+
+    def clone(
+        self,
+        clone_sqls: t.Iterable[str],
+        clone_source: str,
+        clone_target: str,
+    ) -> None:
+        """Clone clone_source into clone_target preserving PRIMARY KEY and FOREIGN KEY.
+
+        Uses SHOW TABLE to obtain the full DDL (including PK/FK constraints) then
+        executes DROP + CREATE + INSERT in a single transaction. Falls back to the
+        server-provided LIKE-based clone_sqls when SHOW TABLE is unavailable.
+        """
+        raw_ddl = self._show_table_ddl(clone_source)
+        if raw_ddl is None:
+            events.fire_debug_event(
+                "SHOW TABLE unavailable for {}; falling back to server-provided clone SQL",
+                clone_source,
+            )
+            for sql in clone_sqls:
+                self.adapter.execute(sql)
+            return
+
+        target_ddl = self._substitute_clone_table_name(raw_ddl, clone_target)
+        for sql in [
+            f"DROP TABLE IF EXISTS {clone_target}",
+            target_ddl,
+            f"INSERT INTO {clone_target} SELECT * FROM {clone_source}",
+        ]:
+            self.adapter.execute(sql)
+
+    def close(self) -> None:
+        for adapter in self._catalog_adapters.values():
+            adapter.cleanup_connections()
+        super().close()
+
+    def _show_table_ddl(self, table_fqn: str) -> t.Optional[str]:
+        """Run SHOW TABLE and return the DDL string, or None if unavailable."""
+        try:
+            result = self.execute(f"SHOW TABLE {table_fqn}", fetch=True)
+            if not result.rows:
+                return None
+            return result.rows[0][0]
+        except Exception as exc:
+            events.fire_debug_event("SHOW TABLE failed for {}: {}", table_fqn, str(exc))
+            return None
+
+    def _substitute_clone_table_name(self, ddl: str, clone_target: str) -> str:
+        """Replace the table name in a SHOW TABLE DDL string with clone_target.
+
+        Tokenizes the DDL and walks the token stream:
+        CREATE → skip LOCAL/TEMPORARY/TEMP → expect TABLE → skip IF NOT EXISTS
+        → record start of first name token, extend end across any DOT+identifier
+            pairs to cover 2- and 3-part names → splice clone_target into that span.
+
+        Accepts unquoted (VAR) and double-quoted (IDENTIFIER) name tokens.
+        Returns ddl unchanged if no CREATE TABLE header is found.
+        """
+
+        _TABLE_MODIFIER_STRINGS = {"LOCAL", "TEMPORARY", "TEMP"}
+        _NAME_TYPES = {TokenType.VAR, TokenType.IDENTIFIER}
+
+        tokens = tokenize(ddl, dialect=self.dialect)
+        len_tokens = len(tokens)
+        i = 0
+
+        while i < len_tokens and tokens[i].token_type != TokenType.CREATE:
+            i += 1
+        if i >= len_tokens:
+            return ddl
+        i += 1
+
+        while i < len_tokens and tokens[i].text.upper() in _TABLE_MODIFIER_STRINGS:
+            i += 1
+
+        if i >= len_tokens or tokens[i].token_type != TokenType.TABLE:
+            return ddl
+        i += 1
+
+        if (
+            i + 2 < len_tokens
+            and tokens[i].text.upper() == "IF"
+            and tokens[i + 1].text.upper() == "NOT"
+            and tokens[i + 2].text.upper() == "EXISTS"
+        ):
+            i += 3
+
+        if i >= len_tokens or tokens[i].token_type not in _NAME_TYPES:
+            return ddl
+
+        name_start = tokens[i].start
+        name_end = tokens[i].end
+        i += 1
+
+        while (
+            i + 1 < len_tokens
+            and tokens[i].token_type == TokenType.DOT
+            and tokens[i + 1].token_type in _NAME_TYPES
+        ):
+            name_end = tokens[i + 1].end
+            i += 2
+
+        return ddl[:name_start] + clone_target + ddl[name_end + 1 :]
+
+    def prefetch_last_modified_epochs(self, table_fqns: t.Collection[str]) -> Future[None]:
+        """Batch-prefetch last modified timestamps for the given FQNs into the cache.
+
+        Runs SHOW TABLES per schema and the sys_query_detail join in parallel background
+        threads. Results are merged (sys_query_detail overrides SHOW TABLES) then stored
+        in the cache, so subsequent get_last_modified_epoch calls are cache hits.
+
+        Args:
+            table_fqns: Fully qualified, quoted table name strings to prefetch.
+
+        Returns:
+            A Future that completes when the cache has been populated.
+        """
+        if not table_fqns:
+            return super().prefetch_last_modified_epochs(table_fqns)
+
+        claimed_fqns: t.List[str] = []
+        tables_by_schema: t.Dict[t.Tuple[str, str], t.List[exp.Table]] = defaultdict(list)
+
+        with self._last_modified_epoch_cache as cache:
+            for raw_fqn in table_fqns:
+                table = self._to_fqn(raw_fqn)
+                fqn = self._sql(table)
+                if cache.claim_if_available(fqn):
+                    claimed_fqns.append(fqn)
+                    tables_by_schema[(table.catalog, table.db)].append(table)
+
+        if not claimed_fqns:
+            return super().prefetch_last_modified_epochs(table_fqns)
+
+        tables_by_catalog: t.Dict[str, t.List[exp.Table]] = defaultdict(list)
+        for (catalog, _), tables in tables_by_schema.items():
+            tables_by_catalog[catalog].extend(tables)
+
+        def _fetch_show_tables_thread(
+            catalog: str, schema: str, tables: t.List[exp.Table]
+        ) -> t.Dict[str, t.Optional[int]]:
+            set_invocation_context()
+            self._ensure_thread_connection("prefetch_last_modified_timestamps")
+            return self._query_show_tables_for_schema(catalog, schema, tables)
+
+        def _fetch_sys_query_detail_thread(
+            catalog: str, catalog_tables: t.List[exp.Table]
+        ) -> t.Dict[str, t.Optional[int]]:
+            set_invocation_context()
+
+            catalog_adapter = self._get_or_create_catalog_adapter(catalog)
+            try:
+                with catalog_adapter.connection_named(catalog):
+                    return self._query_sys_query_detail(catalog_tables, adapter=catalog_adapter)
+            finally:
+                self._thread_local.connection_acquired = False
+
+        show_futures = [
+            self._executor.submit(_fetch_show_tables_thread, catalog, schema, tables)
+            for (catalog, schema), tables in tables_by_schema.items()
+        ]
+        sys_futures = [
+            self._executor.submit(_fetch_sys_query_detail_thread, catalog, catalog_tables)
+            for catalog, catalog_tables in tables_by_catalog.items()
+        ]
+
+        def _wait_all() -> None:
+            merged: t.Dict[str, t.Optional[int]] = {}
+            for f in show_futures:
+                try:
+                    merged.update(f.result())
+                except Exception as exc:
+                    events.fire_warn_event_suboptimal(
+                        "Failed to prefetch SHOW TABLES for Redshift: {}", str(exc)
+                    )
+            for f in sys_futures:
+                try:
+                    merged.update(f.result())
+                except Exception as exc:
+                    events.fire_warn_event_suboptimal(
+                        "Failed to prefetch sys_query_detail for Redshift: {}", str(exc)
+                    )
+            for fqn in claimed_fqns:
+                if fqn not in merged:
+                    merged[fqn] = None
+            with self._last_modified_epoch_cache as cache:
+                cache.fulfill_many(merged)
+
+        return self._executor.submit(_wait_all)
+
+    def _fetch_last_modified_epochs(
+        self, table_batch: t.Collection[exp.Table]
+    ) -> dict[str, t.Optional[int]]:
+        if not table_batch:
+            return {}
+
+        table_fqns = [self._to_fqn(t) for t in table_batch]
+        last_modified_epochs: dict[str, t.Optional[int]] = {
+            self._sql(fqn): None for fqn in table_fqns
+        }
+
+        tables_by_schema: dict[tuple[str, str], list[exp.Table]] = defaultdict(list)
+        tables_by_catalog: dict[str, list[exp.Table]] = defaultdict(list)
+        for fqn in table_fqns:
+            tables_by_schema[(fqn.catalog, fqn.db)].append(fqn)
+            tables_by_catalog[fqn.catalog].append(fqn)
+
+        for (catalog, schema), schema_tables in tables_by_schema.items():
+            last_modified_epochs.update(
+                self._query_show_tables_for_schema(catalog, schema, schema_tables)
+            )
+
+        for catalog, catalog_tables in tables_by_catalog.items():
+            catalog_adapter = self._get_or_create_catalog_adapter(catalog)
+            with catalog_adapter.connection_named(catalog):
+                sys_results = self._query_sys_query_detail(catalog_tables, adapter=catalog_adapter)
+
+            for fqn_key, epoch in sys_results.items():
+                if fqn_key in last_modified_epochs:
+                    last_modified_epochs[fqn_key] = epoch
+
+        return last_modified_epochs
+
+    def _query_show_tables_for_schema(
+        self,
+        catalog: str,
+        schema: str,
+        tables: t.Collection[exp.Table],
+    ) -> t.Dict[str, t.Optional[int]]:
+        """Run SHOW TABLES for one schema and return an FQN→epoch dict for the given tables."""
+        result: t.Dict[str, t.Optional[int]] = {}
+        table_names = {tbl.name.lower() for tbl in tables}
+        show_result = self.execute(f"SHOW TABLES FROM SCHEMA {catalog}.{schema}", fetch=True)
+        fallback_epoch = None
+        for row in show_result:
+            if row["table_name"].lower() not in table_names:
+                continue
+            fqn_key = self._build_fqn_from_row(
+                row["database_name"], row["schema_name"], row["table_name"]
+            )
+            candidates = [
+                ts for ts in (row["last_modified_time"], row["last_altered_time"]) if ts is not None
+            ]
+            if candidates:
+                result[fqn_key] = int(
+                    max(candidates).replace(tzinfo=timezone.utc).timestamp() * 1000
+                )
+            else:
+                # Both timestamps NULL: Redshift's metadata refresh populates these via
+                # a background process with ~20 min lag. _query_sys_query_detail normally
+                # supplements via an OID-based join on insert/delete steps, but Redshift's
+                # ALTER TABLE RENAME assigns a new OID, orphaning the INSERT step's
+                # table_id — so tables built via CREATE + INSERT + RENAME come back empty
+                # from both signals.
+                #
+                # The table exists (it's in SHOW TABLES); skipping it would make downstream
+                # logic treat it as nonexistent. Record a "now" sentinel; sys_query_detail
+                # still overrides this whenever its OID join does match (e.g. direct DML).
+                if fallback_epoch is None:
+                    fallback_epoch = int(self.current_timestamp_utc().timestamp() * 1000)
+                result[fqn_key] = fallback_epoch
+        return result
+
+    def _query_sys_query_detail(
+        self, tables: t.Collection[exp.Table], adapter: SQLAdapter
+    ) -> t.Dict[str, t.Optional[int]]:
+        """Query sys_query_detail for recent writes and return an FQN→epoch dict."""
+        if not tables:
+            return {}
+
+        catalog = next(iter(tables)).catalog or self.default_catalog
+        schema_table_filter = " OR ".join(
+            f"(upper(ns.nspname) = upper('{tbl.db}') AND upper(c.relname) = upper('{tbl.name}'))"
+            for tbl in tables
+        )
+        sys_rows_sql = f"""
+            SELECT
+                ns.nspname AS schema_name,
+                c.relname AS table_name,
+                MAX(qd.end_time) AS last_updated
+            FROM pg_class c
+            JOIN pg_namespace ns ON ns.oid = c.relnamespace
+            JOIN sys_query_detail qd ON qd.table_id = c.oid
+            LEFT JOIN sys_query_history qh ON qh.query_id = qd.query_id
+            LEFT JOIN sys_transaction_history th ON th.transaction_id = qh.transaction_id
+            WHERE qd.step_name IN ('insert', 'delete')
+            AND th.status = 'committed'
+            AND qd.end_time >= DATEADD(minute, -{self._SYS_QUERY_DETAIL_LOOKBACK_MINUTES}, (SYSDATE at time zone CURRENT_SETTING('timezone')) at time zone 'UTC')
+            AND ({schema_table_filter})
+            GROUP BY 1, 2
+            """
+
+        _, agate_result = adapter.execute(sys_rows_sql, fetch=True)
+        sys_rows = agate_result.rows
+
+        result: t.Dict[str, t.Optional[int]] = {}
+        for schema_name, table_name, last_updated in sys_rows:
+            if last_updated is None:
+                continue
+            fqn_key = self._build_fqn_from_row(catalog, schema_name.strip(), table_name.strip())
+            result[fqn_key] = int(last_updated.replace(tzinfo=timezone.utc).timestamp() * 1000)
+        return result
+
+    def _fetch_view_definitions(
+        self, table_batch: t.Collection[exp.Table]
+    ) -> t.Collection[ViewDefinition]:
+        if not table_batch:
+            return []
+
+        queries = []
+        # redshift supports cross-database queries, need to group/query by catalog
+        for catalog, tables in group_tables_by_catalog(table_batch, self.default_catalog).items():
+            filter_expr = build_information_schema_filter(tables, ("table_schema", "table_name"))
+            query = f"""
+            SELECT
+                table_catalog,
+                table_schema,
+                table_name,
+                view_definition
+            FROM {catalog}.information_schema.views
+            WHERE {self._sql(filter_expr)}
+            """
+
+            queries.append(query)
+
+        query = "UNION ALL\n".join(queries)
+
+        view_definitions = []
+        result_rows = self.execute(query, fetch=True).rows
+
+        for catalog, schema, name, view_definition in result_rows:
+            fqn = self._build_fqn_from_row(catalog, schema, name)
+            view_definitions.append(
+                ViewDefinition(
+                    fqn=fqn,
+                    definition=view_definition,
+                    dialect=self.dialect,
+                    default_catalog=catalog,
+                    default_schema=schema,
+                )
+            )
+
+        return view_definitions
+
+    def cache_view_definition(self, table: exp.Table, definition: str, default_schema: str) -> None:
+        # Redshift uses early-binding views by default: at CREATE VIEW time it
+        # fully resolves the view body and stores the resolved form. Reading the
+        # view back from information_schema.views therefore returns the resolved
+        # SQL, not the original — e.g. `select id from upstream as up` becomes
+        # `SELECT up.id FROM upstream up`.
+        # https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_VIEW.html
+
+        # dbt's compiled_code is the unresolved form. If we cache compiled_code
+        # in run 1 and fall through to information_schema.views in run 2 (because
+        # the view isn't in the selected set), the two runs produce different
+        # `stable_sql` for the view dependency, which breaks candidate matching on
+        # the server.
+
+        # Skip the cache write so every run sources view definitions from the
+        # warehouse via `_fetch_view_definitions`, ensuring identical
+        # canonicalization across runs.
+        return
+
+    def _get_or_create_catalog_adapter(self, catalog: str) -> SQLAdapter:
+        if catalog.lower() == self.default_catalog.lower():
+            return self.adapter
+
+        if catalog not in self._catalog_adapters:
+            creds = self.adapter.config.credentials.replace(database=catalog)
+            config = replace(self.adapter.config, credentials=creds)
+            self._catalog_adapters[catalog] = type(self.adapter)(config, get_context("spawn"))
+        return self._catalog_adapters[catalog]

{run_cache-2.3.2 → run_cache-2.4.0}/src/dbt_run_cache/dev_cloner.py

@@ -100,8 +100,11 @@ class DevCloner:
         clone_target: str,
     ) -> None:
         with events.downgrade_adapter_error_events():
-            for sql in clone_sqls:
-                adapter.execute(sql)
+            if self._adapter_ext.IMPLEMENTS_CUSTOM_CLONE:
+                self._adapter_ext.clone(clone_sqls, clone_source, clone_target)
+            else:
+                for sql in clone_sqls:
+                    adapter.execute(sql)
 
         # Add the created relation to the cache
         self._adapter_ext.cache_node_relation(node)

{run_cache-2.3.2 → run_cache-2.4.0}/src/dbt_run_cache/run_cache.py

@@ -656,8 +656,15 @@ class RunCache:
                     should_run_hooks = False
                     return NoRunResult(query_cache_response.request_id, failed_to_clone=True)
             with events.downgrade_adapter_error_events():
-                for sql in query_cache_response.clone_sqls:
-                    self._adapter.execute(sql)
+                if self._adapter_ext.IMPLEMENTS_CUSTOM_CLONE:
+                    self._adapter_ext.clone(
+                        query_cache_response.clone_sqls,
+                        query_cache_response.clone_source,
+                        query_cache_response.clone_target,
+                    )
+                else:
+                    for sql in query_cache_response.clone_sqls:
+                        self._adapter.execute(sql)
             self._commit_if_open()
             # when we clone a table, we put it in the relation cache so that the remainder of the
             # invocation knows it exists without hitting the db

run_cache-2.3.2/src/dbt_run_cache/_version.py

@@ -1 +0,0 @@
-__version__ = "2.3.2"