supertable 2.3.3__tar.gz → 2.3.5__tar.gz

{supertable-2.3.3/supertable.egg-info → supertable-2.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.3.3
+Version: 2.3.5
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.3.3 → supertable-2.3.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.3.3"
+version = "2.3.5"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.3.3 → supertable-2.3.5}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.3.3",
+    version="2.3.5",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.3.3 → supertable-2.3.5}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.3.3"
+__version__ = "2.3.5"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.3.3 → supertable-2.3.5}/supertable/config/defaults.py

@@ -17,6 +17,12 @@ handler.setFormatter(colorlog.ColoredFormatter(
 logging.basicConfig(level=logging.INFO, handlers=[handler])
 logger = logging.getLogger(__name__)
 
+# Quiet noisy third-party HTTP client loggers.  At DEBUG these emit one line
+# per request (connection setup + every HEAD/GET/PUT), which drowns out
+# SuperTable's own logs.  WARNING keeps genuine connection problems visible.
+for _noisy_logger in ("urllib3", "botocore", "boto3", "s3transfer", "boto"):
+    logging.getLogger(_noisy_logger).setLevel(logging.WARNING)
+
 _VALID_LOG_LEVELS = frozenset({"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"})
 
 @dataclass(slots=True)

supertable-2.3.5/supertable/config/homedir.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import tempfile
+
+from supertable.config.settings import settings
+from supertable.config.defaults import logger
+
+# If this file is located in a subdirectory, adjust the path logic as needed.
+# Currently appending ".." from __file__ to add the project root directory
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+# ---------- lazy home directory resolution ----------
+_resolved_home: str | None = None
+
+def _is_writable_dir(path: str) -> bool:
+    """Create *path* if needed and verify we can actually write a file in it.
+
+    ``os.access(..., W_OK)`` is unreliable under containers, ACLs and
+    root-squashed mounts, so probe with a real create+unlink: this is the
+    difference between a home that merely *resolves* and one DuckDB can root
+    its temp/spill, cache and extension dirs under.
+    """
+    try:
+        os.makedirs(path, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=path):
+            pass
+        return True
+    except OSError:
+        return False
+
+def _resolve_app_home() -> str:
+    """
+    Resolve, expand, and normalise the application home directory once.
+
+    The home must be *writable*, not just resolvable: DuckDB roots its
+    temp/spill, external-cache and extension directories here, so a
+    non-writable home silently breaks every write (the probe fails with
+    ``errno 13`` and falls back to the slow full-read path).  We therefore
+    verify writability and, when the configured home is not usable, fall back
+    to ``<tempdir>/supertable`` with a loud warning rather than returning a
+    path that only looks valid.
+    """
+    global _resolved_home
+    if _resolved_home is not None:
+        return _resolved_home
+
+    raw = settings.SUPERTABLE_HOME
+    expanded = os.path.abspath(os.path.expanduser(raw))
+
+    if _is_writable_dir(expanded):
+        logger.debug(f"Ensured app home directory exists: {expanded}")
+        _resolved_home = expanded
+        return _resolved_home
+
+    fallback = os.path.join(tempfile.gettempdir(), "supertable")
+    if _is_writable_dir(fallback):
+        logger.warning(
+            f"SUPERTABLE_HOME={expanded!r} is not writable; falling back to "
+            f"{fallback!r}. Set SUPERTABLE_HOME to a writable directory to "
+            f"silence this — DuckDB temp/spill, cache and extensions live under it."
+        )
+        _resolved_home = fallback
+        return _resolved_home
+
+    raise RuntimeError(
+        f"No writable application home: tried SUPERTABLE_HOME={expanded!r} and "
+        f"fallback {fallback!r}. Set SUPERTABLE_HOME to a writable directory."
+    )
+
+def change_to_app_home(home_dir: str | None = None) -> None:
+    """
+    Attempts to change the current working directory to `home_dir`.
+    If home_dir is not provided, uses the resolved app home.
+    Logs the outcome.
+    """
+    target = home_dir if home_dir else _resolve_app_home()
+    expanded_dir = os.path.expanduser(target)
+    try:
+        os.chdir(expanded_dir)
+        logger.debug(f"Changed working directory to {expanded_dir}")
+    except Exception as e:
+        logger.error(f"Failed to change working directory to {expanded_dir}: {e}")
+
+# ---------- eager init (preserves original import-time behaviour) ----------
+_app_home = _resolve_app_home()
+change_to_app_home(_app_home)
+logger.debug(f"Current working directory: {os.getcwd()}")
+
+# ---------- public API ----------
+
+# Kept for backward compatibility; prefer get_app_home() for the expanded path.
+app_home = _app_home
+
+def get_app_home() -> str:
+    """Return the fully expanded, absolute application home directory."""
+    return _resolve_app_home()

{supertable-2.3.3 → supertable-2.3.5}/supertable/config/settings.py

@@ -157,6 +157,25 @@ class Settings:
     SUPERTABLE_DUCKDB_MATERIALIZE: str = "view"    # SUPERTABLE_DUCKDB_MATERIALIZE
     SUPERTABLE_DUCKDB_PRESIGNED: bool = False      # SUPERTABLE_DUCKDB_PRESIGNED
     SUPERTABLE_DUCKDB_USE_HTTPFS: bool = False     # SUPERTABLE_DUCKDB_USE_HTTPFS
+    # Deletion-vector (tombstone) table cache.  Each entry is a small
+    # `DISTINCT __rowid__` table keyed by the stable tombstone path; the
+    # tombstone view ANTI JOINs it instead of re-reading the parquet every
+    # query.  Eviction is purely per-table — a churny table can never evict a
+    # slow table's cached deletion-vector:
+    #   * Idle TTL (below): every entry, including a table's latest, is dropped
+    #     once it goes unqueried for the TTL window.
+    #   * Per-table cap (this knob): at most N most-recently-used versions are
+    #     kept per table, so a burst of rewrites (e.g. 1000 updates in 5 min)
+    #     retains only the last N rather than all of them.
+    # <= 0 disables the cache entirely (inline read_parquet fallback).
+    SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE: int = 8  # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE
+    # Idle TTL (seconds): a cached deletion-vector is dropped once it has gone
+    # unqueried for this long; every query that uses it refreshes the timer.
+    # Applies to every entry (a table's latest included), so an abandoned table
+    # reclaims its cache instead of lingering until the connection resets.
+    # <= 0 keeps an entry only while a query references it (no persistence).
+    # Defaults to SUPERTABLE_ENGINE_FRESHNESS_SEC (300 s / 5 min).
+    SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC: int = 300  # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC
     SUPERTABLE_DEBUG_TIMINGS: bool = False          # SUPERTABLE_DEBUG_TIMINGS
 
     # ── Engine Routing / Executor ────────────────────────────────────
@@ -418,6 +437,8 @@ def _build_settings() -> Settings:
         SUPERTABLE_DUCKDB_MATERIALIZE=_env_str("SUPERTABLE_DUCKDB_MATERIALIZE", "view"),
         SUPERTABLE_DUCKDB_PRESIGNED=_env_bool("SUPERTABLE_DUCKDB_PRESIGNED", False),
         SUPERTABLE_DUCKDB_USE_HTTPFS=_env_bool("SUPERTABLE_DUCKDB_USE_HTTPFS", False),
+        SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE", 8),
+        SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC", 300),
         SUPERTABLE_DEBUG_TIMINGS=_env_bool("SUPERTABLE_DEBUG_TIMINGS", False),
 
         # ── Engine Routing ───────────────────────────────────────────

{supertable-2.3.3 → supertable-2.3.5}/supertable/data_classes.py

@@ -61,9 +61,17 @@ class TombstoneDef:
 
     - tombstone_path: storage path of the deletion-vector parquet
       (columns ``__file__`` + ``__rowid__``).  ``None`` means no
-      tombstone exists, so no anti-join is applied.
+      tombstone exists, so no anti-join is applied.  This may be a
+      presigned/object-store URL that rotates per request, so it is
+      *not* stable enough to use as a cache key.
+    - cache_key: the bare, stable storage key of the same deletion-vector
+      parquet (no presign).  Stable across pure appends (carry-forward
+      returns the previous tombstone), so DuckDB engines use it to key the
+      materialised deletion-vector table cache.  ``None`` disables caching
+      for this alias (falls back to inline ``read_parquet``).
     """
     tombstone_path: Optional[str] = None
+    cache_key: Optional[str] = None
 
 
 @dataclass

{supertable-2.3.3 → supertable-2.3.5}/supertable/data_reader.py

@@ -322,6 +322,9 @@ class DataReader:
                                 # storage returns the key unchanged.
                                 reflection.tombstone_views[td.alias] = TombstoneDef(
                                     tombstone_path=estimator._to_duckdb_path(tomb_path),
+                                    # Bare key (pre-presign) is stable across
+                                    # appends → safe deletion-vector cache key.
+                                    cache_key=tomb_path,
                                 )
                     except Exception as te:
                         logger.debug(self._lp(f"[tombstone] leaf lookup failed for {td.alias}: {te}"))

{supertable-2.3.3 → supertable-2.3.5}/supertable/data_writer.py

@@ -26,6 +26,7 @@ from supertable.processing import (
     resolve_overwrite_writes,
     identify_all_rowids,
     build_tombstone_file,
+    reclaim_fully_dead_files,
     build_stats_file,
     extract_stats_rows,
     probe_ranges_from_df,
@@ -370,6 +371,14 @@ class DataWriter:
                 profiler=profiler,
             )
             mark("overlap")
+            if overwrite_columns:
+                _snap_files = len(last_simple_table.get("resources") or [])
+                _cand = sum(1 for _, ov, _ in overlapping_files if ov)
+                logger.debug(lp(
+                    f"step[overlap]: {_cand}/{_snap_files} existing file(s) are overwrite "
+                    f"candidates on {overwrite_columns} "
+                    f"(snapshot has no per-file key stats → every file is suspect)"
+                ))
 
             # --- Stats-driven file pruning (consumer 5a) ----------------------
             # Narrow the overwrite/delete candidate set using the external stats
@@ -386,6 +395,10 @@ class DataWriter:
                     stored_stats_df = load_stats(stats_file, allow_cache=True, profiler=profiler)
                     if stored_stats_df is not None and stored_stats_df.height > 0:
                         probe = probe_ranges_from_df(dataframe, overwrite_columns)
+                        _probe_desc = {
+                            c: (f"{v[0]}[{v[1]}..{v[2]}]" if v else "unconstrained(null/unsupported)")
+                            for c, v in probe.items()
+                        }
                         before = len(overlapping_files)
                         overlapping_files = prune_overlapping_files_by_stats(
                             overlapping_files,
@@ -394,8 +407,21 @@ class DataWriter:
                             profiler=profiler,
                         )
                         pruned = before - len(overlapping_files)
+                        logger.debug(lp(
+                            f"step[stats-prune]: df-probe {_probe_desc} vs {stored_stats_df.height} "
+                            f"stored stat row(s) → kept {len(overlapping_files)}/{before}, "
+                            f"pruned {pruned} (no data file opened)"
+                        ))
                         if pruned > 0:
                             logger.info(lp(f"stats pruning: skipped {pruned}/{before} candidate files"))
+                    else:
+                        logger.debug(lp(
+                            "step[stats-prune]: stats artifact empty → no pruning, all candidates retained"
+                        ))
+                else:
+                    logger.debug(lp(
+                        "step[stats-prune]: snapshot has no stats_file → no pruning, all candidates retained"
+                    ))
                 mark("stats_prune")
 
             # File cache: used only by delete_only's identify_all_rowids below.
@@ -419,6 +445,16 @@ class DataWriter:
                     newer_than_col=newer_than,
                     profiler=profiler,
                 )
+                mark("resolve_overwrite")
+                _counts = profiler.counts
+                _fallback = bool(_counts.get("overwrite_resolve_fallback"))
+                logger.debug(lp(
+                    f"step[probe-resolve] via {'polars-fallback' if _fallback else 'duckdb-pushdown'}: "
+                    f"matched {_counts.get('probe_rows_matched', _counts.get('delete_rows_matched', 0))} "
+                    f"existing row(s) on {overwrite_columns} → "
+                    f"{len(resolved_delete_pairs or [])} (file,__rowid__) delete pair(s); "
+                    f"{dataframe.height}/{pre_filter_count} incoming row(s) survive"
+                ))
                 if newer_than:
                     skipped = pre_filter_count - dataframe.height
                     if skipped > 0:
@@ -472,6 +508,17 @@ class DataWriter:
                 new_resources = []
                 sunset_files = set()
 
+                # Load the current deletion-vector once: used both to exclude
+                # already-tombstoned rows from this write's deletes (below) and,
+                # via prev_df, to extend the vector without a second read.
+                prev_dv_df = (
+                    _read_parquet_safe(prev_tombstone_path, profiler=profiler)
+                    if prev_tombstone_path else None
+                )
+                prev_dv_rowids = set()
+                if prev_dv_df is not None and "__rowid__" in prev_dv_df.columns:
+                    prev_dv_rowids = set(prev_dv_df.get_column("__rowid__").to_list())
+
                 # 1. Identify which existing rows this write deletes/replaces.
                 #    overwrite_columns drives the anti-join key (delete + upsert);
                 #    pure appends (no overwrite_columns) tombstone nothing.  The
@@ -487,8 +534,26 @@ class DataWriter:
                         file_cache=file_cache,
                         profiler=profiler,
                     )
+
+                # Never re-tombstone rows already in the deletion-vector.  The
+                # overlap probe (and identify_all_rowids) scan the *physical*
+                # files, which still hold logically-deleted rows until
+                # compaction; without this filter every write re-counts those
+                # already-dead rows — inflating ``deleted`` and forcing a
+                # needless tombstone rewrite even when nothing live was removed.
+                # Excluding them makes ``deleted`` the true count of live rows
+                # removed and lets unchanged writes carry the vector forward.
+                if new_delete_pairs and prev_dv_rowids:
+                    new_delete_pairs = [
+                        (f, rid) for (f, rid) in new_delete_pairs
+                        if rid not in prev_dv_rowids
+                    ]
                 deleted = len(new_delete_pairs)
                 mark("identify_deletes")
+                logger.debug(lp(
+                    f"step[deletes]: tombstoning {deleted} live row(s) this write "
+                    f"(excluded {len(prev_dv_rowids)} row(s) already in the deletion-vector)"
+                ))
 
                 # 2. Write the incoming rows as a new file (insert/upsert side).
                 #    delete_only carries only predicate columns — nothing to insert.
@@ -505,6 +570,10 @@ class DataWriter:
                 else:
                     inserted = 0
                 mark("write_parquet")
+                logger.debug(lp(
+                    f"step[write]: appended {inserted} incoming row(s) as {len(new_resources)} "
+                    f"new immutable file(s) (no existing data file rewritten)"
+                ))
 
                 # 3. Carry forward + extend the deletion-vector tombstone file.
                 #    No new deletes → reuse the previous file (combined_df=None).
@@ -515,6 +584,7 @@ class DataWriter:
                     new_pairs=new_delete_pairs,
                     compression_level=compression_level,
                     profiler=profiler,
+                    prev_df=prev_dv_df,
                 )
 
                 # Track the live deletion-vector row count so meta reads can
@@ -527,6 +597,42 @@ class DataWriter:
                     else int(last_simple_table.get("tombstone_rows", 0) or 0)
                 )
                 mark("build_tombstone")
+                logger.debug(lp(
+                    f"step[tombstone]: deletion-vector now {tombstone_rows} row(s) "
+                    f"({'rewritten' if combined_tombstone_df is not None else 'carried forward unchanged'})"
+                ))
+
+                # 3b. Eager reclamation of fully-dead files.  Any existing data
+                #     file whose every physical row is now tombstoned is 100%
+                #     dead: drop it from the snapshot for free (no rewrite) and
+                #     remove its rowids from the vector.  Without this, fully
+                #     deleted files linger until the compaction threshold,
+                #     bloating the snapshot and getting re-scanned by every later
+                #     overwrite probe.  Only runs when the vector changed this
+                #     write (combined_tombstone_df is not None) — a carry-forward
+                #     can create no newly-dead file.
+                if combined_tombstone_df is not None:
+                    reclaimed_files, reclaimed_tomb_path, reclaimed_dv = (
+                        reclaim_fully_dead_files(
+                            resources=last_simple_table.get("resources") or [],
+                            combined_dv=combined_tombstone_df,
+                            tombstone_dir=tombstone_dir,
+                            compression_level=compression_level,
+                            profiler=profiler,
+                        )
+                    )
+                    if reclaimed_files:
+                        sunset_files |= reclaimed_files
+                        tombstone_path = reclaimed_tomb_path
+                        combined_tombstone_df = reclaimed_dv
+                        tombstone_rows = (
+                            reclaimed_dv.height if reclaimed_dv is not None else 0
+                        )
+                        logger.info(lp(
+                            f"reclaimed {len(reclaimed_files)} fully-deleted "
+                            f"file(s); deletion-vector now {tombstone_rows} rows"
+                        ))
+                mark("reclaim_dead_files")
 
                 # 4. Threshold compaction (two triggers, same physical step):
                 #      (a) the deletion-vector grew past max_tombstone_rows, or
@@ -538,7 +644,9 @@ class DataWriter:
                 #    rows (hidden on read, never reclaimable).  Draining first
                 #    guarantees Phase B only ever sees vector-free survivors.
                 post_write_resources = (
-                    (last_simple_table.get("resources") or []) + new_resources
+                    [r for r in (last_simple_table.get("resources") or [])
+                     if r.get("file") not in sunset_files]
+                    + new_resources
                 )
                 compaction_gate = should_compact_small_files(
                     post_write_resources, table_config
@@ -554,9 +662,17 @@ class DataWriter:
                 if tombstone_threshold_hit or compaction_gate:
                     dv_to_drain = combined_tombstone_df
                     if dv_to_drain is None and tombstone_path:
-                        # Pure carry-forward: load the live vector so the merge
-                        # below never sunsets a file it still references.
-                        dv_to_drain = _read_parquet_safe(tombstone_path, profiler=profiler)
+                        # Pure carry-forward: the pointer is unchanged, so the
+                        # live vector is exactly the one already loaded at the
+                        # top of this block — reuse it instead of a second
+                        # storage read (fall back to a read only if it wasn't
+                        # loaded, which shouldn't happen when tombstone_path is
+                        # set, but stays correct if it ever does).
+                        dv_to_drain = (
+                            prev_dv_df
+                            if prev_dv_df is not None
+                            else _read_parquet_safe(tombstone_path, profiler=profiler)
+                        )
                     if dv_to_drain is not None and dv_to_drain.height > 0:
                         removed, tomb_new, tomb_sunset = compact_tombstones(
                             snapshot=last_simple_table,
@@ -743,7 +859,7 @@ class DataWriter:
                         schema_json = "{}"
                     _org, _sup = self.super_table.organization, self.super_table.super_name
                     self.catalog.r.set(RK.schema(_org, _sup, simple_name), schema_json)
-                    self.catalog.r.sadd(RK.table_names(_org, _sup), simple_name)
+                    self.catalog.r.sadd(RK.meta_table_names(_org, _sup), simple_name)
                 except Exception as e:
                     logger.debug(f"[data-writer] schema/table_names Redis write failed: {e}")
 
@@ -793,7 +909,7 @@ class DataWriter:
                         f"total={total_duration:.3f} | "
                         f"convert={timings.get('convert', 0):.3f} | dedup_ts={timings.get('dedup_ts', 0):.3f} | validate={timings.get('validate', 0):.3f} | "
                         f"lock={timings.get('lock', 0):.3f} | snapshot={timings.get('snapshot', 0):.3f} | "
-                        f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
+                        f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | resolve_overwrite={timings.get('resolve_overwrite', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
                         f"identify_deletes={timings.get('identify_deletes', 0):.3f} | write_parquet={timings.get('write_parquet', 0):.3f} | "
                         f"build_tombstone={timings.get('build_tombstone', 0):.3f} | compact_tombstones={timings.get('compact_tombstones', 0):.3f} | compact_small={timings.get('compact_small', 0):.3f} | build_stats={timings.get('build_stats', 0):.3f} | "
                         f"update_simple={timings.get('update_simple', 0):.3f} | bump_root={timings.get('bump_root', 0):.3f} | "

{supertable-2.3.3 → supertable-2.3.5}/supertable/demo/quickstart/defaults.py

@@ -14,7 +14,9 @@ from enum import Enum
 
 from supertable.config import defaults
 
-logging.getLogger("supertable").setLevel(logging.INFO)
+# Follow the configured SUPERTABLE_LOG_LEVEL (resolved in supertable.config.defaults)
+# instead of hard-pinning INFO, so DEBUG surfaces the detailed write step[...] logs.
+logging.getLogger("supertable").setLevel(defaults.default.LOG_LEVEL)
 
 defaults.default.IS_SHOW_TIMING = True
 

{supertable-2.3.3 → supertable-2.3.5}/supertable/engine/duckdb_lite.py

@@ -10,6 +10,7 @@ import duckdb
 import pandas as pd
 
 from supertable.config.defaults import logger
+from supertable.config.settings import settings
 from supertable.query_plan_manager import QueryPlanManager
 from supertable.utils.sql_parser import SQLParser
 from supertable.data_classes import Reflection
@@ -24,6 +25,7 @@ from supertable.engine.engine_common import (
     apply_runtime_pragmas,
     create_rbac_view,
     create_tombstone_view,
+    TombstoneCache,
 )
 
 
@@ -56,6 +58,13 @@ class DuckDBLite:
         self._lock = threading.Lock()
         self._con: Optional[duckdb.DuckDBPyConnection] = None
         self._httpfs_configured = False
+        # Shared deletion-vector table cache: per-table eviction (idle TTL +
+        # per-table version cap), bounded by config. Tables live on the
+        # persistent connection and are forgotten when it resets.
+        self._tombstone_cache = TombstoneCache(
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
+        )
 
     # ------------------------------------------------------------------
     # Connection lifecycle
@@ -92,6 +101,8 @@ class DuckDBLite:
                 pass
             self._con = None
             self._httpfs_configured = False
+            # Tables died with the connection — just forget the registry.
+            self._tombstone_cache.clear_registry()
             logger.warning("[duckdb.lite] connection reset")
 
     # ------------------------------------------------------------------
@@ -163,6 +174,8 @@ class DuckDBLite:
 
         # Create per-query VIEWs. Dropped in finally regardless of outcome.
         created_views: List[str] = []
+        # Deletion-vector cache keys acquired this query — released in finally.
+        acquired_dv_keys: List[str] = []
         try:
             for alias, table_name in alias_to_table_name.items():
                 files = alias_to_files[alias]
@@ -193,7 +206,15 @@ class DuckDBLite:
                 source = query_alias_to_name[alias]
                 tomb_def = tombstone_views.get(alias)
                 view = f"tomb_{source}_{query_suffix}"
-                create_tombstone_view(con, source, view, tomb_def)
+                # Reuse a materialised deletion-vector table when the cache is
+                # enabled and the alias has a stable key; otherwise the call
+                # falls back to the inline read_parquet path (dv_table=None).
+                cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
+                tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
+                dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
+                if dv_table:
+                    acquired_dv_keys.append(cache_key)
+                create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
                 created_views.append(view)
                 query_alias_to_name[alias] = view
 
@@ -260,4 +281,11 @@ class DuckDBLite:
                     con.execute(f"DROP VIEW IF EXISTS {view};")
                 except Exception:
                     pass
+            # Release deletion-vector refs now the views referencing them are
+            # gone; this may evict + DROP unreferenced DV tables over capacity.
+            for cache_key in acquired_dv_keys:
+                try:
+                    self._tombstone_cache.release(con, cache_key)
+                except Exception:
+                    pass
 

{supertable-2.3.3 → supertable-2.3.5}/supertable/engine/duckdb_pro.py

@@ -29,6 +29,7 @@ from supertable.engine.engine_common import (
     create_rbac_view,
     create_tombstone_view,
     rbac_view_name,
+    TombstoneCache,
 )
 
 
@@ -80,6 +81,14 @@ class DuckDBPro:
         # Multiple entries per key when old version still has in-flight queries.
         self._registry: Dict[Tuple[str, str], List[_ProCacheEntry]] = {}
 
+        # Shared deletion-vector table cache: per-table eviction (idle TTL +
+        # per-table version cap), bounded by config. Tables live on the
+        # persistent connection and are forgotten when it resets.
+        self._tombstone_cache = TombstoneCache(
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
+        )
+
         # Temp dir for spill — set on first query
         self._temp_dir: Optional[str] = None
 
@@ -123,6 +132,8 @@ class DuckDBPro:
             self._con = None
             self._httpfs_configured = False
             self._registry.clear()
+            # DV tables died with the connection — just forget the registry.
+            self._tombstone_cache.clear_registry()
             logger.warning("[duckdb.pro] connection reset — all cached views lost")
 
     # ---------------------------------------------------------
@@ -313,6 +324,8 @@ class DuckDBPro:
         # assignments are reached (which would cause a NameError otherwise).
         rbac_view_names: List[str] = []
         tombstone_view_names: List[str] = []
+        # Deletion-vector cache keys acquired this query — released in finally.
+        acquired_dv_keys: List[str] = []
         try:
             query_alias_to_name = dict(alias_to_table_name)
             # Per-query suffix so concurrent queries never collide on a shared
@@ -328,8 +341,18 @@ class DuckDBPro:
                 source = query_alias_to_name[alias]
                 tomb_def = tombstone_views.get(alias)
                 view = f"tomb_{source}_{query_suffix}"
+                # Reuse a materialised deletion-vector table when the cache is
+                # enabled and the alias has a stable key; otherwise fall back to
+                # the inline read_parquet path (dv_table=None). All DDL — the DV
+                # CREATE TABLE inside acquire() and the view creation — runs
+                # under the connection lock, matching Pro's serialised model.
+                cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
+                tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
                 with self._lock:
-                    create_tombstone_view(con, source, view, tomb_def)
+                    dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
+                    if dv_table:
+                        acquired_dv_keys.append(cache_key)
+                    create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
                 tombstone_view_names.append(view)
                 query_alias_to_name[alias] = view
 
@@ -395,6 +418,16 @@ class DuckDBPro:
                         except Exception:
                             pass
 
+            # Release deletion-vector refs now their views are gone; this may
+            # evict + DROP unreferenced DV tables over capacity.
+            if acquired_dv_keys:
+                with self._lock:
+                    for cache_key in acquired_dv_keys:
+                        try:
+                            self._tombstone_cache.release(con, cache_key)
+                        except Exception:
+                            pass
+
             # Release refs and drop stale tables
             with self._lock:
                 self._release_refs(tables_used)