supertable 2.3.3__tar.gz → 2.3.4__tar.gz

{supertable-2.3.3/supertable.egg-info → supertable-2.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.3.3
+Version: 2.3.4
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.3.3 → supertable-2.3.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.3.3"
+version = "2.3.4"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.3.3 → supertable-2.3.4}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.3.3",
+    version="2.3.4",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.3.3 → supertable-2.3.4}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.3.3"
+__version__ = "2.3.4"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.3.3 → supertable-2.3.4}/supertable/config/settings.py

@@ -157,6 +157,25 @@ class Settings:
     SUPERTABLE_DUCKDB_MATERIALIZE: str = "view"    # SUPERTABLE_DUCKDB_MATERIALIZE
     SUPERTABLE_DUCKDB_PRESIGNED: bool = False      # SUPERTABLE_DUCKDB_PRESIGNED
     SUPERTABLE_DUCKDB_USE_HTTPFS: bool = False     # SUPERTABLE_DUCKDB_USE_HTTPFS
+    # Deletion-vector (tombstone) table cache.  Each entry is a small
+    # `DISTINCT __rowid__` table keyed by the stable tombstone path; the
+    # tombstone view ANTI JOINs it instead of re-reading the parquet every
+    # query.  Eviction is purely per-table — a churny table can never evict a
+    # slow table's cached deletion-vector:
+    #   * Idle TTL (below): every entry, including a table's latest, is dropped
+    #     once it goes unqueried for the TTL window.
+    #   * Per-table cap (this knob): at most N most-recently-used versions are
+    #     kept per table, so a burst of rewrites (e.g. 1000 updates in 5 min)
+    #     retains only the last N rather than all of them.
+    # <= 0 disables the cache entirely (inline read_parquet fallback).
+    SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE: int = 8  # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE
+    # Idle TTL (seconds): a cached deletion-vector is dropped once it has gone
+    # unqueried for this long; every query that uses it refreshes the timer.
+    # Applies to every entry (a table's latest included), so an abandoned table
+    # reclaims its cache instead of lingering until the connection resets.
+    # <= 0 keeps an entry only while a query references it (no persistence).
+    # Defaults to SUPERTABLE_ENGINE_FRESHNESS_SEC (300 s / 5 min).
+    SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC: int = 300  # SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC
     SUPERTABLE_DEBUG_TIMINGS: bool = False          # SUPERTABLE_DEBUG_TIMINGS
 
     # ── Engine Routing / Executor ────────────────────────────────────
@@ -418,6 +437,8 @@ def _build_settings() -> Settings:
         SUPERTABLE_DUCKDB_MATERIALIZE=_env_str("SUPERTABLE_DUCKDB_MATERIALIZE", "view"),
         SUPERTABLE_DUCKDB_PRESIGNED=_env_bool("SUPERTABLE_DUCKDB_PRESIGNED", False),
         SUPERTABLE_DUCKDB_USE_HTTPFS=_env_bool("SUPERTABLE_DUCKDB_USE_HTTPFS", False),
+        SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE", 8),
+        SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC", 300),
         SUPERTABLE_DEBUG_TIMINGS=_env_bool("SUPERTABLE_DEBUG_TIMINGS", False),
 
         # ── Engine Routing ───────────────────────────────────────────

{supertable-2.3.3 → supertable-2.3.4}/supertable/data_classes.py

@@ -61,9 +61,17 @@ class TombstoneDef:
 
     - tombstone_path: storage path of the deletion-vector parquet
       (columns ``__file__`` + ``__rowid__``).  ``None`` means no
-      tombstone exists, so no anti-join is applied.
+      tombstone exists, so no anti-join is applied.  This may be a
+      presigned/object-store URL that rotates per request, so it is
+      *not* stable enough to use as a cache key.
+    - cache_key: the bare, stable storage key of the same deletion-vector
+      parquet (no presign).  Stable across pure appends (carry-forward
+      returns the previous tombstone), so DuckDB engines use it to key the
+      materialised deletion-vector table cache.  ``None`` disables caching
+      for this alias (falls back to inline ``read_parquet``).
     """
     tombstone_path: Optional[str] = None
+    cache_key: Optional[str] = None
 
 
 @dataclass

{supertable-2.3.3 → supertable-2.3.4}/supertable/data_reader.py

@@ -322,6 +322,9 @@ class DataReader:
                                 # storage returns the key unchanged.
                                 reflection.tombstone_views[td.alias] = TombstoneDef(
                                     tombstone_path=estimator._to_duckdb_path(tomb_path),
+                                    # Bare key (pre-presign) is stable across
+                                    # appends → safe deletion-vector cache key.
+                                    cache_key=tomb_path,
                                 )
                     except Exception as te:
                         logger.debug(self._lp(f"[tombstone] leaf lookup failed for {td.alias}: {te}"))

{supertable-2.3.3 → supertable-2.3.4}/supertable/data_writer.py

@@ -26,6 +26,7 @@ from supertable.processing import (
     resolve_overwrite_writes,
     identify_all_rowids,
     build_tombstone_file,
+    reclaim_fully_dead_files,
     build_stats_file,
     extract_stats_rows,
     probe_ranges_from_df,
@@ -472,6 +473,17 @@ class DataWriter:
                 new_resources = []
                 sunset_files = set()
 
+                # Load the current deletion-vector once: used both to exclude
+                # already-tombstoned rows from this write's deletes (below) and,
+                # via prev_df, to extend the vector without a second read.
+                prev_dv_df = (
+                    _read_parquet_safe(prev_tombstone_path, profiler=profiler)
+                    if prev_tombstone_path else None
+                )
+                prev_dv_rowids = set()
+                if prev_dv_df is not None and "__rowid__" in prev_dv_df.columns:
+                    prev_dv_rowids = set(prev_dv_df.get_column("__rowid__").to_list())
+
                 # 1. Identify which existing rows this write deletes/replaces.
                 #    overwrite_columns drives the anti-join key (delete + upsert);
                 #    pure appends (no overwrite_columns) tombstone nothing.  The
@@ -487,6 +499,20 @@ class DataWriter:
                         file_cache=file_cache,
                         profiler=profiler,
                     )
+
+                # Never re-tombstone rows already in the deletion-vector.  The
+                # overlap probe (and identify_all_rowids) scan the *physical*
+                # files, which still hold logically-deleted rows until
+                # compaction; without this filter every write re-counts those
+                # already-dead rows — inflating ``deleted`` and forcing a
+                # needless tombstone rewrite even when nothing live was removed.
+                # Excluding them makes ``deleted`` the true count of live rows
+                # removed and lets unchanged writes carry the vector forward.
+                if new_delete_pairs and prev_dv_rowids:
+                    new_delete_pairs = [
+                        (f, rid) for (f, rid) in new_delete_pairs
+                        if rid not in prev_dv_rowids
+                    ]
                 deleted = len(new_delete_pairs)
                 mark("identify_deletes")
 
@@ -515,6 +541,7 @@ class DataWriter:
                     new_pairs=new_delete_pairs,
                     compression_level=compression_level,
                     profiler=profiler,
+                    prev_df=prev_dv_df,
                 )
 
                 # Track the live deletion-vector row count so meta reads can
@@ -528,6 +555,38 @@ class DataWriter:
                 )
                 mark("build_tombstone")
 
+                # 3b. Eager reclamation of fully-dead files.  Any existing data
+                #     file whose every physical row is now tombstoned is 100%
+                #     dead: drop it from the snapshot for free (no rewrite) and
+                #     remove its rowids from the vector.  Without this, fully
+                #     deleted files linger until the compaction threshold,
+                #     bloating the snapshot and getting re-scanned by every later
+                #     overwrite probe.  Only runs when the vector changed this
+                #     write (combined_tombstone_df is not None) — a carry-forward
+                #     can create no newly-dead file.
+                if combined_tombstone_df is not None:
+                    reclaimed_files, reclaimed_tomb_path, reclaimed_dv = (
+                        reclaim_fully_dead_files(
+                            resources=last_simple_table.get("resources") or [],
+                            combined_dv=combined_tombstone_df,
+                            tombstone_dir=tombstone_dir,
+                            compression_level=compression_level,
+                            profiler=profiler,
+                        )
+                    )
+                    if reclaimed_files:
+                        sunset_files |= reclaimed_files
+                        tombstone_path = reclaimed_tomb_path
+                        combined_tombstone_df = reclaimed_dv
+                        tombstone_rows = (
+                            reclaimed_dv.height if reclaimed_dv is not None else 0
+                        )
+                        logger.info(lp(
+                            f"reclaimed {len(reclaimed_files)} fully-deleted "
+                            f"file(s); deletion-vector now {tombstone_rows} rows"
+                        ))
+                mark("reclaim_dead_files")
+
                 # 4. Threshold compaction (two triggers, same physical step):
                 #      (a) the deletion-vector grew past max_tombstone_rows, or
                 #      (b) the small files tripped the auto-compaction gate.
@@ -538,7 +597,9 @@ class DataWriter:
                 #    rows (hidden on read, never reclaimable).  Draining first
                 #    guarantees Phase B only ever sees vector-free survivors.
                 post_write_resources = (
-                    (last_simple_table.get("resources") or []) + new_resources
+                    [r for r in (last_simple_table.get("resources") or [])
+                     if r.get("file") not in sunset_files]
+                    + new_resources
                 )
                 compaction_gate = should_compact_small_files(
                     post_write_resources, table_config
@@ -554,9 +615,17 @@ class DataWriter:
                 if tombstone_threshold_hit or compaction_gate:
                     dv_to_drain = combined_tombstone_df
                     if dv_to_drain is None and tombstone_path:
-                        # Pure carry-forward: load the live vector so the merge
-                        # below never sunsets a file it still references.
-                        dv_to_drain = _read_parquet_safe(tombstone_path, profiler=profiler)
+                        # Pure carry-forward: the pointer is unchanged, so the
+                        # live vector is exactly the one already loaded at the
+                        # top of this block — reuse it instead of a second
+                        # storage read (fall back to a read only if it wasn't
+                        # loaded, which shouldn't happen when tombstone_path is
+                        # set, but stays correct if it ever does).
+                        dv_to_drain = (
+                            prev_dv_df
+                            if prev_dv_df is not None
+                            else _read_parquet_safe(tombstone_path, profiler=profiler)
+                        )
                     if dv_to_drain is not None and dv_to_drain.height > 0:
                         removed, tomb_new, tomb_sunset = compact_tombstones(
                             snapshot=last_simple_table,

{supertable-2.3.3 → supertable-2.3.4}/supertable/engine/duckdb_lite.py

@@ -10,6 +10,7 @@ import duckdb
 import pandas as pd
 
 from supertable.config.defaults import logger
+from supertable.config.settings import settings
 from supertable.query_plan_manager import QueryPlanManager
 from supertable.utils.sql_parser import SQLParser
 from supertable.data_classes import Reflection
@@ -24,6 +25,7 @@ from supertable.engine.engine_common import (
     apply_runtime_pragmas,
     create_rbac_view,
     create_tombstone_view,
+    TombstoneCache,
 )
 
 
@@ -56,6 +58,13 @@ class DuckDBLite:
         self._lock = threading.Lock()
         self._con: Optional[duckdb.DuckDBPyConnection] = None
         self._httpfs_configured = False
+        # Shared deletion-vector table cache: per-table eviction (idle TTL +
+        # per-table version cap), bounded by config. Tables live on the
+        # persistent connection and are forgotten when it resets.
+        self._tombstone_cache = TombstoneCache(
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
+        )
 
     # ------------------------------------------------------------------
     # Connection lifecycle
@@ -92,6 +101,8 @@ class DuckDBLite:
                 pass
             self._con = None
             self._httpfs_configured = False
+            # Tables died with the connection — just forget the registry.
+            self._tombstone_cache.clear_registry()
             logger.warning("[duckdb.lite] connection reset")
 
     # ------------------------------------------------------------------
@@ -163,6 +174,8 @@ class DuckDBLite:
 
         # Create per-query VIEWs. Dropped in finally regardless of outcome.
         created_views: List[str] = []
+        # Deletion-vector cache keys acquired this query — released in finally.
+        acquired_dv_keys: List[str] = []
         try:
             for alias, table_name in alias_to_table_name.items():
                 files = alias_to_files[alias]
@@ -193,7 +206,15 @@ class DuckDBLite:
                 source = query_alias_to_name[alias]
                 tomb_def = tombstone_views.get(alias)
                 view = f"tomb_{source}_{query_suffix}"
-                create_tombstone_view(con, source, view, tomb_def)
+                # Reuse a materialised deletion-vector table when the cache is
+                # enabled and the alias has a stable key; otherwise the call
+                # falls back to the inline read_parquet path (dv_table=None).
+                cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
+                tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
+                dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
+                if dv_table:
+                    acquired_dv_keys.append(cache_key)
+                create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
                 created_views.append(view)
                 query_alias_to_name[alias] = view
 
@@ -260,4 +281,11 @@ class DuckDBLite:
                     con.execute(f"DROP VIEW IF EXISTS {view};")
                 except Exception:
                     pass
+            # Release deletion-vector refs now the views referencing them are
+            # gone; this may evict + DROP unreferenced DV tables over capacity.
+            for cache_key in acquired_dv_keys:
+                try:
+                    self._tombstone_cache.release(con, cache_key)
+                except Exception:
+                    pass
 

{supertable-2.3.3 → supertable-2.3.4}/supertable/engine/duckdb_pro.py

@@ -29,6 +29,7 @@ from supertable.engine.engine_common import (
     create_rbac_view,
     create_tombstone_view,
     rbac_view_name,
+    TombstoneCache,
 )
 
 
@@ -80,6 +81,14 @@ class DuckDBPro:
         # Multiple entries per key when old version still has in-flight queries.
         self._registry: Dict[Tuple[str, str], List[_ProCacheEntry]] = {}
 
+        # Shared deletion-vector table cache: per-table eviction (idle TTL +
+        # per-table version cap), bounded by config. Tables live on the
+        # persistent connection and are forgotten when it resets.
+        self._tombstone_cache = TombstoneCache(
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE,
+            settings.SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC,
+        )
+
         # Temp dir for spill — set on first query
         self._temp_dir: Optional[str] = None
 
@@ -123,6 +132,8 @@ class DuckDBPro:
             self._con = None
             self._httpfs_configured = False
             self._registry.clear()
+            # DV tables died with the connection — just forget the registry.
+            self._tombstone_cache.clear_registry()
             logger.warning("[duckdb.pro] connection reset — all cached views lost")
 
     # ---------------------------------------------------------
@@ -313,6 +324,8 @@ class DuckDBPro:
         # assignments are reached (which would cause a NameError otherwise).
         rbac_view_names: List[str] = []
         tombstone_view_names: List[str] = []
+        # Deletion-vector cache keys acquired this query — released in finally.
+        acquired_dv_keys: List[str] = []
         try:
             query_alias_to_name = dict(alias_to_table_name)
             # Per-query suffix so concurrent queries never collide on a shared
@@ -328,8 +341,18 @@ class DuckDBPro:
                 source = query_alias_to_name[alias]
                 tomb_def = tombstone_views.get(alias)
                 view = f"tomb_{source}_{query_suffix}"
+                # Reuse a materialised deletion-vector table when the cache is
+                # enabled and the alias has a stable key; otherwise fall back to
+                # the inline read_parquet path (dv_table=None). All DDL — the DV
+                # CREATE TABLE inside acquire() and the view creation — runs
+                # under the connection lock, matching Pro's serialised model.
+                cache_key = getattr(tomb_def, "cache_key", None) if tomb_def else None
+                tomb_path = getattr(tomb_def, "tombstone_path", None) if tomb_def else None
                 with self._lock:
-                    create_tombstone_view(con, source, view, tomb_def)
+                    dv_table = self._tombstone_cache.acquire(con, cache_key, tomb_path)
+                    if dv_table:
+                        acquired_dv_keys.append(cache_key)
+                    create_tombstone_view(con, source, view, tomb_def, dv_table=dv_table)
                 tombstone_view_names.append(view)
                 query_alias_to_name[alias] = view
 
@@ -395,6 +418,16 @@ class DuckDBPro:
                         except Exception:
                             pass
 
+            # Release deletion-vector refs now their views are gone; this may
+            # evict + DROP unreferenced DV tables over capacity.
+            if acquired_dv_keys:
+                with self._lock:
+                    for cache_key in acquired_dv_keys:
+                        try:
+                            self._tombstone_cache.release(con, cache_key)
+                        except Exception:
+                            pass
+
             # Release refs and drop stale tables
             with self._lock:
                 self._release_refs(tables_used)