supertable 2.3.4__tar.gz → 2.3.5__tar.gz

{supertable-2.3.4/supertable.egg-info → supertable-2.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.3.4
+Version: 2.3.5
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.3.4 → supertable-2.3.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.3.4"
+version = "2.3.5"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.3.4 → supertable-2.3.5}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.3.4",
+    version="2.3.5",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.3.4 → supertable-2.3.5}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.3.4"
+__version__ = "2.3.5"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.3.4 → supertable-2.3.5}/supertable/config/defaults.py

@@ -17,6 +17,12 @@ handler.setFormatter(colorlog.ColoredFormatter(
 logging.basicConfig(level=logging.INFO, handlers=[handler])
 logger = logging.getLogger(__name__)
 
+# Quiet noisy third-party HTTP client loggers.  At DEBUG these emit one line
+# per request (connection setup + every HEAD/GET/PUT), which drowns out
+# SuperTable's own logs.  WARNING keeps genuine connection problems visible.
+for _noisy_logger in ("urllib3", "botocore", "boto3", "s3transfer", "boto"):
+    logging.getLogger(_noisy_logger).setLevel(logging.WARNING)
+
 _VALID_LOG_LEVELS = frozenset({"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"})
 
 @dataclass(slots=True)

supertable-2.3.5/supertable/config/homedir.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import tempfile
+
+from supertable.config.settings import settings
+from supertable.config.defaults import logger
+
+# If this file is located in a subdirectory, adjust the path logic as needed.
+# Currently appending ".." from __file__ to add the project root directory
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+# ---------- lazy home directory resolution ----------
+_resolved_home: str | None = None
+
+def _is_writable_dir(path: str) -> bool:
+    """Create *path* if needed and verify we can actually write a file in it.
+
+    ``os.access(..., W_OK)`` is unreliable under containers, ACLs and
+    root-squashed mounts, so probe with a real create+unlink: this is the
+    difference between a home that merely *resolves* and one DuckDB can root
+    its temp/spill, cache and extension dirs under.
+    """
+    try:
+        os.makedirs(path, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=path):
+            pass
+        return True
+    except OSError:
+        return False
+
+def _resolve_app_home() -> str:
+    """
+    Resolve, expand, and normalise the application home directory once.
+
+    The home must be *writable*, not just resolvable: DuckDB roots its
+    temp/spill, external-cache and extension directories here, so a
+    non-writable home silently breaks every write (the probe fails with
+    ``errno 13`` and falls back to the slow full-read path).  We therefore
+    verify writability and, when the configured home is not usable, fall back
+    to ``<tempdir>/supertable`` with a loud warning rather than returning a
+    path that only looks valid.
+    """
+    global _resolved_home
+    if _resolved_home is not None:
+        return _resolved_home
+
+    raw = settings.SUPERTABLE_HOME
+    expanded = os.path.abspath(os.path.expanduser(raw))
+
+    if _is_writable_dir(expanded):
+        logger.debug(f"Ensured app home directory exists: {expanded}")
+        _resolved_home = expanded
+        return _resolved_home
+
+    fallback = os.path.join(tempfile.gettempdir(), "supertable")
+    if _is_writable_dir(fallback):
+        logger.warning(
+            f"SUPERTABLE_HOME={expanded!r} is not writable; falling back to "
+            f"{fallback!r}. Set SUPERTABLE_HOME to a writable directory to "
+            f"silence this — DuckDB temp/spill, cache and extensions live under it."
+        )
+        _resolved_home = fallback
+        return _resolved_home
+
+    raise RuntimeError(
+        f"No writable application home: tried SUPERTABLE_HOME={expanded!r} and "
+        f"fallback {fallback!r}. Set SUPERTABLE_HOME to a writable directory."
+    )
+
+def change_to_app_home(home_dir: str | None = None) -> None:
+    """
+    Attempts to change the current working directory to `home_dir`.
+    If home_dir is not provided, uses the resolved app home.
+    Logs the outcome.
+    """
+    target = home_dir if home_dir else _resolve_app_home()
+    expanded_dir = os.path.expanduser(target)
+    try:
+        os.chdir(expanded_dir)
+        logger.debug(f"Changed working directory to {expanded_dir}")
+    except Exception as e:
+        logger.error(f"Failed to change working directory to {expanded_dir}: {e}")
+
+# ---------- eager init (preserves original import-time behaviour) ----------
+_app_home = _resolve_app_home()
+change_to_app_home(_app_home)
+logger.debug(f"Current working directory: {os.getcwd()}")
+
+# ---------- public API ----------
+
+# Kept for backward compatibility; prefer get_app_home() for the expanded path.
+app_home = _app_home
+
+def get_app_home() -> str:
+    """Return the fully expanded, absolute application home directory."""
+    return _resolve_app_home()

{supertable-2.3.4 → supertable-2.3.5}/supertable/data_writer.py

@@ -371,6 +371,14 @@ class DataWriter:
                 profiler=profiler,
             )
             mark("overlap")
+            if overwrite_columns:
+                _snap_files = len(last_simple_table.get("resources") or [])
+                _cand = sum(1 for _, ov, _ in overlapping_files if ov)
+                logger.debug(lp(
+                    f"step[overlap]: {_cand}/{_snap_files} existing file(s) are overwrite "
+                    f"candidates on {overwrite_columns} "
+                    f"(snapshot has no per-file key stats → every file is suspect)"
+                ))
 
             # --- Stats-driven file pruning (consumer 5a) ----------------------
             # Narrow the overwrite/delete candidate set using the external stats
@@ -387,6 +395,10 @@ class DataWriter:
                     stored_stats_df = load_stats(stats_file, allow_cache=True, profiler=profiler)
                     if stored_stats_df is not None and stored_stats_df.height > 0:
                         probe = probe_ranges_from_df(dataframe, overwrite_columns)
+                        _probe_desc = {
+                            c: (f"{v[0]}[{v[1]}..{v[2]}]" if v else "unconstrained(null/unsupported)")
+                            for c, v in probe.items()
+                        }
                         before = len(overlapping_files)
                         overlapping_files = prune_overlapping_files_by_stats(
                             overlapping_files,
@@ -395,8 +407,21 @@ class DataWriter:
                             profiler=profiler,
                         )
                         pruned = before - len(overlapping_files)
+                        logger.debug(lp(
+                            f"step[stats-prune]: df-probe {_probe_desc} vs {stored_stats_df.height} "
+                            f"stored stat row(s) → kept {len(overlapping_files)}/{before}, "
+                            f"pruned {pruned} (no data file opened)"
+                        ))
                         if pruned > 0:
                             logger.info(lp(f"stats pruning: skipped {pruned}/{before} candidate files"))
+                    else:
+                        logger.debug(lp(
+                            "step[stats-prune]: stats artifact empty → no pruning, all candidates retained"
+                        ))
+                else:
+                    logger.debug(lp(
+                        "step[stats-prune]: snapshot has no stats_file → no pruning, all candidates retained"
+                    ))
                 mark("stats_prune")
 
             # File cache: used only by delete_only's identify_all_rowids below.
@@ -420,6 +445,16 @@ class DataWriter:
                     newer_than_col=newer_than,
                     profiler=profiler,
                 )
+                mark("resolve_overwrite")
+                _counts = profiler.counts
+                _fallback = bool(_counts.get("overwrite_resolve_fallback"))
+                logger.debug(lp(
+                    f"step[probe-resolve] via {'polars-fallback' if _fallback else 'duckdb-pushdown'}: "
+                    f"matched {_counts.get('probe_rows_matched', _counts.get('delete_rows_matched', 0))} "
+                    f"existing row(s) on {overwrite_columns} → "
+                    f"{len(resolved_delete_pairs or [])} (file,__rowid__) delete pair(s); "
+                    f"{dataframe.height}/{pre_filter_count} incoming row(s) survive"
+                ))
                 if newer_than:
                     skipped = pre_filter_count - dataframe.height
                     if skipped > 0:
@@ -515,6 +550,10 @@ class DataWriter:
                     ]
                 deleted = len(new_delete_pairs)
                 mark("identify_deletes")
+                logger.debug(lp(
+                    f"step[deletes]: tombstoning {deleted} live row(s) this write "
+                    f"(excluded {len(prev_dv_rowids)} row(s) already in the deletion-vector)"
+                ))
 
                 # 2. Write the incoming rows as a new file (insert/upsert side).
                 #    delete_only carries only predicate columns — nothing to insert.
@@ -531,6 +570,10 @@ class DataWriter:
                 else:
                     inserted = 0
                 mark("write_parquet")
+                logger.debug(lp(
+                    f"step[write]: appended {inserted} incoming row(s) as {len(new_resources)} "
+                    f"new immutable file(s) (no existing data file rewritten)"
+                ))
 
                 # 3. Carry forward + extend the deletion-vector tombstone file.
                 #    No new deletes → reuse the previous file (combined_df=None).
@@ -554,6 +597,10 @@ class DataWriter:
                     else int(last_simple_table.get("tombstone_rows", 0) or 0)
                 )
                 mark("build_tombstone")
+                logger.debug(lp(
+                    f"step[tombstone]: deletion-vector now {tombstone_rows} row(s) "
+                    f"({'rewritten' if combined_tombstone_df is not None else 'carried forward unchanged'})"
+                ))
 
                 # 3b. Eager reclamation of fully-dead files.  Any existing data
                 #     file whose every physical row is now tombstoned is 100%
@@ -812,7 +859,7 @@ class DataWriter:
                         schema_json = "{}"
                     _org, _sup = self.super_table.organization, self.super_table.super_name
                     self.catalog.r.set(RK.schema(_org, _sup, simple_name), schema_json)
-                    self.catalog.r.sadd(RK.table_names(_org, _sup), simple_name)
+                    self.catalog.r.sadd(RK.meta_table_names(_org, _sup), simple_name)
                 except Exception as e:
                     logger.debug(f"[data-writer] schema/table_names Redis write failed: {e}")
 
@@ -862,7 +909,7 @@ class DataWriter:
                         f"total={total_duration:.3f} | "
                         f"convert={timings.get('convert', 0):.3f} | dedup_ts={timings.get('dedup_ts', 0):.3f} | validate={timings.get('validate', 0):.3f} | "
                         f"lock={timings.get('lock', 0):.3f} | snapshot={timings.get('snapshot', 0):.3f} | "
-                        f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
+                        f"overlap={timings.get('overlap', 0):.3f} | stats_prune={timings.get('stats_prune', 0):.3f} | resolve_overwrite={timings.get('resolve_overwrite', 0):.3f} | newer_than={timings.get('newer_than', 0):.3f} | "
                         f"identify_deletes={timings.get('identify_deletes', 0):.3f} | write_parquet={timings.get('write_parquet', 0):.3f} | "
                         f"build_tombstone={timings.get('build_tombstone', 0):.3f} | compact_tombstones={timings.get('compact_tombstones', 0):.3f} | compact_small={timings.get('compact_small', 0):.3f} | build_stats={timings.get('build_stats', 0):.3f} | "
                         f"update_simple={timings.get('update_simple', 0):.3f} | bump_root={timings.get('bump_root', 0):.3f} | "

{supertable-2.3.4 → supertable-2.3.5}/supertable/demo/quickstart/defaults.py

@@ -14,7 +14,9 @@ from enum import Enum
 
 from supertable.config import defaults
 
-logging.getLogger("supertable").setLevel(logging.INFO)
+# Follow the configured SUPERTABLE_LOG_LEVEL (resolved in supertable.config.defaults)
+# instead of hard-pinning INFO, so DEBUG surfaces the detailed write step[...] logs.
+logging.getLogger("supertable").setLevel(defaults.default.LOG_LEVEL)
 
 defaults.default.IS_SHOW_TIMING = True
 

{supertable-2.3.4 → supertable-2.3.5}/supertable/engine/engine_common.py

@@ -285,39 +285,35 @@ def configure_httpfs_and_s3(
         "true" if meta_cache_on else "false",
     )
 
-    # External file cache — an in-memory cache of remote data blocks so
-    # repeated queries do not re-download the same row groups.  It is only
-    # enabled when a size cap can be enforced: DuckDB builds without
-    # external_file_cache_max_size (e.g. 1.5.x) cannot bound it, and an
-    # uncapped cache grows to memory_limit on the persistent connection.
-    # Without an enforceable cap we keep it OFF to protect memory.
+    # External file cache — an in-memory cache of external (e.g. remote
+    # Parquet) data blocks so repeated queries do not re-download the same row
+    # groups.  Enabled whenever a cache size is configured.  The cache is
+    # bounded by the global memory_limit (DuckDB enforces that bound), which is
+    # the effective cap; a dedicated per-cache cap is applied only on builds
+    # that expose external_file_cache_max_size (no released DuckDB through
+    # 1.5.x does), so in practice memory_limit is the bound.
     cache_size = settings.SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE
     can_cap = "external_file_cache_max_size" in supported
-    if cache_size and can_cap:
+    if cache_size:
         set_if_supported("enable_external_file_cache", "true")
-        set_if_supported("external_file_cache_max_size", f"'{cache_size}'")
-        cache_dir_raw = settings.SUPERTABLE_DUCKDB_EXTERNAL_CACHE_DIR
-        if not cache_dir_raw:
-            # Derive from SUPERTABLE_HOME — single env var controls all paths.
-            cache_dir_raw = os.path.join(get_app_home(), "duckdb_cache")
-        # Expand ~ so DuckDB receives an absolute path.
-        cache_dir = os.path.expanduser(cache_dir_raw)
-        os.makedirs(cache_dir, exist_ok=True)
-        set_if_supported("external_file_cache_directory", f"'{cache_dir}'")
-        logger.info(
+        if can_cap:
+            set_if_supported("external_file_cache_max_size", f"'{cache_size}'")
+            cache_dir_raw = settings.SUPERTABLE_DUCKDB_EXTERNAL_CACHE_DIR
+            if not cache_dir_raw:
+                # Derive from SUPERTABLE_HOME — single env var controls all paths.
+                cache_dir_raw = os.path.join(get_app_home(), "duckdb_cache")
+            # Expand ~ so DuckDB receives an absolute path.
+            cache_dir = os.path.expanduser(cache_dir_raw)
+            os.makedirs(cache_dir, exist_ok=True)
+            set_if_supported("external_file_cache_directory", f"'{cache_dir}'")
+        logger.debug(
             "[duckdb.cache] external file cache enabled"
-            + (f" size={cache_size}" if cache_size else "")
-            + f", dir={cache_dir}"
+            + (f", capped at {cache_size}" if can_cap
+               else f", bounded by memory_limit (size={cache_size}; "
+                    "this DuckDB build has no dedicated cap)")
         )
     else:
-        # Uncappable (or disabled via empty size) — turn it off explicitly so
-        # the DuckDB 1.5.x default-on cache cannot accumulate in memory.
         set_if_supported("enable_external_file_cache", "false")
-        if cache_size and not can_cap:
-            logger.info(
-                "[duckdb.cache] external file cache disabled: this DuckDB build "
-                "cannot cap it (no external_file_cache_max_size)"
-            )
 
 
 # =========================================================
@@ -589,14 +585,13 @@ def rewrite_query_with_hashed_tables(
 # =========================================================
 
 def _external_file_cache_cappable(con: duckdb.DuckDBPyConnection) -> bool:
-    """True when this DuckDB build can bound the external file cache size.
-
-    DuckDB 1.5.x enables ``enable_external_file_cache`` by default but does
-    not expose ``external_file_cache_max_size``.  An enabled-but-uncapped
-    cache is held in memory (not on disk) and grows to ``memory_limit`` on
-    the long-lived persistent connection — a sustained-memory / OOM hazard.
-    When this returns False the cache is disabled outright rather than left
-    running unbounded.
+    """True when this DuckDB build exposes a dedicated external-file-cache cap.
+
+    No released DuckDB (through 1.5.x) exposes ``external_file_cache_max_size``;
+    the cache is in-memory and bounded by ``memory_limit``, which DuckDB
+    enforces.  This predicate gates only the *dedicated* per-cache size cap:
+    when it returns False the cache still runs, bounded by ``memory_limit``
+    rather than a separate cap.
     """
     try:
         return bool(con.execute(
@@ -679,16 +674,19 @@ def init_connection(
     except Exception:
         pass  # older DuckDB builds may not support this setting
 
-    # External file cache baseline.  DuckDB 1.5.x turns the cache ON by
-    # default but cannot cap it, so an uncapped in-memory cache accumulates
-    # remote data up to memory_limit on the persistent connection.  Disable
-    # it here when uncappable; configure_httpfs_and_s3 / apply_runtime_pragmas
-    # re-enable it (capped) only on builds that support a size cap.
-    if not _external_file_cache_cappable(con):
-        try:
-            con.execute("SET enable_external_file_cache=false;")
-        except Exception:
-            pass
+    # External file cache baseline.  DuckDB (>=1.3.0) ships an in-memory cache
+    # of external files, bounded by memory_limit (DuckDB enforces that bound).
+    # Honour the configured default here: enable when a cache size is set,
+    # otherwise off.  configure_httpfs_and_s3 / apply_runtime_pragmas refine
+    # this (and apply a dedicated cap on builds that expose one).
+    try:
+        con.execute(
+            "SET enable_external_file_cache="
+            + ("true" if settings.SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE else "false")
+            + ";"
+        )
+    except Exception:
+        pass
 
     # Thread count.
     # If SUPERTABLE_DUCKDB_THREADS is set explicitly, honour it exactly.
@@ -783,17 +781,19 @@ def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
         except Exception:
             pass
 
-    # External file cache: only run it when the size cap is enforceable.
-    # On DuckDB builds without external_file_cache_max_size (e.g. 1.5.x) an
-    # enabled cache is in-memory and unbounded — it grows to memory_limit on
-    # the persistent connection — so we disable it instead of running uncapped.
+    # External file cache: enable whenever the org configures a cache size.
+    # The cache is in-memory and bounded by memory_limit (DuckDB enforces that
+    # bound), which is the effective cap.  A dedicated per-cache cap is applied
+    # only on builds that expose external_file_cache_max_size (no released
+    # DuckDB through 1.5.x does); otherwise memory_limit is the bound.
     cache_size = normalize_memory_size(cfg.duckdb_external_cache_size, default="")
-    if cache_size and _external_file_cache_cappable(con):
+    if cache_size:
         try:
             con.execute("SET enable_external_file_cache=true;")
-            con.execute(
-                f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';"
-            )
+            if _external_file_cache_cappable(con):
+                con.execute(
+                    f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';"
+                )
         except Exception as e:
             logger.warning(f"[duckdb.pragma] external file cache config failed: {e}")
     else:
@@ -892,7 +892,7 @@ def run_engine_diagnostics(cfg=None, engine: str = "lite") -> Dict[str, Any]:
             ("external_file_cache_max_size supported — the file cache can be capped"
              if cappable else
              "this build has no external_file_cache_max_size — the file cache "
-             "cannot be capped, so it is disabled to stay memory-safe"),
+             "runs bounded by memory_limit rather than a dedicated cap"),
             version)
     except Exception as e:
         add("version", "DuckDB version", "warn", f"version() failed: {e}")
@@ -1039,18 +1039,19 @@ def run_engine_diagnostics(cfg=None, engine: str = "lite") -> Dict[str, Any]:
                 getattr(cfg, "duckdb_external_cache_size", ""), default=""
             )
         if efc_on and not cappable:
-            add("cache", "External file cache", "fail",
-                "Cache is ON but this build cannot cap it — it grows to the memory "
-                "limit and causes OOM", "on · uncapped")
+            add("cache", "External file cache", "ok",
+                "Cache is ON, bounded by memory_limit — this build has no "
+                "dedicated cap (external_file_cache_max_size), so memory_limit "
+                "is the bound", "on · memory_limit")
         elif efc_on and cappable:
             add("cache", "External file cache", "ok",
                 f"Cache is ON and capped at {cache_cfg or 'the configured size'}",
                 "on · capped")
         else:
             add("cache", "External file cache", "ok",
-                "Cache is OFF — memory-safe; remote files are re-fetched per query "
-                "(set a Disk cache size on a cap-capable build to speed up repeats)",
-                "off")
+                "Cache is OFF — remote files are re-fetched per query (set "
+                "SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE to enable, bounded by "
+                "memory_limit)", "off")
     except Exception as e:
         add("cache", "External file cache", "warn", f"Could not read cache state: {e}")
 

{supertable-2.3.4 → supertable-2.3.5}/supertable/processing.py

@@ -225,6 +225,7 @@ def _read_parquet_safe(
         path: str,
         profiler: Optional[Profiler] = None,
         file_size: int = 0,
+        columns: Optional[List[str]] = None,
 ) -> Optional[polars.DataFrame]:
     p = profiler or get_null_profiler()
     if not _safe_exists(path, profiler=p):
@@ -232,7 +233,13 @@ def _read_parquet_safe(
         return None
     try:
         with p.span("io.read_parquet"):
-            tbl = _get_storage().read_parquet(path)  # -> pyarrow.Table
+            # Project to *columns* when given so only those column chunks are
+            # read (memory-bound fallback); gated so storages/test doubles that
+            # only accept ``path`` keep working on the unprojected paths.
+            tbl = (
+                _get_storage().read_parquet(path, columns=columns)
+                if columns else _get_storage().read_parquet(path)
+            )  # -> pyarrow.Table
         with p.span("io.arrow_to_polars"):
             df = polars.from_arrow(tbl)
         p.add("files_read", 1)
@@ -627,11 +634,12 @@ def _write_single_parquet_file(
     rows = write_df.shape[0]
     columns = write_df.shape[1]
 
-    # Ensure target directory exists (no-op on object storage)
+    # Ensure the target directory exists.  makedirs is idempotent on local
+    # storage and a no-op on object storage; calling it directly avoids a
+    # pointless prefix HEAD (which always 404s) on object stores.
     with p.span("write.ensure_dir"):
         try:
-            if not _get_storage().exists(target_dir):
-                _get_storage().makedirs(target_dir)
+            _get_storage().makedirs(target_dir)
         except Exception:
             pass
 
@@ -728,6 +736,7 @@ def filter_stale_incoming_rows(
         newer_than_col: str,
         file_cache: Optional[Dict[str, polars.DataFrame]] = None,
         profiler: Optional[Profiler] = None,
+        read_columns: Optional[List[str]] = None,
 ) -> polars.DataFrame:
     """
     Remove rows from *incoming_df* that are stale or already present in existing data.
@@ -762,7 +771,7 @@ def filter_stale_incoming_rows(
     # Read and collect relevant rows from overlapping files
     existing_parts: List[polars.DataFrame] = []
     for file_path, file_size in overlap_true_files:
-        part = _read_parquet_safe(file_path, profiler=p, file_size=file_size)
+        part = _read_parquet_safe(file_path, profiler=p, file_size=file_size, columns=read_columns)
         if part is None:
             continue
         # Cache the full DataFrame for downstream reuse (avoids double-read)
@@ -880,6 +889,7 @@ def identify_deleted_rowids(
         overwrite_columns: List[str],
         file_cache: Optional[Dict[str, polars.DataFrame]] = None,
         profiler: Optional[Profiler] = None,
+        read_columns: Optional[List[str]] = None,
 ) -> List[Tuple[str, int]]:
     """Find the ``(file, __rowid__)`` pairs of existing rows matching a delete predicate.
 
@@ -911,7 +921,7 @@ def identify_deleted_rowids(
         if file_cache is not None and file in file_cache:
             existing_df = file_cache.get(file)
         else:
-            existing_df = _read_parquet_safe(file, profiler=p, file_size=file_size)
+            existing_df = _read_parquet_safe(file, profiler=p, file_size=file_size, columns=read_columns)
         if existing_df is None:
             continue
         if ROWID_COL not in existing_df.columns:
@@ -1110,6 +1120,10 @@ def _duckdb_probe_overlap_matches(
             f"filename=TRUE, hive_partitioning=FALSE) AS src "
             f"SEMI JOIN {ik_name} AS k ON {join_cond}"
         )
+        logging.debug(
+            f"[write-probe] duckdb scan: {len(paths)} file(s), "
+            f"project={select_cols}, semi-join on {incoming_keys.height} key(s)"
+        )
         with p.span("io.duckdb_probe"):
             return con.execute(sql).pl()
 
@@ -1168,6 +1182,11 @@ def _duckdb_probe_overlap_matches(
         return None
     p.add("probe_files", len(duck_paths))
     p.add("probe_rows_matched", int(matched.height))
+    logging.debug(
+        f"[write-probe] duckdb scan matched {matched.height} existing row(s) "
+        f"across {len(duck_paths)} file(s) (only key/__rowid__ columns read, "
+        f"row groups skipped by footer min/max)"
+    )
     return matched
 
 
@@ -1273,6 +1292,11 @@ def resolve_overwrite_writes(
         return incoming_df, []
 
     incoming_keys = incoming_df.select(overwrite_columns).unique()
+    logging.debug(
+        f"[write-probe] resolve: {len(overlap_true)} overlapping file(s), "
+        f"{incoming_keys.height} unique incoming key(s) on {overwrite_columns}, "
+        f"newer_than={newer_than_col}"
+    )
     matched = _duckdb_probe_overlap_matches(
         overlap_true, overwrite_columns, newer_than_col, incoming_keys, profiler=p,
     )
@@ -1286,6 +1310,19 @@ def resolve_overwrite_writes(
 
     # ---- Fallback: original polars full-read path (semantics oracle) ----
     p.add("overwrite_resolve_fallback", 1)
+    # Project reads to only the columns the fallback consumes — overwrite keys
+    # (+ newer-than for stale filtering) + __rowid__ (for the delete vector) —
+    # so wide tables are not fully materialised into memory.  The shared
+    # file_cache holds this projected union; each consumer selects its subset.
+    read_columns = list(dict.fromkeys(
+        list(overwrite_columns)
+        + ([newer_than_col] if newer_than_col else [])
+        + [ROWID_COL]
+    ))
+    logging.debug(
+        f"[write-probe] polars full-read fallback over {len(overlap_true)} file(s), "
+        f"reading only {read_columns}"
+    )
     file_cache: Dict[str, polars.DataFrame] = {}
     if newer_than_col:
         filtered = filter_stale_incoming_rows(
@@ -1295,12 +1332,13 @@ def resolve_overwrite_writes(
             newer_than_col=newer_than_col,
             file_cache=file_cache,
             profiler=p,
+            read_columns=read_columns,
         )
     else:
         filtered = incoming_df
     pairs = identify_deleted_rowids(
         filtered, overlapping_files, overwrite_columns,
-        file_cache=file_cache, profiler=p,
+        file_cache=file_cache, profiler=p, read_columns=read_columns,
     )
     return filtered, pairs
 
@@ -1354,8 +1392,8 @@ def build_tombstone_file(
     combined = combined.unique(subset=[ROWID_COL], keep="first")
 
     try:
-        if not _get_storage().exists(tombstone_dir):
-            _get_storage().makedirs(tombstone_dir)
+        # Direct makedirs (idempotent local, no-op object) — avoids a 404 prefix HEAD.
+        _get_storage().makedirs(tombstone_dir)
     except Exception:
         pass
 
@@ -1418,8 +1456,8 @@ def reclaim_fully_dead_files(
         return fully_dead, None, None
 
     try:
-        if not _get_storage().exists(tombstone_dir):
-            _get_storage().makedirs(tombstone_dir)
+        # Direct makedirs (idempotent local, no-op object) — avoids a 404 prefix HEAD.
+        _get_storage().makedirs(tombstone_dir)
     except Exception:
         pass
 
@@ -1688,8 +1726,8 @@ def build_stats_file(
         combined = new_df
 
     try:
-        if not _get_storage().exists(stats_dir):
-            _get_storage().makedirs(stats_dir)
+        # Direct makedirs (idempotent local, no-op object) — avoids a 404 prefix HEAD.
+        _get_storage().makedirs(stats_dir)
     except Exception:
         pass
 

{supertable-2.3.4 → supertable-2.3.5}/supertable/simple_table.py

@@ -128,19 +128,10 @@ class SimpleTable:
         self.data_dir = os.path.join(self.simple_dir, "data")
         self.snapshot_dir = os.path.join(self.simple_dir, "snapshots")
 
-        logger.debug(f"simple_dir: {self.simple_dir}")
-        logger.debug(f"data_dir: {self.data_dir}")
-        logger.debug(f"snapshot_dir: {self.snapshot_dir}")
-
         # Fast path: if meta:leaf exists, don't touch storage
         if self.catalog.leaf_exists(
                 self.super_table.organization, self.super_table.super_name, self.simple_name
         ):
-            logger.debug(
-                f"[SimpleTable] Leaf exists in Redis for "
-                f"{self.super_table.organization}/{self.super_table.super_name}/{self.simple_name}; "
-                f"skipping storage mkdirs and bootstrap."
-            )
             return
 
         # Read-only opt-out: refuse to bootstrap as a side effect. The