supertable 2.3.8__tar.gz → 2.4.0__tar.gz

{supertable-2.3.8/supertable.egg-info → supertable-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.3.8
+Version: 2.4.0
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.3.8 → supertable-2.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.3.8"
+version = "2.4.0"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.3.8 → supertable-2.4.0}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.3.8",
+    version="2.4.0",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.3.8 → supertable-2.4.0}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.3.8"
+__version__ = "2.4.0"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.3.8 → supertable-2.4.0}/supertable/config/settings.py

@@ -157,6 +157,14 @@ class Settings:
     SUPERTABLE_DUCKDB_MATERIALIZE: str = "view"    # SUPERTABLE_DUCKDB_MATERIALIZE
     SUPERTABLE_DUCKDB_PRESIGNED: bool = False      # SUPERTABLE_DUCKDB_PRESIGNED
     SUPERTABLE_DUCKDB_USE_HTTPFS: bool = False     # SUPERTABLE_DUCKDB_USE_HTTPFS
+    # Allow DuckDB to DOWNLOAD a missing extension (httpfs) from
+    # extensions.duckdb.org at query time.  Default OFF: httpfs is baked into
+    # the image and seeded into the extension dir, and an implicit network
+    # install on an offline/firewalled node can hang for minutes instead of
+    # failing.  When OFF, a genuinely missing extension raises immediately with
+    # an actionable message; set ON only on a networked box that must
+    # self-install.
+    SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD: bool = False  # SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD
     # Write-path overwrite/delete resolution via the DuckDB pushdown probe.
     # Disabled by default: the polars fallback reads only the projected key
     # columns through the storage SDK and needs no httpfs extension, so it works
@@ -196,6 +204,12 @@ class Settings:
     SUPERTABLE_SPARK_STATEMENT_TIMEOUT: int = 120  # SUPERTABLE_SPARK_STATEMENT_TIMEOUT
     SUPERTABLE_SPARK_CONNECT_TIMEOUT: int = 30     # SUPERTABLE_SPARK_CONNECT_TIMEOUT
     SUPERTABLE_SPARK_BATCH_SIZE: int = 50          # SUPERTABLE_SPARK_BATCH_SIZE
+    # Spark file access.  Default False → Spark scans direct s3a://bucket/key
+    # paths using its own fs.s3a.* credentials, independent of the DuckDB
+    # presign setting that shapes the shared reflection file list.  True →
+    # Spark scans presigned http(s):// URLs minted per request (use when the
+    # cluster cannot reach the object store with its own credentials).
+    SUPERTABLE_SPARK_PRESIGNED: bool = False       # SUPERTABLE_SPARK_PRESIGNED
 
     # ── Redis ────────────────────────────────────────────────────────
     SUPERTABLE_REDIS_URL: str = ""               # SUPERTABLE_REDIS_URL
@@ -291,11 +305,24 @@ class Settings:
     # always read fresh and never cached).  0 disables caching.
     SUPERTABLE_STATS_CACHE_MAX_TABLES: int = 64   # SUPERTABLE_STATS_CACHE_MAX_TABLES
 
+    # Max number of tables whose *latest* tombstone (deletion-vector) artifact is
+    # held in the in-process tombstone cache (one DataFrame per table; older
+    # versions are always read fresh and never cached).  Mirrors the stats cache
+    # so a process writing in a loop skips the carry-forward read.  0 disables.
+    SUPERTABLE_TOMBSTONE_CACHE_MAX_TABLES: int = 64   # SUPERTABLE_TOMBSTONE_CACHE_MAX_TABLES
+
     # Read-path file pruning: when True the estimator uses the stats artifact to
     # drop parquet files that provably can't satisfy a query's WHERE predicates.
     # Conservative (never drops a file that could match); set False to disable.
     SUPERTABLE_READ_PRUNING_ENABLED: bool = True  # SUPERTABLE_READ_PRUNING_ENABLED
 
+    # Projection-aware read sizing: when True the estimator charges a query only
+    # the on-disk (compressed) bytes of the columns it selects — via per-column
+    # ``compressed_bytes`` in the stats artifact, falling back to a type-width
+    # ratio of the whole file. Drives AUTO engine routing (lite/pro/spark). Set
+    # False to fall back to whole-file sizing (every query charged all columns).
+    SUPERTABLE_READ_PROJECTION_SIZING_ENABLED: bool = True
+
     # ── Audit ────────────────────────────────────────────────────────
     # Audit is OFF by default.  Enable per-organization in the WebUI
     # /ui/audit → Compliance tab (persisted at supertable:{org}:system:audit:config),
@@ -444,6 +471,7 @@ def _build_settings() -> Settings:
         SUPERTABLE_DUCKDB_MATERIALIZE=_env_str("SUPERTABLE_DUCKDB_MATERIALIZE", "view"),
         SUPERTABLE_DUCKDB_PRESIGNED=_env_bool("SUPERTABLE_DUCKDB_PRESIGNED", False),
         SUPERTABLE_DUCKDB_USE_HTTPFS=_env_bool("SUPERTABLE_DUCKDB_USE_HTTPFS", False),
+        SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD=_env_bool("SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD", False),
         SUPERTABLE_DUCKDB_WRITE_PROBE=_env_bool("SUPERTABLE_DUCKDB_WRITE_PROBE", False),
         SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_MAX_PER_TABLE", 8),
         SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC=_env_int("SUPERTABLE_DUCKDB_TOMBSTONE_CACHE_TTL_SEC", 300),
@@ -460,6 +488,7 @@ def _build_settings() -> Settings:
         SUPERTABLE_SPARK_STATEMENT_TIMEOUT=_env_int("SUPERTABLE_SPARK_STATEMENT_TIMEOUT", 120),
         SUPERTABLE_SPARK_CONNECT_TIMEOUT=_env_int("SUPERTABLE_SPARK_CONNECT_TIMEOUT", 30),
         SUPERTABLE_SPARK_BATCH_SIZE=_env_int("SUPERTABLE_SPARK_BATCH_SIZE", 50),
+        SUPERTABLE_SPARK_PRESIGNED=_env_bool("SUPERTABLE_SPARK_PRESIGNED", False),
 
         # ── Redis ────────────────────────────────────────────────────
         SUPERTABLE_REDIS_URL=_env_str("SUPERTABLE_REDIS_URL"),
@@ -551,7 +580,9 @@ def _build_settings() -> Settings:
         # ── Meta Reader / Caching ────────────────────────────────────
         SUPERTABLE_SUPER_META_CACHE_TTL_S=meta_ttl,
         SUPERTABLE_STATS_CACHE_MAX_TABLES=_env_int("SUPERTABLE_STATS_CACHE_MAX_TABLES", 64),
+        SUPERTABLE_TOMBSTONE_CACHE_MAX_TABLES=_env_int("SUPERTABLE_TOMBSTONE_CACHE_MAX_TABLES", 64),
         SUPERTABLE_READ_PRUNING_ENABLED=_env_bool("SUPERTABLE_READ_PRUNING_ENABLED", True),
+        SUPERTABLE_READ_PROJECTION_SIZING_ENABLED=_env_bool("SUPERTABLE_READ_PROJECTION_SIZING_ENABLED", True),
 
         # ── Audit ────────────────────────────────────────────────────
         SUPERTABLE_AUDIT_ENABLED=_env_bool("SUPERTABLE_AUDIT_ENABLED", False),

{supertable-2.3.8 → supertable-2.4.0}/supertable/data_writer.py

@@ -34,6 +34,8 @@ from supertable.processing import (
     prune_overlapping_files_by_stats,
     load_stats,
     cache_stats,
+    load_tombstone,
+    cache_tombstone,
     write_parquet_and_collect_resources,
     compact_resources,
     compact_tombstones,
@@ -525,8 +527,13 @@ class DataWriter:
                 # required=True: a DV that exists but cannot be read must abort
                 # the write, never be treated as empty — silently dropping the
                 # carried-forward vector would resurrect previously deleted rows.
+                # Cache-first: if this process wrote the table on a prior loop
+                # iteration, the current deletion-vector is already in memory
+                # (seeded below after the pointer is pinned), so this is a hit
+                # with no storage round-trip.  required=True preserves the
+                # carry-forward safety on a genuine miss (abort, never truncate).
                 prev_dv_df = (
-                    _read_parquet_safe(prev_tombstone_path, profiler=profiler, required=True)
+                    load_tombstone(prev_tombstone_path, allow_cache=True, required=True, profiler=profiler)
                     if prev_tombstone_path else None
                 )
                 # The rowid set is consumed only by the idempotency filter below,
@@ -790,6 +797,15 @@ class DataWriter:
                 #    and its row count.
                 last_simple_table["tombstone"] = tombstone_path
                 last_simple_table["tombstone_rows"] = tombstone_rows
+                # Seed the in-process cache so the NEXT write's carry-forward read
+                # (prev_dv_df above) is a pure memory hit.  The frame is the fresh
+                # build/reclaim result when this write changed the vector, else the
+                # already-loaded prev frame for a pure carry-forward.  No-op when
+                # the vector was fully consumed this write (tombstone_path is None).
+                cache_tombstone(
+                    tombstone_path,
+                    combined_tombstone_df if combined_tombstone_df is not None else prev_dv_df,
+                )
                 mark("compact_tombstones")
 
                 # Phase B — auto small-file compaction.  Merge the accumulated
@@ -1328,7 +1344,10 @@ class DataWriter:
             # new file while the vector kept pointing at the sunset __file__ —
             # leaving them permanently unreclaimable. Failing loud leaves the
             # prior snapshot + vector intact for a retry, and matches the
-            # write-path carry-forward read (required=True) above.
+            # write-path carry-forward read (required=True) above.  Read directly
+            # (not via the in-process cache): compact() always drains the vector,
+            # so there is no carry-forward hit to gain and it never re-seeds the
+            # cache after draining — the loop-caching win lives on the write path.
             tombstone_df = (
                 _read_parquet_safe(tombstone_path, required=True)
                 if tombstone_path else None

{supertable-2.3.8 → supertable-2.4.0}/supertable/engine/data_estimator.py

@@ -7,6 +7,8 @@ from collections import defaultdict
 from typing import Iterable, Set, List, Dict, Optional, Tuple
 from urllib.parse import urlparse
 
+import polars
+
 from supertable.config.defaults import logger
 from supertable.config.settings import settings
 from supertable.data_classes import Reflection, SuperSnapshot
@@ -18,7 +20,12 @@ from supertable.utils.profiler import Profiler
 from supertable.redis_catalog import RedisCatalog  # Redis leaf pointers for snapshots
 
 from supertable.utils.sql_parser import TableDefinition
-from supertable.processing import load_stats, prune_files_by_predicates
+from supertable.processing import (
+    load_stats,
+    prune_files_by_predicates,
+    ROWID_COL,
+    TIMESTAMP_COL,
+)
 
 
 from typing import Dict, List, Optional, Set, Tuple
@@ -306,22 +313,25 @@ class DataEstimator:
         super_name: str,
         simple_name: str,
         raw_keys: List[str],
-        stats_file: Optional[str],
+        stats_df: Optional["polars.DataFrame"],
         profiler: Optional[Profiler] = None,
     ) -> List[str]:
         """Narrow *raw_keys* to those that could satisfy the query predicates.
 
-        Returns *raw_keys* unchanged whenever pruning is disabled, there's no
-        stats artifact, or the query carries no usable constraint for this
-        table — and never raises (a pruning failure must not break a read).
+        Takes the already-loaded *stats_df* (loaded once per table by
+        :meth:`estimate` and shared with projection sizing) rather than a path,
+        so the artifact is read at most once.  Returns *raw_keys* unchanged
+        whenever pruning is disabled, there's no stats artifact, or the query
+        carries no usable constraint for this table — and never raises (a
+        pruning failure must not break a read).
 
         *profiler*, when supplied, accumulates the same IO/pruning counters the
-        write path emits (``stats_cache_hit``/``stats_cache_miss``,
-        ``read_pruned_files``) so the read monitoring payload can surface them.
+        write path emits (``read_pruned_files``) so the read monitoring payload
+        can surface them.
         """
         if not settings.SUPERTABLE_READ_PRUNING_ENABLED:
             return raw_keys
-        if not stats_file or not raw_keys:
+        if stats_df is None or not raw_keys:
             return raw_keys
         occurrences = self.predicate_constraints.get(
             (super_name.lower(), simple_name.lower())
@@ -329,7 +339,6 @@ class DataEstimator:
         if not occurrences:
             return raw_keys
         try:
-            stats_df = load_stats(stats_file, allow_cache=True, profiler=profiler)
             return prune_files_by_predicates(
                 raw_keys, stats_df, occurrences, profiler=profiler,
             )
@@ -337,6 +346,138 @@ class DataEstimator:
             logger.warning(f"[estimate.prune] pruning skipped for {super_name}.{simple_name}: {e}")
             return raw_keys
 
+    # ----------------------- projection-aware sizing -----------------------
+
+    # Rough per-value byte widths for the type-width *fallback* (used only when
+    # per-column ``compressed_bytes`` are unavailable — e.g. a stats file that
+    # predates the column, or no stats artifact at all).  Substring-matched
+    # against the stored (DuckDB/polars) type name, most-specific first.
+    _STRING_AVG_WIDTH = 16
+
+    def _type_width(self, type_name: Optional[str]) -> int:
+        """Estimated on-disk bytes-per-value for a column type (fallback only)."""
+        t = (type_name or "").strip().lower()
+        if not t:
+            return 8
+        if "bool" in t:
+            return 1
+        if "timestamp" in t or "datetime" in t:
+            return 8
+        if "date" in t:
+            return 4
+        if any(s in t for s in ("varchar", "char", "utf8", "string", "text",
+                                "json", "blob", "binary", "bytea")):
+            return self._STRING_AVG_WIDTH
+        if "double" in t or "float64" in t:
+            return 8
+        if "float" in t or "real" in t:
+            return 4
+        if "decimal" in t or "numeric" in t:
+            return 8
+        if "bigint" in t or "int64" in t or "long" in t:
+            return 8
+        if "smallint" in t or "int16" in t:
+            return 2
+        if "tinyint" in t or "int8" in t:
+            return 1
+        if "int" in t:
+            return 4
+        return 8
+
+    def _selected_columns(self, super_name: str, simple_name: str) -> Optional[Set[str]]:
+        """Lowercased set of user columns this query projects from the table.
+
+        Returns ``None`` to mean "every column" — i.e. ``SELECT *`` / ``t.*``
+        (an empty ``TableDefinition.columns``), an unrecognised table, or a
+        projection that resolves to only system columns.  In every ``None`` case
+        the caller uses the whole-file size (no projection savings), which is
+        the safe over-estimate.  System columns (``__rowid__`` / ``__timestamp__``)
+        are stripped: the read view hides them, so they're never scanned.
+        """
+        key = (super_name.lower(), simple_name.lower())
+        selected: Set[str] = set()
+        matched = False
+        for t in self.tables:
+            if (t.super_name.lower(), t.simple_name.lower()) != key:
+                continue
+            matched = True
+            if not t.columns:          # [] => SELECT * / t.* => whole table
+                return None
+            for c in t.columns:
+                cl = c.lower()
+                if cl not in (ROWID_COL, TIMESTAMP_COL):
+                    selected.add(cl)
+        if not matched or not selected:
+            return None
+        return selected
+
+    def _projected_bytes_index(
+        self,
+        stats_df: Optional["polars.DataFrame"],
+        selected_cols: Set[str],
+    ) -> Tuple[Set[str], Dict[str, int]]:
+        """Sum per-column ``compressed_bytes`` for the selected columns per file.
+
+        Returns ``(tier3_files, proj)`` where *proj* maps a file path to the
+        summed on-disk bytes of its selected columns, and *tier3_files* is the
+        subset of files for which that sum is *trustworthy* — every matched row
+        carried a non-NULL ``compressed_bytes``.  A file with any NULL (an older
+        carried-forward row) is omitted so the caller falls back to a whole-file
+        ratio rather than under-counting it as zero.
+        """
+        if (
+            stats_df is None
+            or stats_df.height == 0
+            or "compressed_bytes" not in stats_df.columns
+        ):
+            return set(), {}
+        sel = (
+            stats_df.select(["file_path", "column_name", "compressed_bytes"])
+            .with_columns(polars.col("column_name").str.to_lowercase().alias("__cn"))
+            .filter(polars.col("__cn").is_in(list(selected_cols)))
+        )
+        if sel.height == 0:
+            return set(), {}
+        agg = sel.group_by("file_path").agg(
+            [
+                polars.col("compressed_bytes").sum().alias("__b"),
+                polars.col("compressed_bytes").is_null().sum().alias("__nulls"),
+            ]
+        )
+        tier3_files: Set[str] = set()
+        proj: Dict[str, int] = {}
+        for r in agg.iter_rows(named=True):
+            if int(r["__nulls"] or 0) == 0:
+                fp = r["file_path"]
+                tier3_files.add(fp)
+                proj[fp] = int(r["__b"] or 0)
+        return tier3_files, proj
+
+    def _ratio_bytes(
+        self,
+        file_key: str,
+        key_size: Dict[str, int],
+        selected_cols: Set[str],
+        schema_types: Dict[str, str],
+    ) -> int:
+        """Fallback size: scale the whole-file bytes by the selected columns'
+        type-width share of the table schema.  Used only when precise
+        per-column ``compressed_bytes`` are unavailable for *file_key*."""
+        full = int(key_size.get(file_key, 0))
+        if full <= 0 or not schema_types:
+            return full
+        all_cols = {
+            c: ty for c, ty in schema_types.items()
+            if c not in (ROWID_COL, TIMESTAMP_COL)
+        }
+        total_w = sum(self._type_width(ty) for ty in all_cols.values())
+        if total_w <= 0:
+            return full
+        sel_w = sum(self._type_width(all_cols[c]) for c in selected_cols if c in all_cols)
+        if sel_w <= 0:
+            return full
+        return int(full * sel_w / total_w)
+
     # ----------------------- main API -----------------------
     def estimate(self) -> Reflection:
         """
@@ -352,7 +493,8 @@ class DataEstimator:
         prune_profiler = Profiler()
 
         supers: List[SuperSnapshot] = []
-        reflection_file_size = 0
+        reflection_file_size = 0        # projected (selected-column) bytes — routing
+        reflection_file_size_raw = 0    # whole-file bytes — physical footprint
         max_freshness_ms = 0
         files_before_prune = 0
         files_pruned = 0
@@ -378,6 +520,7 @@ class DataEstimator:
                 )
 
                 schema: Set[str] = set()
+                schema_types: Dict[str, str] = {}
                 raw_keys: List[str] = []
                 key_size: Dict[str, int] = {}
                 stats_file: Optional[str] = None
@@ -395,7 +538,12 @@ class DataEstimator:
 
                     current_version = current_snapshot_data.get("snapshot_version", 0)
                     current_schema = self._schema_to_dict(current_snapshot_data.get("schema", {}))
-                    schema.update(dict_keys_to_lowercase(current_schema).keys())
+                    lowered_schema = dict_keys_to_lowercase(current_schema)
+                    schema.update(lowered_schema.keys())
+                    # Retain name->type for the projection ratio fallback. First
+                    # writer wins for a given column (schemas are stable per table).
+                    for _cname, _ctype in lowered_schema.items():
+                        schema_types.setdefault(_cname, _ctype)
                     sf = current_snapshot_data.get("stats_file")
                     if sf:
                         stats_file = sf
@@ -408,23 +556,62 @@ class DataEstimator:
                         raw_keys.append(file_key)
                         key_size[file_key] = int(resource.get("file_size", 0))
 
+                # Which columns does the query actually read? None => SELECT *
+                # (whole table, no projection savings).
+                selected_cols = self._selected_columns(super_name, simple_name)
+                need_projection = (
+                    selected_cols is not None
+                    and settings.SUPERTABLE_READ_PROJECTION_SIZING_ENABLED
+                )
+                has_predicate = bool(
+                    self.predicate_constraints.get((super_name.lower(), simple_name.lower()))
+                )
+
+                # Load the stats artifact ONCE per table and reuse it for BOTH
+                # predicate pruning and projection sizing (cache-backed: a repeated
+                # read of the same table is a memory hit). Only load when at least
+                # one consumer needs it, so a SELECT * with no WHERE stays free.
+                stats_df: Optional["polars.DataFrame"] = None
+                if stats_file and (need_projection or has_predicate):
+                    stats_df = load_stats(stats_file, allow_cache=True, profiler=prune_profiler)
+
                 # Read-path pruning: drop raw keys whose stats prove they cannot
                 # satisfy the query's WHERE before resolving them to scan URLs.
                 # The span accumulates the wall-clock of the whole pruning step
-                # (stats load + predicate eval) across every table in the query.
+                # (predicate eval) across every table in the query.
                 with prune_profiler.span("read.prune"):
                     survivors = self._prune_files(
-                        super_name, simple_name, raw_keys, stats_file,
+                        super_name, simple_name, raw_keys, stats_df,
                         profiler=prune_profiler,
                     )
                 files_before_prune += len(raw_keys)
                 files_pruned += len(raw_keys) - len(survivors)
                 files_kept += len(survivors)
 
+                # Projection-aware size: a query selecting specific columns scans
+                # only those columns' on-disk (compressed) chunks, not the whole
+                # multi-column file. Precise path sums per-column compressed_bytes
+                # from the stats artifact; files predating that column (or with no
+                # stats) fall back to a type-width ratio of the whole-file size.
+                # SELECT * keeps every column (full file).
+                tier3_files: Set[str] = set()
+                proj: Dict[str, int] = {}
+                if need_projection:
+                    tier3_files, proj = self._projected_bytes_index(stats_df, selected_cols)
+
                 parquet_files: List[str] = []
                 for file_key in survivors:
                     parquet_files.append(self._to_duckdb_path(file_key))
-                    reflection_file_size += key_size.get(file_key, 0)
+                    full = int(key_size.get(file_key, 0))
+                    reflection_file_size_raw += full
+                    if not need_projection:
+                        reflection_file_size += full
+                    elif file_key in tier3_files:
+                        reflection_file_size += proj.get(file_key, 0)
+                    else:
+                        reflection_file_size += self._ratio_bytes(
+                            file_key, key_size, selected_cols, schema_types
+                        )
 
                 # SuperSnapshot is created ONCE per (super_name, simple_name) after
                 # all snapshot iterations have accumulated their files and schema.
@@ -465,7 +652,11 @@ class DataEstimator:
         self.timer.capture_and_reset_timing(event="ESTIMATE")
 
         self.plan_stats.add_stat({"REFLECTIONS": total_reflections})
+        # REFLECTION_SIZE is the projected (selected-column) size that drives
+        # engine routing; REFLECTION_SIZE_RAW is the whole-file footprint, kept
+        # for observability so the two are comparable in the plans payload.
         self.plan_stats.add_stat({"REFLECTION_SIZE": reflection_file_size})
+        self.plan_stats.add_stat({"REFLECTION_SIZE_RAW": reflection_file_size_raw})
 
         # Read-path pruning observability — only when pruning is engaged, so a
         # disabled-pruning read doesn't litter the payload with noise. Mirrors

{supertable-2.3.8 → supertable-2.4.0}/supertable/engine/engine_common.py

@@ -215,12 +215,43 @@ def configure_httpfs_and_s3(
     if not for_paths:
         return
 
-    # Load httpfs; install only when not already available.
+    # Load httpfs.  It is baked into the image and seeded into the DuckDB
+    # extension dir (see the container entrypoint), so LOAD normally succeeds
+    # with no network access.
+    #
+    # Why this is NOT a blind ``INSTALL`` fallback: ``INSTALL httpfs`` performs
+    # an HTTP GET to extensions.duckdb.org.  On an offline / firewalled node
+    # that socket can stall for minutes — or hang indefinitely on a blackholed
+    # route — turning a should-be-instant failure into an unbounded query hang.
+    # So we fail fast instead:
+    #   * SET autoinstall_known_extensions=false makes LOAD raise immediately
+    #     when the extension is absent, rather than silently downloading it;
+    #   * a network INSTALL is attempted ONLY when explicitly opted in via
+    #     SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD;
+    #   * otherwise we raise a clear, actionable error the caller returns.
     try:
-        con.execute("LOAD httpfs;")
+        con.execute("SET autoinstall_known_extensions=false;")
     except Exception:
-        con.execute("INSTALL httpfs;")
+        pass
+
+    try:
         con.execute("LOAD httpfs;")
+    except Exception as load_err:
+        if settings.SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD:
+            # Operator explicitly allowed reaching the network for a one-off
+            # install (e.g. an online dev box without a baked extension).
+            con.execute("INSTALL httpfs;")
+            con.execute("LOAD httpfs;")
+        else:
+            raise RuntimeError(
+                "DuckDB 'httpfs' extension is not available locally and network "
+                "auto-download is disabled, so this query cannot run. Bake/seed "
+                "httpfs into "
+                f"'{get_app_home()}/.duckdb/extensions/v<duckdb_version>/<platform>/' "
+                "(the container entrypoint restores it from /opt/duckdb-extensions), "
+                "or set SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD=true to permit a "
+                f"one-time online install. Underlying DuckDB error: {load_err}"
+            ) from load_err
 
     any_s3 = any(str(p).lower().startswith("s3://") for p in for_paths)
     any_http = any(str(p).lower().startswith(("http://", "https://")) for p in for_paths)
@@ -634,6 +665,17 @@ def init_connection(
     except Exception as e:
         logger.warning(f"[duckdb.init] home_directory pin failed: {e}")
 
+    # Never let DuckDB auto-DOWNLOAD an extension.  Everything we need (httpfs)
+    # is baked/seeded into the local extension dir; an implicit network install
+    # would reach out to extensions.duckdb.org and can hang for minutes on an
+    # offline/firewalled node — turning a should-be-instant error into an
+    # unbounded query hang.  configure_httpfs_and_s3() owns the explicit,
+    # opt-in install path (SUPERTABLE_DUCKDB_ALLOW_EXTENSION_DOWNLOAD).
+    try:
+        con.execute("SET autoinstall_known_extensions=false;")
+    except Exception as e:
+        logger.debug(f"[duckdb.init] disabling extension auto-install failed: {e}")
+
     # Resolve memory limit.
     # Single env var SUPERTABLE_DUCKDB_MEMORY_LIMIT controls both executors.
     # The `memory_limit` argument is the caller's fallback when the env var is absent.

{supertable-2.3.8 → supertable-2.4.0}/supertable/engine/spark_thrift.py

@@ -93,6 +93,53 @@ def _to_s3a_path(file_path: str) -> str:
     return file_path
 
 
+def _resolve_spark_file(storage, file_path: str) -> str:
+    """Resolve one snapshot file path to the form Spark should scan.
+
+    ``sup.files`` are resolved once by the estimator using the *DuckDB* presign
+    setting (``SUPERTABLE_DUCKDB_PRESIGNED``) and shared by every engine, so
+    Spark re-resolves here to make its access method depend solely on
+    ``SUPERTABLE_SPARK_PRESIGNED``:
+
+      * default (False) → a direct ``s3a://bucket/key`` path that Spark reads
+        with its own ``fs.s3a.*`` credentials (``_to_s3a_path`` already
+        normalises ``s3://``, presigned ``http(s)://`` and ``s3a://`` inputs);
+      * opt-in (True)   → a freshly minted presigned ``http(s)://`` URL, so
+        Spark works even when the cluster has no object-store credentials — and
+        regardless of whether ``sup.files`` were presigned for DuckDB.
+
+    Presigning is best-effort: a missing ``presign``, a local path, or an
+    unparseable key falls back to the s3a form.
+    """
+    s3a = _to_s3a_path(file_path)
+    if not settings.SUPERTABLE_SPARK_PRESIGNED or not s3a.startswith("s3a://"):
+        return s3a
+
+    presign_fn = getattr(storage, "presign", None)
+    if not callable(presign_fn):
+        return s3a
+
+    # s3a://bucket/full_key → object key.  storage.presign() re-applies
+    # base_prefix (like read_bytes), so strip it here — mirroring
+    # _read_parquet_schema — to avoid doubling it.
+    full_key = s3a[len("s3a://"):].partition("/")[2]
+    if not full_key:
+        return s3a
+    base = (getattr(storage, "base_prefix", "") or "").strip("/")
+    rel_key = (
+        full_key[len(base) + 1:]
+        if base and full_key.startswith(base + "/")
+        else full_key
+    )
+    try:
+        url = presign_fn(rel_key)
+        if isinstance(url, str) and url:
+            return url
+    except Exception as e:  # pragma: no cover - defensive
+        logger.debug(f"[spark.thrift] presign failed for {rel_key!r}; using s3a: {e}")
+    return s3a
+
+
 # =========================================================
 # Spark SQL helpers
 # =========================================================
@@ -209,6 +256,7 @@ def _spark_create_tombstone_view(
         source_table: str,
         view_name: str,
         tombstone_def,
+        storage=None,
 ) -> None:
     """Create a view that hides system columns and drops tombstoned rows.
 
@@ -239,11 +287,12 @@ def _spark_create_tombstone_view(
     has_rowid = "__rowid__" in src_cols
 
     if tomb_path and has_rowid:
-        # The data files are converted to s3a:// for Spark (see the data-file
-        # loop in the executor); apply the same conversion to the already
-        # resolved deletion-vector pointer so Spark reads it via its own S3A
-        # client instead of choking on a presigned HTTP URL or bare key.
-        escaped = _to_s3a_path(tomb_path).replace("'", "''")
+        # Resolve the deletion-vector pointer the same way as the data files
+        # (see the data-file loop in the executor) so Spark reads it through the
+        # same access method — direct s3a:// by default, presigned when
+        # SUPERTABLE_SPARK_PRESIGNED is on — instead of a bare key or a
+        # DuckDB-shaped presigned URL.
+        escaped = _resolve_spark_file(storage, tomb_path).replace("'", "''")
         sql = (
             f"CREATE OR REPLACE TEMPORARY VIEW {view_name} AS "
             f"SELECT {select_cols} FROM {source_table} AS src "
@@ -806,8 +855,10 @@ class SparkThriftExecutor:
                 if sup.files and table_name not in table_repr_file:
                     table_repr_file[table_name] = sup.files[0]
 
-                # Convert to s3a:// paths for Spark (handles s3://, presigned HTTP URLs, etc.)
-                files = [_to_s3a_path(f) for f in sup.files]
+                # Resolve to Spark's access form: direct s3a:// by default, or
+                # presigned URLs when SUPERTABLE_SPARK_PRESIGNED is on (handles
+                # s3://, presigned HTTP URLs and bare keys either way).
+                files = [_resolve_spark_file(self.storage, f) for f in sup.files]
 
                 logger.debug(
                     f"{log_prefix}[spark.thrift] creating view {table_name} "
@@ -898,7 +949,7 @@ class SparkThriftExecutor:
                 source = query_alias_to_name[alias]
                 tomb_def = tombstone_views.get(alias)
                 tomb_view = f"tomb_{source}_{query_suffix}"
-                _spark_create_tombstone_view(cursor, source, tomb_view, tomb_def)
+                _spark_create_tombstone_view(cursor, source, tomb_view, tomb_def, self.storage)
                 created_views.append(tomb_view)
                 query_alias_to_name[alias] = tomb_view